openstack
/
sahara-plugin-spark
Code Issues Proposed changes

Update hacking for Python3 

The repo is Python 3 now, so update hacking to version 3.0 which
supports Python 3.

Fix problems found.

Update local hacking checks for new flake8.

Change-Id: I332da9a6c3753547f84051c80baf5994188d3eb2
This commit is contained in:
Andreas Jaeger 2020-03-30 18:08:10 +02:00
parent 22caae2614
commit cabd8909a4
4 changed files with 6 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
sahara_plugin_spark/plugins/spark/config_helper.py
View File

@ -228,6 +228,7 @@ def _initialise_configs():
return configs
# Initialise plugin Hadoop configurations
PLUGIN_CONFIGS = _initialise_configs()
@ -423,7 +424,7 @@ def generate_hadoop_setup_script(storage_paths, env_configs):
script_lines.append("cat /etc/hadoop/hadoop-env.sh >> /tmp/hadoop-env.sh")
script_lines.append("cp /tmp/hadoop-env.sh /etc/hadoop/hadoop-env.sh")
hadoop_log = storage_paths[0] + "/log/hadoop/\$USER/"
hadoop_log = storage_paths[0] + "/log/hadoop/\\$USER/"
script_lines.append('sed -i "s,export HADOOP_LOG_DIR=.*,'
'export HADOOP_LOG_DIR=%s," /etc/hadoop/hadoop-env.sh'
% hadoop_log)

4
sahara_plugin_spark/plugins/spark/run_scripts.py
View File

@ -83,8 +83,8 @@ def _check_datanodes_count(remote, count):
LOG.debug("Checking DataNodes count")
ex_code, stdout = remote.execute_command(
'sudo su -lc "hdfs dfsadmin -report" hdfs | '
'grep \'Live datanodes\|Datanodes available:\' | '
'grep -o \'[0-9]\+\' | head -n 1')
r'grep \'Live datanodes\|Datanodes available:\' | '
r'grep -o \'[0-9]\+\' | head -n 1')
LOG.debug("DataNodes count='{count}'".format(count=stdout.strip()))
return stdout and int(stdout) == count

2
sahara_plugin_spark/tests/unit/plugins/spark/test_config_helper.py
View File

@ -33,7 +33,7 @@ class ConfigHelperUtilsTest(test_base.SaharaTestCase):
@mock.patch('sahara.plugins.utils.get_config_value_or_default')
def test_cleanup_configs(self, get_config_value):
getter = lambda plugin, key, cluster: plugin_configs[key]
getter = lambda plugin, key, cluster: plugin_configs[key] # noqa: E731
get_config_value.side_effect = getter
plugin_configs = {"Minimum cleanup megabytes": 4096,
"Minimum cleanup seconds": 86400,

2
test-requirements.txt
View File

@ -2,7 +2,7 @@
# of appearance. Changing the order has an impact on the overall integration
# process, which may cause wedges in the gate later.
hacking!=0.13.0,<0.14,>=0.12.0 # Apache-2.0
hacking>=3.0,<3.1.0 # Apache-2.0
bandit>=1.1.0 # Apache-2.0
bashate>=0.5.1 # Apache-2.0