diff --git a/fuelweb_test/helpers/checkers.py b/fuelweb_test/helpers/checkers.py index b1a7f72cc..d9f4c87a4 100644 --- a/fuelweb_test/helpers/checkers.py +++ b/fuelweb_test/helpers/checkers.py @@ -837,9 +837,12 @@ def verify_bootstrap_on_node(ip, os_type, uuid=None): if not uuid: return - cmd = "cat /etc/nailgun-agent/config.yaml" - output = yaml.load(ssh_manager.execute_on_remote(ip, cmd)['stdout_str']) - actual_uuid = output.get("runtime_uuid") + with ssh_manager.open_on_remote( + ip=ip, + path='/etc/nailgun-agent/config.yaml') as f: + data = yaml.safe_load(f) + + actual_uuid = data.get("runtime_uuid") assert_equal(actual_uuid, uuid, "Actual uuid {0} is not the same as expected {1}" .format(actual_uuid, uuid)) diff --git a/fuelweb_test/helpers/fuel_actions.py b/fuelweb_test/helpers/fuel_actions.py index 040ca521c..a95956e33 100644 --- a/fuelweb_test/helpers/fuel_actions.py +++ b/fuelweb_test/helpers/fuel_actions.py @@ -19,10 +19,7 @@ from devops.helpers.helpers import wait from devops.models import DiskDevice from devops.models import Node from devops.models import Volume -from proboscis.asserts import assert_equal from proboscis.asserts import assert_true -# noinspection PyUnresolvedReferences -from six.moves import cStringIO import yaml from core.helpers.log_helpers import logwrap @@ -221,23 +218,17 @@ class AdminActions(BaseActions): ) def get_fuel_settings(self): - result = self.ssh_manager.execute_on_remote( - ip=self.admin_ip, - cmd='cat {cfg_file}'.format(cfg_file=FUEL_SETTINGS_YAML) - ) - return yaml.load(result['stdout_str']) + return YamlEditor( + file_path=FUEL_SETTINGS_YAML, + ip=self.admin_ip + ).get_content() def save_fuel_settings(self, settings): - cmd = 'echo \'{0}\' > {1}'.format(yaml.dump(settings, - default_style='"', - default_flow_style=False), - FUEL_SETTINGS_YAML) - result = self.ssh_manager.execute( - ip=self.admin_ip, - cmd=cmd - ) - assert_equal(result['exit_code'], 0, - "Saving Fuel settings failed: {0}!".format(result)) + with YamlEditor( + file_path=FUEL_SETTINGS_YAML, + ip=self.admin_ip + ) as data: + data.content = settings @logwrap def get_tasks_description(self, release=None): @@ -249,8 +240,7 @@ class AdminActions(BaseActions): if not release: release = '' cmd = "cat `find /etc/puppet/{} -name tasks.yaml`".format(release) - data = self.ssh_manager.execute_on_remote(self.admin_ip, cmd) - return yaml.load(cStringIO(''.join(data['stdout']))) + return self.ssh_manager.check_call(self.admin_ip, cmd).stdout_yaml class NailgunActions(BaseActions): @@ -258,15 +248,11 @@ class NailgunActions(BaseActions): def update_nailgun_settings(self, settings): cfg_file = '/etc/nailgun/settings.yaml' - with self.ssh_manager.open_on_remote(self.admin_ip, cfg_file) as f: - ng_settings = yaml.load(f) + with YamlEditor(file_path=cfg_file, ip=self.admin_ip) as ng_settings: + ng_settings.content.update(settings) - ng_settings.update(settings) - logger.debug('Uploading new nailgun settings: {!r}'.format( - ng_settings)) - with self.ssh_manager.open_on_remote( - self.admin_ip, cfg_file, "w") as f: - yaml.dump(ng_settings, f) + logger.debug('Uploading new nailgun settings: {}'.format( + ng_settings)) self.restart_service("nailgun") def set_collector_address(self, host, port, ssl=False): diff --git a/fuelweb_test/helpers/utils.py b/fuelweb_test/helpers/utils.py index 2ccc9cdfe..4ee0c2ecd 100644 --- a/fuelweb_test/helpers/utils.py +++ b/fuelweb_test/helpers/utils.py @@ -1081,7 +1081,7 @@ def get_quantity_of_numa(ip): numa = int(SSHManager().check_call( ip=ip, - command="lstopo | grep NUMANode| wc -l" + command="lstopo | grep -c NUMANode" ).stdout[0]) if not numa: @@ -1505,18 +1505,43 @@ class YamlEditor(object): """ def __init__(self, file_path, ip=None, port=None): - self.file_path = file_path + """YAML files editor + + :type file_path: str + :type ip: str + :type port: int + """ + self.__file_path = file_path self.ip = ip self.port = port or 22 - self.content = None - self.original_content = None + self.__content = None + self.__original_content = None + + @property + def file_path(self): + """Open file path + + :rtype: str + """ + return self.__file_path + + @property + def content(self): + if self.__content is None: + self.__content = self.get_content() + return self.__content + + @content.setter + def content(self, new_content): + self.__content = new_content def __get_file(self, mode="r"): if self.ip: - return SSHManager().open_on_remote(self.ip, self.file_path, - mode=mode, port=self.port) + return SSHManager().open_on_remote( + self.ip, self.__file_path, + mode=mode, port=self.port) else: - return open(self.file_path, mode) + return open(self.__file_path, mode) def get_content(self): with self.__get_file() as file_obj: @@ -1531,12 +1556,12 @@ class YamlEditor(object): default_style='"') def __enter__(self): - self.content = self.get_content() - self.original_content = copy.deepcopy(self.content) + self.__content = self.get_content() + self.__original_content = copy.deepcopy(self.content) return self def __exit__(self, x, y, z): - if self.content == self.original_content: + if self.content == self.__original_content: return self.write_content() diff --git a/fuelweb_test/models/environment.py b/fuelweb_test/models/environment.py index 895954a1d..7e7376e11 100644 --- a/fuelweb_test/models/environment.py +++ b/fuelweb_test/models/environment.py @@ -693,28 +693,26 @@ class EnvironmentModel(six.with_metaclass(SingletonMeta, object)): if nameservers is None: nameservers = [] - resolv_conf = self.ssh_manager.execute( - ip=self.ssh_manager.admin_ip, - cmd='cat /etc/resolv.conf' - ) - assert_equal(0, resolv_conf['exit_code'], - 'Executing "{0}" on the admin node has failed with: {1}' - .format('cat /etc/resolv.conf', resolv_conf['stderr'])) + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path='/etc/resolv.conf', + ) as f: + resolv_conf = f.readlines() + if merge: - nameservers.extend(resolv_conf['stdout']) + nameservers.extend(resolv_conf) resolv_keys = ['search', 'domain', 'nameserver'] - resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers - if any(x in ns for x in resolv_keys)) - logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new)) - echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new) - echo_result = self.ssh_manager.execute( - ip=self.ssh_manager.admin_ip, - cmd=echo_cmd - ) - assert_equal(0, echo_result['exit_code'], - 'Executing "{0}" on the admin node has failed with: {1}' - .format(echo_cmd, echo_result['stderr'])) - return resolv_conf['stdout'] + resolv_new = "".join( + '{0}\n'.format(ns) for ns in nameservers + if any(x in ns for x in resolv_keys)) + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path='/etc/resolv.conf', + mode='w' + ) as f: + f.write(resolv_new) + + return resolv_conf @logwrap def describe_other_admin_interfaces(self, admin): diff --git a/fuelweb_test/models/fuel_web_client.py b/fuelweb_test/models/fuel_web_client.py index 932d966b7..d4387dbdf 100644 --- a/fuelweb_test/models/fuel_web_client.py +++ b/fuelweb_test/models/fuel_web_client.py @@ -455,8 +455,8 @@ class FuelWebClient29(object): def get_pcm_nodes(self, ctrl_node, pure=False): nodes = {} with self.get_ssh_for_node(ctrl_node) as remote: - pcs_status = remote.execute('pcs status nodes')['stdout'] - pcm_nodes = yaml.load(''.join(pcs_status).strip()) + pcm_nodes = remote.execute('pcs status nodes').stdout_yaml + for status in ('Online', 'Offline', 'Standby'): list_nodes = (pcm_nodes['Pacemaker Nodes'][status] or '').split() if not pure: @@ -2677,8 +2677,8 @@ class FuelWebClient29(object): def get_nailgun_primary_node(self, slave, role='primary-controller'): # returns controller or mongo that is primary in nailgun with self.get_ssh_for_node(slave.name) as remote: - data = yaml.load(''.join( - remote.execute('cat /etc/astute.yaml')['stdout'])) + with remote.open('/etc/astute.yaml') as f: + data = yaml.safe_load(f) nodes = data['network_metadata']['nodes'] node_name = [node['fqdn'] for node in nodes.values() if role in node['node_roles']][0] diff --git a/fuelweb_test/tests/plugins/plugin_emc/test_plugin_emc.py b/fuelweb_test/tests/plugins/plugin_emc/test_plugin_emc.py index 01d4de732..52f25a2a2 100644 --- a/fuelweb_test/tests/plugins/plugin_emc/test_plugin_emc.py +++ b/fuelweb_test/tests/plugins/plugin_emc/test_plugin_emc.py @@ -18,8 +18,6 @@ from proboscis import test # pylint: disable=import-error # noinspection PyUnresolvedReferences from six.moves import configparser -# noinspection PyUnresolvedReferences -from six.moves import cStringIO # pylint: enable=import-error from fuelweb_test.helpers import utils @@ -43,11 +41,12 @@ class EMCPlugin(TestBasic): @classmethod def check_emc_cinder_config(cls, ip, path): - command = 'cat {0}'.format(path) - conf_data = SSHManager().execute_on_remote(ip, command)['stdout_str'] - conf_data = cStringIO(conf_data) - cinder_conf = configparser.ConfigParser() - cinder_conf.readfp(conf_data) + with SSHManager().open_on_remote( + ip=ip, + path=path + ) as f: + cinder_conf = configparser.ConfigParser() + cinder_conf.readfp(f) asserts.assert_equal( cinder_conf.get('DEFAULT', 'volume_driver'), diff --git a/fuelweb_test/tests/test_cli_base.py b/fuelweb_test/tests/test_cli_base.py index 22bb89faf..483b4a8e3 100644 --- a/fuelweb_test/tests/test_cli_base.py +++ b/fuelweb_test/tests/test_cli_base.py @@ -30,6 +30,7 @@ from fuelweb_test.helpers.checkers import fail_deploy from fuelweb_test.helpers.checkers import incomplete_deploy from fuelweb_test.helpers.checkers import incomplete_tasks from fuelweb_test.helpers.ssl_helpers import change_cluster_ssl_config +from fuelweb_test.helpers import utils from fuelweb_test.tests.base_test_case import TestBasic from fuelweb_test import logger from fuelweb_test.helpers.utils import hiera_json_out @@ -81,25 +82,24 @@ class CommandLine(TestBasic): @logwrap def get_networks(self, cluster_id): net_file = self.get_network_filename(cluster_id) - out = self.ssh_manager.execute_on_remote( - ip=self.ssh_manager.admin_ip, - cmd='cat {0}'.format(net_file), - jsonify=True - )['stdout_json'] - return out + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path=net_file + ) as f: + return json.load(f) @logwrap def update_network(self, cluster_id, net_config): net_file = self.get_network_filename(cluster_id) - data = json.dumps(net_config) - cmd = 'echo {data} > {net_file}'.format(data=json.dumps(data), - net_file=net_file) - self.ssh_manager.execute_on_remote( - ip=self.ssh_manager.admin_ip, - cmd=cmd - ) - cmd = ('cd /tmp; fuel --env {0} network --upload --json' - .format(cluster_id)) + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path=net_file, + mode='w' + ) as f: + json.dump(net_config, f) + + cmd = 'cd /tmp; fuel --env {0} network --upload --json'.format( + cluster_id) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd @@ -270,24 +270,22 @@ class CommandLine(TestBasic): ip=self.ssh_manager.admin_ip, cmd=cmd ) - out = self.ssh_manager.execute_on_remote( - ip=self.ssh_manager.admin_ip, - cmd='cd /tmp && cat settings_{0}.json'.format(cluster_id), - jsonify=True - )['stdout_json'] - return out + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path='/tmp/settings_{0}.json'.format(cluster_id) + ) as f: + return json.load(f) def upload_settings(self, cluster_id, settings): - data = json.dumps(settings) - cmd = 'cd /tmp && echo {data} > settings_{id}.json'.format( - data=json.dumps(data), - id=cluster_id) - self.ssh_manager.execute_on_remote( - ip=self.ssh_manager.admin_ip, - cmd=cmd - ) - cmd = ('fuel --env {0} settings --upload --dir /tmp --json'.format( - cluster_id)) + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path='/tmp/settings_{id}.json'.format(id=cluster_id), + mode='w' + ) as f: + json.dump(settings, f) + + cmd = 'fuel --env {0} settings --upload --dir /tmp --json'.format( + cluster_id) self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd @@ -402,22 +400,24 @@ class CommandLine(TestBasic): ip=self.ssh_manager.admin_ip, cmd=cmd ) - out = self.ssh_manager.execute_on_remote( - ip=self.ssh_manager.admin_ip, - cmd='cd /tmp && cat node_{}/interfaces.json'.format(node_id), - jsonify=True - )['stdout_json'] - return out + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path='/tmp/node_{}/interfaces.json'.format(node_id) + ) as f: + return json.load(f) def upload_node_interfaces(self, node_id, interfaces): - data = json.dumps(interfaces) - cmd = 'cd /tmp && echo {data} > node_{id}/interfaces.json'.format( - data=json.dumps(data), - id=node_id) - self.ssh_manager.execute_on_remote( + self.ssh_manager.mkdir_on_remote( ip=self.ssh_manager.admin_ip, - cmd=cmd + path='/tmp/node_{id}'.format(id=node_id) ) + with self.ssh_manager.open_on_remote( + ip=self.ssh_manager.admin_ip, + path='/tmp/node_{id}/interfaces.json'.format(id=node_id), + mode='w' + ) as f: + json.dump(interfaces, f) + cmd = ('fuel node --node-id {} --network --upload --dir /tmp' ' --json'.format(node_id)) self.ssh_manager.execute_on_remote( @@ -457,44 +457,38 @@ class CommandLine(TestBasic): task_id = all_deployment_tasks[0]['id'] cmd = 'fuel2 task network-configuration download {0}'.format(task_id) - out = self.ssh_manager.execute_on_remote( + settings_download = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd - )['stdout'] - settings_download = ''.join(out) + )['stdout_str'] settings_file = settings_download.split()[-1] - out = self.ssh_manager.execute_on_remote( - ip=self.ssh_manager.admin_ip, - cmd='cat {0}'.format(settings_file), - yamlify=True)['stdout_yaml'] - return out + return utils.YamlEditor( + file_path=settings_file, + ip=self.ssh_manager.admin_ip + ).get_content() @logwrap def get_cluster_config_cli(self, task_id): cmd = 'fuel2 task settings download {0}'.format(task_id) - out = self.ssh_manager.execute_on_remote( + settings_download = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd - )['stdout'] - settings_download = ''.join(out) + )['stdout_str'] settings_file = settings_download.split()[-1] - out = self.ssh_manager.execute_on_remote( + return utils.YamlEditor( + file_path=settings_file, ip=self.ssh_manager.admin_ip, - cmd='cat {0}'.format(settings_file), - yamlify=True)['stdout_yaml'] - return out + ).get_content() @logwrap def get_deployment_info_cli(self, task_id): cmd = 'fuel2 task deployment-info download {0}'.format(task_id) - out = self.ssh_manager.execute_on_remote( + settings_download = self.ssh_manager.execute_on_remote( ip=self.ssh_manager.admin_ip, cmd=cmd - )['stdout'] - settings_download = ''.join(out) + )['stdout_str'] settings_file = settings_download.split()[-1] - out = self.ssh_manager.execute_on_remote( + return utils.YamlEditor( + file_path=settings_file, ip=self.ssh_manager.admin_ip, - cmd='cat {0}'.format(settings_file), - yamlify=True)['stdout_yaml'] - return out + ).get_content() diff --git a/fuelweb_test/tests/test_multipath_devices.py b/fuelweb_test/tests/test_multipath_devices.py index b52583f84..0d07aaca9 100644 --- a/fuelweb_test/tests/test_multipath_devices.py +++ b/fuelweb_test/tests/test_multipath_devices.py @@ -104,7 +104,7 @@ class TestMultipath(base_test_case.TestBasic): :rtype: int """ - cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep lvm | wc -l" + cmd = "lsblk -lo NAME,TYPE,MOUNTPOINT | grep '/$' | grep -c lvm" ssh_manager.update_connection(ip, SSH_FUEL_CREDENTIALS['login'], SSH_FUEL_CREDENTIALS['password'], diff --git a/fuelweb_test/tests/test_reduced_footprint.py b/fuelweb_test/tests/test_reduced_footprint.py index 460a0007f..16e6a55eb 100644 --- a/fuelweb_test/tests/test_reduced_footprint.py +++ b/fuelweb_test/tests/test_reduced_footprint.py @@ -285,7 +285,7 @@ class TestVirtRoleBaremetal(TestBasic): with self.ssh_manager.get_remote(self.ssh_manager.admin_ip) as admin: result = admin.execute_through_host( slave_ip, - "cat /proc/cpuinfo | grep processor | wc -l", + "grep -c processor /proc/cpuinfo", auth=self.ssh_auth, timeout=60) asserts.assert_equal( diff --git a/fuelweb_test/tests/tests_strength/test_failover_base.py b/fuelweb_test/tests/tests_strength/test_failover_base.py index 811367b9c..616c1a8eb 100644 --- a/fuelweb_test/tests/tests_strength/test_failover_base.py +++ b/fuelweb_test/tests/tests_strength/test_failover_base.py @@ -28,7 +28,6 @@ from proboscis import SkipTest # noinspection PyUnresolvedReferences from six.moves import xrange # pylint: enable=redefined-builtin -import yaml from core.helpers.log_helpers import logwrap @@ -1116,8 +1115,7 @@ class TestHaFailoverBase(TestBasic): @logwrap def _get_pcm_nodes(remote, pure=False): nodes = {} - pcs_status = remote.execute('pcs status nodes')['stdout'] - pcm_nodes = yaml.load(''.join(pcs_status).strip()) + pcm_nodes = remote.execute('pcs status nodes').stdout_yaml for status in ('Online', 'Offline', 'Standby'): list_nodes = (pcm_nodes['Pacemaker Nodes'] [status] or '').split()