diff --git a/doc/system_tests.rst b/doc/system_tests.rst index ea7e93bf6..38d3b56b9 100644 --- a/doc/system_tests.rst +++ b/doc/system_tests.rst @@ -146,6 +146,11 @@ vCenter/DVS failover .. automodule:: system_test.tests.vcenter.test_vcenter_failover :members: +vCenter/DVS cluster actions +--------------------------- +.. automodule:: system_test.tests.vcenter.test_vcenter_cluster_actions + :members: + Helpers ======= diff --git a/fuelweb_test/settings.py b/fuelweb_test/settings.py index 99228663a..dc1cd608d 100644 --- a/fuelweb_test/settings.py +++ b/fuelweb_test/settings.py @@ -702,6 +702,9 @@ REPEAT_COUNT = int(os.environ.get("REPEAT_COUNT", 2)) # in the 'repetitive_restart' test group RESTART_COUNT = int(os.environ.get("RESTART_COUNT", 10)) +# is using in stop on deploy test +PROGRESS_TO_STOP = os.environ.get("PROGRESS_TO_STOP", 60) + # RH-related variables # Need to update these variables, when image with RH for # MOS will be available. diff --git a/system_test/actions/base.py b/system_test/actions/base.py index 550a0907b..78cb32c4e 100644 --- a/system_test/actions/base.py +++ b/system_test/actions/base.py @@ -359,7 +359,8 @@ class BaseActions(PrepareActions, HealthCheckActions, PluginsActions, ) cluster_id = self.cluster_id - self.fuel_web.deploy_cluster_wait_progress(cluster_id, progress=60) + self.fuel_web.deploy_cluster_wait_progress( + cluster_id, progress=settings.PROGRESS_TO_STOP) self.fuel_web.stop_deployment_wait(cluster_id) self.fuel_web.wait_nodes_get_online_state( self.env.d_env.get_nodes(name__in=list(self.assigned_slaves)), diff --git a/system_test/actions/plugins_actions.py b/system_test/actions/plugins_actions.py index baa4ef944..369554760 100644 --- a/system_test/actions/plugins_actions.py +++ b/system_test/actions/plugins_actions.py @@ -14,7 +14,8 @@ import os -from proboscis.asserts import assert_true, assert_equal +from proboscis.asserts import assert_true +from proboscis.asserts import assert_equal from fuelweb_test.helpers import utils from system_test import logger diff --git a/system_test/actions/vcenter_actions.py b/system_test/actions/vcenter_actions.py index b5744213f..7c79f8818 100644 --- a/system_test/actions/vcenter_actions.py +++ b/system_test/actions/vcenter_actions.py @@ -87,33 +87,59 @@ class VMwareActions(object): self.cluster_id, self.plugin_name, self.plugin_version, options, enabled=True) - @deferred_decorator([make_snapshot_if_step_fail]) - @action - def configure_vcenter(self): - """Configure vCenter settings.""" - vmware_vcenter = self.env_settings['vmware_vcenter'] + @staticmethod + def config_attr_vcenter(vmware_attr, vc_user, vc_host, vc_az, vc_pwd): + """Update and return the dictionary with vCenter attributes.""" + logger.info('Configuring vCenter...') + + vc_values = vmware_attr['editable']['value'] + computes = vc_values['availability_zones'][0]['nova_computes'][:] + vcenter_value = { + "availability_zones": [{ + "vcenter_username": vc_user, + "nova_computes": computes, + "vcenter_host": vc_host, + "az_name": vc_az, + "vcenter_password": vc_pwd + }] + } + vmware_attr['editable']['value'].update(vcenter_value) + return vmware_attr + + def config_attr_glance(self, vmware_attr, host, user, pwd, dc, ds): + """Update and return the dictionary with Glance attributes.""" + cluster_attr = self.fuel_web.client.get_cluster_attributes( + self.cluster_id) + cluster_attr['editable']['storage']['images_vcenter']['value'] = True + self.fuel_web.client.update_cluster_attributes(self.cluster_id, + cluster_attr) vcenter_value = { - "glance": {"vcenter_username": "", - "datacenter": "", - "vcenter_host": "", - "vcenter_password": "", - "datastore": "" - }, - "availability_zones": [ - {"vcenter_username": vmware_vcenter['settings']['user'], - "nova_computes": [], - "vcenter_host": vmware_vcenter['settings']['host'], - "az_name": vmware_vcenter['settings']['az'], - "vcenter_password": vmware_vcenter['settings']['pwd'] - }] + "glance": { + "vcenter_host": host, + "vcenter_username": user, + "vcenter_password": pwd, + "datacenter": dc, + "datastore": ds + } } - clusters = vmware_vcenter['nova-compute'] + vmware_attr['editable']['value'].update(vcenter_value) + return vmware_attr + + def config_attr_computes(self, vmware_attr, clusters): + """Configure Nova Computes for VMware. + + :param clusters: dictionary with string keys: cluster name (cluster), + service name (srv_name), datastore regex (datastore), + target node (target_node) + """ nodes = self.fuel_web.client.list_cluster_nodes(self.cluster_id) - roles = ['compute-vmware'] - comp_vmware_nodes = [n for n in nodes if set(roles) <= - set(n['pending_roles'])] + comp_vmware_nodes = [node for node in nodes if {'compute-vmware'} <= + set(node['pending_roles'] + node['roles'])] + + vc_values = vmware_attr['editable']['value'] + vc_values["availability_zones"][0]["nova_computes"] = [] for cluster in clusters: cluster_name = cluster['cluster'] @@ -125,49 +151,95 @@ class VMwareActions(object): else: target_node = cluster['target_node'] - vcenter_value["availability_zones"][0]["nova_computes"].append( - {"vsphere_cluster": cluster_name, - "service_name": srv_name, - "datastore_regex": datastore, - "target_node": { - "current": {"id": target_node, - "label": target_node}, - "options": [{"id": target_node, - "label": target_node}, ]}, - } - ) + vc_values["availability_zones"][0]["nova_computes"].append({ + "vsphere_cluster": cluster_name, + "service_name": srv_name, + "datastore_regex": datastore, + "target_node": { + "current": { + "id": target_node, + "label": target_node + }, + "options": [{ + "id": target_node, + "label": target_node + }] + } + }) - if vmware_vcenter['glance']['enable']: - attributes = self.fuel_web.client.get_cluster_attributes( - self.cluster_id) - attributes['editable']['storage']['images_vcenter']['value'] =\ - vmware_vcenter['glance']['enable'] - self.fuel_web.client.update_cluster_attributes(self.cluster_id, - attributes) + vmware_attr.update(vc_values) + return vmware_attr - vcenter_value["glance"]["vcenter_host"] = vmware_vcenter[ - 'glance']['host'] - vcenter_value["glance"]["vcenter_username"] = vmware_vcenter[ - 'glance']['user'] - vcenter_value["glance"]["vcenter_password"] = vmware_vcenter[ - 'glance']['pwd'] - vcenter_value["glance"]["datacenter"] = vmware_vcenter[ - 'glance']['datacenter'] - vcenter_value["glance"]["datastore"] = vmware_vcenter[ - 'glance']['datastore'] + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def configure_vcenter(self): + """Configure vCenter settings.""" + vmware_vcenter = self.env_settings['vmware_vcenter'] + vmware_attr = self.fuel_web.client.get_cluster_vmware_attributes( + self.cluster_id) - logger.info('Configuring vCenter...') + settings = vmware_vcenter['settings'] + vmware_attr = self.config_attr_vcenter(vmware_attr=vmware_attr, + vc_user=settings['user'], + vc_host=settings['host'], + vc_az=settings['az'], + vc_pwd=settings['pwd']) - vmware_attr = \ - self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id) - vcenter_data = vmware_attr['editable'] - vcenter_data['value'] = vcenter_value - logger.debug("Try to update cluster with next " - "vmware_attributes {0}".format(vmware_attr)) - self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id, - vmware_attr) + glance = vmware_vcenter['glance'] + if glance['enable']: + vmware_attr = self.config_attr_glance(vmware_attr=vmware_attr, + host=glance['host'], + user=glance['user'], + pwd=glance['pwd'], + dc=glance['datacenter'], + ds=glance['datastore']) - logger.debug("Attributes of cluster have been updated") + vmware_attr = self.config_attr_computes( + vmware_attr=vmware_attr, clusters=vmware_vcenter['nova-compute']) + + self.fuel_web.client.update_cluster_vmware_attributes( + self.cluster_id, vmware_attr) + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def configure_vcenter_incorrect(self): + """Configure vCenter settings with incorrect data.""" + vmware_vcenter = self.env_settings['vmware_vcenter'] + vmware_attr = self.fuel_web.client.get_cluster_vmware_attributes( + self.cluster_id) + + vmware_attr = self.config_attr_vcenter(vmware_attr=vmware_attr, + vc_user='user', + vc_host='8.8.8.8', + vc_az='az', + vc_pwd='pwd') + + glance = vmware_vcenter['glance'] + if glance['enable']: + vmware_attr = self.config_attr_glance(vmware_attr=vmware_attr, + host='8.8.8.8', + user='user', + pwd='pwd', + dc='dc', + ds='!@#$%^&*()') + + clusters = [{ + 'cluster': 'Cluster1!', + 'srv_name': 'any', + 'datastore': '!@#$%^&*()', + 'target_node': 'controllers' + }, { + 'cluster': 'Cluster2!', + 'srv_name': 'any2', + 'datastore': '!@#$%^&*()', + 'target_node': 'compute-vmware' + }] + + vmware_attr = self.config_attr_computes(vmware_attr=vmware_attr, + clusters=clusters) + + self.fuel_web.client.update_cluster_vmware_attributes( + self.cluster_id, vmware_attr) @deferred_decorator([make_snapshot_if_step_fail]) @action @@ -573,11 +645,15 @@ class VMwareActions(object): @action def config_idatastore(self): """Reconfigure vCenter settings with incorrect regex of Datastore.""" + vmware_vcenter = self.env_settings['vmware_vcenter'] + instances = vmware_vcenter['nova-compute'] + vmware_attr = \ self.fuel_web.client.get_cluster_vmware_attributes(self.cluster_id) vcenter_data = vmware_attr['editable'] - vcenter_data['value']['availability_zones'][0]['nova_computes'][0][ - 'datastore_regex'] = '!@#$%^&*()' + for i in range(len(instances)): + vcenter_data['value']['availability_zones'][0]['nova_computes'][i][ + 'datastore_regex'] = '!@#$%^&*()' self.fuel_web.client.update_cluster_vmware_attributes(self.cluster_id, vmware_attr) @@ -675,7 +751,7 @@ class VMwareActions(object): def _get_controller_with_vip(self): """Return name of controller with VIPs.""" for node in self.env.d_env.nodes().slaves: - ng_node = self.env.fuel_web.get_nailgun_node_by_devops_node(node) + ng_node = self.fuel_web.get_nailgun_node_by_devops_node(node) if ng_node['online'] and 'controller' in ng_node['roles']: hosts_vip = self.fuel_web.get_pacemaker_resource_location( ng_node['devops_name'], 'vip__management') @@ -709,7 +785,7 @@ class VMwareActions(object): logger.info('Wait offline status for ' '{ctrlr}'.format(ctrlr=self.primary_ctlr_ng.name)) - helpers.wait(lambda: self.env.fuel_web.get_nailgun_node_by_devops_node( + helpers.wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( self.primary_ctlr_ng)['online'] is not True, timeout=timeout, timeout_msg="Primary controller is still online") @@ -748,7 +824,7 @@ class VMwareActions(object): logger.info('Wait online status for ' '{name}'.format(name=self.primary_ctlr_ng.name)) - helpers.wait(lambda: self.env.fuel_web.get_nailgun_node_by_devops_node( + helpers.wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node( self.primary_ctlr_ng)['online'], timeout=timeout, timeout_msg="Primary controller is still offline") logger.info('Primary controller is online') @@ -767,7 +843,7 @@ class VMwareActions(object): @action def ostf_with_haproxy_fail(self): """Run OSTF tests (one should fail).""" - self.env.fuel_web.run_ostf( + self.fuel_web.run_ostf( self.cluster_id, test_sets=['sanity', 'smoke', 'ha'], should_fail=1, @@ -779,7 +855,7 @@ class VMwareActions(object): def fail_ostf(self): """Run OSTF tests (must fail).""" try: - self.env.fuel_web.run_ostf( + self.fuel_web.run_ostf( self.cluster_id, test_sets=['sanity', 'smoke', 'ha']) failed = False @@ -862,3 +938,28 @@ class VMwareActions(object): def wait_ha_services(self): """Wait for HA services.""" self.fuel_web.assert_ha_services_ready(self.cluster_id) + + def mcollective_nodes_online(self): + nodes_uids = set( + [str(node['id']) for node in + self.fuel_web.client.list_cluster_nodes(self.cluster_id)] + ) + ssh_manager = SSHManager() + out = ssh_manager.execute_on_remote( + ip=ssh_manager.admin_ip, + cmd='mco find', + assert_ec_equal=[0, 1] + )['stdout_str'] + ready_nodes_uids = set(out.split('\n')) + unavailable_nodes = nodes_uids - ready_nodes_uids + logger.debug('Nodes {0} are not reacheable via' + ' mcollective'.format(unavailable_nodes)) + return not unavailable_nodes + + @deferred_decorator([make_snapshot_if_step_fail]) + @action + def wait_mcollective(self): + """Wait for mcollective online status of nodes.""" + helpers.wait(lambda: self.mcollective_nodes_online(), timeout=60 * 5, + timeout_msg="Cluster nodes don't become available " + "via mcollective in allotted time.") diff --git a/system_test/tests/vcenter/test_vcenter_cluster_actions.py b/system_test/tests/vcenter/test_vcenter_cluster_actions.py new file mode 100644 index 000000000..8dba5f782 --- /dev/null +++ b/system_test/tests/vcenter/test_vcenter_cluster_actions.py @@ -0,0 +1,146 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE_2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from fuelweb_test.settings import DVS_PLUGIN_PATH +from fuelweb_test.settings import DVS_PLUGIN_VERSION + +from system_test import testcase +from system_test.actions import BaseActions +from system_test.actions import VMwareActions +from system_test.tests import ActionTest + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_redeploy_successful_cluster']) +class RedeploySuccessfulWithVMware(ActionTest, BaseActions, VMwareActions): + """Reset and redeploy cluster with vCenter after successful deployment. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings (depends on yaml config) + 7. Deploy the cluster + 8. Run OSTF + 9. Reset cluster + 10. Check networks + 11. Redeploy cluster + 12. Run OSTF + + Duration 3h 00min + Snapshot cluster_actions_redeploy_successful + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter', + 'deploy_cluster', + 'health_check', + 'reset_cluster', + 'wait_mcollective', + 'network_check', + 'deploy_cluster', + 'health_check' + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_redeploy_stopped_cluster']) +class RedeployAfterStopWithVMware(ActionTest, BaseActions, VMwareActions): + """Stop and redeploy cluster with vCenter with new parameters. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings with incorrect values + 7. Stop on cluster deploy (needs env variable PROGRESS_TO_STOP=50) + 8. Configure vmware settings (depends on yaml config) + 9. Check networks + 10. Deploy cluster + 11. Run OSTF + + Duration 3h 00min + Snapshot cluster_actions_redeploy_stopped + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter_incorrect', + 'stop_on_deploy', + 'wait_mcollective', + 'configure_vcenter', + 'network_check', + 'deploy_cluster', + 'health_check', + ] + + +@testcase(groups=['system_test', + 'system_test.vcenter', + 'system_test.vcenter.vcenter_redeploy_failed_cluster']) +class RedeployFailedWithVMware(ActionTest, BaseActions, VMwareActions): + """Redeploy cluster with vCenter after failed deployment. + + Scenario: + 1. Upload plugin to the master node + 2. Install plugin + 3. Create cluster + 4. Configure dvs settings (depends on yaml config) + 5. Add nodes (depends on yaml config) + 6. Configure vmware settings with incorrect values + 7. Deploy the cluster (deploy should fail) + 8. Configure vmware settings (depends on yaml config) + 9. Redeploy cluster + 10. Run OSTF + + Duration 3h 00min + Snapshot cluster_actions_redeploy_failed + """ + + plugin_name = "fuel-plugin-vmware-dvs" + plugin_path = DVS_PLUGIN_PATH + plugin_version = DVS_PLUGIN_VERSION + + actions_order = [ + 'prepare_env_with_plugin', + 'create_env', + 'configure_dvs_plugin', + 'add_nodes', + 'configure_vcenter_incorrect', + 'fail_deploy_cluster', + 'configure_vcenter', + 'deploy_cluster', + 'health_check' + ] diff --git a/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_glance_vmware_roles.yaml b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_glance_vmware_roles.yaml new file mode 100644 index 000000000..ed3765173 --- /dev/null +++ b/system_test/tests_templates/tests_configs/vcenter_dvs/vcenter_glance_vmware_roles.yaml @@ -0,0 +1,38 @@ +--- +template: + name: 1 controller, 1 cinder-vmware, 1 compute-vmware on Neutron/VLAN with DVS plugin and Glance + slaves: 3 + cluster_template: + name: vcenter_glance_vmware_roles + release: ubuntu + network: + !include cluster_configs/networks/neutron_vlan.yaml + settings: + components: + !include cluster_configs/settings/components/wo_components.yaml + storages: + !include cluster_configs/settings/storages/cinder_only.yaml + vmware_vcenter: + settings: + !include cluster_configs/settings/vmware/vcenter_main.yaml + nova-compute: + !include cluster_configs/settings/vmware/nova_compute/2clusters_ctrl_comp-vmware.yaml + glance: + !include cluster_configs/settings/vmware/vcenter_glance.yaml + vmware_dvs: + !include cluster_configs/settings/vmware/dvs/dvs_main.yaml + nodes: + - roles: + - controller + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 + + - roles: + - cinder-vmware + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1 + + - roles: + - compute-vmware + iface: !include cluster_configs/settings/vmware/vcenter_ifaces.yaml + count: 1