# Copyright 2015 Mirantis, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. import time from proboscis.asserts import assert_equal from proboscis.asserts import assert_false from proboscis.asserts import assert_true from proboscis import test from proboscis import SkipTest from fuelweb_test.helpers import checkers from fuelweb_test.helpers.decorators import log_snapshot_after_test from fuelweb_test.helpers.utils import install_pkg from fuelweb_test.tests import base_test_case from fuelweb_test import settings @test(groups=["prepare_os_upgrade"]) class PrepareOSupgrade(base_test_case.TestBasic): @test(depends_on=[base_test_case.SetupEnvironment.prepare_slaves_9], groups=["ha_ceph_for_all_ubuntu_neutron_vlan"]) @log_snapshot_after_test def ha_ceph_for_all_ubuntu_neutron_vlan(self): """Deploy cluster with ha mode, ceph for all, neutron vlan Scenario: 1. Create cluster 2. Add 3 nodes with controller role 3. Add 3 nodes with compute and ceph OSD roles 4. Deploy the cluster 5. Run ostf 6. Make snapshot Duration 50m Snapshot ha_ceph_for_all_ubuntu_neutron_vlan """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('ha_ceph_for_all_ubuntu_neutron_vlan') self.env.revert_snapshot("ready_with_9_slaves") data = { 'volumes_ceph': True, 'images_ceph': True, 'ephemeral_ceph': True, 'objects_ceph': True, 'volumes_lvm': False, 'net_provider': 'neutron', 'net_segment_type': settings.NEUTRON_SEGMENT['vlan'] } cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE_HA, settings=data ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute', 'ceph-osd'], 'slave-05': ['compute', 'ceph-osd'], 'slave-06': ['compute', 'ceph-osd'] } ) self.fuel_web.deploy_cluster_wait(cluster_id) self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ha_ceph_for_all_ubuntu_neutron_vlan", is_make=True) @test(groups=["os_upgrade"], enabled=False) class TestOSupgrade(base_test_case.TestBasic): @test(groups=["upgrade_ha_ceph_for_all_ubuntu_neutron_vlan"]) @log_snapshot_after_test def upgrade_ha_ceph_for_all_ubuntu_neutron_vlan(self): """Upgrade master node ha mode, ceph for all, neutron vlan Scenario: 1. Revert snapshot with ha mode, ceph for all, neutron vlan env 2. Run upgrade on master 3. Check that upgrade was successful """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('upgrade_ha_ceph_for_all_ubuntu_neutron_vlan') self.env.revert_snapshot('ha_ceph_for_all_ubuntu_neutron_vlan') cluster_id = self.fuel_web.get_last_created_cluster() # TODO: Upgrade procedure were changed and upgrade to 9.0 not # implemented yet. This should be replaced with actual code when it # will be ready # self.env.admin_actions.upgrade_master_node() self.fuel_web.assert_nodes_in_ready_state(cluster_id) self.fuel_web.wait_nodes_get_online_state( self.env.d_env.nodes().slaves[:6]) self.fuel_web.assert_fuel_version(settings.UPGRADE_FUEL_TO) self.fuel_web.assert_nailgun_upgrade_migration() self.env.make_snapshot("upgrade_ha_ceph_for_all_ubuntu_neutron_vlan", is_make=True) @test(depends_on=[upgrade_ha_ceph_for_all_ubuntu_neutron_vlan], groups=["prepare_before_os_upgrade"]) @log_snapshot_after_test def prepare_before_os_upgrade(self): """Make prepare actions before os upgrade Scenario: 1. Revert snapshot upgraded with ceph, neutron vlan 2. yum update 3. pip install pyzabbix 4. yum install fuel-octane 5. Create mirrors """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('prepare_before_os_upgrade') self.env.revert_snapshot("upgrade_ha_ceph_for_all_ubuntu_neutron_vlan") with self.env.d_env.get_admin_remote() as remote: remote.execute("yum -y update") remote.execute("pip install pyzabbix") install_pkg(remote, "fuel-octane") cmd = ( "sed -i 's/DEBUG=\"no\"/DEBUG=\"yes\"/' {}".format( '/etc/fuel-createmirror/config/ubuntu.cfg' ) ) remote.execute(cmd) remote.execute("/usr/bin/fuel-createmirror") self.env.make_snapshot("prepare_before_os_upgrade", is_make=True) @test(depends_on=[prepare_before_os_upgrade], groups=["os_upgrade_env"]) @log_snapshot_after_test def os_upgrade_env(self): """Octane clone target environment Scenario: 1. Revert snapshot prepare_before_os_upgrade 2. run octane upgrade-env """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('os_upgrade_env') self.env.revert_snapshot("prepare_before_os_upgrade") cluster_id = self.fuel_web.get_last_created_cluster() with self.env.d_env.get_admin_remote() as remote: octane_upgrade_env = remote.execute( "octane upgrade-env {0}".format(cluster_id) ) cluster_id = self.fuel_web.get_last_created_cluster() assert_equal(0, octane_upgrade_env['exit_code']) assert_equal(cluster_id, int(octane_upgrade_env['stdout'][0].split()[0])) self.env.make_snapshot("os_upgrade_env", is_make=True) @test(depends_on=[os_upgrade_env], groups=["upgrade_first_cic"]) @log_snapshot_after_test def upgrade_first_cic(self): """Upgrade first controller Scenario: 1. Revert snapshot os_upgrade_env 2. run octane upgrade-node --isolated """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('upgrade_first_cic') self.env.revert_snapshot("os_upgrade_env") target_cluster_id = self.fuel_web.client.get_cluster_id( 'TestOSupgrade' ) seed_cluster_id = self.fuel_web.get_last_created_cluster() controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( target_cluster_id, ["controller"] ) with self.env.d_env.get_admin_remote() as remote: octane_upgrade_node = remote.execute( "octane upgrade-node --isolated {0} {1}".format( seed_cluster_id, controllers[-1]["id"]) ) assert_equal(0, octane_upgrade_node['exit_code'], "octane upgrade-node returns non zero" "status code," "current result {}".format(octane_upgrade_node)) tasks_started_by_octane = [ task for task in self.fuel_web.client.get_tasks() if task['cluster'] == seed_cluster_id ] for task in tasks_started_by_octane: self.fuel_web.assert_task_success(task) self.env.make_snapshot("upgrade_first_cic", is_make=True) @test(depends_on=[upgrade_first_cic], groups=["upgrade_db"]) @log_snapshot_after_test def upgrade_db(self): """Move and upgrade mysql db from target cluster to seed cluster Scenario: 1. Revert snapshot upgrade_first_cic 2. run octane upgrade-db """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('upgrade_db') self.env.revert_snapshot("upgrade_first_cic") target_cluster_id = self.fuel_web.client.get_cluster_id( 'TestOSupgrade' ) target_controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( target_cluster_id, ["controller"] )[0] seed_cluster_id = self.fuel_web.get_last_created_cluster() controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( seed_cluster_id, ["controller"] )[0] with self.env.d_env.get_ssh_to_remote( target_controller["ip"]) as remote: target_ids = remote.execute( 'mysql cinder <<< "select id from volumes;"; ' 'mysql glance <<< "select id from images"; ' 'mysql neutron <<< "(select id from networks) ' 'UNION (select id from routers) ' 'UNION (select id from subnets)"; ' 'mysql keystone <<< "(select id from project) ' 'UNION (select id from user)"' )["stdout"] with self.env.d_env.get_admin_remote() as remote: octane_upgrade_db = remote.execute( "octane upgrade-db {0} {1}".format( target_cluster_id, seed_cluster_id) ) assert_equal(0, octane_upgrade_db['exit_code'], "octane upgrade-db returns non zero" "status code," "current result is {}".format(octane_upgrade_db)) with self.env.d_env.get_ssh_to_remote(controller["ip"]) as remote: stdout = remote.execute("crm resource status")["stdout"] seed_ids = remote.execute( 'mysql cinder <<< "select id from volumes;"; ' 'mysql glance <<< "select id from images"; ' 'mysql neutron <<< "(select id from networks) ' 'UNION (select id from routers) ' 'UNION (select id from subnets)"; ' 'mysql keystone <<< "(select id from project) ' 'UNION (select id from user)"' )["stdout"] while stdout: current = stdout.pop(0) if "vip" in current: assert_true("Started" in current) elif "master_p" in current: next_element = stdout.pop(0) assert_true("Masters: [ node-" in next_element) elif any(x in current for x in ["ntp", "mysql", "dns"]): next_element = stdout.pop(0) assert_true("Started" in next_element) elif any(x in current for x in ["nova", "cinder", "keystone", "heat", "neutron", "glance"]): next_element = stdout.pop(0) assert_true("Stopped" in next_element) assert_equal(sorted(target_ids), sorted(seed_ids), "Objects in target and seed dbs are different") self.env.make_snapshot("upgrade_db", is_make=True) @test(depends_on=[upgrade_db], groups=["upgrade_ceph"]) @log_snapshot_after_test def upgrade_ceph(self): """Upgrade ceph Scenario: 1. Revert snapshot upgrade_db 2. run octane upgrade-ceph """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('upgrade_ceph') self.env.revert_snapshot("upgrade_db") target_cluster_id = self.fuel_web.client.get_cluster_id( 'TestOSupgrade' ) seed_cluster_id = self.fuel_web.get_last_created_cluster() controller = self.fuel_web.get_nailgun_cluster_nodes_by_roles( seed_cluster_id, ["controller"] )[0] with self.env.d_env.get_admin_remote() as remote: octane_upgrade_ceph = remote.execute( "octane upgrade-ceph {0} {1}".format( target_cluster_id, seed_cluster_id) ) assert_equal(0, octane_upgrade_ceph['exit_code'], "octane upgrade-ceph returns non zero status code," "current result is {}".format(octane_upgrade_ceph)) with self.env.d_env.get_ssh_to_remote(controller["ip"]) as remote: ceph_health = remote.execute("ceph health")["stdout"][0][:-1] assert_equal("HEALTH_OK", ceph_health) self.env.make_snapshot("upgrade_ceph", is_make=True) @test(depends_on=[upgrade_ceph], groups=["upgrade_control_plane"]) @log_snapshot_after_test def upgrade_control_plane(self): """Upgrade control plane Scenario: 1. Revert snapshot upgrade_ceph 2. run octane upgrade-control 3. run octane upgrade-node """ if settings.OPENSTACK_RELEASE_UBUNTU not in settings.OPENSTACK_RELEASE: raise SkipTest('Openstack release is not Ubuntu') self.check_run('upgrade_control_plane') self.env.revert_snapshot("upgrade_ceph") target_cluster_id = self.fuel_web.client.get_cluster_id( 'TestOSupgrade' ) seed_cluster_id = self.fuel_web.get_last_created_cluster() with self.env.d_env.get_admin_remote() as remote: octane_upgrade_control = remote.execute( "octane upgrade-control {0} {1}".format( target_cluster_id, seed_cluster_id) ) assert_equal(0, octane_upgrade_control['exit_code'], "octane upgrade-control returns non zero status code," "current result is {}".format(octane_upgrade_control)) controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( seed_cluster_id, ["controller"] ) old_computes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( target_cluster_id, ["compute"] ) old_controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( target_cluster_id, ["controller"] ) ping_ips = [] for node in controllers + old_computes: for data in node["network_data"]: if data["name"] == "management": ping_ips.append(data["ip"].split("/")[0]) ping_ips.append(self.fuel_web.get_mgmt_vip(seed_cluster_id)) non_ping_ips = [] for node in old_controllers: for data in node["network_data"]: if data["name"] == "management": non_ping_ips.append(data["ip"].split("/")[0]) for node in controllers + old_computes: self.ssh_manager.execute(node["ip"], "ip -s -s neigh flush all") for ip in ping_ips: assert_true(checkers.check_ping(node["ip"], ip), "Can not ping {0} from {1}" "need to check network" " connectivity".format(ip, node["ip"])) for ip in non_ping_ips: assert_false(checkers.check_ping(node["ip"], ip), "Patch ports from old controllers" "isn't removed") time.sleep(180) # TODO need to remove # after fix of https://bugs.launchpad.net/fuel/+bug/1499696 with self.env.d_env.get_ssh_to_remote(controllers[0]["ip"]) as remote: stdout = remote.execute("crm resource status")["stdout"] while stdout: current = stdout.pop(0) if "vip" in current: assert_true("Started" in current) elif "master_p" in current: next_element = stdout.pop(0) assert_true("Masters: [ node-" in next_element) elif any(x in current for x in ["ntp", "mysql", "dns"]): next_element = stdout.pop(0) assert_true("Started" in next_element) elif any(x in current for x in ["nova", "cinder", "keystone", "heat", "neutron", "glance"]): next_element = stdout.pop(0) assert_true("Started" in next_element) with self.env.d_env.get_admin_remote() as remote: octane_upgrade_node = remote.execute( "octane upgrade-node {0} {1} {2}".format( seed_cluster_id, old_controllers[0]["id"], old_controllers[1]["id"]) ) assert_equal(0, octane_upgrade_node['exit_code'], "octane upgrade-node returns non zero" "status code," "current result {}".format(octane_upgrade_node)) tasks_started_by_octane = [ task for task in self.fuel_web.client.get_tasks() if task['cluster'] == seed_cluster_id ] for task in tasks_started_by_octane: self.fuel_web.assert_task_success(task) self.env.make_snapshot("upgrade_control_plane", is_make=True)