diff --git a/fuel_ccp_tests/fixtures/os_fixtures.py b/fuel_ccp_tests/fixtures/os_fixtures.py index 27d221e..bb7cb9f 100644 --- a/fuel_ccp_tests/fixtures/os_fixtures.py +++ b/fuel_ccp_tests/fixtures/os_fixtures.py @@ -12,10 +12,14 @@ # License for the specific language governing permissions and limitations # under the License. +from copy import deepcopy +import os import pytest from fuel_ccp_tests import logger +from fuel_ccp_tests import settings from fuel_ccp_tests.helpers import ext +from fuel_ccp_tests.helpers import post_os_deploy_checks from fuel_ccp_tests.managers.osmanager import OSManager LOG = logger.logger @@ -43,3 +47,49 @@ def os_deployed(ccpcluster, else: LOG.info("Openstack allready installed and running...") osmanager.check_os_ready() + + +@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_galera_deployed) +@pytest.fixture(scope='function') +def galera_deployed(ccpcluster, + hardware, + underlay, + revert_snapshot, + config, + k8s_actions): + """Deploy galera cluster + """ + # If no snapshot was reverted, then try to revert the snapshot + # that belongs to the fixture. + # Note: keep fixtures in strict dependences from each other! + if not config.os.running: + general_config = deepcopy(settings.CCP_CONF) + if settings.BUILD_IMAGES: + k8s_actions.create_registry() + ccpcluster.build() + topology_path = \ + os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \ + '3galera_1comp.yaml' + remote = underlay.remote(host=config.k8s.kube_host) + remote.upload(topology_path, '/tmp') + ccpcluster.put_yaml_config('./config_1.yaml', general_config) + ccpcluster.add_includes('./config_1.yaml', [ + settings.CCP_DEPLOY_CONFIG, + settings.CCP_SOURCES_CONFIG, + '/tmp/3galera_1comp.yaml']) + + underlay.sudo_check_call("pip install python-openstackclient", + host=config.k8s.kube_host) + ccpcluster.deploy(params={"config-file": "./config_1.yaml"}, + use_cli_params=True) + post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000) + post_os_deploy_checks.check_pods_status(k8s_actions.api) + # todo: add invocation of galera checker script + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600) + + config.os.running = True + hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed) diff --git a/fuel_ccp_tests/helpers/ext.py b/fuel_ccp_tests/helpers/ext.py index 21ab2c4..b131b9d 100644 --- a/fuel_ccp_tests/helpers/ext.py +++ b/fuel_ccp_tests/helpers/ext.py @@ -42,7 +42,8 @@ SNAPSHOT = enum( 'k8s_deployed', 'ccp_deployed', 'os_deployed', - 'os_deployed_stacklight' + 'os_deployed_stacklight', + 'os_galera_deployed', ) LOG_LEVELS = enum( diff --git a/fuel_ccp_tests/managers/ccpmanager.py b/fuel_ccp_tests/managers/ccpmanager.py index 04b8fa8..fb5edb5 100644 --- a/fuel_ccp_tests/managers/ccpmanager.py +++ b/fuel_ccp_tests/managers/ccpmanager.py @@ -198,10 +198,10 @@ class CCPManager(object): components=components, params=params, suppress_output=suppress_output) - def deploy(self, components=None, params=None): + def deploy(self, components=None, params=None, use_cli_params=False): self.run('deploy', components=components, - params=params) + params=params, use_cli_params=use_cli_params) def dry_deploy(self, export_dir, components=None, params=None): self.run('deploy --dry-run --export-dir={export_dir}'.format( diff --git a/fuel_ccp_tests/managers/envmanager_devops.py b/fuel_ccp_tests/managers/envmanager_devops.py index 69111a1..bb5bac1 100644 --- a/fuel_ccp_tests/managers/envmanager_devops.py +++ b/fuel_ccp_tests/managers/envmanager_devops.py @@ -470,3 +470,47 @@ class EnvironmentManager(object): self.__config.underlay.nameservers = [self.nameserver] if not self.__config.underlay.upstream_dns_servers: self.__config.underlay.upstream_dns_servers = [self.nameserver] + + def get_node_by_ip(self, node_ip): + nodes = self._env.get_nodes() + node = [node for node in nodes if self.node_ip(node) == node_ip] + assert len(node) == 1, "Node with {} ip isn't found".format(node_ip) + return node[0] + + def shutdown_node_by_ip(self, node_ip): + """Shutdown hardware node by ip address + + """ + node = self.get_node_by_ip(node_ip) + node.shutdown() + + def start_node_by_ip(self, node_ip): + """Start hardware node by ip address + + """ + node = self.get_node_by_ip(node_ip) + node.start() + + def wait_node_is_offline(self, node_ip, timeout): + """Wait node is shutdown and doesn't respond + + """ + helpers.wait( + lambda: not helpers.tcp_ping(node_ip, 22), + timeout=timeout, + timeout_msg="Node '{}' didn't go offline after {} sec".format( + node_ip, timeout + ) + ) + + def wait_node_is_online(self, node_ip, timeout): + """Wait node is online after starting + + """ + helpers.wait( + lambda: helpers.tcp_ping(node_ip, 22), + timeout=timeout, + timeout_msg="Node '{}' didn't become online after {} sec".format( + node_ip, timeout + ) + ) diff --git a/fuel_ccp_tests/managers/k8smanager.py b/fuel_ccp_tests/managers/k8smanager.py index e4a9c98..fd15a8d 100644 --- a/fuel_ccp_tests/managers/k8smanager.py +++ b/fuel_ccp_tests/managers/k8smanager.py @@ -343,3 +343,8 @@ class K8SManager(object): self.__config.underlay.upstream_dns_servers LOG.info('Added custom upstream DNS servers (dnsmasq) to the ' 'settings: {0}'.format(k8s_settings['nameservers'])) + + def get_pods_number(self, pod_name, namespace=None): + pods = [pod for pod in self.api.pods.list(namespace=namespace) + if pod_name in pod.name] + return len(pods) diff --git a/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml b/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml new file mode 100644 index 0000000..8d81efd --- /dev/null +++ b/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml @@ -0,0 +1,41 @@ +nodes: + node1: + roles: + - openvswitch + - controller-net-host + - controller-net-bridge + node[2-3]: + roles: + - openvswitch + - controller-net-bridge + - compute + node[1-3]: + roles: + - galera +roles: + controller-net-host: + - neutron-dhcp-agent + - neutron-l3-agent + - neutron-metadata-agent + controller-net-bridge: + - etcd + - glance-api + - glance-registry + - keystone + - memcached + - neutron-server + - nova-api + - nova-conductor + - nova-consoleauth + - nova-novncproxy + - nova-scheduler + - rabbitmq + compute: + - nova-compute + - nova-libvirt + openvswitch: + - neutron-openvswitch-agent + - openvswitch-db + - openvswitch-vswitchd + galera: + - galera \ No newline at end of file diff --git a/fuel_ccp_tests/tests/system/test_galera.py b/fuel_ccp_tests/tests/system/test_galera.py new file mode 100644 index 0000000..3b0450b --- /dev/null +++ b/fuel_ccp_tests/tests/system/test_galera.py @@ -0,0 +1,261 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pytest + +import base_test +from fuel_ccp_tests import logger +from fuel_ccp_tests import settings +from fuel_ccp_tests.helpers import post_os_deploy_checks + +LOG = logger.logger + + +class TestGalera(base_test.SystemBaseTest): + """ Galera scale and destructive scenarios + + """ + @pytest.mark.fail_snapshot + @pytest.mark.galera_shutdown + @pytest.mark.galera + def test_galera_shutdown_node(self, hardware, underlay, config, + ccpcluster, k8s_actions, show_step, + galera_deployed): + """Shutdown galera node + + Scenario: + 1. Revert snapshot with deployed galera + 2. Shutdown one galera node + 3. Check galera state + 4. Create 2 vms + + Duration 30 min + """ + show_step(2) + hardware.shutdown_node_by_ip(underlay.node_names()[1]) + show_step(3) + # todo: add wait for galera to assemble when galera_checker is ready + remote = underlay.remote(host=config.k8s.kube_host) + show_step(4) + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600) + + @pytest.mark.fail_snapshot + @pytest.mark.galera_cold_restart + @pytest.mark.galera + def test_galera_cold_restart_node(self, hardware, underlay, config, + ccpcluster, k8s_actions, show_step, + galera_deployed): + """Cold restart galera node + + Scenario: + 1. Revert snapshot with deployed galera + 2. Cold restart one galera node + 3. Check galera state + 4. Create 2 vms + + Duration 30 min + """ + show_step(2) + hardware.shutdown_node_by_ip(underlay.node_names()[1]) + hardware.start_node_by_ip(underlay.host_by_node_name('slave-0')) + show_step(3) + # todo: add wait for galera to assemble when galera_checker is ready + remote = underlay.remote(host=config.k8s.kube_host) + show_step(4) + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600) + + @pytest.mark.fail_snapshot + @pytest.mark.galera_poweroff + @pytest.mark.galera + def test_galera_poweroff_node(self, hardware, underlay, config, + ccpcluster, k8s_actions, show_step, + galera_deployed): + """Poweroff galera node + + Scenario: + 1. Revert snapshot with deployed galera + 2. Poweroff one galera node + 3. Check galera state + 4. Create 2 vms + + Duration 30 min + """ + galera_node = underlay.node_names()[1] + galera_node_ip = underlay.host_by_node_name(galera_node) + show_step(2) + underlay.sudo_check_call('shutdown +1', node_name=galera_node) + hardware.shutdown_node_by_ip(galera_node_ip) + hardware.wait_node_is_offline(galera_node_ip, 90) + show_step(3) + # todo: add wait for galera to assemble when galera_checker is ready + remote = underlay.remote(host=config.k8s.kube_host) + show_step(4) + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600) + + @pytest.mark.fail_snapshot + @pytest.mark.galera_soft_reboot + @pytest.mark.galera + def test_galera_soft_reboot_node(self, hardware, underlay, config, + ccpcluster, k8s_actions, show_step, + galera_deployed): + """Soft reboot galera node + + Scenario: + 1. Revert snapshot with deployed galera + 2. Soft reboot one galera node + 3. Check galera state + 4. Create 2 vms + + Duration 30 min + """ + galera_node = underlay.node_names()[1] + galera_node_ip = underlay.host_by_node_name(galera_node) + show_step(2) + underlay.sudo_check_call('shutdown +1', node_name=galera_node) + hardware.shutdown_node_by_ip(galera_node_ip) + hardware.wait_node_is_offline(galera_node_ip, 90) + hardware.start_node_by_ip(galera_node_ip) + hardware.wait_node_is_online(galera_node_ip, 180) + show_step(3) + # todo: add wait for galera to assemble when galera_checker is ready + remote = underlay.remote(host=config.k8s.kube_host) + show_step(4) + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600) + + @pytest.mark.fail_snapshot + @pytest.mark.galera_cluster_shutdown + @pytest.mark.galera + def test_galera_cluster_shutdown(self, hardware, underlay, config, + ccpcluster, k8s_actions, show_step, + galera_deployed): + """Galera cluster shutdown + + Scenario: + 1. Revert snapshot with deployed galera + 2. Shutdown all galera nodes and start them one by one + 3. Check galera state + 4. Create 2 vms + + Duration 30 min + """ + galera_nodes = underlay.node_names()[:3] + galera_node_ips = [] + show_step(2) + for galera_node in galera_nodes: + galera_node_ip = underlay.host_by_node_name(galera_node) + galera_node_ips.append(galera_node_ip) + hardware.shutdown_node_by_ip(galera_node_ip) + hardware.wait_node_is_offline(galera_node_ip, 90) + for galera_ip in galera_node_ips: + hardware.start_node_by_ip(galera_ip) + hardware.wait_node_is_online(galera_ip, 180) + show_step(3) + # todo: add wait for galera to assemble when galera_checker is ready + remote = underlay.remote(host=config.k8s.kube_host) + show_step(4) + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600) + + @pytest.mark.fail_snapshot + @pytest.mark.galera_scale_up_down + @pytest.mark.galera + def test_galera_scale(self, hardware, underlay, config, + ccpcluster, k8s_actions, show_step, + galera_deployed): + """Galera cluster scale + + Scenario: + 1. Revert snapshot with deployed galera + 2. Scale up galera to 5 replicas + 3. Check galera state + 4. Check number of galera pods + 5. Create 2 vms + 6. Scale down galera to 3 replicas + 7. Check galera state + 8. Check number of galera pods + 9. Create 2 vms + + Duration 30 min + """ + show_step(2) + with underlay.yaml_editor('/tmp/3galera_1comp.yaml', + host=config.k8s.kube_host) as editor: + del editor.content['nodes']['node[1-3]'] + editor.content['nodes']['node[1-5]'] = {'roles': ['galera']} + + ccpcluster.deploy(params={"config-file": "./config_1.yaml"}, + use_cli_params=True) + post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000) + post_os_deploy_checks.check_pods_status(k8s_actions.api) + show_step(3) + # todo: add invocation of galera checker script + show_step(4) + galera_pods = \ + k8s_actions.get_pods_number('galera', + settings. + CCP_CONF['kubernetes']['namespace']) + assert galera_pods == 5,\ + "Expcted tp have 5 galera pods, got {}".format(galera_pods) + + show_step(5) + remote = underlay.remote(host=config.k8s.kube_host) + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600) + show_step(6) + with underlay.yaml_editor('/tmp/3galera_1comp.yaml', + host=config.k8s.kube_host) as editor: + del editor.content['nodes']['node[1-5]'] + editor.content['nodes']['node[1-3]'] = {'roles': ['galera']} + + ccpcluster.deploy(params={"config-file": "./config_1.yaml"}, + use_cli_params=True) + post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000) + post_os_deploy_checks.check_pods_status(k8s_actions.api) + show_step(7) + # todo: add invocation of galera checker script + show_step(8) + galera_pods = \ + k8s_actions.get_pods_number('galera', + settings. + CCP_CONF['kubernetes']['namespace']) + assert galera_pods == 3,\ + "Expcted tp have 3 galera pods, got {}".format(galera_pods) + show_step(9) + remote.check_call( + "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a" + " create".format( + settings.CCP_CONF["kubernetes"]["namespace"]), + timeout=600)