Add galera tests

- Add tests on galera deployment, galera node reboot, shutdown,
poweroff, cold reboot, cluster shutdown
- Add test on scale up and down galera
- Galera checkers will be added after merging patch
https://review.openstack.org/#/c/399999/8/service/files/galera_checker.py

Change-Id: I31e1fe214e558497bcec7661f9e204ee645a56c1
This commit is contained in:
asledzinskiy 2016-12-01 15:21:02 +02:00
parent b1fb1706d2
commit 8457ee9280
7 changed files with 405 additions and 3 deletions

View File

@ -12,10 +12,14 @@
# License for the specific language governing permissions and limitations
# under the License.
from copy import deepcopy
import os
import pytest
from fuel_ccp_tests import logger
from fuel_ccp_tests import settings
from fuel_ccp_tests.helpers import ext
from fuel_ccp_tests.helpers import post_os_deploy_checks
from fuel_ccp_tests.managers.osmanager import OSManager
LOG = logger.logger
@ -43,3 +47,49 @@ def os_deployed(ccpcluster,
else:
LOG.info("Openstack allready installed and running...")
osmanager.check_os_ready()
@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_galera_deployed)
@pytest.fixture(scope='function')
def galera_deployed(ccpcluster,
hardware,
underlay,
revert_snapshot,
config,
k8s_actions):
"""Deploy galera cluster
"""
# If no snapshot was reverted, then try to revert the snapshot
# that belongs to the fixture.
# Note: keep fixtures in strict dependences from each other!
if not config.os.running:
general_config = deepcopy(settings.CCP_CONF)
if settings.BUILD_IMAGES:
k8s_actions.create_registry()
ccpcluster.build()
topology_path = \
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
'3galera_1comp.yaml'
remote = underlay.remote(host=config.k8s.kube_host)
remote.upload(topology_path, '/tmp')
ccpcluster.put_yaml_config('./config_1.yaml', general_config)
ccpcluster.add_includes('./config_1.yaml', [
settings.CCP_DEPLOY_CONFIG,
settings.CCP_SOURCES_CONFIG,
'/tmp/3galera_1comp.yaml'])
underlay.sudo_check_call("pip install python-openstackclient",
host=config.k8s.kube_host)
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
use_cli_params=True)
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
post_os_deploy_checks.check_pods_status(k8s_actions.api)
# todo: add invocation of galera checker script
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)
config.os.running = True
hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)

View File

@ -42,7 +42,8 @@ SNAPSHOT = enum(
'k8s_deployed',
'ccp_deployed',
'os_deployed',
'os_deployed_stacklight'
'os_deployed_stacklight',
'os_galera_deployed',
)
LOG_LEVELS = enum(

View File

@ -198,10 +198,10 @@ class CCPManager(object):
components=components,
params=params, suppress_output=suppress_output)
def deploy(self, components=None, params=None):
def deploy(self, components=None, params=None, use_cli_params=False):
self.run('deploy',
components=components,
params=params)
params=params, use_cli_params=use_cli_params)
def dry_deploy(self, export_dir, components=None, params=None):
self.run('deploy --dry-run --export-dir={export_dir}'.format(

View File

@ -470,3 +470,47 @@ class EnvironmentManager(object):
self.__config.underlay.nameservers = [self.nameserver]
if not self.__config.underlay.upstream_dns_servers:
self.__config.underlay.upstream_dns_servers = [self.nameserver]
def get_node_by_ip(self, node_ip):
nodes = self._env.get_nodes()
node = [node for node in nodes if self.node_ip(node) == node_ip]
assert len(node) == 1, "Node with {} ip isn't found".format(node_ip)
return node[0]
def shutdown_node_by_ip(self, node_ip):
"""Shutdown hardware node by ip address
"""
node = self.get_node_by_ip(node_ip)
node.shutdown()
def start_node_by_ip(self, node_ip):
"""Start hardware node by ip address
"""
node = self.get_node_by_ip(node_ip)
node.start()
def wait_node_is_offline(self, node_ip, timeout):
"""Wait node is shutdown and doesn't respond
"""
helpers.wait(
lambda: not helpers.tcp_ping(node_ip, 22),
timeout=timeout,
timeout_msg="Node '{}' didn't go offline after {} sec".format(
node_ip, timeout
)
)
def wait_node_is_online(self, node_ip, timeout):
"""Wait node is online after starting
"""
helpers.wait(
lambda: helpers.tcp_ping(node_ip, 22),
timeout=timeout,
timeout_msg="Node '{}' didn't become online after {} sec".format(
node_ip, timeout
)
)

View File

@ -343,3 +343,8 @@ class K8SManager(object):
self.__config.underlay.upstream_dns_servers
LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
'settings: {0}'.format(k8s_settings['nameservers']))
def get_pods_number(self, pod_name, namespace=None):
pods = [pod for pod in self.api.pods.list(namespace=namespace)
if pod_name in pod.name]
return len(pods)

View File

@ -0,0 +1,41 @@
nodes:
node1:
roles:
- openvswitch
- controller-net-host
- controller-net-bridge
node[2-3]:
roles:
- openvswitch
- controller-net-bridge
- compute
node[1-3]:
roles:
- galera
roles:
controller-net-host:
- neutron-dhcp-agent
- neutron-l3-agent
- neutron-metadata-agent
controller-net-bridge:
- etcd
- glance-api
- glance-registry
- keystone
- memcached
- neutron-server
- nova-api
- nova-conductor
- nova-consoleauth
- nova-novncproxy
- nova-scheduler
- rabbitmq
compute:
- nova-compute
- nova-libvirt
openvswitch:
- neutron-openvswitch-agent
- openvswitch-db
- openvswitch-vswitchd
galera:
- galera

View File

@ -0,0 +1,261 @@
# Copyright 2016 Mirantis, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import pytest
import base_test
from fuel_ccp_tests import logger
from fuel_ccp_tests import settings
from fuel_ccp_tests.helpers import post_os_deploy_checks
LOG = logger.logger
class TestGalera(base_test.SystemBaseTest):
""" Galera scale and destructive scenarios
"""
@pytest.mark.fail_snapshot
@pytest.mark.galera_shutdown
@pytest.mark.galera
def test_galera_shutdown_node(self, hardware, underlay, config,
ccpcluster, k8s_actions, show_step,
galera_deployed):
"""Shutdown galera node
Scenario:
1. Revert snapshot with deployed galera
2. Shutdown one galera node
3. Check galera state
4. Create 2 vms
Duration 30 min
"""
show_step(2)
hardware.shutdown_node_by_ip(underlay.node_names()[1])
show_step(3)
# todo: add wait for galera to assemble when galera_checker is ready
remote = underlay.remote(host=config.k8s.kube_host)
show_step(4)
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)
@pytest.mark.fail_snapshot
@pytest.mark.galera_cold_restart
@pytest.mark.galera
def test_galera_cold_restart_node(self, hardware, underlay, config,
ccpcluster, k8s_actions, show_step,
galera_deployed):
"""Cold restart galera node
Scenario:
1. Revert snapshot with deployed galera
2. Cold restart one galera node
3. Check galera state
4. Create 2 vms
Duration 30 min
"""
show_step(2)
hardware.shutdown_node_by_ip(underlay.node_names()[1])
hardware.start_node_by_ip(underlay.host_by_node_name('slave-0'))
show_step(3)
# todo: add wait for galera to assemble when galera_checker is ready
remote = underlay.remote(host=config.k8s.kube_host)
show_step(4)
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)
@pytest.mark.fail_snapshot
@pytest.mark.galera_poweroff
@pytest.mark.galera
def test_galera_poweroff_node(self, hardware, underlay, config,
ccpcluster, k8s_actions, show_step,
galera_deployed):
"""Poweroff galera node
Scenario:
1. Revert snapshot with deployed galera
2. Poweroff one galera node
3. Check galera state
4. Create 2 vms
Duration 30 min
"""
galera_node = underlay.node_names()[1]
galera_node_ip = underlay.host_by_node_name(galera_node)
show_step(2)
underlay.sudo_check_call('shutdown +1', node_name=galera_node)
hardware.shutdown_node_by_ip(galera_node_ip)
hardware.wait_node_is_offline(galera_node_ip, 90)
show_step(3)
# todo: add wait for galera to assemble when galera_checker is ready
remote = underlay.remote(host=config.k8s.kube_host)
show_step(4)
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)
@pytest.mark.fail_snapshot
@pytest.mark.galera_soft_reboot
@pytest.mark.galera
def test_galera_soft_reboot_node(self, hardware, underlay, config,
ccpcluster, k8s_actions, show_step,
galera_deployed):
"""Soft reboot galera node
Scenario:
1. Revert snapshot with deployed galera
2. Soft reboot one galera node
3. Check galera state
4. Create 2 vms
Duration 30 min
"""
galera_node = underlay.node_names()[1]
galera_node_ip = underlay.host_by_node_name(galera_node)
show_step(2)
underlay.sudo_check_call('shutdown +1', node_name=galera_node)
hardware.shutdown_node_by_ip(galera_node_ip)
hardware.wait_node_is_offline(galera_node_ip, 90)
hardware.start_node_by_ip(galera_node_ip)
hardware.wait_node_is_online(galera_node_ip, 180)
show_step(3)
# todo: add wait for galera to assemble when galera_checker is ready
remote = underlay.remote(host=config.k8s.kube_host)
show_step(4)
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)
@pytest.mark.fail_snapshot
@pytest.mark.galera_cluster_shutdown
@pytest.mark.galera
def test_galera_cluster_shutdown(self, hardware, underlay, config,
ccpcluster, k8s_actions, show_step,
galera_deployed):
"""Galera cluster shutdown
Scenario:
1. Revert snapshot with deployed galera
2. Shutdown all galera nodes and start them one by one
3. Check galera state
4. Create 2 vms
Duration 30 min
"""
galera_nodes = underlay.node_names()[:3]
galera_node_ips = []
show_step(2)
for galera_node in galera_nodes:
galera_node_ip = underlay.host_by_node_name(galera_node)
galera_node_ips.append(galera_node_ip)
hardware.shutdown_node_by_ip(galera_node_ip)
hardware.wait_node_is_offline(galera_node_ip, 90)
for galera_ip in galera_node_ips:
hardware.start_node_by_ip(galera_ip)
hardware.wait_node_is_online(galera_ip, 180)
show_step(3)
# todo: add wait for galera to assemble when galera_checker is ready
remote = underlay.remote(host=config.k8s.kube_host)
show_step(4)
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)
@pytest.mark.fail_snapshot
@pytest.mark.galera_scale_up_down
@pytest.mark.galera
def test_galera_scale(self, hardware, underlay, config,
ccpcluster, k8s_actions, show_step,
galera_deployed):
"""Galera cluster scale
Scenario:
1. Revert snapshot with deployed galera
2. Scale up galera to 5 replicas
3. Check galera state
4. Check number of galera pods
5. Create 2 vms
6. Scale down galera to 3 replicas
7. Check galera state
8. Check number of galera pods
9. Create 2 vms
Duration 30 min
"""
show_step(2)
with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
host=config.k8s.kube_host) as editor:
del editor.content['nodes']['node[1-3]']
editor.content['nodes']['node[1-5]'] = {'roles': ['galera']}
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
use_cli_params=True)
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
post_os_deploy_checks.check_pods_status(k8s_actions.api)
show_step(3)
# todo: add invocation of galera checker script
show_step(4)
galera_pods = \
k8s_actions.get_pods_number('galera',
settings.
CCP_CONF['kubernetes']['namespace'])
assert galera_pods == 5,\
"Expcted tp have 5 galera pods, got {}".format(galera_pods)
show_step(5)
remote = underlay.remote(host=config.k8s.kube_host)
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)
show_step(6)
with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
host=config.k8s.kube_host) as editor:
del editor.content['nodes']['node[1-5]']
editor.content['nodes']['node[1-3]'] = {'roles': ['galera']}
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
use_cli_params=True)
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
post_os_deploy_checks.check_pods_status(k8s_actions.api)
show_step(7)
# todo: add invocation of galera checker script
show_step(8)
galera_pods = \
k8s_actions.get_pods_number('galera',
settings.
CCP_CONF['kubernetes']['namespace'])
assert galera_pods == 3,\
"Expcted tp have 3 galera pods, got {}".format(galera_pods)
show_step(9)
remote.check_call(
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
" create".format(
settings.CCP_CONF["kubernetes"]["namespace"]),
timeout=600)