Add galera tests
- Add tests on galera deployment, galera node reboot, shutdown, poweroff, cold reboot, cluster shutdown - Add test on scale up and down galera - Galera checkers will be added after merging patch https://review.openstack.org/#/c/399999/8/service/files/galera_checker.py Change-Id: I31e1fe214e558497bcec7661f9e204ee645a56c1
This commit is contained in:
parent
b1fb1706d2
commit
8457ee9280
|
@ -12,10 +12,14 @@
|
|||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
from copy import deepcopy
|
||||
import os
|
||||
import pytest
|
||||
|
||||
from fuel_ccp_tests import logger
|
||||
from fuel_ccp_tests import settings
|
||||
from fuel_ccp_tests.helpers import ext
|
||||
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
||||
from fuel_ccp_tests.managers.osmanager import OSManager
|
||||
|
||||
LOG = logger.logger
|
||||
|
@ -43,3 +47,49 @@ def os_deployed(ccpcluster,
|
|||
else:
|
||||
LOG.info("Openstack allready installed and running...")
|
||||
osmanager.check_os_ready()
|
||||
|
||||
|
||||
@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_galera_deployed)
|
||||
@pytest.fixture(scope='function')
|
||||
def galera_deployed(ccpcluster,
|
||||
hardware,
|
||||
underlay,
|
||||
revert_snapshot,
|
||||
config,
|
||||
k8s_actions):
|
||||
"""Deploy galera cluster
|
||||
"""
|
||||
# If no snapshot was reverted, then try to revert the snapshot
|
||||
# that belongs to the fixture.
|
||||
# Note: keep fixtures in strict dependences from each other!
|
||||
if not config.os.running:
|
||||
general_config = deepcopy(settings.CCP_CONF)
|
||||
if settings.BUILD_IMAGES:
|
||||
k8s_actions.create_registry()
|
||||
ccpcluster.build()
|
||||
topology_path = \
|
||||
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
||||
'3galera_1comp.yaml'
|
||||
remote = underlay.remote(host=config.k8s.kube_host)
|
||||
remote.upload(topology_path, '/tmp')
|
||||
ccpcluster.put_yaml_config('./config_1.yaml', general_config)
|
||||
ccpcluster.add_includes('./config_1.yaml', [
|
||||
settings.CCP_DEPLOY_CONFIG,
|
||||
settings.CCP_SOURCES_CONFIG,
|
||||
'/tmp/3galera_1comp.yaml'])
|
||||
|
||||
underlay.sudo_check_call("pip install python-openstackclient",
|
||||
host=config.k8s.kube_host)
|
||||
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
|
||||
use_cli_params=True)
|
||||
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||
# todo: add invocation of galera checker script
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
||||
|
||||
config.os.running = True
|
||||
hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)
|
||||
|
|
|
@ -42,7 +42,8 @@ SNAPSHOT = enum(
|
|||
'k8s_deployed',
|
||||
'ccp_deployed',
|
||||
'os_deployed',
|
||||
'os_deployed_stacklight'
|
||||
'os_deployed_stacklight',
|
||||
'os_galera_deployed',
|
||||
)
|
||||
|
||||
LOG_LEVELS = enum(
|
||||
|
|
|
@ -198,10 +198,10 @@ class CCPManager(object):
|
|||
components=components,
|
||||
params=params, suppress_output=suppress_output)
|
||||
|
||||
def deploy(self, components=None, params=None):
|
||||
def deploy(self, components=None, params=None, use_cli_params=False):
|
||||
self.run('deploy',
|
||||
components=components,
|
||||
params=params)
|
||||
params=params, use_cli_params=use_cli_params)
|
||||
|
||||
def dry_deploy(self, export_dir, components=None, params=None):
|
||||
self.run('deploy --dry-run --export-dir={export_dir}'.format(
|
||||
|
|
|
@ -470,3 +470,47 @@ class EnvironmentManager(object):
|
|||
self.__config.underlay.nameservers = [self.nameserver]
|
||||
if not self.__config.underlay.upstream_dns_servers:
|
||||
self.__config.underlay.upstream_dns_servers = [self.nameserver]
|
||||
|
||||
def get_node_by_ip(self, node_ip):
|
||||
nodes = self._env.get_nodes()
|
||||
node = [node for node in nodes if self.node_ip(node) == node_ip]
|
||||
assert len(node) == 1, "Node with {} ip isn't found".format(node_ip)
|
||||
return node[0]
|
||||
|
||||
def shutdown_node_by_ip(self, node_ip):
|
||||
"""Shutdown hardware node by ip address
|
||||
|
||||
"""
|
||||
node = self.get_node_by_ip(node_ip)
|
||||
node.shutdown()
|
||||
|
||||
def start_node_by_ip(self, node_ip):
|
||||
"""Start hardware node by ip address
|
||||
|
||||
"""
|
||||
node = self.get_node_by_ip(node_ip)
|
||||
node.start()
|
||||
|
||||
def wait_node_is_offline(self, node_ip, timeout):
|
||||
"""Wait node is shutdown and doesn't respond
|
||||
|
||||
"""
|
||||
helpers.wait(
|
||||
lambda: not helpers.tcp_ping(node_ip, 22),
|
||||
timeout=timeout,
|
||||
timeout_msg="Node '{}' didn't go offline after {} sec".format(
|
||||
node_ip, timeout
|
||||
)
|
||||
)
|
||||
|
||||
def wait_node_is_online(self, node_ip, timeout):
|
||||
"""Wait node is online after starting
|
||||
|
||||
"""
|
||||
helpers.wait(
|
||||
lambda: helpers.tcp_ping(node_ip, 22),
|
||||
timeout=timeout,
|
||||
timeout_msg="Node '{}' didn't become online after {} sec".format(
|
||||
node_ip, timeout
|
||||
)
|
||||
)
|
||||
|
|
|
@ -343,3 +343,8 @@ class K8SManager(object):
|
|||
self.__config.underlay.upstream_dns_servers
|
||||
LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
|
||||
'settings: {0}'.format(k8s_settings['nameservers']))
|
||||
|
||||
def get_pods_number(self, pod_name, namespace=None):
|
||||
pods = [pod for pod in self.api.pods.list(namespace=namespace)
|
||||
if pod_name in pod.name]
|
||||
return len(pods)
|
||||
|
|
|
@ -0,0 +1,41 @@
|
|||
nodes:
|
||||
node1:
|
||||
roles:
|
||||
- openvswitch
|
||||
- controller-net-host
|
||||
- controller-net-bridge
|
||||
node[2-3]:
|
||||
roles:
|
||||
- openvswitch
|
||||
- controller-net-bridge
|
||||
- compute
|
||||
node[1-3]:
|
||||
roles:
|
||||
- galera
|
||||
roles:
|
||||
controller-net-host:
|
||||
- neutron-dhcp-agent
|
||||
- neutron-l3-agent
|
||||
- neutron-metadata-agent
|
||||
controller-net-bridge:
|
||||
- etcd
|
||||
- glance-api
|
||||
- glance-registry
|
||||
- keystone
|
||||
- memcached
|
||||
- neutron-server
|
||||
- nova-api
|
||||
- nova-conductor
|
||||
- nova-consoleauth
|
||||
- nova-novncproxy
|
||||
- nova-scheduler
|
||||
- rabbitmq
|
||||
compute:
|
||||
- nova-compute
|
||||
- nova-libvirt
|
||||
openvswitch:
|
||||
- neutron-openvswitch-agent
|
||||
- openvswitch-db
|
||||
- openvswitch-vswitchd
|
||||
galera:
|
||||
- galera
|
|
@ -0,0 +1,261 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import pytest
|
||||
|
||||
import base_test
|
||||
from fuel_ccp_tests import logger
|
||||
from fuel_ccp_tests import settings
|
||||
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
||||
|
||||
LOG = logger.logger
|
||||
|
||||
|
||||
class TestGalera(base_test.SystemBaseTest):
|
||||
""" Galera scale and destructive scenarios
|
||||
|
||||
"""
|
||||
@pytest.mark.fail_snapshot
|
||||
@pytest.mark.galera_shutdown
|
||||
@pytest.mark.galera
|
||||
def test_galera_shutdown_node(self, hardware, underlay, config,
|
||||
ccpcluster, k8s_actions, show_step,
|
||||
galera_deployed):
|
||||
"""Shutdown galera node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot with deployed galera
|
||||
2. Shutdown one galera node
|
||||
3. Check galera state
|
||||
4. Create 2 vms
|
||||
|
||||
Duration 30 min
|
||||
"""
|
||||
show_step(2)
|
||||
hardware.shutdown_node_by_ip(underlay.node_names()[1])
|
||||
show_step(3)
|
||||
# todo: add wait for galera to assemble when galera_checker is ready
|
||||
remote = underlay.remote(host=config.k8s.kube_host)
|
||||
show_step(4)
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
||||
|
||||
@pytest.mark.fail_snapshot
|
||||
@pytest.mark.galera_cold_restart
|
||||
@pytest.mark.galera
|
||||
def test_galera_cold_restart_node(self, hardware, underlay, config,
|
||||
ccpcluster, k8s_actions, show_step,
|
||||
galera_deployed):
|
||||
"""Cold restart galera node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot with deployed galera
|
||||
2. Cold restart one galera node
|
||||
3. Check galera state
|
||||
4. Create 2 vms
|
||||
|
||||
Duration 30 min
|
||||
"""
|
||||
show_step(2)
|
||||
hardware.shutdown_node_by_ip(underlay.node_names()[1])
|
||||
hardware.start_node_by_ip(underlay.host_by_node_name('slave-0'))
|
||||
show_step(3)
|
||||
# todo: add wait for galera to assemble when galera_checker is ready
|
||||
remote = underlay.remote(host=config.k8s.kube_host)
|
||||
show_step(4)
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
||||
|
||||
@pytest.mark.fail_snapshot
|
||||
@pytest.mark.galera_poweroff
|
||||
@pytest.mark.galera
|
||||
def test_galera_poweroff_node(self, hardware, underlay, config,
|
||||
ccpcluster, k8s_actions, show_step,
|
||||
galera_deployed):
|
||||
"""Poweroff galera node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot with deployed galera
|
||||
2. Poweroff one galera node
|
||||
3. Check galera state
|
||||
4. Create 2 vms
|
||||
|
||||
Duration 30 min
|
||||
"""
|
||||
galera_node = underlay.node_names()[1]
|
||||
galera_node_ip = underlay.host_by_node_name(galera_node)
|
||||
show_step(2)
|
||||
underlay.sudo_check_call('shutdown +1', node_name=galera_node)
|
||||
hardware.shutdown_node_by_ip(galera_node_ip)
|
||||
hardware.wait_node_is_offline(galera_node_ip, 90)
|
||||
show_step(3)
|
||||
# todo: add wait for galera to assemble when galera_checker is ready
|
||||
remote = underlay.remote(host=config.k8s.kube_host)
|
||||
show_step(4)
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
||||
|
||||
@pytest.mark.fail_snapshot
|
||||
@pytest.mark.galera_soft_reboot
|
||||
@pytest.mark.galera
|
||||
def test_galera_soft_reboot_node(self, hardware, underlay, config,
|
||||
ccpcluster, k8s_actions, show_step,
|
||||
galera_deployed):
|
||||
"""Soft reboot galera node
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot with deployed galera
|
||||
2. Soft reboot one galera node
|
||||
3. Check galera state
|
||||
4. Create 2 vms
|
||||
|
||||
Duration 30 min
|
||||
"""
|
||||
galera_node = underlay.node_names()[1]
|
||||
galera_node_ip = underlay.host_by_node_name(galera_node)
|
||||
show_step(2)
|
||||
underlay.sudo_check_call('shutdown +1', node_name=galera_node)
|
||||
hardware.shutdown_node_by_ip(galera_node_ip)
|
||||
hardware.wait_node_is_offline(galera_node_ip, 90)
|
||||
hardware.start_node_by_ip(galera_node_ip)
|
||||
hardware.wait_node_is_online(galera_node_ip, 180)
|
||||
show_step(3)
|
||||
# todo: add wait for galera to assemble when galera_checker is ready
|
||||
remote = underlay.remote(host=config.k8s.kube_host)
|
||||
show_step(4)
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
||||
|
||||
@pytest.mark.fail_snapshot
|
||||
@pytest.mark.galera_cluster_shutdown
|
||||
@pytest.mark.galera
|
||||
def test_galera_cluster_shutdown(self, hardware, underlay, config,
|
||||
ccpcluster, k8s_actions, show_step,
|
||||
galera_deployed):
|
||||
"""Galera cluster shutdown
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot with deployed galera
|
||||
2. Shutdown all galera nodes and start them one by one
|
||||
3. Check galera state
|
||||
4. Create 2 vms
|
||||
|
||||
Duration 30 min
|
||||
"""
|
||||
galera_nodes = underlay.node_names()[:3]
|
||||
galera_node_ips = []
|
||||
show_step(2)
|
||||
for galera_node in galera_nodes:
|
||||
galera_node_ip = underlay.host_by_node_name(galera_node)
|
||||
galera_node_ips.append(galera_node_ip)
|
||||
hardware.shutdown_node_by_ip(galera_node_ip)
|
||||
hardware.wait_node_is_offline(galera_node_ip, 90)
|
||||
for galera_ip in galera_node_ips:
|
||||
hardware.start_node_by_ip(galera_ip)
|
||||
hardware.wait_node_is_online(galera_ip, 180)
|
||||
show_step(3)
|
||||
# todo: add wait for galera to assemble when galera_checker is ready
|
||||
remote = underlay.remote(host=config.k8s.kube_host)
|
||||
show_step(4)
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
||||
|
||||
@pytest.mark.fail_snapshot
|
||||
@pytest.mark.galera_scale_up_down
|
||||
@pytest.mark.galera
|
||||
def test_galera_scale(self, hardware, underlay, config,
|
||||
ccpcluster, k8s_actions, show_step,
|
||||
galera_deployed):
|
||||
"""Galera cluster scale
|
||||
|
||||
Scenario:
|
||||
1. Revert snapshot with deployed galera
|
||||
2. Scale up galera to 5 replicas
|
||||
3. Check galera state
|
||||
4. Check number of galera pods
|
||||
5. Create 2 vms
|
||||
6. Scale down galera to 3 replicas
|
||||
7. Check galera state
|
||||
8. Check number of galera pods
|
||||
9. Create 2 vms
|
||||
|
||||
Duration 30 min
|
||||
"""
|
||||
show_step(2)
|
||||
with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
|
||||
host=config.k8s.kube_host) as editor:
|
||||
del editor.content['nodes']['node[1-3]']
|
||||
editor.content['nodes']['node[1-5]'] = {'roles': ['galera']}
|
||||
|
||||
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
|
||||
use_cli_params=True)
|
||||
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||
show_step(3)
|
||||
# todo: add invocation of galera checker script
|
||||
show_step(4)
|
||||
galera_pods = \
|
||||
k8s_actions.get_pods_number('galera',
|
||||
settings.
|
||||
CCP_CONF['kubernetes']['namespace'])
|
||||
assert galera_pods == 5,\
|
||||
"Expcted tp have 5 galera pods, got {}".format(galera_pods)
|
||||
|
||||
show_step(5)
|
||||
remote = underlay.remote(host=config.k8s.kube_host)
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
||||
show_step(6)
|
||||
with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
|
||||
host=config.k8s.kube_host) as editor:
|
||||
del editor.content['nodes']['node[1-5]']
|
||||
editor.content['nodes']['node[1-3]'] = {'roles': ['galera']}
|
||||
|
||||
ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
|
||||
use_cli_params=True)
|
||||
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||
show_step(7)
|
||||
# todo: add invocation of galera checker script
|
||||
show_step(8)
|
||||
galera_pods = \
|
||||
k8s_actions.get_pods_number('galera',
|
||||
settings.
|
||||
CCP_CONF['kubernetes']['namespace'])
|
||||
assert galera_pods == 3,\
|
||||
"Expcted tp have 3 galera pods, got {}".format(galera_pods)
|
||||
show_step(9)
|
||||
remote.check_call(
|
||||
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||
" create".format(
|
||||
settings.CCP_CONF["kubernetes"]["namespace"]),
|
||||
timeout=600)
|
Loading…
Reference in New Issue