summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2016-12-08 12:51:45 +0000
committerGerrit Code Review <review@openstack.org>2016-12-08 12:51:45 +0000
commit2cc97d84abcd381990e02e23556cf11377569e2a (patch)
treeec02b01b46f5280e44d8e7369b33727a43626dbc
parent493dc8b27aac4bd1c75f7cd33d82797d37a2b9a1 (diff)
parent8457ee9280abaafe5150191cdc2252e83e756681 (diff)
Merge "Add galera tests"
-rw-r--r--fuel_ccp_tests/fixtures/os_fixtures.py50
-rw-r--r--fuel_ccp_tests/helpers/ext.py3
-rw-r--r--fuel_ccp_tests/managers/ccpmanager.py4
-rw-r--r--fuel_ccp_tests/managers/envmanager_devops.py44
-rw-r--r--fuel_ccp_tests/managers/k8smanager.py5
-rw-r--r--fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml41
-rw-r--r--fuel_ccp_tests/tests/system/test_galera.py261
7 files changed, 405 insertions, 3 deletions
diff --git a/fuel_ccp_tests/fixtures/os_fixtures.py b/fuel_ccp_tests/fixtures/os_fixtures.py
index 27d221e..bb7cb9f 100644
--- a/fuel_ccp_tests/fixtures/os_fixtures.py
+++ b/fuel_ccp_tests/fixtures/os_fixtures.py
@@ -12,10 +12,14 @@
12# License for the specific language governing permissions and limitations 12# License for the specific language governing permissions and limitations
13# under the License. 13# under the License.
14 14
15from copy import deepcopy
16import os
15import pytest 17import pytest
16 18
17from fuel_ccp_tests import logger 19from fuel_ccp_tests import logger
20from fuel_ccp_tests import settings
18from fuel_ccp_tests.helpers import ext 21from fuel_ccp_tests.helpers import ext
22from fuel_ccp_tests.helpers import post_os_deploy_checks
19from fuel_ccp_tests.managers.osmanager import OSManager 23from fuel_ccp_tests.managers.osmanager import OSManager
20 24
21LOG = logger.logger 25LOG = logger.logger
@@ -43,3 +47,49 @@ def os_deployed(ccpcluster,
43 else: 47 else:
44 LOG.info("Openstack allready installed and running...") 48 LOG.info("Openstack allready installed and running...")
45 osmanager.check_os_ready() 49 osmanager.check_os_ready()
50
51
52@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_galera_deployed)
53@pytest.fixture(scope='function')
54def galera_deployed(ccpcluster,
55 hardware,
56 underlay,
57 revert_snapshot,
58 config,
59 k8s_actions):
60 """Deploy galera cluster
61 """
62 # If no snapshot was reverted, then try to revert the snapshot
63 # that belongs to the fixture.
64 # Note: keep fixtures in strict dependences from each other!
65 if not config.os.running:
66 general_config = deepcopy(settings.CCP_CONF)
67 if settings.BUILD_IMAGES:
68 k8s_actions.create_registry()
69 ccpcluster.build()
70 topology_path = \
71 os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
72 '3galera_1comp.yaml'
73 remote = underlay.remote(host=config.k8s.kube_host)
74 remote.upload(topology_path, '/tmp')
75 ccpcluster.put_yaml_config('./config_1.yaml', general_config)
76 ccpcluster.add_includes('./config_1.yaml', [
77 settings.CCP_DEPLOY_CONFIG,
78 settings.CCP_SOURCES_CONFIG,
79 '/tmp/3galera_1comp.yaml'])
80
81 underlay.sudo_check_call("pip install python-openstackclient",
82 host=config.k8s.kube_host)
83 ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
84 use_cli_params=True)
85 post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
86 post_os_deploy_checks.check_pods_status(k8s_actions.api)
87 # todo: add invocation of galera checker script
88 remote.check_call(
89 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
90 " create".format(
91 settings.CCP_CONF["kubernetes"]["namespace"]),
92 timeout=600)
93
94 config.os.running = True
95 hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)
diff --git a/fuel_ccp_tests/helpers/ext.py b/fuel_ccp_tests/helpers/ext.py
index 21ab2c4..b131b9d 100644
--- a/fuel_ccp_tests/helpers/ext.py
+++ b/fuel_ccp_tests/helpers/ext.py
@@ -42,7 +42,8 @@ SNAPSHOT = enum(
42 'k8s_deployed', 42 'k8s_deployed',
43 'ccp_deployed', 43 'ccp_deployed',
44 'os_deployed', 44 'os_deployed',
45 'os_deployed_stacklight' 45 'os_deployed_stacklight',
46 'os_galera_deployed',
46) 47)
47 48
48LOG_LEVELS = enum( 49LOG_LEVELS = enum(
diff --git a/fuel_ccp_tests/managers/ccpmanager.py b/fuel_ccp_tests/managers/ccpmanager.py
index 04b8fa8..fb5edb5 100644
--- a/fuel_ccp_tests/managers/ccpmanager.py
+++ b/fuel_ccp_tests/managers/ccpmanager.py
@@ -198,10 +198,10 @@ class CCPManager(object):
198 components=components, 198 components=components,
199 params=params, suppress_output=suppress_output) 199 params=params, suppress_output=suppress_output)
200 200
201 def deploy(self, components=None, params=None): 201 def deploy(self, components=None, params=None, use_cli_params=False):
202 self.run('deploy', 202 self.run('deploy',
203 components=components, 203 components=components,
204 params=params) 204 params=params, use_cli_params=use_cli_params)
205 205
206 def dry_deploy(self, export_dir, components=None, params=None): 206 def dry_deploy(self, export_dir, components=None, params=None):
207 self.run('deploy --dry-run --export-dir={export_dir}'.format( 207 self.run('deploy --dry-run --export-dir={export_dir}'.format(
diff --git a/fuel_ccp_tests/managers/envmanager_devops.py b/fuel_ccp_tests/managers/envmanager_devops.py
index 69111a1..bb5bac1 100644
--- a/fuel_ccp_tests/managers/envmanager_devops.py
+++ b/fuel_ccp_tests/managers/envmanager_devops.py
@@ -470,3 +470,47 @@ class EnvironmentManager(object):
470 self.__config.underlay.nameservers = [self.nameserver] 470 self.__config.underlay.nameservers = [self.nameserver]
471 if not self.__config.underlay.upstream_dns_servers: 471 if not self.__config.underlay.upstream_dns_servers:
472 self.__config.underlay.upstream_dns_servers = [self.nameserver] 472 self.__config.underlay.upstream_dns_servers = [self.nameserver]
473
474 def get_node_by_ip(self, node_ip):
475 nodes = self._env.get_nodes()
476 node = [node for node in nodes if self.node_ip(node) == node_ip]
477 assert len(node) == 1, "Node with {} ip isn't found".format(node_ip)
478 return node[0]
479
480 def shutdown_node_by_ip(self, node_ip):
481 """Shutdown hardware node by ip address
482
483 """
484 node = self.get_node_by_ip(node_ip)
485 node.shutdown()
486
487 def start_node_by_ip(self, node_ip):
488 """Start hardware node by ip address
489
490 """
491 node = self.get_node_by_ip(node_ip)
492 node.start()
493
494 def wait_node_is_offline(self, node_ip, timeout):
495 """Wait node is shutdown and doesn't respond
496
497 """
498 helpers.wait(
499 lambda: not helpers.tcp_ping(node_ip, 22),
500 timeout=timeout,
501 timeout_msg="Node '{}' didn't go offline after {} sec".format(
502 node_ip, timeout
503 )
504 )
505
506 def wait_node_is_online(self, node_ip, timeout):
507 """Wait node is online after starting
508
509 """
510 helpers.wait(
511 lambda: helpers.tcp_ping(node_ip, 22),
512 timeout=timeout,
513 timeout_msg="Node '{}' didn't become online after {} sec".format(
514 node_ip, timeout
515 )
516 )
diff --git a/fuel_ccp_tests/managers/k8smanager.py b/fuel_ccp_tests/managers/k8smanager.py
index e4a9c98..fd15a8d 100644
--- a/fuel_ccp_tests/managers/k8smanager.py
+++ b/fuel_ccp_tests/managers/k8smanager.py
@@ -343,3 +343,8 @@ class K8SManager(object):
343 self.__config.underlay.upstream_dns_servers 343 self.__config.underlay.upstream_dns_servers
344 LOG.info('Added custom upstream DNS servers (dnsmasq) to the ' 344 LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
345 'settings: {0}'.format(k8s_settings['nameservers'])) 345 'settings: {0}'.format(k8s_settings['nameservers']))
346
347 def get_pods_number(self, pod_name, namespace=None):
348 pods = [pod for pod in self.api.pods.list(namespace=namespace)
349 if pod_name in pod.name]
350 return len(pods)
diff --git a/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml b/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml
new file mode 100644
index 0000000..8d81efd
--- /dev/null
+++ b/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml
@@ -0,0 +1,41 @@
1nodes:
2 node1:
3 roles:
4 - openvswitch
5 - controller-net-host
6 - controller-net-bridge
7 node[2-3]:
8 roles:
9 - openvswitch
10 - controller-net-bridge
11 - compute
12 node[1-3]:
13 roles:
14 - galera
15roles:
16 controller-net-host:
17 - neutron-dhcp-agent
18 - neutron-l3-agent
19 - neutron-metadata-agent
20 controller-net-bridge:
21 - etcd
22 - glance-api
23 - glance-registry
24 - keystone
25 - memcached
26 - neutron-server
27 - nova-api
28 - nova-conductor
29 - nova-consoleauth
30 - nova-novncproxy
31 - nova-scheduler
32 - rabbitmq
33 compute:
34 - nova-compute
35 - nova-libvirt
36 openvswitch:
37 - neutron-openvswitch-agent
38 - openvswitch-db
39 - openvswitch-vswitchd
40 galera:
41 - galera \ No newline at end of file
diff --git a/fuel_ccp_tests/tests/system/test_galera.py b/fuel_ccp_tests/tests/system/test_galera.py
new file mode 100644
index 0000000..3b0450b
--- /dev/null
+++ b/fuel_ccp_tests/tests/system/test_galera.py
@@ -0,0 +1,261 @@
1# Copyright 2016 Mirantis, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import pytest
16
17import base_test
18from fuel_ccp_tests import logger
19from fuel_ccp_tests import settings
20from fuel_ccp_tests.helpers import post_os_deploy_checks
21
22LOG = logger.logger
23
24
25class TestGalera(base_test.SystemBaseTest):
26 """ Galera scale and destructive scenarios
27
28 """
29 @pytest.mark.fail_snapshot
30 @pytest.mark.galera_shutdown
31 @pytest.mark.galera
32 def test_galera_shutdown_node(self, hardware, underlay, config,
33 ccpcluster, k8s_actions, show_step,
34 galera_deployed):
35 """Shutdown galera node
36
37 Scenario:
38 1. Revert snapshot with deployed galera
39 2. Shutdown one galera node
40 3. Check galera state
41 4. Create 2 vms
42
43 Duration 30 min
44 """
45 show_step(2)
46 hardware.shutdown_node_by_ip(underlay.node_names()[1])
47 show_step(3)
48 # todo: add wait for galera to assemble when galera_checker is ready
49 remote = underlay.remote(host=config.k8s.kube_host)
50 show_step(4)
51 remote.check_call(
52 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
53 " create".format(
54 settings.CCP_CONF["kubernetes"]["namespace"]),
55 timeout=600)
56
57 @pytest.mark.fail_snapshot
58 @pytest.mark.galera_cold_restart
59 @pytest.mark.galera
60 def test_galera_cold_restart_node(self, hardware, underlay, config,
61 ccpcluster, k8s_actions, show_step,
62 galera_deployed):
63 """Cold restart galera node
64
65 Scenario:
66 1. Revert snapshot with deployed galera
67 2. Cold restart one galera node
68 3. Check galera state
69 4. Create 2 vms
70
71 Duration 30 min
72 """
73 show_step(2)
74 hardware.shutdown_node_by_ip(underlay.node_names()[1])
75 hardware.start_node_by_ip(underlay.host_by_node_name('slave-0'))
76 show_step(3)
77 # todo: add wait for galera to assemble when galera_checker is ready
78 remote = underlay.remote(host=config.k8s.kube_host)
79 show_step(4)
80 remote.check_call(
81 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
82 " create".format(
83 settings.CCP_CONF["kubernetes"]["namespace"]),
84 timeout=600)
85
86 @pytest.mark.fail_snapshot
87 @pytest.mark.galera_poweroff
88 @pytest.mark.galera
89 def test_galera_poweroff_node(self, hardware, underlay, config,
90 ccpcluster, k8s_actions, show_step,
91 galera_deployed):
92 """Poweroff galera node
93
94 Scenario:
95 1. Revert snapshot with deployed galera
96 2. Poweroff one galera node
97 3. Check galera state
98 4. Create 2 vms
99
100 Duration 30 min
101 """
102 galera_node = underlay.node_names()[1]
103 galera_node_ip = underlay.host_by_node_name(galera_node)
104 show_step(2)
105 underlay.sudo_check_call('shutdown +1', node_name=galera_node)
106 hardware.shutdown_node_by_ip(galera_node_ip)
107 hardware.wait_node_is_offline(galera_node_ip, 90)
108 show_step(3)
109 # todo: add wait for galera to assemble when galera_checker is ready
110 remote = underlay.remote(host=config.k8s.kube_host)
111 show_step(4)
112 remote.check_call(
113 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
114 " create".format(
115 settings.CCP_CONF["kubernetes"]["namespace"]),
116 timeout=600)
117
118 @pytest.mark.fail_snapshot
119 @pytest.mark.galera_soft_reboot
120 @pytest.mark.galera
121 def test_galera_soft_reboot_node(self, hardware, underlay, config,
122 ccpcluster, k8s_actions, show_step,
123 galera_deployed):
124 """Soft reboot galera node
125
126 Scenario:
127 1. Revert snapshot with deployed galera
128 2. Soft reboot one galera node
129 3. Check galera state
130 4. Create 2 vms
131
132 Duration 30 min
133 """
134 galera_node = underlay.node_names()[1]
135 galera_node_ip = underlay.host_by_node_name(galera_node)
136 show_step(2)
137 underlay.sudo_check_call('shutdown +1', node_name=galera_node)
138 hardware.shutdown_node_by_ip(galera_node_ip)
139 hardware.wait_node_is_offline(galera_node_ip, 90)
140 hardware.start_node_by_ip(galera_node_ip)
141 hardware.wait_node_is_online(galera_node_ip, 180)
142 show_step(3)
143 # todo: add wait for galera to assemble when galera_checker is ready
144 remote = underlay.remote(host=config.k8s.kube_host)
145 show_step(4)
146 remote.check_call(
147 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
148 " create".format(
149 settings.CCP_CONF["kubernetes"]["namespace"]),
150 timeout=600)
151
152 @pytest.mark.fail_snapshot
153 @pytest.mark.galera_cluster_shutdown
154 @pytest.mark.galera
155 def test_galera_cluster_shutdown(self, hardware, underlay, config,
156 ccpcluster, k8s_actions, show_step,
157 galera_deployed):
158 """Galera cluster shutdown
159
160 Scenario:
161 1. Revert snapshot with deployed galera
162 2. Shutdown all galera nodes and start them one by one
163 3. Check galera state
164 4. Create 2 vms
165
166 Duration 30 min
167 """
168 galera_nodes = underlay.node_names()[:3]
169 galera_node_ips = []
170 show_step(2)
171 for galera_node in galera_nodes:
172 galera_node_ip = underlay.host_by_node_name(galera_node)
173 galera_node_ips.append(galera_node_ip)
174 hardware.shutdown_node_by_ip(galera_node_ip)
175 hardware.wait_node_is_offline(galera_node_ip, 90)
176 for galera_ip in galera_node_ips:
177 hardware.start_node_by_ip(galera_ip)
178 hardware.wait_node_is_online(galera_ip, 180)
179 show_step(3)
180 # todo: add wait for galera to assemble when galera_checker is ready
181 remote = underlay.remote(host=config.k8s.kube_host)
182 show_step(4)
183 remote.check_call(
184 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
185 " create".format(
186 settings.CCP_CONF["kubernetes"]["namespace"]),
187 timeout=600)
188
189 @pytest.mark.fail_snapshot
190 @pytest.mark.galera_scale_up_down
191 @pytest.mark.galera
192 def test_galera_scale(self, hardware, underlay, config,
193 ccpcluster, k8s_actions, show_step,
194 galera_deployed):
195 """Galera cluster scale
196
197 Scenario:
198 1. Revert snapshot with deployed galera
199 2. Scale up galera to 5 replicas
200 3. Check galera state
201 4. Check number of galera pods
202 5. Create 2 vms
203 6. Scale down galera to 3 replicas
204 7. Check galera state
205 8. Check number of galera pods
206 9. Create 2 vms
207
208 Duration 30 min
209 """
210 show_step(2)
211 with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
212 host=config.k8s.kube_host) as editor:
213 del editor.content['nodes']['node[1-3]']
214 editor.content['nodes']['node[1-5]'] = {'roles': ['galera']}
215
216 ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
217 use_cli_params=True)
218 post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
219 post_os_deploy_checks.check_pods_status(k8s_actions.api)
220 show_step(3)
221 # todo: add invocation of galera checker script
222 show_step(4)
223 galera_pods = \
224 k8s_actions.get_pods_number('galera',
225 settings.
226 CCP_CONF['kubernetes']['namespace'])
227 assert galera_pods == 5,\
228 "Expcted tp have 5 galera pods, got {}".format(galera_pods)
229
230 show_step(5)
231 remote = underlay.remote(host=config.k8s.kube_host)
232 remote.check_call(
233 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
234 " create".format(
235 settings.CCP_CONF["kubernetes"]["namespace"]),
236 timeout=600)
237 show_step(6)
238 with underlay.yaml_editor('/tmp/3galera_1comp.yaml',
239 host=config.k8s.kube_host) as editor:
240 del editor.content['nodes']['node[1-5]']
241 editor.content['nodes']['node[1-3]'] = {'roles': ['galera']}
242
243 ccpcluster.deploy(params={"config-file": "./config_1.yaml"},
244 use_cli_params=True)
245 post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
246 post_os_deploy_checks.check_pods_status(k8s_actions.api)
247 show_step(7)
248 # todo: add invocation of galera checker script
249 show_step(8)
250 galera_pods = \
251 k8s_actions.get_pods_number('galera',
252 settings.
253 CCP_CONF['kubernetes']['namespace'])
254 assert galera_pods == 3,\
255 "Expcted tp have 3 galera pods, got {}".format(galera_pods)
256 show_step(9)
257 remote.check_call(
258 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
259 " create".format(
260 settings.CCP_CONF["kubernetes"]["namespace"]),
261 timeout=600)