summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJenkins <jenkins@review.openstack.org>2016-12-22 14:03:39 +0000
committerGerrit Code Review <review@openstack.org>2016-12-22 14:03:39 +0000
commit5d3db0e0d45566c7a7af6839a334f63247dd1e63 (patch)
treeb8c6cb5ee21c7f3317774b623925c218336412f9
parent89c8319b9b463bc70c0467868dc209b6e3276e20 (diff)
parentca1ad32f207a214e76de702d8a7ee0848bf8392d (diff)
Merge "Add tests to check rabbit cluster"
-rw-r--r--README.rst7
-rw-r--r--fuel_ccp_tests/fixtures/ccp_fixtures.py2
-rw-r--r--fuel_ccp_tests/fixtures/os_fixtures.py25
-rw-r--r--fuel_ccp_tests/helpers/rabbit.py56
-rw-r--r--fuel_ccp_tests/helpers/utils.py10
-rw-r--r--fuel_ccp_tests/managers/k8smanager.py8
-rw-r--r--fuel_ccp_tests/managers/osmanager.py20
-rw-r--r--fuel_ccp_tests/requirements.txt1
-rw-r--r--fuel_ccp_tests/settings.py4
-rw-r--r--fuel_ccp_tests/settings_oslo.py7
-rw-r--r--fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl.yaml (renamed from fuel_ccp_tests/templates/k8s_templates/1ctrl.yaml)0
-rw-r--r--fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp.yaml (renamed from fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp.yaml)0
-rw-r--r--fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_diff_ctrl.yaml (renamed from fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_diff_ctrl.yaml)0
-rw-r--r--fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_same_ctrl.yaml (renamed from fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_same_ctrl.yaml)0
-rw-r--r--fuel_ccp_tests/templates/ccp_deploy_topology/3galera_1comp.yaml (renamed from fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml)0
-rw-r--r--fuel_ccp_tests/templates/ccp_deploy_topology/default_deploy_topology.yaml (renamed from fuel_ccp_tests/templates/k8s_templates/k8s_topology.yaml)6
-rw-r--r--fuel_ccp_tests/templates/ccp_deploy_topology/stacklight_topology.yaml (renamed from fuel_ccp_tests/templates/k8s_templates/stacklight_topology.yaml)0
-rw-r--r--fuel_ccp_tests/templates/k8s_templates/build-deploy_cluster.yaml17
-rw-r--r--fuel_ccp_tests/tests/system/pre_commit/test_glance_api.py3
-rw-r--r--fuel_ccp_tests/tests/system/pre_commit/test_horizon.py3
-rw-r--r--fuel_ccp_tests/tests/system/pre_commit/test_stacklight.py3
-rw-r--r--fuel_ccp_tests/tests/system/test_few_os.py10
-rw-r--r--fuel_ccp_tests/tests/system/test_rabbitmq.py487
23 files changed, 610 insertions, 59 deletions
diff --git a/README.rst b/README.rst
index 02f9f09..8ed7c3c 100644
--- a/README.rst
+++ b/README.rst
@@ -12,7 +12,7 @@ fuel_ccp_tests
12├── managers 12├── managers
13│ └── k8s 13│ └── k8s
14├── templates 14├── templates
15│ ├── k8s_templates 15│ ├── ccp_deploy_topology
16│ ├── misc 16│ ├── misc
17│ └── registry_templates 17│ └── registry_templates
18└── tests 18└── tests
@@ -47,7 +47,7 @@ Contains .yaml templates with environment configuration(virtual machines, networ
47 47
48Tests Overview 48Tests Overview
49================ 49================
50The fuel-ccp-test are performed to verify that the completed software (ccp) functions according to the expectations defined by the requirements. 50The fuel-ccp-test are performed to verify that the completed software (ccp) functions according to the expectations defined by the requirements.
51 51
52The tests depended on purpose are divided on several categories. 52The tests depended on purpose are divided on several categories.
53 53
@@ -67,7 +67,7 @@ Consists from 2 categories **precommit** and **system**. The purpose of the sys
67 67
68Test execution 68Test execution
69-------------- 69--------------
70To execute tests necessary to add value to several variables via *export* or in the test command. Variables: 70To execute tests necessary to add value to several variables via *export* or in the test command. Variables:
71 71
72- ENV_NAME - prefix name for the env and VMs 72- ENV_NAME - prefix name for the env and VMs
73- IMAGE_PATH - qcow2 image path 73- IMAGE_PATH - qcow2 image path
@@ -78,4 +78,3 @@ To execute tests necessary to add value to several variables via *export* or in
78After exporting execute the command: 78After exporting execute the command:
79 79
80py.test -vvv -s -k <test_name> or py.test -vvv -s -m <test_mark> 80py.test -vvv -s -k <test_name> or py.test -vvv -s -m <test_mark>
81
diff --git a/fuel_ccp_tests/fixtures/ccp_fixtures.py b/fuel_ccp_tests/fixtures/ccp_fixtures.py
index ef4a822..3e18b0d 100644
--- a/fuel_ccp_tests/fixtures/ccp_fixtures.py
+++ b/fuel_ccp_tests/fixtures/ccp_fixtures.py
@@ -76,7 +76,7 @@ def ccpcluster(revert_snapshot, config, hardware,
76 path=settings.CCP_FETCH_CONFIG, 76 path=settings.CCP_FETCH_CONFIG,
77 config=settings.CCP_FETCH_PARAMS) 77 config=settings.CCP_FETCH_PARAMS)
78 78
79 with open(settings.TOPOLOGY_PATH, 'r') as f: 79 with open(config.ccp_deploy.topology_path, 'r') as f:
80 ccp_actions.put_raw_config( 80 ccp_actions.put_raw_config(
81 path=settings.CCP_DEPLOY_TOPOLOGY, 81 path=settings.CCP_DEPLOY_TOPOLOGY,
82 content=f.read()) 82 content=f.read())
diff --git a/fuel_ccp_tests/fixtures/os_fixtures.py b/fuel_ccp_tests/fixtures/os_fixtures.py
index a6f9537..a300898 100644
--- a/fuel_ccp_tests/fixtures/os_fixtures.py
+++ b/fuel_ccp_tests/fixtures/os_fixtures.py
@@ -15,11 +15,13 @@
15from copy import deepcopy 15from copy import deepcopy
16import os 16import os
17import pytest 17import pytest
18from devops.helpers import helpers
18 19
19from fuel_ccp_tests import logger 20from fuel_ccp_tests import logger
20from fuel_ccp_tests import settings 21from fuel_ccp_tests import settings
21from fuel_ccp_tests.helpers import ext 22from fuel_ccp_tests.helpers import ext
22from fuel_ccp_tests.helpers import post_os_deploy_checks 23from fuel_ccp_tests.helpers import post_os_deploy_checks
24from fuel_ccp_tests.helpers import rabbit
23from fuel_ccp_tests.managers.osmanager import OSManager 25from fuel_ccp_tests.managers.osmanager import OSManager
24 26
25LOG = logger.logger 27LOG = logger.logger
@@ -37,12 +39,7 @@ def os_deployed(ccpcluster,
37 """ 39 """
38 osmanager = OSManager(config, underlay, k8s_actions, ccpcluster) 40 osmanager = OSManager(config, underlay, k8s_actions, ccpcluster)
39 if not config.os.running: 41 if not config.os.running:
40 LOG.info("Preparing openstack log collector fixture...") 42 osmanager.install_os()
41 topology = None
42 if config.os_deploy.stacklight_enable:
43 topology = ('/fuel_ccp_tests/templates/k8s_templates/'
44 'stacklight_topology.yaml')
45 osmanager.install_os(topology=topology)
46 hardware.create_snapshot(ext.SNAPSHOT.os_deployed) 43 hardware.create_snapshot(ext.SNAPSHOT.os_deployed)
47 else: 44 else:
48 LOG.info("Openstack allready installed and running...") 45 LOG.info("Openstack allready installed and running...")
@@ -68,7 +65,7 @@ def galera_deployed(ccpcluster,
68 k8s_actions.create_registry() 65 k8s_actions.create_registry()
69 ccpcluster.build() 66 ccpcluster.build()
70 topology_path = \ 67 topology_path = \
71 os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \ 68 os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
72 '3galera_1comp.yaml' 69 '3galera_1comp.yaml'
73 remote = underlay.remote(host=config.k8s.kube_host) 70 remote = underlay.remote(host=config.k8s.kube_host)
74 remote.upload(topology_path, '/tmp') 71 remote.upload(topology_path, '/tmp')
@@ -93,3 +90,17 @@ def galera_deployed(ccpcluster,
93 90
94 config.os.running = True 91 config.os.running = True
95 hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed) 92 hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)
93
94
95@pytest.fixture(scope='function')
96def rabbit_client(underlay, config, os_deployed):
97 """Deploy openstack
98 """
99 host = config.k8s.kube_host
100 remote = underlay.remote(host=host)
101 rabbit_port = ''.join(remote.execute(
102 "kubectl get service --namespace ccp rabbitmq -o yaml |"
103 " awk '/nodePort: / {print $NF}'")['stdout'])
104 client = helpers.wait_pass(lambda: rabbit.RabbitClient(host, rabbit_port),
105 interval=60, timeout=360)
106 return client
diff --git a/fuel_ccp_tests/helpers/rabbit.py b/fuel_ccp_tests/helpers/rabbit.py
new file mode 100644
index 0000000..8623791
--- /dev/null
+++ b/fuel_ccp_tests/helpers/rabbit.py
@@ -0,0 +1,56 @@
1import kombu
2
3from fuel_ccp_tests import logger
4from fuel_ccp_tests.helpers import utils
5
6LOG = logger.logger
7
8
9class RabbitClient(object):
10
11 def __init__(self, ip, port, user='rabbitmq', password='password'):
12 c = kombu.Connection("amqp://{0}:{1}@{2}:{3}//".format(user, password,
13 ip, port))
14 c.connect()
15 self.ch = c.channel()
16
17 def list_nodes(self, remote, pod, namespace):
18 output = ''.join(
19 remote.execute("kubectl exec -i {} --namespace={}"
20 " -- rabbitmqctl"
21 " cluster_status".format(pod,
22 namespace))['stdout'])
23 substring_ind = output.find('{running_nodes')
24 sub_end_ind = output.find('cluster_name')
25 result_str = output[substring_ind: sub_end_ind]
26 num_node = result_str.count("rabbit@")
27 return num_node
28
29 def check_queue_replicated(self, queue, remote, pod, namespace):
30 remote.check_call("kubectl exec -i {} --namespace={}"
31 " -- rabbitmqctl list_queues |"
32 " grep {}".format(pod, namespace,
33 queue))
34
35 def create_queue(self):
36 test_queue = 'test-rabbit-{}'.format(utils.rand_name())
37 q = kombu.Queue(test_queue, channel=self.ch, durable=False,
38 queue_arguments={"x-expires": 15 * 60 * 1000})
39 q.declare()
40 return test_queue
41
42 def publish_message_to_queue(self, queue):
43 uid = utils.generate_uuid()
44 producer = kombu.Producer(channel=self.ch, routing_key=queue)
45 producer.publish(uid)
46 return {'queue': queue, 'id': uid}
47
48 def check_queue_message(self, message):
49 q = kombu.Queue(message['queue'], channel=self.ch)
50 msg = q.get(True)
51 assert msg.body in message['id'],\
52 "Message body is {}, expected {}".format(msg.body, message['id'])
53
54 def delete_queue(self, queue):
55 q = kombu.Queue(queue, channel=self.ch)
56 q.delete()
diff --git a/fuel_ccp_tests/helpers/utils.py b/fuel_ccp_tests/helpers/utils.py
index 8cd40f7..0106405 100644
--- a/fuel_ccp_tests/helpers/utils.py
+++ b/fuel_ccp_tests/helpers/utils.py
@@ -14,10 +14,12 @@
14 14
15import copy 15import copy
16import os 16import os
17import random
17import shutil 18import shutil
18import tempfile 19import tempfile
19import time 20import time
20import traceback 21import traceback
22import uuid
21 23
22import paramiko 24import paramiko
23import yaml 25import yaml
@@ -483,3 +485,11 @@ def get_top_fixtures_marks(request, mark_name):
483 .format(top_fixtures_marks)) 485 .format(top_fixtures_marks))
484 486
485 return top_fixtures_marks 487 return top_fixtures_marks
488
489
490def rand_name():
491 return str(random.randint(1, 0x7fffffff))
492
493
494def generate_uuid():
495 return uuid.uuid4().hex
diff --git a/fuel_ccp_tests/managers/k8smanager.py b/fuel_ccp_tests/managers/k8smanager.py
index fd15a8d..0cb3e1f 100644
--- a/fuel_ccp_tests/managers/k8smanager.py
+++ b/fuel_ccp_tests/managers/k8smanager.py
@@ -344,7 +344,11 @@ class K8SManager(object):
344 LOG.info('Added custom upstream DNS servers (dnsmasq) to the ' 344 LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
345 'settings: {0}'.format(k8s_settings['nameservers'])) 345 'settings: {0}'.format(k8s_settings['nameservers']))
346 346
347 def get_pods_number(self, pod_name, namespace=None): 347 def get_running_pods(self, pod_name, namespace=None):
348 pods = [pod for pod in self.api.pods.list(namespace=namespace) 348 pods = [pod for pod in self.api.pods.list(namespace=namespace)
349 if pod_name in pod.name] 349 if pod_name in pod.name and pod.status.phase == 'Running']
350 return pods
351
352 def get_pods_number(self, pod_name, namespace=None):
353 pods = self.get_running_pods(pod_name, namespace)
350 return len(pods) 354 return len(pods)
diff --git a/fuel_ccp_tests/managers/osmanager.py b/fuel_ccp_tests/managers/osmanager.py
index c8c0c6d..d1bd5ab 100644
--- a/fuel_ccp_tests/managers/osmanager.py
+++ b/fuel_ccp_tests/managers/osmanager.py
@@ -11,7 +11,6 @@
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the 11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations 12# License for the specific language governing permissions and limitations
13# under the License. 13# under the License.
14import os
15 14
16from fuel_ccp_tests import logger 15from fuel_ccp_tests import logger
17from fuel_ccp_tests import settings 16from fuel_ccp_tests import settings
@@ -32,8 +31,7 @@ class OSManager(object):
32 self.__k8s_actions = k8s_actions 31 self.__k8s_actions = k8s_actions
33 self.__ccpcluster = ccpcluster 32 self.__ccpcluster = ccpcluster
34 33
35 def install_os(self, topology=None, 34 def install_os(self, check_os_ready=True):
36 check_os_ready=True):
37 """Action to deploy openstack by ccp tool 35 """Action to deploy openstack by ccp tool
38 36
39 Additional steps: 37 Additional steps:
@@ -47,7 +45,7 @@ class OSManager(object):
47 LOG.info("Trying to install k8s") 45 LOG.info("Trying to install k8s")
48 46
49 """ 47 """
50 Deploy openstack with stacklight topology 48 Deploy openstack with provided topology
51 """ 49 """
52 LOG.info("Preparing openstack log collector fixture...") 50 LOG.info("Preparing openstack log collector fixture...")
53 if settings.REGISTRY == "127.0.0.1:31500": 51 if settings.REGISTRY == "127.0.0.1:31500":
@@ -55,16 +53,6 @@ class OSManager(object):
55 self.__k8s_actions.create_registry() 53 self.__k8s_actions.create_registry()
56 LOG.info("Building images...") 54 LOG.info("Building images...")
57 self.__ccpcluster.build() 55 self.__ccpcluster.build()
58 if topology:
59 LOG.info("Pushing topology yaml...")
60 LOG.warn(
61 "Patched topology used, workaround until kube 1.4 released")
62 topology_path = \
63 os.getcwd() + topology
64 self.__underlay.remote(
65 host=self.__config.k8s.kube_host).upload(
66 topology_path,
67 settings.DEPLOY_CONFIG)
68 LOG.info("Deploy openstack") 56 LOG.info("Deploy openstack")
69 self.__ccpcluster.deploy() 57 self.__ccpcluster.deploy()
70 if check_os_ready: 58 if check_os_ready:
@@ -77,8 +65,8 @@ class OSManager(object):
77 if check_jobs_ready: 65 if check_jobs_ready:
78 LOG.info("Checking openstack jobs statuses...") 66 LOG.info("Checking openstack jobs statuses...")
79 post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api, 67 post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api,
80 timeout=3600) 68 timeout=4500)
81 if check_pods_ready: 69 if check_pods_ready:
82 LOG.info("Checking openstack pods statuses...") 70 LOG.info("Checking openstack pods statuses...")
83 post_os_deploy_checks.check_pods_status(self.__k8s_actions.api, 71 post_os_deploy_checks.check_pods_status(self.__k8s_actions.api,
84 timeout=3600) 72 timeout=4500)
diff --git a/fuel_ccp_tests/requirements.txt b/fuel_ccp_tests/requirements.txt
index ea3fba7..e05ff7e 100644
--- a/fuel_ccp_tests/requirements.txt
+++ b/fuel_ccp_tests/requirements.txt
@@ -11,3 +11,4 @@ psycopg2
11python-k8sclient==0.3.0 11python-k8sclient==0.3.0
12junit-xml 12junit-xml
13elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0 13elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0
14kombu>=3.0.25 # BSD
diff --git a/fuel_ccp_tests/settings.py b/fuel_ccp_tests/settings.py
index a51c636..3fd2421 100644
--- a/fuel_ccp_tests/settings.py
+++ b/fuel_ccp_tests/settings.py
@@ -123,8 +123,8 @@ CCP_DEPLOY_CONFIG = '~/.ccp.deploy-config.yaml'
123CCP_DEPLOY_TOPOLOGY = '~/.ccp.deploy-topology.yaml' 123CCP_DEPLOY_TOPOLOGY = '~/.ccp.deploy-topology.yaml'
124TOPOLOGY_PATH = os.environ.get('TOPOLOGY_PATH', 124TOPOLOGY_PATH = os.environ.get('TOPOLOGY_PATH',
125 os.getcwd() + '/fuel_ccp_tests/templates/' 125 os.getcwd() + '/fuel_ccp_tests/templates/'
126 'k8s_templates/k8s_topology.yaml') 126 'ccp_deploy_topology/'
127 127 'default_deploy_topology.yaml')
128 128
129FUEL_CCP_KEYSTONE_LOCAL_REPO = os.environ.get('FUEL_CCP_KEYSTONE_LOCAL_REPO', 129FUEL_CCP_KEYSTONE_LOCAL_REPO = os.environ.get('FUEL_CCP_KEYSTONE_LOCAL_REPO',
130 None) 130 None)
diff --git a/fuel_ccp_tests/settings_oslo.py b/fuel_ccp_tests/settings_oslo.py
index 0c332e9..276499d 100644
--- a/fuel_ccp_tests/settings_oslo.py
+++ b/fuel_ccp_tests/settings_oslo.py
@@ -26,6 +26,9 @@ from fuel_ccp_tests import settings
26_default_conf = pkg_resources.resource_filename( 26_default_conf = pkg_resources.resource_filename(
27 __name__, 'templates/default.yaml') 27 __name__, 'templates/default.yaml')
28 28
29_default_topology = pkg_resources.resource_filename(
30 __name__, 'templates/ccp_deploy_topology/default_deploy_topology.yaml')
31
29 32
30hardware_opts = [ 33hardware_opts = [
31 ct.Cfg('manager', ct.String(), 34 ct.Cfg('manager', ct.String(),
@@ -103,7 +106,9 @@ ccp_deploy_opts = [
103 ct.Cfg('ccp_globals', ct.JSONDict(), 106 ct.Cfg('ccp_globals', ct.JSONDict(),
104 help="", default=None), 107 help="", default=None),
105 ct.Cfg('ccp_params', ct.JSONDict(), 108 ct.Cfg('ccp_params', ct.JSONDict(),
106 help="", default=None) 109 help="", default=None),
110 ct.Cfg('topology_path', ct.String(),
111 help="", default=_default_topology),
107] 112]
108 113
109# Access credentials to a ready CCP 114# Access credentials to a ready CCP
diff --git a/fuel_ccp_tests/templates/k8s_templates/1ctrl.yaml b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl.yaml
index 5fbaa07..5fbaa07 100644
--- a/fuel_ccp_tests/templates/k8s_templates/1ctrl.yaml
+++ b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl.yaml
diff --git a/fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp.yaml b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp.yaml
index 792eada..792eada 100644
--- a/fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp.yaml
+++ b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp.yaml
diff --git a/fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_diff_ctrl.yaml b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_diff_ctrl.yaml
index fb0c5ad..fb0c5ad 100644
--- a/fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_diff_ctrl.yaml
+++ b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_diff_ctrl.yaml
diff --git a/fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_same_ctrl.yaml b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_same_ctrl.yaml
index 21109cb..21109cb 100644
--- a/fuel_ccp_tests/templates/k8s_templates/1ctrl_1comp_same_ctrl.yaml
+++ b/fuel_ccp_tests/templates/ccp_deploy_topology/1ctrl_1comp_same_ctrl.yaml
diff --git a/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml b/fuel_ccp_tests/templates/ccp_deploy_topology/3galera_1comp.yaml
index 8d81efd..8d81efd 100644
--- a/fuel_ccp_tests/templates/k8s_templates/3galera_1comp.yaml
+++ b/fuel_ccp_tests/templates/ccp_deploy_topology/3galera_1comp.yaml
diff --git a/fuel_ccp_tests/templates/k8s_templates/k8s_topology.yaml b/fuel_ccp_tests/templates/ccp_deploy_topology/default_deploy_topology.yaml
index 7039899..05598b8 100644
--- a/fuel_ccp_tests/templates/k8s_templates/k8s_topology.yaml
+++ b/fuel_ccp_tests/templates/ccp_deploy_topology/default_deploy_topology.yaml
@@ -7,6 +7,9 @@ nodes:
7 roles: 7 roles:
8 - compute 8 - compute
9 - openvswitch 9 - openvswitch
10 node[1-3]:
11 roles:
12 - rabbitmq
10roles: 13roles:
11 controller: 14 controller:
12 - etcd 15 - etcd
@@ -27,7 +30,6 @@ roles:
27 - nova-consoleauth 30 - nova-consoleauth
28 - nova-novncproxy 31 - nova-novncproxy
29 - nova-scheduler 32 - nova-scheduler
30 - rabbitmq
31 compute: 33 compute:
32 - nova-compute 34 - nova-compute
33 - nova-libvirt 35 - nova-libvirt
@@ -35,3 +37,5 @@ roles:
35 - neutron-openvswitch-agent 37 - neutron-openvswitch-agent
36 - openvswitch-db 38 - openvswitch-db
37 - openvswitch-vswitchd 39 - openvswitch-vswitchd
40 rabbitmq:
41 - rabbitmq \ No newline at end of file
diff --git a/fuel_ccp_tests/templates/k8s_templates/stacklight_topology.yaml b/fuel_ccp_tests/templates/ccp_deploy_topology/stacklight_topology.yaml
index b81130f..b81130f 100644
--- a/fuel_ccp_tests/templates/k8s_templates/stacklight_topology.yaml
+++ b/fuel_ccp_tests/templates/ccp_deploy_topology/stacklight_topology.yaml
diff --git a/fuel_ccp_tests/templates/k8s_templates/build-deploy_cluster.yaml b/fuel_ccp_tests/templates/k8s_templates/build-deploy_cluster.yaml
deleted file mode 100644
index 1e1cbf9..0000000
--- a/fuel_ccp_tests/templates/k8s_templates/build-deploy_cluster.yaml
+++ /dev/null
@@ -1,17 +0,0 @@
1ccp-microservices-options:
2- images-base-distro: debian
3- images-maintainer: mos-microservices@mirantis.com
4- repositories-protocol: https
5- repositories-port: 443
6- builder-push
7- registry-address: {registry_address}
8- logfile: /var/log/microservices.log
9- verbose
10- debug
11- builder-workers: 1
12- registry-insecure
13- images-tag: {images_tag}
14- deploy-config: {deploy_config}
15- images-namespace: {images_namespace}
16dry_run_options:
17 export_dir: {export_dir}
diff --git a/fuel_ccp_tests/tests/system/pre_commit/test_glance_api.py b/fuel_ccp_tests/tests/system/pre_commit/test_glance_api.py
index 177f91e..2356c43 100644
--- a/fuel_ccp_tests/tests/system/pre_commit/test_glance_api.py
+++ b/fuel_ccp_tests/tests/system/pre_commit/test_glance_api.py
@@ -51,7 +51,8 @@ class TestServiceGlance(object):
51 ccpcluster.build() 51 ccpcluster.build()
52 52
53 topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \ 53 topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
54 'k8s_templates/k8s_topology.yaml' 54 'ccp_deploy_topology/' \
55 'default_deploy_topology.yaml'
55 remote.upload(topology_path, './') 56 remote.upload(topology_path, './')
56 with remote.get_sudo(remote): 57 with remote.get_sudo(remote):
57 ccpcluster.deploy() 58 ccpcluster.deploy()
diff --git a/fuel_ccp_tests/tests/system/pre_commit/test_horizon.py b/fuel_ccp_tests/tests/system/pre_commit/test_horizon.py
index 8298a48..a98c15d 100644
--- a/fuel_ccp_tests/tests/system/pre_commit/test_horizon.py
+++ b/fuel_ccp_tests/tests/system/pre_commit/test_horizon.py
@@ -63,7 +63,8 @@ class TestServiceHorizon(object):
63 ccpcluster.build() 63 ccpcluster.build()
64 64
65 topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \ 65 topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
66 'k8s_templates/k8s_topology.yaml' 66 'ccp_deploy_topology/' \
67 'default_deploy_topology.yaml'
67 remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config']) 68 remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
68 ccpcluster.deploy() 69 ccpcluster.deploy()
69 post_os_deploy_checks.check_jobs_status(k8sclient, timeout=1500, 70 post_os_deploy_checks.check_jobs_status(k8sclient, timeout=1500,
diff --git a/fuel_ccp_tests/tests/system/pre_commit/test_stacklight.py b/fuel_ccp_tests/tests/system/pre_commit/test_stacklight.py
index 5dff016..7ecd6ee 100644
--- a/fuel_ccp_tests/tests/system/pre_commit/test_stacklight.py
+++ b/fuel_ccp_tests/tests/system/pre_commit/test_stacklight.py
@@ -64,7 +64,8 @@ class TestPreStackLight(object):
64 ccpcluster.build(suppress_output=False) 64 ccpcluster.build(suppress_output=False)
65 65
66 topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \ 66 topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
67 'k8s_templates/stacklight_topology.yaml' 67 'ccp_deploy_topology/' \
68 'stacklight_topology.yaml'
68 69
69 remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config']) 70 remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
70 ccpcluster.deploy() 71 ccpcluster.deploy()
diff --git a/fuel_ccp_tests/tests/system/test_few_os.py b/fuel_ccp_tests/tests/system/test_few_os.py
index dc4a02b..c309b37 100644
--- a/fuel_ccp_tests/tests/system/test_few_os.py
+++ b/fuel_ccp_tests/tests/system/test_few_os.py
@@ -49,7 +49,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
49 k8s_actions.create_registry() 49 k8s_actions.create_registry()
50 ccpcluster.build() 50 ccpcluster.build()
51 topology_path = \ 51 topology_path = \
52 os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \ 52 os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
53 '1ctrl_1comp.yaml' 53 '1ctrl_1comp.yaml'
54 remote = underlay.remote(host=config.k8s.kube_host) 54 remote = underlay.remote(host=config.k8s.kube_host)
55 remote.upload(topology_path, '/tmp') 55 remote.upload(topology_path, '/tmp')
@@ -71,7 +71,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
71 timeout=600) 71 timeout=600)
72 72
73 topology_path = \ 73 topology_path = \
74 os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \ 74 os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
75 '1ctrl_1comp_diff.yaml' 75 '1ctrl_1comp_diff.yaml'
76 remote.upload(topology_path, '/tmp') 76 remote.upload(topology_path, '/tmp')
77 conf = copy.deepcopy(settings.CCP_CONF) 77 conf = copy.deepcopy(settings.CCP_CONF)
@@ -117,7 +117,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
117 k8s_actions.create_registry() 117 k8s_actions.create_registry()
118 ccpcluster.build() 118 ccpcluster.build()
119 topology_path = \ 119 topology_path = \
120 os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \ 120 os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
121 '1ctrl_1comp.yaml' 121 '1ctrl_1comp.yaml'
122 remote = underlay.remote(host=config.k8s.kube_host) 122 remote = underlay.remote(host=config.k8s.kube_host)
123 remote.upload(topology_path, '/tmp') 123 remote.upload(topology_path, '/tmp')
@@ -139,7 +139,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
139 timeout=600) 139 timeout=600)
140 140
141 topology_path = \ 141 topology_path = \
142 os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \ 142 os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
143 '1ctrl_1comp_same.yaml' 143 '1ctrl_1comp_same.yaml'
144 remote.upload(topology_path, '/tmp') 144 remote.upload(topology_path, '/tmp')
145 conf = copy.deepcopy(settings.CCP_CONF) 145 conf = copy.deepcopy(settings.CCP_CONF)
@@ -186,7 +186,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
186 k8s_actions.create_registry() 186 k8s_actions.create_registry()
187 ccpcluster.build() 187 ccpcluster.build()
188 topology_path = \ 188 topology_path = \
189 os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \ 189 os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
190 '1ctrl.yaml' 190 '1ctrl.yaml'
191 remote = underlay.remote(host=config.k8s.kube_host) 191 remote = underlay.remote(host=config.k8s.kube_host)
192 remote.upload(topology_path, '/tmp') 192 remote.upload(topology_path, '/tmp')
diff --git a/fuel_ccp_tests/tests/system/test_rabbitmq.py b/fuel_ccp_tests/tests/system/test_rabbitmq.py
new file mode 100644
index 0000000..b54ed43
--- /dev/null
+++ b/fuel_ccp_tests/tests/system/test_rabbitmq.py
@@ -0,0 +1,487 @@
1# Copyright 2016 Mirantis, Inc.
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may
4# not use this file except in compliance with the License. You may obtain
5# a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations
13# under the License.
14
15import pytest
16import time
17from devops.helpers import helpers
18
19import base_test
20from fuel_ccp_tests import logger
21from fuel_ccp_tests import settings
22from fuel_ccp_tests.helpers import post_os_deploy_checks
23from fuel_ccp_tests.helpers import rabbit
24
25LOG = logger.logger
26
27
28class TestRabbitmq(base_test.SystemBaseTest):
29 """ Galera scale and destructive scenarios
30
31 """
32 def get_pods(self, k8s):
33 return \
34 k8s.get_running_pods('rabbit',
35 settings.CCP_CONF['kubernetes']['namespace'])
36
37 @pytest.mark.fail_snapshot
38 @pytest.mark.rabbitmq_deploy
39 @pytest.mark.rabbitmq
40 def test_rabbitmq(self, underlay, config,
41 k8s_actions, show_step,
42 os_deployed, rabbit_client):
43 """Deploy rabbit cluster
44
45 Scenario:
46 1. Revert snapshot with deployed rabbit
47 2. Check rabbit cluster state
48 3. Check queue, messages replication
49
50 Duration 30 min
51 """
52 show_step(2)
53 show_step(3)
54 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
55 remote = underlay.remote(host=config.k8s.kube_host)
56 rabbit_pods = self.get_pods(k8s_actions)
57 queue = rabbit_client.create_queue()
58 message = rabbit_client.publish_message_to_queue(queue)
59 rabbit_client.check_queue_message(message)
60 for pod in rabbit_pods:
61 rabbit_cluster_nodes = rabbit_client.list_nodes(
62 remote, pod.name, namespace)
63 assert rabbit_cluster_nodes == len(rabbit_pods),\
64 "Expected to have {} nodes in cluster," \
65 " got {}".format(len(rabbit_pods), rabbit_cluster_nodes)
66 rabbit_client.check_queue_replicated(queue, remote,
67 pod.name, namespace)
68 rabbit_client.delete_queue(queue)
69
70 @pytest.mark.fail_snapshot
71 @pytest.mark.rabbitmq_shutdown
72 @pytest.mark.rabbitmq
73 def test_rabbitmq_shutdown_node(self, hardware, underlay, config,
74 ccpcluster, k8s_actions, show_step,
75 os_deployed, rabbit_client):
76 """Shutdown rabbitmq node
77
78 Scenario:
79 1. Revert snapshot with deployed rabbit
80 2. Shutdown one rabbit node
81 3. Check rabbit cluster state
82 4. Check queue, messages replication
83 5. Create 2 vms
84
85 Duration 30 min
86 """
87 rabbit_node = underlay.node_names()[1]
88 rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
89 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
90 rabbit_pods = self.get_pods(k8s_actions)
91 show_step(2)
92 hardware.shutdown_node_by_ip(rabbit_node_ip)
93 show_step(3)
94
95 helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
96 len(rabbit_pods) - 1),
97 timeout=600,
98 timeout_msg='Timeout waiting for rabbit pod'
99 ' to be terminated')
100 pods_after_shutdown = self.get_pods(k8s_actions)
101 remote = underlay.remote(host=config.k8s.kube_host)
102 show_step(4)
103 queue = rabbit_client.create_queue()
104 message = rabbit_client.publish_message_to_queue(queue)
105 rabbit_client.check_queue_message(message)
106 for pod in pods_after_shutdown:
107 rabbit_cluster_nodes = rabbit_client.list_nodes(
108 remote, pod.name, namespace)
109 assert rabbit_cluster_nodes == len(pods_after_shutdown),\
110 "Expected to have {} nodes in cluster," \
111 " got {}".format(len(pods_after_shutdown),
112 rabbit_cluster_nodes)
113 rabbit_client.check_queue_replicated(queue, remote,
114 pod.name, namespace)
115 rabbit_client.delete_queue(queue)
116 show_step(5)
117 remote.check_call(
118 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
119 " create".format(namespace), timeout=600)
120
121 @pytest.mark.fail_snapshot
122 @pytest.mark.rabbitmq_cold_restart
123 @pytest.mark.rabbitmq
124 def test_rabbitmq_cold_restart_node(self, hardware, underlay, config,
125 ccpcluster, k8s_actions, show_step,
126 os_deployed, rabbit_client):
127 """Cold restart rabbitmq node
128
129 Scenario:
130 1. Revert snapshot with deployed rabbit
131 2. Cold restart one rabbit node
132 3. Check rabbit cluster state
133 4. Check queue, messages replication
134 5. Create 2 vms
135
136 Duration 30 min
137 """
138 rabbit_node = underlay.node_names()[1]
139 rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
140 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
141 rabbit_pods = self.get_pods(k8s_actions)
142
143 show_step(2)
144 hardware.shutdown_node_by_ip(rabbit_node_ip)
145 hardware.wait_node_is_offline(rabbit_node_ip, 90)
146 time.sleep(15)
147 hardware.start_node_by_ip(rabbit_node_ip)
148 hardware.wait_node_is_online(rabbit_node_ip, 180)
149 show_step(3)
150
151 helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
152 len(rabbit_pods) - 1),
153 timeout=1200,
154 timeout_msg='Expected to have one pod destroyed'
155 ' after reboot')
156
157 helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
158 len(rabbit_pods)),
159 timeout=1200,
160 timeout_msg='Expected pod to come back after reboot')
161
162 pods_after_reboot = self.get_pods(k8s_actions)
163 remote = underlay.remote(host=config.k8s.kube_host)
164 show_step(4)
165 queue = rabbit_client.create_queue()
166 message = rabbit_client.publish_message_to_queue(queue)
167 rabbit_client.check_queue_message(message)
168 for pod in pods_after_reboot:
169 rabbit_cluster_nodes = rabbit_client.list_nodes(
170 remote, pod.name, namespace)
171 assert rabbit_cluster_nodes == len(pods_after_reboot),\
172 "Expected to have {} nodes in cluster," \
173 " got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
174 rabbit_client.check_queue_replicated(queue, remote,
175 pod.name, namespace)
176 rabbit_client.delete_queue(queue)
177 show_step(5)
178 remote.check_call(
179 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
180 " create".format(namespace), timeout=600)
181
182 @pytest.mark.fail_snapshot
183 @pytest.mark.rabbitmq_poweroff
184 @pytest.mark.rabbitmq
185 def test_rabbitmq_poweroff_node(self, hardware, underlay, config,
186 ccpcluster, k8s_actions, show_step,
187 os_deployed, rabbit_client):
188 """Poweroff rabbit node
189
190 Scenario:
191 1. Revert snapshot with deployed rabbit
192 2. Poweroff one rabbit node
193 3. Check rabbit cluster state
194 4. Check queue, messages replication
195 5. Create 2 vms
196
197 Duration 30 min
198 """
199 rabbit_node = underlay.node_names()[1]
200 rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
201 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
202 rabbit_pods = self.get_pods(k8s_actions)
203 show_step(2)
204 underlay.sudo_check_call('shutdown +1', node_name=rabbit_node)
205 hardware.shutdown_node_by_ip(rabbit_node_ip)
206 hardware.wait_node_is_offline(rabbit_node_ip, 90)
207 show_step(3)
208
209 helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
210 len(rabbit_pods) - 1),
211 timeout=600,
212 timeout_msg='Timeout waiting for rabbit pod'
213 ' to be terminated')
214 pods_after_reboot = self.get_pods(k8s_actions)
215 remote = underlay.remote(host=config.k8s.kube_host)
216 show_step(4)
217 queue = rabbit_client.create_queue()
218 message = rabbit_client.publish_message_to_queue(queue)
219 rabbit_client.check_queue_message(message)
220 for pod in pods_after_reboot:
221 rabbit_cluster_nodes = rabbit_client.list_nodes(
222 remote, pod.name, namespace)
223 assert rabbit_cluster_nodes == len(pods_after_reboot),\
224 "Expected to have {} nodes in cluster," \
225 " got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
226 rabbit_client.check_queue_replicated(queue, remote,
227 pod.name, namespace)
228 rabbit_client.delete_queue(queue)
229 show_step(5)
230 remote.check_call(
231 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
232 " create".format(namespace), timeout=600)
233
234 @pytest.mark.fail_snapshot
235 @pytest.mark.rabbitmq_soft_reboot
236 @pytest.mark.rabbitmq
237 def test_rabbitmq_soft_reboot_node(self, hardware, underlay, config,
238 ccpcluster, k8s_actions, show_step,
239 os_deployed, rabbit_client):
240 """Soft reboot rabbitmq node
241
242 Scenario:
243 1. Revert snapshot with deployed rabbit
244 2. Reboot one rabbit node
245 3. Check rabbit cluster state
246 4. Check queue, messages replication
247 5. Create 2 vms
248
249 Duration 30 min
250 """
251 rabbit_node = underlay.node_names()[1]
252 rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
253 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
254 rabbit_pods = self.get_pods(k8s_actions)
255 show_step(2)
256 underlay.sudo_check_call('shutdown +1', node_name=rabbit_node)
257 hardware.shutdown_node_by_ip(rabbit_node_ip)
258 hardware.wait_node_is_offline(rabbit_node_ip, 90)
259 time.sleep(15)
260 hardware.start_node_by_ip(rabbit_node_ip)
261 hardware.wait_node_is_online(rabbit_node_ip, 180)
262 show_step(3)
263 helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
264 len(rabbit_pods)),
265 timeout=600,
266 timeout_msg='Timeout waiting for rabbit pod'
267 ' to be terminated')
268 pods_after_reboot = self.get_pods(k8s_actions)
269 remote = underlay.remote(host=config.k8s.kube_host)
270 show_step(4)
271 queue = rabbit_client.create_queue()
272 message = rabbit_client.publish_message_to_queue(queue)
273 rabbit_client.check_queue_message(message)
274 for pod in pods_after_reboot:
275 rabbit_cluster_nodes = rabbit_client.list_nodes(
276 remote, pod.name, namespace)
277 assert rabbit_cluster_nodes == len(pods_after_reboot),\
278 "Expected to have {} nodes in cluster," \
279 " got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
280 rabbit_client.check_queue_replicated(queue, remote,
281 pod.name, namespace)
282 rabbit_client.delete_queue(queue)
283 show_step(5)
284 remote.check_call(
285 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
286 " create".format(namespace), timeout=600)
287
288 @pytest.mark.fail_snapshot
289 @pytest.mark.rabbitmq_cluster_shutdown
290 @pytest.mark.rabbitmq
291 def test_rabbitmq_cluster_shutdown(self, hardware, underlay, config,
292 ccpcluster, k8s_actions, show_step,
293 os_deployed, rabbit_client):
294 """Rabbitmq cluster shutdown
295
296 Scenario:
297 1. Revert snapshot with deployed rabbit
298 2. Shutdown all rabbit nodes and start them one by one
299 3. Check rabbit cluster state
300 4. Check queue, messages replication
301 5. Create 2 vms
302
303 Duration 30 min
304 """
305 rabbit_nodes = underlay.node_names()[:3]
306 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
307 rabbit_pods = self.get_pods(k8s_actions)
308 rabbit_node_ips = []
309 show_step(2)
310 for rabbit_node in rabbit_nodes:
311 rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
312 rabbit_node_ips.append(rabbit_node_ip)
313 hardware.shutdown_node_by_ip(rabbit_node_ip)
314 hardware.wait_node_is_offline(rabbit_node_ip, 90)
315 for rabbit_ip in rabbit_node_ips:
316 hardware.start_node_by_ip(rabbit_ip)
317 hardware.wait_node_is_online(rabbit_ip, 180)
318 show_step(3)
319 post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
320 post_os_deploy_checks.check_pods_status(k8s_actions.api)
321 pods_after_shutdown = self.get_pods(k8s_actions)
322 assert len(rabbit_pods) == len(pods_after_shutdown),\
323 "Different number of pods after shutdown, was {}," \
324 " now {}".format(len(rabbit_pods), len(pods_after_shutdown))
325 remote = underlay.remote(host=config.k8s.kube_host)
326 show_step(4)
327 host = config.k8s.kube_host
328 rabbit_port = ''.join(remote.execute(
329 "kubectl get service --namespace ccp rabbitmq -o yaml |"
330 " awk '/nodePort: / {print $NF}'")['stdout'])
331 rabbit_client = rabbit.RabbitClient(host, rabbit_port)
332 queue = rabbit_client.create_queue()
333 message = rabbit_client.publish_message_to_queue(queue)
334 rabbit_client.check_queue_message(message)
335 for pod in pods_after_shutdown:
336 rabbit_cluster_nodes = rabbit_client.list_nodes(
337 remote, pod.name, namespace)
338 assert rabbit_cluster_nodes == len(pods_after_shutdown),\
339 "Expected to have {} nodes in cluster," \
340 " got {}".format(len(pods_after_shutdown),
341 rabbit_cluster_nodes)
342 rabbit_client.check_queue_replicated(queue, remote,
343 pod.name, namespace)
344 rabbit_client.delete_queue(queue)
345 remote.check_call(
346 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
347 " create".format(namespace), timeout=600)
348
349 @pytest.mark.fail_snapshot
350 @pytest.mark.rabbitmq_scale_up_down
351 @pytest.mark.rabbitmq
352 def test_rabbitmq_scale(self, hardware, underlay, config,
353 ccpcluster, k8s_actions, show_step,
354 os_deployed, rabbit_client):
355 """Rabbit cluster scale
356
357 Scenario:
358 1. Revert snapshot with deployed rabbit
359 2. Scale up rabbit to 5 replicas
360 3. Check rabbit state
361 4. Check number of rabbit pods
362 5. Create 2 vms
363 6. Scale down rabbit to 3 replicas
364 7. Check rabbit state
365 8. Check number of rabbit pods
366 9. Create 2 vms
367
368 Duration 30 min
369 """
370 show_step(2)
371 with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
372 host=config.k8s.kube_host) as editor:
373 del editor.content['nodes']['node[1-3]']
374 editor.content['nodes']['node[1-5]'] = {'roles': ['rabbitmq']}
375
376 ccpcluster.deploy()
377 post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
378 post_os_deploy_checks.check_pods_status(k8s_actions.api)
379 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
380 remote = underlay.remote(host=config.k8s.kube_host)
381 rabbit_pods = self.get_pods(k8s_actions)
382 queue = rabbit_client.create_queue()
383 message = rabbit_client.publish_message_to_queue(queue)
384 rabbit_client.check_queue_message(message)
385 for pod in rabbit_pods:
386 rabbit_cluster_nodes = rabbit_client.list_nodes(
387 remote, pod.name, namespace)
388 assert rabbit_cluster_nodes == len(rabbit_pods),\
389 "Expected to have {} nodes in cluster," \
390 " got {}".format(len(rabbit_pods), rabbit_cluster_nodes)
391 rabbit_client.check_queue_replicated(queue, remote,
392 pod.name, namespace)
393 rabbit_client.delete_queue(queue)
394
395 show_step(4)
396 rabbit_pods = \
397 k8s_actions.get_pods_number('rabbit', namespace)
398 assert rabbit_pods == 5,\
399 "Expcted to have 5 rabbit pods, got {}".format(rabbit_pods)
400
401 show_step(5)
402 remote.check_call(
403 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
404 " create".format(namespace), timeout=600)
405 show_step(6)
406 with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
407 host=config.k8s.kube_host) as editor:
408 del editor.content['nodes']['node[1-5]']
409 editor.content['nodes']['node[1-3]'] = {'roles': ['rabbitmq']}
410
411 ccpcluster.deploy()
412 post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
413 post_os_deploy_checks.check_pods_status(k8s_actions.api)
414 show_step(7)
415 show_step(8)
416 rabbit_pods = \
417 k8s_actions.get_pods_number('rabbit', namespace)
418 assert rabbit_pods == 3,\
419 "Expcted to have 3 rabbit pods, got {}".format(rabbit_pods)
420 show_step(9)
421 remote.check_call(
422 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
423 " create".format(namespace), timeout=600)
424
425 @pytest.mark.fail_snapshot
426 @pytest.mark.rabbitmq_node_replacement
427 @pytest.mark.rabbitmq
428 def test_rabbitmq_node_replacement(self, hardware, underlay, config,
429 ccpcluster, k8s_actions, show_step,
430 os_deployed, rabbit_client):
431 """Rabbitmq node replacement
432
433 Scenario:
434 1. Revert snapshot with deployed rabbit
435 2. Shutdown one rabbit node
436 3. Add new rabbit node to config
437 4. Re-deploy cluster
438 5. Check rabbit cluster state
439 6. Check queue, messages replication
440 7. Create 2 vms
441
442 Duration 30 min
443 """
444 rabbit_node = underlay.node_names()[1]
445 rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
446 namespace = settings.CCP_CONF["kubernetes"]["namespace"]
447 rabbit_pods = self.get_pods(k8s_actions)
448 show_step(2)
449 hardware.shutdown_node_by_ip(rabbit_node_ip)
450
451 helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
452 len(rabbit_pods) - 1),
453 timeout=600,
454 timeout_msg='Timeout waiting for rabbit pod'
455 ' to be terminated')
456 show_step(3)
457 with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
458 host=config.k8s.kube_host) as editor:
459 del editor.content['nodes']['node[1-3]']
460 editor.content['nodes']['node[1-2]'] = {'roles': ['rabbitmq']}
461 editor.content['nodes']['node4'] = {'roles': ['rabbitmq', 'etcd']}
462 show_step(4)
463 ccpcluster.deploy()
464 post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
465 post_os_deploy_checks.check_pods_status(k8s_actions.api)
466
467 pods_after_replacement = self.get_pods(k8s_actions)
468 remote = underlay.remote(host=config.k8s.kube_host)
469 show_step(6)
470 queue = rabbit_client.create_queue()
471 message = rabbit_client.publish_message_to_queue(queue)
472 rabbit_client.check_queue_message(message)
473 show_step(5)
474 for pod in pods_after_replacement:
475 rabbit_cluster_nodes = rabbit_client.list_nodes(
476 remote, pod.name, namespace)
477 assert rabbit_cluster_nodes == len(pods_after_replacement),\
478 "Expected to have {} nodes in cluster," \
479 " got {}".format(len(pods_after_replacement),
480 rabbit_cluster_nodes)
481 rabbit_client.check_queue_replicated(queue, remote,
482 pod.name, namespace)
483 rabbit_client.delete_queue(queue)
484 show_step(7)
485 remote.check_call(
486 "source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
487 " create".format(namespace), timeout=600)