Merge "Add tests to check rabbit cluster"
This commit is contained in:
commit
5d3db0e0d4
|
@ -12,7 +12,7 @@ fuel_ccp_tests
|
||||||
├── managers
|
├── managers
|
||||||
│ └── k8s
|
│ └── k8s
|
||||||
├── templates
|
├── templates
|
||||||
│ ├── k8s_templates
|
│ ├── ccp_deploy_topology
|
||||||
│ ├── misc
|
│ ├── misc
|
||||||
│ └── registry_templates
|
│ └── registry_templates
|
||||||
└── tests
|
└── tests
|
||||||
|
@ -78,4 +78,3 @@ To execute tests necessary to add value to several variables via *export* or in
|
||||||
After exporting execute the command:
|
After exporting execute the command:
|
||||||
|
|
||||||
py.test -vvv -s -k <test_name> or py.test -vvv -s -m <test_mark>
|
py.test -vvv -s -k <test_name> or py.test -vvv -s -m <test_mark>
|
||||||
|
|
||||||
|
|
|
@ -76,7 +76,7 @@ def ccpcluster(revert_snapshot, config, hardware,
|
||||||
path=settings.CCP_FETCH_CONFIG,
|
path=settings.CCP_FETCH_CONFIG,
|
||||||
config=settings.CCP_FETCH_PARAMS)
|
config=settings.CCP_FETCH_PARAMS)
|
||||||
|
|
||||||
with open(settings.TOPOLOGY_PATH, 'r') as f:
|
with open(config.ccp_deploy.topology_path, 'r') as f:
|
||||||
ccp_actions.put_raw_config(
|
ccp_actions.put_raw_config(
|
||||||
path=settings.CCP_DEPLOY_TOPOLOGY,
|
path=settings.CCP_DEPLOY_TOPOLOGY,
|
||||||
content=f.read())
|
content=f.read())
|
||||||
|
|
|
@ -15,11 +15,13 @@
|
||||||
from copy import deepcopy
|
from copy import deepcopy
|
||||||
import os
|
import os
|
||||||
import pytest
|
import pytest
|
||||||
|
from devops.helpers import helpers
|
||||||
|
|
||||||
from fuel_ccp_tests import logger
|
from fuel_ccp_tests import logger
|
||||||
from fuel_ccp_tests import settings
|
from fuel_ccp_tests import settings
|
||||||
from fuel_ccp_tests.helpers import ext
|
from fuel_ccp_tests.helpers import ext
|
||||||
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
||||||
|
from fuel_ccp_tests.helpers import rabbit
|
||||||
from fuel_ccp_tests.managers.osmanager import OSManager
|
from fuel_ccp_tests.managers.osmanager import OSManager
|
||||||
|
|
||||||
LOG = logger.logger
|
LOG = logger.logger
|
||||||
|
@ -37,12 +39,7 @@ def os_deployed(ccpcluster,
|
||||||
"""
|
"""
|
||||||
osmanager = OSManager(config, underlay, k8s_actions, ccpcluster)
|
osmanager = OSManager(config, underlay, k8s_actions, ccpcluster)
|
||||||
if not config.os.running:
|
if not config.os.running:
|
||||||
LOG.info("Preparing openstack log collector fixture...")
|
osmanager.install_os()
|
||||||
topology = None
|
|
||||||
if config.os_deploy.stacklight_enable:
|
|
||||||
topology = ('/fuel_ccp_tests/templates/k8s_templates/'
|
|
||||||
'stacklight_topology.yaml')
|
|
||||||
osmanager.install_os(topology=topology)
|
|
||||||
hardware.create_snapshot(ext.SNAPSHOT.os_deployed)
|
hardware.create_snapshot(ext.SNAPSHOT.os_deployed)
|
||||||
else:
|
else:
|
||||||
LOG.info("Openstack allready installed and running...")
|
LOG.info("Openstack allready installed and running...")
|
||||||
|
@ -68,7 +65,7 @@ def galera_deployed(ccpcluster,
|
||||||
k8s_actions.create_registry()
|
k8s_actions.create_registry()
|
||||||
ccpcluster.build()
|
ccpcluster.build()
|
||||||
topology_path = \
|
topology_path = \
|
||||||
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
|
||||||
'3galera_1comp.yaml'
|
'3galera_1comp.yaml'
|
||||||
remote = underlay.remote(host=config.k8s.kube_host)
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
remote.upload(topology_path, '/tmp')
|
remote.upload(topology_path, '/tmp')
|
||||||
|
@ -93,3 +90,17 @@ def galera_deployed(ccpcluster,
|
||||||
|
|
||||||
config.os.running = True
|
config.os.running = True
|
||||||
hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)
|
hardware.create_snapshot(ext.SNAPSHOT.os_galera_deployed)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(scope='function')
|
||||||
|
def rabbit_client(underlay, config, os_deployed):
|
||||||
|
"""Deploy openstack
|
||||||
|
"""
|
||||||
|
host = config.k8s.kube_host
|
||||||
|
remote = underlay.remote(host=host)
|
||||||
|
rabbit_port = ''.join(remote.execute(
|
||||||
|
"kubectl get service --namespace ccp rabbitmq -o yaml |"
|
||||||
|
" awk '/nodePort: / {print $NF}'")['stdout'])
|
||||||
|
client = helpers.wait_pass(lambda: rabbit.RabbitClient(host, rabbit_port),
|
||||||
|
interval=60, timeout=360)
|
||||||
|
return client
|
||||||
|
|
|
@ -0,0 +1,56 @@
|
||||||
|
import kombu
|
||||||
|
|
||||||
|
from fuel_ccp_tests import logger
|
||||||
|
from fuel_ccp_tests.helpers import utils
|
||||||
|
|
||||||
|
LOG = logger.logger
|
||||||
|
|
||||||
|
|
||||||
|
class RabbitClient(object):
|
||||||
|
|
||||||
|
def __init__(self, ip, port, user='rabbitmq', password='password'):
|
||||||
|
c = kombu.Connection("amqp://{0}:{1}@{2}:{3}//".format(user, password,
|
||||||
|
ip, port))
|
||||||
|
c.connect()
|
||||||
|
self.ch = c.channel()
|
||||||
|
|
||||||
|
def list_nodes(self, remote, pod, namespace):
|
||||||
|
output = ''.join(
|
||||||
|
remote.execute("kubectl exec -i {} --namespace={}"
|
||||||
|
" -- rabbitmqctl"
|
||||||
|
" cluster_status".format(pod,
|
||||||
|
namespace))['stdout'])
|
||||||
|
substring_ind = output.find('{running_nodes')
|
||||||
|
sub_end_ind = output.find('cluster_name')
|
||||||
|
result_str = output[substring_ind: sub_end_ind]
|
||||||
|
num_node = result_str.count("rabbit@")
|
||||||
|
return num_node
|
||||||
|
|
||||||
|
def check_queue_replicated(self, queue, remote, pod, namespace):
|
||||||
|
remote.check_call("kubectl exec -i {} --namespace={}"
|
||||||
|
" -- rabbitmqctl list_queues |"
|
||||||
|
" grep {}".format(pod, namespace,
|
||||||
|
queue))
|
||||||
|
|
||||||
|
def create_queue(self):
|
||||||
|
test_queue = 'test-rabbit-{}'.format(utils.rand_name())
|
||||||
|
q = kombu.Queue(test_queue, channel=self.ch, durable=False,
|
||||||
|
queue_arguments={"x-expires": 15 * 60 * 1000})
|
||||||
|
q.declare()
|
||||||
|
return test_queue
|
||||||
|
|
||||||
|
def publish_message_to_queue(self, queue):
|
||||||
|
uid = utils.generate_uuid()
|
||||||
|
producer = kombu.Producer(channel=self.ch, routing_key=queue)
|
||||||
|
producer.publish(uid)
|
||||||
|
return {'queue': queue, 'id': uid}
|
||||||
|
|
||||||
|
def check_queue_message(self, message):
|
||||||
|
q = kombu.Queue(message['queue'], channel=self.ch)
|
||||||
|
msg = q.get(True)
|
||||||
|
assert msg.body in message['id'],\
|
||||||
|
"Message body is {}, expected {}".format(msg.body, message['id'])
|
||||||
|
|
||||||
|
def delete_queue(self, queue):
|
||||||
|
q = kombu.Queue(queue, channel=self.ch)
|
||||||
|
q.delete()
|
|
@ -14,10 +14,12 @@
|
||||||
|
|
||||||
import copy
|
import copy
|
||||||
import os
|
import os
|
||||||
|
import random
|
||||||
import shutil
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
import time
|
import time
|
||||||
import traceback
|
import traceback
|
||||||
|
import uuid
|
||||||
|
|
||||||
import paramiko
|
import paramiko
|
||||||
import yaml
|
import yaml
|
||||||
|
@ -483,3 +485,11 @@ def get_top_fixtures_marks(request, mark_name):
|
||||||
.format(top_fixtures_marks))
|
.format(top_fixtures_marks))
|
||||||
|
|
||||||
return top_fixtures_marks
|
return top_fixtures_marks
|
||||||
|
|
||||||
|
|
||||||
|
def rand_name():
|
||||||
|
return str(random.randint(1, 0x7fffffff))
|
||||||
|
|
||||||
|
|
||||||
|
def generate_uuid():
|
||||||
|
return uuid.uuid4().hex
|
||||||
|
|
|
@ -344,7 +344,11 @@ class K8SManager(object):
|
||||||
LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
|
LOG.info('Added custom upstream DNS servers (dnsmasq) to the '
|
||||||
'settings: {0}'.format(k8s_settings['nameservers']))
|
'settings: {0}'.format(k8s_settings['nameservers']))
|
||||||
|
|
||||||
def get_pods_number(self, pod_name, namespace=None):
|
def get_running_pods(self, pod_name, namespace=None):
|
||||||
pods = [pod for pod in self.api.pods.list(namespace=namespace)
|
pods = [pod for pod in self.api.pods.list(namespace=namespace)
|
||||||
if pod_name in pod.name]
|
if pod_name in pod.name and pod.status.phase == 'Running']
|
||||||
|
return pods
|
||||||
|
|
||||||
|
def get_pods_number(self, pod_name, namespace=None):
|
||||||
|
pods = self.get_running_pods(pod_name, namespace)
|
||||||
return len(pods)
|
return len(pods)
|
||||||
|
|
|
@ -11,7 +11,6 @@
|
||||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
# License for the specific language governing permissions and limitations
|
# License for the specific language governing permissions and limitations
|
||||||
# under the License.
|
# under the License.
|
||||||
import os
|
|
||||||
|
|
||||||
from fuel_ccp_tests import logger
|
from fuel_ccp_tests import logger
|
||||||
from fuel_ccp_tests import settings
|
from fuel_ccp_tests import settings
|
||||||
|
@ -32,8 +31,7 @@ class OSManager(object):
|
||||||
self.__k8s_actions = k8s_actions
|
self.__k8s_actions = k8s_actions
|
||||||
self.__ccpcluster = ccpcluster
|
self.__ccpcluster = ccpcluster
|
||||||
|
|
||||||
def install_os(self, topology=None,
|
def install_os(self, check_os_ready=True):
|
||||||
check_os_ready=True):
|
|
||||||
"""Action to deploy openstack by ccp tool
|
"""Action to deploy openstack by ccp tool
|
||||||
|
|
||||||
Additional steps:
|
Additional steps:
|
||||||
|
@ -47,7 +45,7 @@ class OSManager(object):
|
||||||
LOG.info("Trying to install k8s")
|
LOG.info("Trying to install k8s")
|
||||||
|
|
||||||
"""
|
"""
|
||||||
Deploy openstack with stacklight topology
|
Deploy openstack with provided topology
|
||||||
"""
|
"""
|
||||||
LOG.info("Preparing openstack log collector fixture...")
|
LOG.info("Preparing openstack log collector fixture...")
|
||||||
if settings.REGISTRY == "127.0.0.1:31500":
|
if settings.REGISTRY == "127.0.0.1:31500":
|
||||||
|
@ -55,16 +53,6 @@ class OSManager(object):
|
||||||
self.__k8s_actions.create_registry()
|
self.__k8s_actions.create_registry()
|
||||||
LOG.info("Building images...")
|
LOG.info("Building images...")
|
||||||
self.__ccpcluster.build()
|
self.__ccpcluster.build()
|
||||||
if topology:
|
|
||||||
LOG.info("Pushing topology yaml...")
|
|
||||||
LOG.warn(
|
|
||||||
"Patched topology used, workaround until kube 1.4 released")
|
|
||||||
topology_path = \
|
|
||||||
os.getcwd() + topology
|
|
||||||
self.__underlay.remote(
|
|
||||||
host=self.__config.k8s.kube_host).upload(
|
|
||||||
topology_path,
|
|
||||||
settings.DEPLOY_CONFIG)
|
|
||||||
LOG.info("Deploy openstack")
|
LOG.info("Deploy openstack")
|
||||||
self.__ccpcluster.deploy()
|
self.__ccpcluster.deploy()
|
||||||
if check_os_ready:
|
if check_os_ready:
|
||||||
|
@ -77,8 +65,8 @@ class OSManager(object):
|
||||||
if check_jobs_ready:
|
if check_jobs_ready:
|
||||||
LOG.info("Checking openstack jobs statuses...")
|
LOG.info("Checking openstack jobs statuses...")
|
||||||
post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api,
|
post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api,
|
||||||
timeout=3600)
|
timeout=4500)
|
||||||
if check_pods_ready:
|
if check_pods_ready:
|
||||||
LOG.info("Checking openstack pods statuses...")
|
LOG.info("Checking openstack pods statuses...")
|
||||||
post_os_deploy_checks.check_pods_status(self.__k8s_actions.api,
|
post_os_deploy_checks.check_pods_status(self.__k8s_actions.api,
|
||||||
timeout=3600)
|
timeout=4500)
|
||||||
|
|
|
@ -11,3 +11,4 @@ psycopg2
|
||||||
python-k8sclient==0.3.0
|
python-k8sclient==0.3.0
|
||||||
junit-xml
|
junit-xml
|
||||||
elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0
|
elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0
|
||||||
|
kombu>=3.0.25 # BSD
|
||||||
|
|
|
@ -123,8 +123,8 @@ CCP_DEPLOY_CONFIG = '~/.ccp.deploy-config.yaml'
|
||||||
CCP_DEPLOY_TOPOLOGY = '~/.ccp.deploy-topology.yaml'
|
CCP_DEPLOY_TOPOLOGY = '~/.ccp.deploy-topology.yaml'
|
||||||
TOPOLOGY_PATH = os.environ.get('TOPOLOGY_PATH',
|
TOPOLOGY_PATH = os.environ.get('TOPOLOGY_PATH',
|
||||||
os.getcwd() + '/fuel_ccp_tests/templates/'
|
os.getcwd() + '/fuel_ccp_tests/templates/'
|
||||||
'k8s_templates/k8s_topology.yaml')
|
'ccp_deploy_topology/'
|
||||||
|
'default_deploy_topology.yaml')
|
||||||
|
|
||||||
FUEL_CCP_KEYSTONE_LOCAL_REPO = os.environ.get('FUEL_CCP_KEYSTONE_LOCAL_REPO',
|
FUEL_CCP_KEYSTONE_LOCAL_REPO = os.environ.get('FUEL_CCP_KEYSTONE_LOCAL_REPO',
|
||||||
None)
|
None)
|
||||||
|
|
|
@ -26,6 +26,9 @@ from fuel_ccp_tests import settings
|
||||||
_default_conf = pkg_resources.resource_filename(
|
_default_conf = pkg_resources.resource_filename(
|
||||||
__name__, 'templates/default.yaml')
|
__name__, 'templates/default.yaml')
|
||||||
|
|
||||||
|
_default_topology = pkg_resources.resource_filename(
|
||||||
|
__name__, 'templates/ccp_deploy_topology/default_deploy_topology.yaml')
|
||||||
|
|
||||||
|
|
||||||
hardware_opts = [
|
hardware_opts = [
|
||||||
ct.Cfg('manager', ct.String(),
|
ct.Cfg('manager', ct.String(),
|
||||||
|
@ -103,7 +106,9 @@ ccp_deploy_opts = [
|
||||||
ct.Cfg('ccp_globals', ct.JSONDict(),
|
ct.Cfg('ccp_globals', ct.JSONDict(),
|
||||||
help="", default=None),
|
help="", default=None),
|
||||||
ct.Cfg('ccp_params', ct.JSONDict(),
|
ct.Cfg('ccp_params', ct.JSONDict(),
|
||||||
help="", default=None)
|
help="", default=None),
|
||||||
|
ct.Cfg('topology_path', ct.String(),
|
||||||
|
help="", default=_default_topology),
|
||||||
]
|
]
|
||||||
|
|
||||||
# Access credentials to a ready CCP
|
# Access credentials to a ready CCP
|
||||||
|
|
|
@ -7,6 +7,9 @@ nodes:
|
||||||
roles:
|
roles:
|
||||||
- compute
|
- compute
|
||||||
- openvswitch
|
- openvswitch
|
||||||
|
node[1-3]:
|
||||||
|
roles:
|
||||||
|
- rabbitmq
|
||||||
roles:
|
roles:
|
||||||
controller:
|
controller:
|
||||||
- etcd
|
- etcd
|
||||||
|
@ -27,7 +30,6 @@ roles:
|
||||||
- nova-consoleauth
|
- nova-consoleauth
|
||||||
- nova-novncproxy
|
- nova-novncproxy
|
||||||
- nova-scheduler
|
- nova-scheduler
|
||||||
- rabbitmq
|
|
||||||
compute:
|
compute:
|
||||||
- nova-compute
|
- nova-compute
|
||||||
- nova-libvirt
|
- nova-libvirt
|
||||||
|
@ -35,3 +37,5 @@ roles:
|
||||||
- neutron-openvswitch-agent
|
- neutron-openvswitch-agent
|
||||||
- openvswitch-db
|
- openvswitch-db
|
||||||
- openvswitch-vswitchd
|
- openvswitch-vswitchd
|
||||||
|
rabbitmq:
|
||||||
|
- rabbitmq
|
|
@ -1,17 +0,0 @@
|
||||||
ccp-microservices-options:
|
|
||||||
- images-base-distro: debian
|
|
||||||
- images-maintainer: mos-microservices@mirantis.com
|
|
||||||
- repositories-protocol: https
|
|
||||||
- repositories-port: 443
|
|
||||||
- builder-push
|
|
||||||
- registry-address: {registry_address}
|
|
||||||
- logfile: /var/log/microservices.log
|
|
||||||
- verbose
|
|
||||||
- debug
|
|
||||||
- builder-workers: 1
|
|
||||||
- registry-insecure
|
|
||||||
- images-tag: {images_tag}
|
|
||||||
- deploy-config: {deploy_config}
|
|
||||||
- images-namespace: {images_namespace}
|
|
||||||
dry_run_options:
|
|
||||||
export_dir: {export_dir}
|
|
|
@ -51,7 +51,8 @@ class TestServiceGlance(object):
|
||||||
ccpcluster.build()
|
ccpcluster.build()
|
||||||
|
|
||||||
topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
|
topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
|
||||||
'k8s_templates/k8s_topology.yaml'
|
'ccp_deploy_topology/' \
|
||||||
|
'default_deploy_topology.yaml'
|
||||||
remote.upload(topology_path, './')
|
remote.upload(topology_path, './')
|
||||||
with remote.get_sudo(remote):
|
with remote.get_sudo(remote):
|
||||||
ccpcluster.deploy()
|
ccpcluster.deploy()
|
||||||
|
|
|
@ -63,7 +63,8 @@ class TestServiceHorizon(object):
|
||||||
ccpcluster.build()
|
ccpcluster.build()
|
||||||
|
|
||||||
topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
|
topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
|
||||||
'k8s_templates/k8s_topology.yaml'
|
'ccp_deploy_topology/' \
|
||||||
|
'default_deploy_topology.yaml'
|
||||||
remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
|
remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
|
||||||
ccpcluster.deploy()
|
ccpcluster.deploy()
|
||||||
post_os_deploy_checks.check_jobs_status(k8sclient, timeout=1500,
|
post_os_deploy_checks.check_jobs_status(k8sclient, timeout=1500,
|
||||||
|
|
|
@ -64,7 +64,8 @@ class TestPreStackLight(object):
|
||||||
ccpcluster.build(suppress_output=False)
|
ccpcluster.build(suppress_output=False)
|
||||||
|
|
||||||
topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
|
topology_path = os.getcwd() + '/fuel_ccp_tests/templates/' \
|
||||||
'k8s_templates/stacklight_topology.yaml'
|
'ccp_deploy_topology/' \
|
||||||
|
'stacklight_topology.yaml'
|
||||||
|
|
||||||
remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
|
remote.upload(topology_path, settings.CCP_CLI_PARAMS['deploy-config'])
|
||||||
ccpcluster.deploy()
|
ccpcluster.deploy()
|
||||||
|
|
|
@ -49,7 +49,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
|
||||||
k8s_actions.create_registry()
|
k8s_actions.create_registry()
|
||||||
ccpcluster.build()
|
ccpcluster.build()
|
||||||
topology_path = \
|
topology_path = \
|
||||||
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
|
||||||
'1ctrl_1comp.yaml'
|
'1ctrl_1comp.yaml'
|
||||||
remote = underlay.remote(host=config.k8s.kube_host)
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
remote.upload(topology_path, '/tmp')
|
remote.upload(topology_path, '/tmp')
|
||||||
|
@ -71,7 +71,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
|
||||||
timeout=600)
|
timeout=600)
|
||||||
|
|
||||||
topology_path = \
|
topology_path = \
|
||||||
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
|
||||||
'1ctrl_1comp_diff.yaml'
|
'1ctrl_1comp_diff.yaml'
|
||||||
remote.upload(topology_path, '/tmp')
|
remote.upload(topology_path, '/tmp')
|
||||||
conf = copy.deepcopy(settings.CCP_CONF)
|
conf = copy.deepcopy(settings.CCP_CONF)
|
||||||
|
@ -117,7 +117,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
|
||||||
k8s_actions.create_registry()
|
k8s_actions.create_registry()
|
||||||
ccpcluster.build()
|
ccpcluster.build()
|
||||||
topology_path = \
|
topology_path = \
|
||||||
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
|
||||||
'1ctrl_1comp.yaml'
|
'1ctrl_1comp.yaml'
|
||||||
remote = underlay.remote(host=config.k8s.kube_host)
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
remote.upload(topology_path, '/tmp')
|
remote.upload(topology_path, '/tmp')
|
||||||
|
@ -139,7 +139,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
|
||||||
timeout=600)
|
timeout=600)
|
||||||
|
|
||||||
topology_path = \
|
topology_path = \
|
||||||
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
|
||||||
'1ctrl_1comp_same.yaml'
|
'1ctrl_1comp_same.yaml'
|
||||||
remote.upload(topology_path, '/tmp')
|
remote.upload(topology_path, '/tmp')
|
||||||
conf = copy.deepcopy(settings.CCP_CONF)
|
conf = copy.deepcopy(settings.CCP_CONF)
|
||||||
|
@ -186,7 +186,7 @@ class TestDeployTwoOS(base_test.SystemBaseTest):
|
||||||
k8s_actions.create_registry()
|
k8s_actions.create_registry()
|
||||||
ccpcluster.build()
|
ccpcluster.build()
|
||||||
topology_path = \
|
topology_path = \
|
||||||
os.getcwd() + '/fuel_ccp_tests/templates/k8s_templates/' \
|
os.getcwd() + '/fuel_ccp_tests/templates/ccp_deploy_topology/' \
|
||||||
'1ctrl.yaml'
|
'1ctrl.yaml'
|
||||||
remote = underlay.remote(host=config.k8s.kube_host)
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
remote.upload(topology_path, '/tmp')
|
remote.upload(topology_path, '/tmp')
|
||||||
|
|
|
@ -0,0 +1,487 @@
|
||||||
|
# Copyright 2016 Mirantis, Inc.
|
||||||
|
#
|
||||||
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||||
|
# not use this file except in compliance with the License. You may obtain
|
||||||
|
# a copy of the License at
|
||||||
|
#
|
||||||
|
# http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
#
|
||||||
|
# Unless required by applicable law or agreed to in writing, software
|
||||||
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||||
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||||
|
# License for the specific language governing permissions and limitations
|
||||||
|
# under the License.
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
import time
|
||||||
|
from devops.helpers import helpers
|
||||||
|
|
||||||
|
import base_test
|
||||||
|
from fuel_ccp_tests import logger
|
||||||
|
from fuel_ccp_tests import settings
|
||||||
|
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
||||||
|
from fuel_ccp_tests.helpers import rabbit
|
||||||
|
|
||||||
|
LOG = logger.logger
|
||||||
|
|
||||||
|
|
||||||
|
class TestRabbitmq(base_test.SystemBaseTest):
|
||||||
|
""" Galera scale and destructive scenarios
|
||||||
|
|
||||||
|
"""
|
||||||
|
def get_pods(self, k8s):
|
||||||
|
return \
|
||||||
|
k8s.get_running_pods('rabbit',
|
||||||
|
settings.CCP_CONF['kubernetes']['namespace'])
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_deploy
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq(self, underlay, config,
|
||||||
|
k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Deploy rabbit cluster
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Check rabbit cluster state
|
||||||
|
3. Check queue, messages replication
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
show_step(2)
|
||||||
|
show_step(3)
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
for pod in rabbit_pods:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(rabbit_pods),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(rabbit_pods), rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_shutdown
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq_shutdown_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Shutdown rabbitmq node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Shutdown one rabbit node
|
||||||
|
3. Check rabbit cluster state
|
||||||
|
4. Check queue, messages replication
|
||||||
|
5. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
rabbit_node = underlay.node_names()[1]
|
||||||
|
rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
show_step(2)
|
||||||
|
hardware.shutdown_node_by_ip(rabbit_node_ip)
|
||||||
|
show_step(3)
|
||||||
|
|
||||||
|
helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
|
||||||
|
len(rabbit_pods) - 1),
|
||||||
|
timeout=600,
|
||||||
|
timeout_msg='Timeout waiting for rabbit pod'
|
||||||
|
' to be terminated')
|
||||||
|
pods_after_shutdown = self.get_pods(k8s_actions)
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
for pod in pods_after_shutdown:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(pods_after_shutdown),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(pods_after_shutdown),
|
||||||
|
rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
show_step(5)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_cold_restart
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq_cold_restart_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Cold restart rabbitmq node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Cold restart one rabbit node
|
||||||
|
3. Check rabbit cluster state
|
||||||
|
4. Check queue, messages replication
|
||||||
|
5. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
rabbit_node = underlay.node_names()[1]
|
||||||
|
rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
|
||||||
|
show_step(2)
|
||||||
|
hardware.shutdown_node_by_ip(rabbit_node_ip)
|
||||||
|
hardware.wait_node_is_offline(rabbit_node_ip, 90)
|
||||||
|
time.sleep(15)
|
||||||
|
hardware.start_node_by_ip(rabbit_node_ip)
|
||||||
|
hardware.wait_node_is_online(rabbit_node_ip, 180)
|
||||||
|
show_step(3)
|
||||||
|
|
||||||
|
helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
|
||||||
|
len(rabbit_pods) - 1),
|
||||||
|
timeout=1200,
|
||||||
|
timeout_msg='Expected to have one pod destroyed'
|
||||||
|
' after reboot')
|
||||||
|
|
||||||
|
helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
|
||||||
|
len(rabbit_pods)),
|
||||||
|
timeout=1200,
|
||||||
|
timeout_msg='Expected pod to come back after reboot')
|
||||||
|
|
||||||
|
pods_after_reboot = self.get_pods(k8s_actions)
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
for pod in pods_after_reboot:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(pods_after_reboot),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
show_step(5)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_poweroff
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq_poweroff_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Poweroff rabbit node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Poweroff one rabbit node
|
||||||
|
3. Check rabbit cluster state
|
||||||
|
4. Check queue, messages replication
|
||||||
|
5. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
rabbit_node = underlay.node_names()[1]
|
||||||
|
rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
show_step(2)
|
||||||
|
underlay.sudo_check_call('shutdown +1', node_name=rabbit_node)
|
||||||
|
hardware.shutdown_node_by_ip(rabbit_node_ip)
|
||||||
|
hardware.wait_node_is_offline(rabbit_node_ip, 90)
|
||||||
|
show_step(3)
|
||||||
|
|
||||||
|
helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
|
||||||
|
len(rabbit_pods) - 1),
|
||||||
|
timeout=600,
|
||||||
|
timeout_msg='Timeout waiting for rabbit pod'
|
||||||
|
' to be terminated')
|
||||||
|
pods_after_reboot = self.get_pods(k8s_actions)
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
for pod in pods_after_reboot:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(pods_after_reboot),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
show_step(5)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_soft_reboot
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq_soft_reboot_node(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Soft reboot rabbitmq node
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Reboot one rabbit node
|
||||||
|
3. Check rabbit cluster state
|
||||||
|
4. Check queue, messages replication
|
||||||
|
5. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
rabbit_node = underlay.node_names()[1]
|
||||||
|
rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
show_step(2)
|
||||||
|
underlay.sudo_check_call('shutdown +1', node_name=rabbit_node)
|
||||||
|
hardware.shutdown_node_by_ip(rabbit_node_ip)
|
||||||
|
hardware.wait_node_is_offline(rabbit_node_ip, 90)
|
||||||
|
time.sleep(15)
|
||||||
|
hardware.start_node_by_ip(rabbit_node_ip)
|
||||||
|
hardware.wait_node_is_online(rabbit_node_ip, 180)
|
||||||
|
show_step(3)
|
||||||
|
helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
|
||||||
|
len(rabbit_pods)),
|
||||||
|
timeout=600,
|
||||||
|
timeout_msg='Timeout waiting for rabbit pod'
|
||||||
|
' to be terminated')
|
||||||
|
pods_after_reboot = self.get_pods(k8s_actions)
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
for pod in pods_after_reboot:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(pods_after_reboot),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(pods_after_reboot), rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
show_step(5)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_cluster_shutdown
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq_cluster_shutdown(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Rabbitmq cluster shutdown
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Shutdown all rabbit nodes and start them one by one
|
||||||
|
3. Check rabbit cluster state
|
||||||
|
4. Check queue, messages replication
|
||||||
|
5. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
rabbit_nodes = underlay.node_names()[:3]
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
rabbit_node_ips = []
|
||||||
|
show_step(2)
|
||||||
|
for rabbit_node in rabbit_nodes:
|
||||||
|
rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
|
||||||
|
rabbit_node_ips.append(rabbit_node_ip)
|
||||||
|
hardware.shutdown_node_by_ip(rabbit_node_ip)
|
||||||
|
hardware.wait_node_is_offline(rabbit_node_ip, 90)
|
||||||
|
for rabbit_ip in rabbit_node_ips:
|
||||||
|
hardware.start_node_by_ip(rabbit_ip)
|
||||||
|
hardware.wait_node_is_online(rabbit_ip, 180)
|
||||||
|
show_step(3)
|
||||||
|
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||||
|
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||||
|
pods_after_shutdown = self.get_pods(k8s_actions)
|
||||||
|
assert len(rabbit_pods) == len(pods_after_shutdown),\
|
||||||
|
"Different number of pods after shutdown, was {}," \
|
||||||
|
" now {}".format(len(rabbit_pods), len(pods_after_shutdown))
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(4)
|
||||||
|
host = config.k8s.kube_host
|
||||||
|
rabbit_port = ''.join(remote.execute(
|
||||||
|
"kubectl get service --namespace ccp rabbitmq -o yaml |"
|
||||||
|
" awk '/nodePort: / {print $NF}'")['stdout'])
|
||||||
|
rabbit_client = rabbit.RabbitClient(host, rabbit_port)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
for pod in pods_after_shutdown:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(pods_after_shutdown),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(pods_after_shutdown),
|
||||||
|
rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_scale_up_down
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq_scale(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Rabbit cluster scale
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Scale up rabbit to 5 replicas
|
||||||
|
3. Check rabbit state
|
||||||
|
4. Check number of rabbit pods
|
||||||
|
5. Create 2 vms
|
||||||
|
6. Scale down rabbit to 3 replicas
|
||||||
|
7. Check rabbit state
|
||||||
|
8. Check number of rabbit pods
|
||||||
|
9. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
show_step(2)
|
||||||
|
with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
|
||||||
|
host=config.k8s.kube_host) as editor:
|
||||||
|
del editor.content['nodes']['node[1-3]']
|
||||||
|
editor.content['nodes']['node[1-5]'] = {'roles': ['rabbitmq']}
|
||||||
|
|
||||||
|
ccpcluster.deploy()
|
||||||
|
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||||
|
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
for pod in rabbit_pods:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(rabbit_pods),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(rabbit_pods), rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
|
||||||
|
show_step(4)
|
||||||
|
rabbit_pods = \
|
||||||
|
k8s_actions.get_pods_number('rabbit', namespace)
|
||||||
|
assert rabbit_pods == 5,\
|
||||||
|
"Expcted to have 5 rabbit pods, got {}".format(rabbit_pods)
|
||||||
|
|
||||||
|
show_step(5)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
||||||
|
show_step(6)
|
||||||
|
with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
|
||||||
|
host=config.k8s.kube_host) as editor:
|
||||||
|
del editor.content['nodes']['node[1-5]']
|
||||||
|
editor.content['nodes']['node[1-3]'] = {'roles': ['rabbitmq']}
|
||||||
|
|
||||||
|
ccpcluster.deploy()
|
||||||
|
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||||
|
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||||
|
show_step(7)
|
||||||
|
show_step(8)
|
||||||
|
rabbit_pods = \
|
||||||
|
k8s_actions.get_pods_number('rabbit', namespace)
|
||||||
|
assert rabbit_pods == 3,\
|
||||||
|
"Expcted to have 3 rabbit pods, got {}".format(rabbit_pods)
|
||||||
|
show_step(9)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
||||||
|
|
||||||
|
@pytest.mark.fail_snapshot
|
||||||
|
@pytest.mark.rabbitmq_node_replacement
|
||||||
|
@pytest.mark.rabbitmq
|
||||||
|
def test_rabbitmq_node_replacement(self, hardware, underlay, config,
|
||||||
|
ccpcluster, k8s_actions, show_step,
|
||||||
|
os_deployed, rabbit_client):
|
||||||
|
"""Rabbitmq node replacement
|
||||||
|
|
||||||
|
Scenario:
|
||||||
|
1. Revert snapshot with deployed rabbit
|
||||||
|
2. Shutdown one rabbit node
|
||||||
|
3. Add new rabbit node to config
|
||||||
|
4. Re-deploy cluster
|
||||||
|
5. Check rabbit cluster state
|
||||||
|
6. Check queue, messages replication
|
||||||
|
7. Create 2 vms
|
||||||
|
|
||||||
|
Duration 30 min
|
||||||
|
"""
|
||||||
|
rabbit_node = underlay.node_names()[1]
|
||||||
|
rabbit_node_ip = underlay.host_by_node_name(rabbit_node)
|
||||||
|
namespace = settings.CCP_CONF["kubernetes"]["namespace"]
|
||||||
|
rabbit_pods = self.get_pods(k8s_actions)
|
||||||
|
show_step(2)
|
||||||
|
hardware.shutdown_node_by_ip(rabbit_node_ip)
|
||||||
|
|
||||||
|
helpers.wait(lambda: (len(self.get_pods(k8s_actions)) ==
|
||||||
|
len(rabbit_pods) - 1),
|
||||||
|
timeout=600,
|
||||||
|
timeout_msg='Timeout waiting for rabbit pod'
|
||||||
|
' to be terminated')
|
||||||
|
show_step(3)
|
||||||
|
with underlay.yaml_editor(settings.CCP_DEPLOY_TOPOLOGY,
|
||||||
|
host=config.k8s.kube_host) as editor:
|
||||||
|
del editor.content['nodes']['node[1-3]']
|
||||||
|
editor.content['nodes']['node[1-2]'] = {'roles': ['rabbitmq']}
|
||||||
|
editor.content['nodes']['node4'] = {'roles': ['rabbitmq', 'etcd']}
|
||||||
|
show_step(4)
|
||||||
|
ccpcluster.deploy()
|
||||||
|
post_os_deploy_checks.check_jobs_status(k8s_actions.api, timeout=2000)
|
||||||
|
post_os_deploy_checks.check_pods_status(k8s_actions.api)
|
||||||
|
|
||||||
|
pods_after_replacement = self.get_pods(k8s_actions)
|
||||||
|
remote = underlay.remote(host=config.k8s.kube_host)
|
||||||
|
show_step(6)
|
||||||
|
queue = rabbit_client.create_queue()
|
||||||
|
message = rabbit_client.publish_message_to_queue(queue)
|
||||||
|
rabbit_client.check_queue_message(message)
|
||||||
|
show_step(5)
|
||||||
|
for pod in pods_after_replacement:
|
||||||
|
rabbit_cluster_nodes = rabbit_client.list_nodes(
|
||||||
|
remote, pod.name, namespace)
|
||||||
|
assert rabbit_cluster_nodes == len(pods_after_replacement),\
|
||||||
|
"Expected to have {} nodes in cluster," \
|
||||||
|
" got {}".format(len(pods_after_replacement),
|
||||||
|
rabbit_cluster_nodes)
|
||||||
|
rabbit_client.check_queue_replicated(queue, remote,
|
||||||
|
pod.name, namespace)
|
||||||
|
rabbit_client.delete_queue(queue)
|
||||||
|
show_step(7)
|
||||||
|
remote.check_call(
|
||||||
|
"source openrc-{}; bash fuel-ccp/tools/deploy-test-vms.sh -a"
|
||||||
|
" create".format(namespace), timeout=600)
|
Loading…
Reference in New Issue