195 lines
7.8 KiB
Python
195 lines
7.8 KiB
Python
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
|
|
from __future__ import print_function
|
|
from cloudpulse.operator.ansible.ansible_runner import ansible_runner
|
|
from cloudpulse.operator.ansible.openstack_node_info_reader import \
|
|
openstack_node_info_reader
|
|
from cloudpulse.scenario import base
|
|
import json
|
|
from oslo_config import cfg
|
|
import re
|
|
|
|
TESTS_OPTS = [
|
|
cfg.StrOpt('operator_setup_file',
|
|
default='/etc/cloudpulse/openstack_config.yaml',
|
|
help='Setup File for the operator'),
|
|
cfg.BoolOpt('containerized',
|
|
default=True,
|
|
help='enable if the processes are running as containers'),
|
|
cfg.StrOpt('rabbit_container',
|
|
default='rabbitmq_v1',
|
|
help='name of the rabitmq container'),
|
|
cfg.StrOpt('galera_container',
|
|
default='mariadb_v1',
|
|
help='name of the galera cluster container'),
|
|
|
|
]
|
|
|
|
PERIODIC_TESTS_OPTS = [
|
|
cfg.IntOpt('rabbitmq_check',
|
|
default=0,
|
|
help='The rabbitmq periodic check'),
|
|
cfg.IntOpt('galera_check',
|
|
default=0,
|
|
help='The galera periodic check'),
|
|
cfg.IntOpt('ceph_check',
|
|
default=0,
|
|
help='The ceph periodic check'),
|
|
cfg.IntOpt('docker_check',
|
|
default=0,
|
|
help='The docker periodic check')
|
|
]
|
|
|
|
CONF = cfg.CONF
|
|
|
|
operator_test_group = cfg.OptGroup(name='operator_test',
|
|
title='Options for the Operators')
|
|
CONF.register_group(operator_test_group)
|
|
CONF.register_opts(TESTS_OPTS, operator_test_group)
|
|
|
|
periodic_test_group = cfg.OptGroup(name='periodic_tests',
|
|
title='Periodic tests to be run')
|
|
CONF.register_opts(PERIODIC_TESTS_OPTS, periodic_test_group)
|
|
|
|
|
|
class operator_scenario(base.Scenario):
|
|
|
|
def load(self):
|
|
self.os_node_info_obj = openstack_node_info_reader(
|
|
cfg.CONF.operator_test.operator_setup_file)
|
|
openstack_node_list = self.os_node_info_obj.get_host_list()
|
|
self.ans_runner = ansible_runner(openstack_node_list)
|
|
inventory = self.ans_runner.init_ansible_inventory(openstack_node_list)
|
|
self.ans_runner.set_ansible_inventory(inventory)
|
|
|
|
@base.scenario(admin_only=False, operator=True)
|
|
def rabbitmq_check(self):
|
|
self.load()
|
|
cmd = "rabbitmqctl cluster_status -q"
|
|
|
|
is_containerized = cfg.CONF.operator_test.containerized
|
|
if is_containerized:
|
|
rabbit_container = cfg.CONF.operator_test.rabbit_container
|
|
cmd = ("docker exec %s %s" % (rabbit_container, cmd))
|
|
|
|
out = self.ans_runner.execute(cmd)
|
|
res, output = self.ans_runner.validate_results(out)
|
|
|
|
if res['status'] is 'PASS':
|
|
node_status = res['contacted'][
|
|
res['contacted'].keys()[0]]['stdout']
|
|
node_status_string = node_status.replace('\n', '')
|
|
|
|
nodes = []
|
|
running = []
|
|
mathobj = re.search(
|
|
r'nodes,\[{disc,\[(.*?)\]', node_status_string, re.M | re.I)
|
|
if mathobj:
|
|
nodes = [x.rstrip(" ").lstrip(" ").rstrip("'").lstrip("'")
|
|
for x in mathobj.group(1).split(",")]
|
|
|
|
mathobj = re.search(
|
|
r'running_nodes,\[(.*?)\]}', node_status_string, re.M | re.I)
|
|
|
|
if mathobj:
|
|
running = [x.rstrip(" ").lstrip(" ").rstrip("'").lstrip("'")
|
|
for x in mathobj.group(1).split(",")]
|
|
|
|
diffnodes = list(set(nodes) - set(running))
|
|
if diffnodes:
|
|
return(404, ("Failed Nodes : %s" %
|
|
str(diffnodes)))
|
|
else:
|
|
return (200, "Running Nodes : %s" % str(nodes),
|
|
['RabbitMQ-server Running'])
|
|
else:
|
|
return (404, ("RabbitMQ-server test failed :%s" %
|
|
res['status_message']), [])
|
|
|
|
@base.scenario(admin_only=False, operator=True)
|
|
def galera_check(self):
|
|
self.load()
|
|
galera = self.os_node_info_obj.get_galera_details()
|
|
|
|
cmd = ((r"mysql -u %s -p%s -e 'SHOW STATUS;'|grep "
|
|
"wsrep_incoming_addresses") %
|
|
(galera['username'], galera['password']))
|
|
|
|
is_containerized = cfg.CONF.operator_test.containerized
|
|
if is_containerized:
|
|
galera_container = cfg.CONF.operator_test.galera_container
|
|
cmd = ("docker exec %s %s" % (galera_container, cmd))
|
|
|
|
out = self.ans_runner.execute(cmd)
|
|
results, failed_hosts = self.ans_runner.validate_results(out)
|
|
|
|
if results['status'] is 'PASS':
|
|
galera_status = results['contacted'][
|
|
results['contacted'].keys()[0]]['stdout']
|
|
galera_status_string = galera_status.replace('\n', '')
|
|
mathobj = re.search(r'wsrep_incoming_addresses\s+(.*?)$',
|
|
galera_status_string, re.M | re.I)
|
|
nodes = mathobj.group(1)
|
|
return (200, "Active Nodes : %s" % nodes,
|
|
['Galera Cluster Test Passed'])
|
|
else:
|
|
return (404, ("Galera Cluster Test Failed: %s" %
|
|
results['status_message']), [])
|
|
|
|
@base.scenario(admin_only=False, operator=True)
|
|
def docker_check(self):
|
|
self.load()
|
|
cmd = "docker ps -aq --filter 'status=exited'"
|
|
out = self.ans_runner.execute(cmd)
|
|
|
|
results, failed_hosts = self.ans_runner.validate_results(out)
|
|
if results['status'] is 'PASS':
|
|
docker_failed = {key: results['contacted'][key]['stdout']
|
|
for key in results['contacted']
|
|
if results['contacted'][key]['stdout']}
|
|
if docker_failed:
|
|
docker_str = " ".join(["Containers failed in %s : %s" % (
|
|
key, docker_failed[key]) for key in docker_failed])
|
|
return (404, docker_str, [])
|
|
else:
|
|
return (200, "All docker containers are up",
|
|
['Docker container Test Passed'])
|
|
else:
|
|
return (404, ("Docker Check Failed: %s" %
|
|
results['status_message']), [])
|
|
|
|
@base.scenario(admin_only=False, operator=True)
|
|
def ceph_check(self):
|
|
self.load()
|
|
cmd = (r"ceph -f json status")
|
|
out = self.ans_runner.execute(cmd)
|
|
results, failed_hosts = self.ans_runner.validate_results(out)
|
|
|
|
if results['status'] is 'PASS':
|
|
ceph_status = results['contacted'][
|
|
results['contacted'].keys()[0]]['stdout']
|
|
ceph_status_string = ceph_status.replace('\n', '')
|
|
ceph_json = json.loads(ceph_status_string)
|
|
overall_status = ceph_json['health']['overall_status']
|
|
num_of_osd = ceph_json['osdmap']['osdmap']['num_osds']
|
|
num_up_osds = ceph_json['osdmap']['osdmap']['num_up_osds']
|
|
if overall_status == 'HEALTH_OK':
|
|
return (200, "Overall Status = %s, Cluster status = %s/%s" %
|
|
(overall_status, num_up_osds, num_of_osd))
|
|
else:
|
|
return (404, "Overall Status = %s, Cluster status = %s/%s" %
|
|
(overall_status, num_up_osds, num_of_osd))
|
|
else:
|
|
return (404, ("Ceph cluster Test Failed: %s" %
|
|
results['status_message']), [])
|