From 52ddfb73e339051b9d043b60017bc72a55993999 Mon Sep 17 00:00:00 2001 From: Jenkins Date: Mon, 26 Sep 2016 12:05:37 +0000 Subject: [PATCH] Test logging aggregation - to run this test, the following env variable should be exported: export STACKLIGHT_ENABLE=true - os manager - os deployed fixture - os deployed with stacklight fixture - assertion messages added - moved to component folder - list of files switched from ls to find - elastic client return None if not items found - show_step applied to all scenarios Change-Id: I4ded419730889761ffa102a23e97d655a07d20fa --- fuel_ccp_tests/fixtures/os_fixtures.py | 52 ++ fuel_ccp_tests/helpers/ext.py | 45 ++ fuel_ccp_tests/helpers/utils.py | 102 ++- fuel_ccp_tests/managers/osmanager.py | 84 +++ fuel_ccp_tests/requirements.txt | 1 + fuel_ccp_tests/settings_oslo.py | 21 + .../tests/component/ccp/test_ccp_logging.py | 706 ++++++++++++++++++ fuel_ccp_tests/tests/component/conftest.py | 3 +- fuel_ccp_tests/tests/system/conftest.py | 3 +- 9 files changed, 1013 insertions(+), 4 deletions(-) create mode 100644 fuel_ccp_tests/fixtures/os_fixtures.py create mode 100644 fuel_ccp_tests/managers/osmanager.py create mode 100644 fuel_ccp_tests/tests/component/ccp/test_ccp_logging.py diff --git a/fuel_ccp_tests/fixtures/os_fixtures.py b/fuel_ccp_tests/fixtures/os_fixtures.py new file mode 100644 index 0000000..b23d14b --- /dev/null +++ b/fuel_ccp_tests/fixtures/os_fixtures.py @@ -0,0 +1,52 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pytest + +from fuel_ccp_tests import logger +from fuel_ccp_tests.helpers import ext +from fuel_ccp_tests.managers.osmanager import OSManager + +LOG = logger.logger + + +@pytest.fixture(scope='function') +def os_deployed(ccpcluster, + hardware, + underlay, + revert_snapshot, + config, + k8s_actions): + """Deploy openstack + """ + # If no snapshot was reverted, then try to revert the snapshot + # that belongs to the fixture. + # Note: keep fixtures in strict dependences from each other! + if not revert_snapshot: + if hardware.has_snapshot(ext.SNAPSHOT.os_deployed) and \ + hardware.has_snapshot_config(ext.SNAPSHOT.os_deployed): + hardware.revert_snapshot(ext.SNAPSHOT.os_deployed) + + osmanager = OSManager(config, underlay, k8s_actions, ccpcluster) + if not config.os.running: + LOG.info("Preparing openstack log collector fixture...") + topology = None + if config.os_deploy.stacklight_enable: + topology = ('/fuel_ccp_tests/templates/k8s_templates/' + 'stacklight_topology.yaml') + osmanager.install_os(topology=topology) + hardware.create_snapshot(ext.SNAPSHOT.os_deployed) + else: + LOG.info("Openstack allready installed and running...") + osmanager.check_os_ready() diff --git a/fuel_ccp_tests/helpers/ext.py b/fuel_ccp_tests/helpers/ext.py index 2125ea8..ae28790 100644 --- a/fuel_ccp_tests/helpers/ext.py +++ b/fuel_ccp_tests/helpers/ext.py @@ -14,6 +14,8 @@ import collections +from enum import IntEnum + def enum(*values, **kwargs): names = kwargs.get('names') @@ -38,4 +40,47 @@ SNAPSHOT = enum( 'underlay', 'k8s_deployed', 'ccp_deployed', + 'os_deployed', + 'os_deployed_stacklight' ) + +LOG_LEVELS = enum( + 'INFO', + 'WARNING', + 'ERROR', + 'CRITICAL', + 'DEBUG', + 'NOTE' +) + + +class ExitCodes(IntEnum): + EX_OK = 0 # successful termination + EX_INVALID = 0xDEADBEEF # uint32 debug value. Impossible for POSIX + EX_ERROR = 1 # general failure + EX_BUILTIN = 2 # Misuse of shell builtins (according to Bash) + EX_USAGE = 64 # command line usage error + EX_DATAERR = 65 # data format error + EX_NOINPUT = 66 # cannot open input + EX_NOUSER = 67 # addressee unknown + EX_NOHOST = 68 # host name unknown + EX_UNAVAILABLE = 69 # service unavailable + EX_SOFTWARE = 70 # internal software error + EX_OSERR = 71 # system error (e.g., can't fork) + EX_OSFILE = 72 # critical OS file missing + EX_CANTCREAT = 73 # can't create (user) output file + EX_IOERR = 74 # input/output error + EX_TEMPFAIL = 75 # temp failure; user is invited to retry + EX_PROTOCOL = 76 # remote error in protocol + EX_NOPERM = 77 # permission denied + EX_CONFIG = 78 # configuration error + EX_NOEXEC = 126 # If a command is found but is not executable + EX_NOCMD = 127 # If a command is not found + + +class HttpCodes(enumerate): + OK = '200' + + +class Namespace(enumerate): + BASE_NAMESPACE = 'ccp' diff --git a/fuel_ccp_tests/helpers/utils.py b/fuel_ccp_tests/helpers/utils.py index 9e5db0e..89b8215 100644 --- a/fuel_ccp_tests/helpers/utils.py +++ b/fuel_ccp_tests/helpers/utils.py @@ -14,16 +14,19 @@ import os -import time -import paramiko import shutil import tempfile +import time import traceback +import paramiko import yaml +from devops.helpers import helpers +from elasticsearch import Elasticsearch from fuel_ccp_tests import logger from fuel_ccp_tests import settings +from fuel_ccp_tests.helpers import ext LOG = logger.logger @@ -179,3 +182,98 @@ def retry(tries_number=3, exception=Exception): iter_number += 1 return wrapper return _retry + + +class ElasticClient(object): + def __init__(self, host='localhost', port=9200): + self.es = Elasticsearch([{'host': '{}'.format(host), + 'port': port}]) + self.host = host + self.port = port + + def find(self, key, value): + LOG.info('Search for {} for {}'.format(key, value)) + search_request_body = '{' +\ + ' "query": {' +\ + ' "simple_query_string": {' +\ + ' "query": "{}",'.format(value) +\ + ' "analyze_wildcard" : "true",' +\ + ' "fields" : ["{}"],'.format(key) +\ + ' "default_operator": "AND"' +\ + ' }' +\ + ' },' +\ + ' "size": 1' +\ + '}' + LOG.info('Search by {}'.format(search_request_body)) + + def is_found(): + def temporary_status(): + res = self.es.search(index='_all', body=search_request_body) + return res['hits']['total'] != 0 + return temporary_status + + predicate = is_found() + helpers.wait(predicate, timeout=300, + timeout_msg='Timeout waiting, result from elastic') + + es_raw = self.es.search(index='_all', body=search_request_body) + if es_raw['timed_out']: + raise RuntimeError('Elastic search timeout exception') + + return ElasticSearchResult(key, value, es_raw['hits']['total'], es_raw) + + +class ElasticSearchResult(object): + def __init__(self, key, value, count, raw): + self.key = key + self.value = value + self.count = count + self.raw = raw + if self.count != 0: + self.items = raw['hits']['hits'] + + def get(self, index): + if self.count != 0: + return self.items[index]['_source'] + else: + None + + +def create_file(node, pod, path, size, + namespace=ext.Namespace.BASE_NAMESPACE): + node.check_call( + 'kubectl exec {} --namespace={} {}'.format( + pod.name, + namespace, + 'dd -- if=/dev/zero -- of={} bs=1MB count={}'.format(path, size)), + expected=[ext.ExitCodes.EX_OK]) + + +def run_daily_cron(node, pod, task, + namespace=ext.Namespace.BASE_NAMESPACE): + node.check_call( + 'kubectl exec {} --namespace={} {}'.format( + pod.name, + namespace, + '/etc/cron.daily/{}'.format(task)), + expected=[ext.ExitCodes.EX_OK]) + + +def list_files(node, pod, path, mask, + namespace=ext.Namespace.BASE_NAMESPACE): + return "".join(node.check_call( + 'kubectl exec {} --namespace={} {}'.format( + pod.name, + namespace, + 'find {} -- -iname {}'.format(path, mask)), + expected=[ext.ExitCodes.EX_OK])['stdout']) \ + .replace('\n', ' ').strip().split(" ") + + +def rm_files(node, pod, path, + namespace=ext.Namespace.BASE_NAMESPACE): + node.execute( + 'kubectl exec {} --namespace={} {}'.format( + pod.name, + namespace, + 'rm -- {}'.format(path))) diff --git a/fuel_ccp_tests/managers/osmanager.py b/fuel_ccp_tests/managers/osmanager.py new file mode 100644 index 0000000..9e7e5b9 --- /dev/null +++ b/fuel_ccp_tests/managers/osmanager.py @@ -0,0 +1,84 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import os + +from fuel_ccp_tests import logger +from fuel_ccp_tests import settings +from fuel_ccp_tests.helpers import post_os_deploy_checks + +LOG = logger.logger + + +class OSManager(object): + """docstring for K8SManager""" + + __config = None + __underlay = None + + def __init__(self, config, underlay, k8s_actions, ccpcluster): + self.__config = config + self.__underlay = underlay + self.__k8s_actions = k8s_actions + self.__ccpcluster = ccpcluster + + def install_os(self, topology=None, + check_os_ready=True): + """Action to deploy openstack by ccp tool + + Additional steps: + TODO + + :param env: EnvManager + :param custom_yaml: False if deploy with kargo default, None if deploy + with environment settings, or put you own + :rtype: None + """ + LOG.info("Trying to install k8s") + + """ + Deploy openstack with stacklight topology + """ + LOG.info("Preparing openstack log collector fixture...") + if settings.BUILD_IMAGES: + LOG.info("Creating registry...") + self.__k8s_actions.create_registry() + LOG.info("Building images...") + self.__ccpcluster.build() + if topology: + LOG.info("Pushing topology yaml...") + LOG.warn( + "Patched topology used, workaround until kube 1.4 released") + topology_path = \ + os.getcwd() + topology + self.__underlay.remote( + host=self.__config.k8s.kube_host).upload( + topology_path, + settings.DEPLOY_CONFIG) + LOG.info("Deploy openstack") + self.__ccpcluster.deploy() + if check_os_ready: + self.check_os_ready() + self.__config.os.running = True + + def check_os_ready(self, + check_jobs_ready=True, + check_pods_ready=True): + if check_jobs_ready: + LOG.info("Checking openstack jobs statuses...") + post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api, + timeout=3600) + if check_pods_ready: + LOG.info("Checking openstack pods statuses...") + post_os_deploy_checks.check_pods_status(self.__k8s_actions.api, + timeout=3600) diff --git a/fuel_ccp_tests/requirements.txt b/fuel_ccp_tests/requirements.txt index f0d0d67..ea3fba7 100644 --- a/fuel_ccp_tests/requirements.txt +++ b/fuel_ccp_tests/requirements.txt @@ -10,3 +10,4 @@ urllib3 psycopg2 python-k8sclient==0.3.0 junit-xml +elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0 diff --git a/fuel_ccp_tests/settings_oslo.py b/fuel_ccp_tests/settings_oslo.py index 55ecf8f..0c332e9 100644 --- a/fuel_ccp_tests/settings_oslo.py +++ b/fuel_ccp_tests/settings_oslo.py @@ -113,6 +113,16 @@ ccp_opts = [ help="", default='0.0.0.0'), ] +os_deploy_opts = [ + ct.Cfg('stacklight_enable', ct.Boolean(), + help="", default=False), +] + +os_opts = [ + ct.Cfg('running', ct.Boolean(), + help="", default=False), +] + _group_opts = [ ('hardware', hardware_opts), @@ -121,6 +131,8 @@ _group_opts = [ ('k8s', k8s_opts), ('ccp_deploy', ccp_deploy_opts), ('ccp', ccp_opts), + ('os_deploy', os_deploy_opts), + ('os', os_opts), ] @@ -148,6 +160,15 @@ def register_opts(config): config.register_group(cfg.OptGroup(name='ccp', title="CCP config and credentials", help="")) config.register_opts(group='ccp', opts=ccp_opts) + + config.register_group(cfg.OptGroup(name='os', + title="Openstack config and credentials", help="")) + config.register_opts(group='os', opts=os_opts) + config.register_group( + cfg.OptGroup(name='os_deploy', + title="Openstack deploy config and credentials", + help="")) + config.register_opts(group='os_deploy', opts=os_deploy_opts) return config diff --git a/fuel_ccp_tests/tests/component/ccp/test_ccp_logging.py b/fuel_ccp_tests/tests/component/ccp/test_ccp_logging.py new file mode 100644 index 0000000..4822cf0 --- /dev/null +++ b/fuel_ccp_tests/tests/component/ccp/test_ccp_logging.py @@ -0,0 +1,706 @@ +# Copyright 2016 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import json +import uuid +from datetime import datetime +from time import sleep + +import pytest + +from fuel_ccp_tests.helpers import ext +from fuel_ccp_tests.helpers import utils +from fuel_ccp_tests.logger import logger + + +@pytest.yield_fixture(scope='function') +def admin_node(config, underlay, ccpcluster): + logger.info("Get SSH access to admin node") + with underlay.remote(host=config.k8s.kube_host) as remote: + yield remote + + +@pytest.yield_fixture(scope='function') +def elastic_client_public(os_deployed, k8s_actions, config): + """ + Discover elasticsearch on the cluster and return simple elastic client + initialized with public endpoint + :param os_deployed: + :param k8s_actions: + :param config: + :return: utils.ElasticClient + """ + service_list = k8s_actions.api.services.list( + namespace=ext.Namespace.BASE_NAMESPACE) + service = [service for service in + service_list if 'elasticsearch' in service.name][0] + + elastic_search_public_port = service.spec.ports[0].node_port + elastic_search_public_host = config.k8s.kube_host + + yield utils.ElasticClient(elastic_search_public_host, + elastic_search_public_port) + + +@pytest.yield_fixture(scope='function') +def elastic_client_private(os_deployed, + k8s_actions, + admin_node, + config): + """ + Discover elasticsearch on the cluster and return simple elastic client + initialized with pod ip endpoint + :param os_deployed: + :param k8s_actions: + :param config: + :return: utils.ElasticClient + """ + service_list = k8s_actions.api.services.list( + namespace=ext.Namespace.BASE_NAMESPACE) + service = [service for service in + service_list if 'elasticsearch' in service.name][0] + + elastic_search_service_host = service.spec.cluster_ip + elastic_search_service_port = service.spec.ports[0].name + + yield utils.ElasticClient(elastic_search_service_host, + elastic_search_service_port) + + +@pytest.fixture(scope='function') +def kibana_public_endpoint(os_deployed, + k8s_actions, + admin_node, + config): + """ + Discover kibana on the cluster and return kibana endpoint + :param os_deployed: + :param k8s_actions: + :param admin_node: + :param config: + :return: host:port + """ + service_list = k8s_actions.api.services.list( + namespace=ext.Namespace.BASE_NAMESPACE) + service = [service for service in + service_list if 'kibana' in service.name][0] + + kibana_public_port = service.spec.ports[0].node_port + kibana_public_address = config.k8s.kube_host + + return '{}:{}'.format(kibana_public_address, kibana_public_port) + + +@pytest.mark.ccp_logging +@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_deployed) +class TestCppLogging(object): + """Check logging aggregation""" + def test_logging_connection_to_elasticsearch_public( + self, admin_node, elastic_client_public, show_step): + """Elasticsearch api test + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Test elasticsearch is accessible on public ip + """ + show_step(1) + elastic_call = 'curl -s -o /dev/null -w "%{{http_code}}" http://{}:{}/' + elastic_http_response = admin_node.execute(elastic_call.format( + elastic_client_public.host, + elastic_client_public.port))['stdout'] + + assert ext.HttpCodes.OK in elastic_http_response, \ + "Elastic respond with unexpected " \ + "HTTP_RESPONSE on public endpoint Expected {} Actual {}".format( + ext.HttpCodes.OK, elastic_http_response) + + def test_logging_connection_to_elasticsearch_private( + self, + admin_node, + elastic_client_private, + show_step): + """Elasticsearch api test + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Test elasticsearch is accessibile on private ip + """ + show_step(1) + elastic_call = 'curl -s -o /dev/null -w "%{{http_code}}" http://{}:{}/' + elastic_http_response = admin_node.execute( + elastic_call.format( + elastic_client_private.host, + elastic_client_private.port))['stdout'] + assert ext.HttpCodes.OK in elastic_http_response, \ + "Elastic respond with unexpected " \ + "HTTP_RESPONSE on private endpoint Expected {} Actual {}".format( + ext.HttpCodes.OK, elastic_http_response) + + def test_logging_search_for_logs_from_all_running_heka_instances( + self, admin_node, k8scluster, elastic_client_public, show_step): + """Heka connection test + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Find logs from all heka nodes + 2. Test that logs from each heka node exist + """ + + show_step(1) + ec = elastic_client_public + k8sclient = k8scluster.api + # get all nodes + nodes = k8sclient.nodes.list() + # get all heka instances + hekas = [pod for pod + in k8sclient.pods.list(namespace=ext.Namespace.BASE_NAMESPACE) + if 'heka' in pod.name] + # ensure heka is running on each node + assert len(nodes) == len(hekas) + + show_step(2) + for heka_job in hekas: + logger.info('Checking presense in aggregated log messages from {}' + .format(heka_job.name)) + assert ec.find('Hostname', heka_job.name).count > 0, \ + "Log message from heka node {} not found on elastic".format( + heka_job.name) + + def test_logging_trigger_event_into_mysql(self, + admin_node, + k8scluster, + elastic_client_public, + show_step): + """Trigger event in mysql container + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Get mysql pod + 2. Trigger mysql log event inside the container + 3. Test that triggered event pushed to elasticsearch + """ + ec = elastic_client_public + k8sclient = k8scluster.api + + show_step(1) + mysql_pod = [pod for pod in + k8sclient.pods.list( + namespace=ext.Namespace.BASE_NAMESPACE) + if 'mariadb' in pod.name][0] + + show_step(2) + mysql_id = str(uuid.uuid4()).replace('-', '') + mysql_template = \ + '{} 140115909998528 ' \ + '[Note] mysqld: ready for connections. {}\n'.format( + datetime.today().strftime('%Y-%m-%d %H:%M:%S'), + mysql_id) + + admin_node.check_call( + 'kubectl exec {} --namespace={} -- {}'.format( + mysql_pod.name, + ext.Namespace.BASE_NAMESPACE, + '\'/bin/bash\' -xc \'(echo \"{}\" >> ' + '/var/log/ccp/mysql/mysql.log)\''.format( + mysql_template) + ), + expected=[ext.ExitCodes.EX_OK]) + + show_step(3) + injected = ec.find('Payload', mysql_id) + assert injected.count == 1, \ + "New log message from mysql from {} not picked by heka".format( + mysql_pod) + + def test_logging_trigger_event_into_rabbitmq(self, + admin_node, + k8scluster, + elastic_client_public, + show_step): + """Trigger event in rabbitmq container + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Get rabbitmq pod + 2. Trigger rabbitmq log event inside the container + 3. Test that triggered event pushed to elasticsearch + """ + show_step(1) + ec = elastic_client_public + k8sclient = k8scluster.api + rabbitmq_pod = [pod for pod in + k8sclient.pods.list( + namespace=ext.Namespace.BASE_NAMESPACE) + if 'rabbitmq' in pod.name][0] + + show_step(2) + rabbitmq_id = str(uuid.uuid4()).replace('-', '') + rabbitmq_template = "=INFO REPORT==== {} ===\n" \ + "accepting AMQP connection <0.580.0> " \ + "(10.233.83.7 -> 10.233.83.89:5672):\n" \ + "{}".format(datetime.today() + .strftime("%d-%b-%Y::%H:%M:%S"), + rabbitmq_id) + admin_node.check_call( + 'kubectl exec {} --namespace={} -- {}'.format( + rabbitmq_pod.name, + ext.Namespace.BASE_NAMESPACE, + '\'/bin/bash\' -xc \'(echo -e \"{}\n\" >> ' + '/var/log/ccp/rabbitmq/rabbitmq.log)\''.format( + rabbitmq_template)), + expected=[ext.ExitCodes.EX_OK]) + + show_step(3) + injected = ec.find('Payload', rabbitmq_id) + assert injected.count == 1,\ + "New log message from mysql from {} not picked by heka".format( + rabbitmq_pod) + + def test_logging_attributes_for_mysql_message(self, + elastic_client_public, + show_step): + """Test attibute population consistency for mysql + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search mysql log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'mysql').get(0) + + show_step(2) + self.check_message_format(event, 'mysql') + + def test_logging_attributes_for_rabbitmq_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for rabbitmq + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search rabbitmq log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'rabbitmq').get(0) + + show_step(2) + self.check_message_format(event, 'rabbitmq') + + def test_logging_attributes_for_openstack_horizon_apache_message( + self, + elastic_client_public, + k8s_actions, + admin_node, + show_step): + """Test attibute population consistency for horizon-apache + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Trigger horizon to produce logs + 2. Search horizon-apache log event in the elasticsearch + 3. Test logged message consistency + """ + show_step(1) + service_list = k8s_actions.api.services.list( + ext.Namespace.BASE_NAMESPACE) + service = [service for service in + service_list if 'horizon' in service.name][0] + horizon_service_host = service.spec.cluster_ip + horizon_service_port = service.spec.ports[0].name + admin_node.check_call( + 'curl http://{}:{}'.format(horizon_service_host, + horizon_service_port), + expected=[ext.ExitCodes.EX_OK]) + + show_step(2) + ec = elastic_client_public + event = ec.find('Logger', 'openstack.horizon-apache').get(0) + + show_step(3) + self.check_message_format(event, 'openstack.horizon-apache') + + def test_logging_attributes_for_openstack_keystone_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for keystone + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search keystone log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'openstack.keystone').get(0) + + show_step(2) + self.check_message_format(event, 'openstack.keystone') + + def test_logging_attributes_for_openstack_keystone_apache_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for keystone-apache + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search keystone-apache log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'openstack.keystone-apache').get(0) + + show_step(2) + self.check_message_format(event, 'openstack.keystone-apache') + + def test_logging_attributes_for_openstack_nova_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for nova + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search nova log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'openstack.nova').get(0) + + show_step(2) + self.check_message_format(event, 'openstack.nova') + + def test_logging_attributes_for_openstack_neutron_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for neutron + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search neutron log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'openstack.neutron').get(0) + + show_step(2) + self.check_message_format(event, 'openstack.neutron') + + def test_logging_attributes_for_openstack_glance_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for glance + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search glance log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'openstack.glance').get(0) + + show_step(2) + self.check_message_format(event, 'openstack.glance') + + def test_logging_attributes_for_openstack_heat_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for heat + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search heat log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'openstack.heat').get(0) + + show_step(2) + self.check_message_format(event, 'openstack.heat') + + def test_logging_attributes_for_openvswitch_message( + self, + elastic_client_public, + show_step): + """Test attibute population consistency for openvswitch + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Search openvswitch log event in the elasticsearch + 2. Test logged message consistency + """ + show_step(1) + ec = elastic_client_public + event = ec.find('Logger', 'openvswitch').get(0) + + show_step(2) + self.check_message_format(event, 'openvswitch') + + def test_logging_explore_indexes_with_kibana( + self, + admin_node, + kibana_public_endpoint, + show_step): + """Test index availability from kibana + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Get kibana service status + 2. Test overal status should be green + """ + show_step(1) + status_full = json.loads( + "".join(admin_node.execute('curl http://{}/api/status'.format( + kibana_public_endpoint))['stdout'])) + status_overall = status_full['status']['overall']['state'] + + show_step(2) + assert status_overall == 'green', "Kibaba service have issues. " \ + "Status {}".format(status_overall) + + def test_logging_kibana_running_single_node( + self, + admin_node, + os_deployed, + k8s_actions, + show_step): + """Test kibana running single node + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Check kibana running single node + """ + show_step(1) + pods_list = k8s_actions.api.pods.list( + namespace=ext.Namespace.BASE_NAMESPACE) + kibana_count = len( + [pod for pod in pods_list + if 'kibana' in pod.name]) + assert kibana_count == 1, "Unexpected count of kibana instances" + + def test_logging_elastic_running_single_node( + self, + admin_node, + os_deployed, + k8s_actions, + show_step): + """Test elasticsearch running single node + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Check elasticsearch running single node + """ + show_step(1) + pods_list = k8s_actions.api.pods.list( + namespace=ext.Namespace.BASE_NAMESPACE) + elastic_count = len( + [pod for pod in pods_list + if 'elasticsearch' in pod.name]) + assert elastic_count == 1, \ + "Unexpected count of elasticsearch instances" + + def test_logging_heka_running_all_nodes( + self, + admin_node, + os_deployed, + k8s_actions, + show_step): + """Test heka running all nodes + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Check heka running on each node + """ + show_step(1) + pods_list = k8s_actions.api.pods.list( + namespace=ext.Namespace.BASE_NAMESPACE) + nodes_list = [node.name for node in k8s_actions.api.nodes.list()] + heka_nodes = [ + pod.spec.node_name for pod in pods_list if 'heka' in pod.name] + + assert sorted(nodes_list) == sorted(heka_nodes),\ + "Unexpected count of heka running on nodes instances" + + def test_logging_log_rotate_for_mysql( + self, + admin_node, + k8s_actions, + show_step, + os_deployed): + """Test log rotate for mysql + Precondition: + 1. Install k8s + 2. Install microservices + 3. Fetch all repos + 4. Build images or use external registry + 5. Deploy openstack + + Scenario: + 1. Clean mysql log on cron pod + 2. Simulate 8 days log rotation + 3. Ensure that count of log files is equal to 7(week rotation) + """ + logger.info('Log rotate for mysql') + log_path = '/var/log/ccp/mysql/' + log_file = 'mysql.log' + + show_step(1) + # get cron pod + cron_pod = [pod for pod in k8s_actions.api.pods.list( + namespace=ext.Namespace.BASE_NAMESPACE) + if 'cron-' in pod.name][0] + # clean files + utils.rm_files(admin_node, cron_pod, log_path + log_file + '*') + + show_step(2) + for day in range(0, 8): + utils.create_file(admin_node, cron_pod, log_path + log_file, 110) + utils.run_daily_cron(admin_node, cron_pod, 'logrotate') + sleep(5) + + show_step(3) + log_files = utils.list_files( + admin_node, cron_pod, log_path, log_file + '*') + assert len(log_files) == 7,\ + "Count of log files after rotation is wrong. " \ + "Expected {} Actual {}".format(log_files, 7) + + @staticmethod + def check_message_format(event, logger): + """Event consistency validator""" + assert event is not None, "No result found for {}".format(logger) + assert len(event['Timestamp']) > 0, \ + "Logger {} have wrong [timestamp] field".format(logger) + assert event['Type'] == 'log', \ + "Logger {} have wrong [Type] field".format(logger) + assert len(event['Payload']) > 0, \ + "Logger {} have wrong [Payload] field".format(logger) + assert isinstance(event['Pid'], int), \ + "Logger {} have wrong [Pid] field".format(logger) + assert len(event['Hostname']) > 0, \ + "Logger {} have wrong [Hostname] field".format(logger) + assert event['severity_label'] in ext.LOG_LEVELS, \ + "Logger {} have wrong [severity_label] field".format(logger) + assert len(event['programname']) > 0, \ + "Logger {} have wrong [programname] field".format(logger) + assert isinstance(event['Severity'], int), \ + "Logger {} have wrong [Severity] field".format(logger) diff --git a/fuel_ccp_tests/tests/component/conftest.py b/fuel_ccp_tests/tests/component/conftest.py index 3da7af5..1f5408d 100644 --- a/fuel_ccp_tests/tests/component/conftest.py +++ b/fuel_ccp_tests/tests/component/conftest.py @@ -18,4 +18,5 @@ pytest_plugins = ['fuel_ccp_tests.fixtures.common_fixtures', 'fuel_ccp_tests.fixtures.k8s_fixtures', 'fuel_ccp_tests.fixtures.rally_fixtures', 'fuel_ccp_tests.fixtures.ccp_fixtures', - 'fuel_ccp_tests.fixtures.influxdb_fixtures'] + 'fuel_ccp_tests.fixtures.influxdb_fixtures', + 'fuel_ccp_tests.fixtures.os_fixtures'] diff --git a/fuel_ccp_tests/tests/system/conftest.py b/fuel_ccp_tests/tests/system/conftest.py index 49b0a59..e806ea8 100644 --- a/fuel_ccp_tests/tests/system/conftest.py +++ b/fuel_ccp_tests/tests/system/conftest.py @@ -17,4 +17,5 @@ pytest_plugins = ['fuel_ccp_tests.fixtures.common_fixtures', 'fuel_ccp_tests.fixtures.underlay_fixtures', 'fuel_ccp_tests.fixtures.k8s_fixtures', 'fuel_ccp_tests.fixtures.rally_fixtures', - 'fuel_ccp_tests.fixtures.ccp_fixtures'] + 'fuel_ccp_tests.fixtures.ccp_fixtures', + 'fuel_ccp_tests.fixtures.os_fixtures']