Test logging aggregation

- to run this test, the following env variable should be exported:
  export STACKLIGHT_ENABLE=true
- os manager
- os deployed fixture
- os deployed with stacklight fixture
- assertion messages added
- moved to component folder
- list of files switched from ls to find
- elastic client return None if not items found
- show_step applied to all scenarios

Change-Id: I4ded419730889761ffa102a23e97d655a07d20fa

fuel_ccp_tests/fixtures/os_fixtures.py

@@ -0,0 +1,52 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import pytest
+
+from fuel_ccp_tests import logger
+from fuel_ccp_tests.helpers import ext
+from fuel_ccp_tests.managers.osmanager import OSManager
+
+LOG = logger.logger
+
+
+@pytest.fixture(scope='function')
+def os_deployed(ccpcluster,
+                hardware,
+                underlay,
+                revert_snapshot,
+                config,
+                k8s_actions):
+    """Deploy openstack
+    """
+    # If no snapshot was reverted, then try to revert the snapshot
+    # that belongs to the fixture.
+    # Note: keep fixtures in strict dependences from each other!
+    if not revert_snapshot:
+        if hardware.has_snapshot(ext.SNAPSHOT.os_deployed) and \
+                hardware.has_snapshot_config(ext.SNAPSHOT.os_deployed):
+            hardware.revert_snapshot(ext.SNAPSHOT.os_deployed)
+
+    osmanager = OSManager(config, underlay, k8s_actions, ccpcluster)
+    if not config.os.running:
+        LOG.info("Preparing openstack log collector fixture...")
+        topology = None
+        if config.os_deploy.stacklight_enable:
+            topology = ('/fuel_ccp_tests/templates/k8s_templates/'
+                        'stacklight_topology.yaml')
+        osmanager.install_os(topology=topology)
+        hardware.create_snapshot(ext.SNAPSHOT.os_deployed)
+    else:
+        LOG.info("Openstack allready installed and running...")
+        osmanager.check_os_ready()

fuel_ccp_tests/helpers/ext.py

@@ -14,6 +14,8 @@
 
 import collections
 
+from enum import IntEnum
+
 
 def enum(*values, **kwargs):
     names = kwargs.get('names')
@@ -38,4 +40,47 @@ SNAPSHOT = enum(
     'underlay',
     'k8s_deployed',
     'ccp_deployed',
+    'os_deployed',
+    'os_deployed_stacklight'
 )
+
+LOG_LEVELS = enum(
+    'INFO',
+    'WARNING',
+    'ERROR',
+    'CRITICAL',
+    'DEBUG',
+    'NOTE'
+)
+
+
+class ExitCodes(IntEnum):
+    EX_OK = 0  # successful termination
+    EX_INVALID = 0xDEADBEEF  # uint32 debug value. Impossible for POSIX
+    EX_ERROR = 1  # general failure
+    EX_BUILTIN = 2  # Misuse of shell builtins (according to Bash)
+    EX_USAGE = 64  # command line usage error
+    EX_DATAERR = 65  # data format error
+    EX_NOINPUT = 66  # cannot open input
+    EX_NOUSER = 67  # addressee unknown
+    EX_NOHOST = 68  # host name unknown
+    EX_UNAVAILABLE = 69  # service unavailable
+    EX_SOFTWARE = 70  # internal software error
+    EX_OSERR = 71  # system error (e.g., can't fork)
+    EX_OSFILE = 72  # critical OS file missing
+    EX_CANTCREAT = 73  # can't create (user) output file
+    EX_IOERR = 74  # input/output error
+    EX_TEMPFAIL = 75  # temp failure; user is invited to retry
+    EX_PROTOCOL = 76  # remote error in protocol
+    EX_NOPERM = 77  # permission denied
+    EX_CONFIG = 78  # configuration error
+    EX_NOEXEC = 126  # If a command is found but is not executable
+    EX_NOCMD = 127  # If a command is not found
+
+
+class HttpCodes(enumerate):
+    OK = '200'
+
+
+class Namespace(enumerate):
+    BASE_NAMESPACE = 'ccp'

fuel_ccp_tests/helpers/utils.py

@@ -14,16 +14,19 @@
 
 
 import os
-import time
-import paramiko
 import shutil
 import tempfile
+import time
 import traceback
 
+import paramiko
 import yaml
+from devops.helpers import helpers
+from elasticsearch import Elasticsearch
 
 from fuel_ccp_tests import logger
 from fuel_ccp_tests import settings
+from fuel_ccp_tests.helpers import ext
 
 LOG = logger.logger
 
@@ -179,3 +182,98 @@ def retry(tries_number=3, exception=Exception):
                 iter_number += 1
         return wrapper
     return _retry
+
+
+class ElasticClient(object):
+    def __init__(self, host='localhost', port=9200):
+        self.es = Elasticsearch([{'host': '{}'.format(host),
+                                  'port': port}])
+        self.host = host
+        self.port = port
+
+    def find(self, key, value):
+        LOG.info('Search for {} for {}'.format(key, value))
+        search_request_body = '{' +\
+            '  "query": {' +\
+            '   "simple_query_string": {' +\
+            '     "query": "{}",'.format(value) +\
+            '     "analyze_wildcard" : "true",' +\
+            '     "fields" : ["{}"],'.format(key) +\
+            '     "default_operator": "AND"' +\
+            '     }' +\
+            ' },' +\
+            '  "size": 1' +\
+            '}'
+        LOG.info('Search by {}'.format(search_request_body))
+
+        def is_found():
+            def temporary_status():
+                res = self.es.search(index='_all', body=search_request_body)
+                return res['hits']['total'] != 0
+            return temporary_status
+
+        predicate = is_found()
+        helpers.wait(predicate, timeout=300,
+                     timeout_msg='Timeout waiting, result from elastic')
+
+        es_raw = self.es.search(index='_all', body=search_request_body)
+        if es_raw['timed_out']:
+            raise RuntimeError('Elastic search timeout exception')
+
+        return ElasticSearchResult(key, value, es_raw['hits']['total'], es_raw)
+
+
+class ElasticSearchResult(object):
+    def __init__(self, key, value, count, raw):
+        self.key = key
+        self.value = value
+        self.count = count
+        self.raw = raw
+        if self.count != 0:
+            self.items = raw['hits']['hits']
+
+    def get(self, index):
+        if self.count != 0:
+            return self.items[index]['_source']
+        else:
+            None
+
+
+def create_file(node, pod, path, size,
+                namespace=ext.Namespace.BASE_NAMESPACE):
+    node.check_call(
+        'kubectl exec {} --namespace={} {}'.format(
+            pod.name,
+            namespace,
+            'dd -- if=/dev/zero -- of={} bs=1MB count={}'.format(path, size)),
+        expected=[ext.ExitCodes.EX_OK])
+
+
+def run_daily_cron(node, pod, task,
+                   namespace=ext.Namespace.BASE_NAMESPACE):
+    node.check_call(
+        'kubectl exec {} --namespace={} {}'.format(
+            pod.name,
+            namespace,
+            '/etc/cron.daily/{}'.format(task)),
+        expected=[ext.ExitCodes.EX_OK])
+
+
+def list_files(node, pod, path, mask,
+               namespace=ext.Namespace.BASE_NAMESPACE):
+    return "".join(node.check_call(
+        'kubectl exec {} --namespace={} {}'.format(
+            pod.name,
+            namespace,
+            'find {} -- -iname {}'.format(path, mask)),
+        expected=[ext.ExitCodes.EX_OK])['stdout']) \
+        .replace('\n', ' ').strip().split(" ")
+
+
+def rm_files(node, pod, path,
+             namespace=ext.Namespace.BASE_NAMESPACE):
+    node.execute(
+        'kubectl exec {} --namespace={} {}'.format(
+            pod.name,
+            namespace,
+            'rm -- {}'.format(path)))

fuel_ccp_tests/managers/osmanager.py

@@ -0,0 +1,84 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import os
+
+from fuel_ccp_tests import logger
+from fuel_ccp_tests import settings
+from fuel_ccp_tests.helpers import post_os_deploy_checks
+
+LOG = logger.logger
+
+
+class OSManager(object):
+    """docstring for K8SManager"""
+
+    __config = None
+    __underlay = None
+
+    def __init__(self, config, underlay, k8s_actions, ccpcluster):
+        self.__config = config
+        self.__underlay = underlay
+        self.__k8s_actions = k8s_actions
+        self.__ccpcluster = ccpcluster
+
+    def install_os(self, topology=None,
+                   check_os_ready=True):
+        """Action to deploy openstack by ccp tool
+
+        Additional steps:
+            TODO
+
+        :param env: EnvManager
+        :param custom_yaml: False if deploy with kargo default, None if deploy
+            with environment settings, or put you own
+        :rtype: None
+        """
+        LOG.info("Trying to install k8s")
+
+        """
+        Deploy openstack with stacklight topology
+        """
+        LOG.info("Preparing openstack log collector fixture...")
+        if settings.BUILD_IMAGES:
+            LOG.info("Creating registry...")
+            self.__k8s_actions.create_registry()
+            LOG.info("Building images...")
+            self.__ccpcluster.build()
+        if topology:
+            LOG.info("Pushing topology yaml...")
+            LOG.warn(
+                "Patched topology used, workaround until kube 1.4 released")
+            topology_path = \
+                os.getcwd() + topology
+            self.__underlay.remote(
+                host=self.__config.k8s.kube_host).upload(
+                topology_path,
+                settings.DEPLOY_CONFIG)
+        LOG.info("Deploy openstack")
+        self.__ccpcluster.deploy()
+        if check_os_ready:
+            self.check_os_ready()
+        self.__config.os.running = True
+
+    def check_os_ready(self,
+                       check_jobs_ready=True,
+                       check_pods_ready=True):
+        if check_jobs_ready:
+            LOG.info("Checking openstack jobs statuses...")
+            post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api,
+                                                    timeout=3600)
+        if check_pods_ready:
+            LOG.info("Checking openstack pods statuses...")
+            post_os_deploy_checks.check_pods_status(self.__k8s_actions.api,
+                                                    timeout=3600)

fuel_ccp_tests/requirements.txt

@@ -10,3 +10,4 @@ urllib3
 psycopg2
 python-k8sclient==0.3.0
 junit-xml
+elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0

fuel_ccp_tests/settings_oslo.py

@@ -113,6 +113,16 @@ ccp_opts = [
            help="", default='0.0.0.0'),
 ]
 
+os_deploy_opts = [
+    ct.Cfg('stacklight_enable', ct.Boolean(),
+           help="", default=False),
+]
+
+os_opts = [
+    ct.Cfg('running', ct.Boolean(),
+           help="", default=False),
+]
+
 
 _group_opts = [
     ('hardware', hardware_opts),
@@ -121,6 +131,8 @@ _group_opts = [
     ('k8s', k8s_opts),
     ('ccp_deploy', ccp_deploy_opts),
     ('ccp', ccp_opts),
+    ('os_deploy', os_deploy_opts),
+    ('os', os_opts),
 ]
 
 
@@ -148,6 +160,15 @@ def register_opts(config):
     config.register_group(cfg.OptGroup(name='ccp',
                           title="CCP config and credentials", help=""))
     config.register_opts(group='ccp', opts=ccp_opts)
+
+    config.register_group(cfg.OptGroup(name='os',
+                          title="Openstack config and credentials", help=""))
+    config.register_opts(group='os', opts=os_opts)
+    config.register_group(
+        cfg.OptGroup(name='os_deploy',
+                     title="Openstack deploy config and credentials",
+                     help=""))
+    config.register_opts(group='os_deploy', opts=os_deploy_opts)
     return config
 
 

fuel_ccp_tests/tests/component/ccp/test_ccp_logging.py

@@ -0,0 +1,706 @@
+#    Copyright 2016 Mirantis, Inc.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+import json
+import uuid
+from datetime import datetime
+from time import sleep
+
+import pytest
+
+from fuel_ccp_tests.helpers import ext
+from fuel_ccp_tests.helpers import utils
+from fuel_ccp_tests.logger import logger
+
+
+@pytest.yield_fixture(scope='function')
+def admin_node(config, underlay, ccpcluster):
+    logger.info("Get SSH access to admin node")
+    with underlay.remote(host=config.k8s.kube_host) as remote:
+        yield remote
+
+
+@pytest.yield_fixture(scope='function')
+def elastic_client_public(os_deployed, k8s_actions, config):
+    """
+    Discover elasticsearch on the cluster and return simple elastic client
+    initialized with public endpoint
+    :param os_deployed:
+    :param k8s_actions:
+    :param config:
+    :return: utils.ElasticClient
+    """
+    service_list = k8s_actions.api.services.list(
+        namespace=ext.Namespace.BASE_NAMESPACE)
+    service = [service for service in
+               service_list if 'elasticsearch' in service.name][0]
+
+    elastic_search_public_port = service.spec.ports[0].node_port
+    elastic_search_public_host = config.k8s.kube_host
+
+    yield utils.ElasticClient(elastic_search_public_host,
+                              elastic_search_public_port)
+
+
+@pytest.yield_fixture(scope='function')
+def elastic_client_private(os_deployed,
+                           k8s_actions,
+                           admin_node,
+                           config):
+    """
+    Discover elasticsearch on the cluster and return simple elastic client
+    initialized with pod ip endpoint
+    :param os_deployed:
+    :param k8s_actions:
+    :param config:
+    :return: utils.ElasticClient
+    """
+    service_list = k8s_actions.api.services.list(
+        namespace=ext.Namespace.BASE_NAMESPACE)
+    service = [service for service in
+               service_list if 'elasticsearch' in service.name][0]
+
+    elastic_search_service_host = service.spec.cluster_ip
+    elastic_search_service_port = service.spec.ports[0].name
+
+    yield utils.ElasticClient(elastic_search_service_host,
+                              elastic_search_service_port)
+
+
+@pytest.fixture(scope='function')
+def kibana_public_endpoint(os_deployed,
+                           k8s_actions,
+                           admin_node,
+                           config):
+    """
+    Discover kibana on the cluster and return kibana endpoint
+    :param os_deployed:
+    :param k8s_actions:
+    :param admin_node:
+    :param config:
+    :return: host:port
+    """
+    service_list = k8s_actions.api.services.list(
+        namespace=ext.Namespace.BASE_NAMESPACE)
+    service = [service for service in
+               service_list if 'kibana' in service.name][0]
+
+    kibana_public_port = service.spec.ports[0].node_port
+    kibana_public_address = config.k8s.kube_host
+
+    return '{}:{}'.format(kibana_public_address, kibana_public_port)
+
+
+@pytest.mark.ccp_logging
+@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_deployed)
+class TestCppLogging(object):
+    """Check logging aggregation"""
+    def test_logging_connection_to_elasticsearch_public(
+            self, admin_node, elastic_client_public, show_step):
+        """Elasticsearch api test
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Test elasticsearch is accessible on public ip
+        """
+        show_step(1)
+        elastic_call = 'curl -s -o /dev/null -w "%{{http_code}}" http://{}:{}/'
+        elastic_http_response = admin_node.execute(elastic_call.format(
+            elastic_client_public.host,
+            elastic_client_public.port))['stdout']
+
+        assert ext.HttpCodes.OK in elastic_http_response, \
+            "Elastic respond with unexpected " \
+            "HTTP_RESPONSE on public endpoint Expected {} Actual {}".format(
+                ext.HttpCodes.OK, elastic_http_response)
+
+    def test_logging_connection_to_elasticsearch_private(
+            self,
+            admin_node,
+            elastic_client_private,
+            show_step):
+        """Elasticsearch api test
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Test elasticsearch is accessibile on private ip
+        """
+        show_step(1)
+        elastic_call = 'curl -s -o /dev/null -w "%{{http_code}}" http://{}:{}/'
+        elastic_http_response = admin_node.execute(
+            elastic_call.format(
+                elastic_client_private.host,
+                elastic_client_private.port))['stdout']
+        assert ext.HttpCodes.OK in elastic_http_response, \
+            "Elastic respond with unexpected " \
+            "HTTP_RESPONSE on private endpoint Expected {} Actual {}".format(
+                ext.HttpCodes.OK, elastic_http_response)
+
+    def test_logging_search_for_logs_from_all_running_heka_instances(
+            self, admin_node, k8scluster, elastic_client_public, show_step):
+        """Heka connection test
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Find logs from all heka nodes
+        2. Test that logs from each heka node exist
+        """
+
+        show_step(1)
+        ec = elastic_client_public
+        k8sclient = k8scluster.api
+        # get all nodes
+        nodes = k8sclient.nodes.list()
+        # get all heka instances
+        hekas = [pod for pod
+                 in k8sclient.pods.list(namespace=ext.Namespace.BASE_NAMESPACE)
+                 if 'heka' in pod.name]
+        # ensure heka is running on each node
+        assert len(nodes) == len(hekas)
+
+        show_step(2)
+        for heka_job in hekas:
+            logger.info('Checking presense in aggregated log messages from {}'
+                        .format(heka_job.name))
+            assert ec.find('Hostname', heka_job.name).count > 0, \
+                "Log message from heka node {} not found on elastic".format(
+                    heka_job.name)
+
+    def test_logging_trigger_event_into_mysql(self,
+                                              admin_node,
+                                              k8scluster,
+                                              elastic_client_public,
+                                              show_step):
+        """Trigger event in mysql container
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Get mysql pod
+        2. Trigger mysql log event inside the container
+        3. Test that triggered event pushed to elasticsearch
+        """
+        ec = elastic_client_public
+        k8sclient = k8scluster.api
+
+        show_step(1)
+        mysql_pod = [pod for pod in
+                     k8sclient.pods.list(
+                         namespace=ext.Namespace.BASE_NAMESPACE)
+                     if 'mariadb' in pod.name][0]
+
+        show_step(2)
+        mysql_id = str(uuid.uuid4()).replace('-', '')
+        mysql_template = \
+            '{} 140115909998528 ' \
+            '[Note] mysqld: ready for connections. {}\n'.format(
+                datetime.today().strftime('%Y-%m-%d %H:%M:%S'),
+                mysql_id)
+
+        admin_node.check_call(
+            'kubectl exec {} --namespace={} -- {}'.format(
+                mysql_pod.name,
+                ext.Namespace.BASE_NAMESPACE,
+                '\'/bin/bash\' -xc \'(echo \"{}\" >> '
+                '/var/log/ccp/mysql/mysql.log)\''.format(
+                    mysql_template)
+            ),
+            expected=[ext.ExitCodes.EX_OK])
+
+        show_step(3)
+        injected = ec.find('Payload', mysql_id)
+        assert injected.count == 1, \
+            "New log message from mysql from {} not picked by heka".format(
+                mysql_pod)
+
+    def test_logging_trigger_event_into_rabbitmq(self,
+                                                 admin_node,
+                                                 k8scluster,
+                                                 elastic_client_public,
+                                                 show_step):
+        """Trigger event in rabbitmq container
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Get rabbitmq pod
+        2. Trigger rabbitmq log event inside the container
+        3. Test that triggered event pushed to elasticsearch
+        """
+        show_step(1)
+        ec = elastic_client_public
+        k8sclient = k8scluster.api
+        rabbitmq_pod = [pod for pod in
+                        k8sclient.pods.list(
+                            namespace=ext.Namespace.BASE_NAMESPACE)
+                        if 'rabbitmq' in pod.name][0]
+
+        show_step(2)
+        rabbitmq_id = str(uuid.uuid4()).replace('-', '')
+        rabbitmq_template = "=INFO REPORT==== {} ===\n" \
+                            "accepting AMQP connection <0.580.0> " \
+                            "(10.233.83.7 -> 10.233.83.89:5672):\n" \
+                            "{}".format(datetime.today()
+                                        .strftime("%d-%b-%Y::%H:%M:%S"),
+                                        rabbitmq_id)
+        admin_node.check_call(
+            'kubectl exec {} --namespace={} -- {}'.format(
+                rabbitmq_pod.name,
+                ext.Namespace.BASE_NAMESPACE,
+                '\'/bin/bash\' -xc \'(echo -e \"{}\n\" >> '
+                '/var/log/ccp/rabbitmq/rabbitmq.log)\''.format(
+                    rabbitmq_template)),
+            expected=[ext.ExitCodes.EX_OK])
+
+        show_step(3)
+        injected = ec.find('Payload', rabbitmq_id)
+        assert injected.count == 1,\
+            "New log message from mysql from {} not picked by heka".format(
+                rabbitmq_pod)
+
+    def test_logging_attributes_for_mysql_message(self,
+                                                  elastic_client_public,
+                                                  show_step):
+        """Test attibute population consistency for mysql
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search mysql log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'mysql').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'mysql')
+
+    def test_logging_attributes_for_rabbitmq_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for rabbitmq
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search rabbitmq log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'rabbitmq').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'rabbitmq')
+
+    def test_logging_attributes_for_openstack_horizon_apache_message(
+            self,
+            elastic_client_public,
+            k8s_actions,
+            admin_node,
+            show_step):
+        """Test attibute population consistency for horizon-apache
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Trigger horizon to produce logs
+        2. Search horizon-apache log event in the elasticsearch
+        3. Test logged message consistency
+        """
+        show_step(1)
+        service_list = k8s_actions.api.services.list(
+            ext.Namespace.BASE_NAMESPACE)
+        service = [service for service in
+                   service_list if 'horizon' in service.name][0]
+        horizon_service_host = service.spec.cluster_ip
+        horizon_service_port = service.spec.ports[0].name
+        admin_node.check_call(
+            'curl http://{}:{}'.format(horizon_service_host,
+                                       horizon_service_port),
+            expected=[ext.ExitCodes.EX_OK])
+
+        show_step(2)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openstack.horizon-apache').get(0)
+
+        show_step(3)
+        self.check_message_format(event, 'openstack.horizon-apache')
+
+    def test_logging_attributes_for_openstack_keystone_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for keystone
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search keystone log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openstack.keystone').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'openstack.keystone')
+
+    def test_logging_attributes_for_openstack_keystone_apache_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for keystone-apache
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search keystone-apache log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openstack.keystone-apache').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'openstack.keystone-apache')
+
+    def test_logging_attributes_for_openstack_nova_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for nova
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search nova log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openstack.nova').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'openstack.nova')
+
+    def test_logging_attributes_for_openstack_neutron_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for neutron
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search neutron log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openstack.neutron').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'openstack.neutron')
+
+    def test_logging_attributes_for_openstack_glance_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for glance
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search glance log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openstack.glance').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'openstack.glance')
+
+    def test_logging_attributes_for_openstack_heat_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for heat
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search heat log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openstack.heat').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'openstack.heat')
+
+    def test_logging_attributes_for_openvswitch_message(
+            self,
+            elastic_client_public,
+            show_step):
+        """Test attibute population consistency for openvswitch
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Search openvswitch log event in the elasticsearch
+        2. Test logged message consistency
+        """
+        show_step(1)
+        ec = elastic_client_public
+        event = ec.find('Logger', 'openvswitch').get(0)
+
+        show_step(2)
+        self.check_message_format(event, 'openvswitch')
+
+    def test_logging_explore_indexes_with_kibana(
+            self,
+            admin_node,
+            kibana_public_endpoint,
+            show_step):
+        """Test index availability from kibana
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Get kibana service status
+        2. Test overal status should be green
+        """
+        show_step(1)
+        status_full = json.loads(
+            "".join(admin_node.execute('curl http://{}/api/status'.format(
+                kibana_public_endpoint))['stdout']))
+        status_overall = status_full['status']['overall']['state']
+
+        show_step(2)
+        assert status_overall == 'green', "Kibaba service have issues. " \
+                                          "Status {}".format(status_overall)
+
+    def test_logging_kibana_running_single_node(
+            self,
+            admin_node,
+            os_deployed,
+            k8s_actions,
+            show_step):
+        """Test kibana running single node
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Check kibana running single node
+        """
+        show_step(1)
+        pods_list = k8s_actions.api.pods.list(
+            namespace=ext.Namespace.BASE_NAMESPACE)
+        kibana_count = len(
+            [pod for pod in pods_list
+             if 'kibana' in pod.name])
+        assert kibana_count == 1, "Unexpected count of kibana instances"
+
+    def test_logging_elastic_running_single_node(
+            self,
+            admin_node,
+            os_deployed,
+            k8s_actions,
+            show_step):
+        """Test elasticsearch running single node
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Check elasticsearch running single node
+        """
+        show_step(1)
+        pods_list = k8s_actions.api.pods.list(
+            namespace=ext.Namespace.BASE_NAMESPACE)
+        elastic_count = len(
+            [pod for pod in pods_list
+             if 'elasticsearch' in pod.name])
+        assert elastic_count == 1, \
+            "Unexpected count of elasticsearch instances"
+
+    def test_logging_heka_running_all_nodes(
+            self,
+            admin_node,
+            os_deployed,
+            k8s_actions,
+            show_step):
+        """Test heka running all nodes
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Check heka running on each node
+        """
+        show_step(1)
+        pods_list = k8s_actions.api.pods.list(
+            namespace=ext.Namespace.BASE_NAMESPACE)
+        nodes_list = [node.name for node in k8s_actions.api.nodes.list()]
+        heka_nodes = [
+            pod.spec.node_name for pod in pods_list if 'heka' in pod.name]
+
+        assert sorted(nodes_list) == sorted(heka_nodes),\
+            "Unexpected count of heka running on nodes instances"
+
+    def test_logging_log_rotate_for_mysql(
+            self,
+            admin_node,
+            k8s_actions,
+            show_step,
+            os_deployed):
+        """Test log rotate for mysql
+        Precondition:
+        1. Install k8s
+        2. Install microservices
+        3. Fetch all repos
+        4. Build images or use external registry
+        5. Deploy openstack
+
+        Scenario:
+        1. Clean mysql log on cron pod
+        2. Simulate 8 days log rotation
+        3. Ensure that count of log files is equal to 7(week rotation)
+        """
+        logger.info('Log rotate for mysql')
+        log_path = '/var/log/ccp/mysql/'
+        log_file = 'mysql.log'
+
+        show_step(1)
+        # get cron pod
+        cron_pod = [pod for pod in k8s_actions.api.pods.list(
+            namespace=ext.Namespace.BASE_NAMESPACE)
+            if 'cron-' in pod.name][0]
+        # clean files
+        utils.rm_files(admin_node, cron_pod, log_path + log_file + '*')
+
+        show_step(2)
+        for day in range(0, 8):
+            utils.create_file(admin_node, cron_pod, log_path + log_file, 110)
+            utils.run_daily_cron(admin_node, cron_pod, 'logrotate')
+            sleep(5)
+
+        show_step(3)
+        log_files = utils.list_files(
+            admin_node, cron_pod, log_path, log_file + '*')
+        assert len(log_files) == 7,\
+            "Count of log files after rotation is wrong. " \
+            "Expected {} Actual {}".format(log_files, 7)
+
+    @staticmethod
+    def check_message_format(event, logger):
+        """Event consistency validator"""
+        assert event is not None, "No result found for {}".format(logger)
+        assert len(event['Timestamp']) > 0, \
+            "Logger {} have wrong [timestamp] field".format(logger)
+        assert event['Type'] == 'log', \
+            "Logger {} have wrong [Type] field".format(logger)
+        assert len(event['Payload']) > 0, \
+            "Logger {} have wrong [Payload] field".format(logger)
+        assert isinstance(event['Pid'], int), \
+            "Logger {} have wrong [Pid] field".format(logger)
+        assert len(event['Hostname']) > 0, \
+            "Logger {} have wrong [Hostname] field".format(logger)
+        assert event['severity_label'] in ext.LOG_LEVELS, \
+            "Logger {} have wrong [severity_label] field".format(logger)
+        assert len(event['programname']) > 0, \
+            "Logger {} have wrong [programname] field".format(logger)
+        assert isinstance(event['Severity'], int), \
+            "Logger {} have wrong [Severity] field".format(logger)

fuel_ccp_tests/tests/component/conftest.py

@@ -18,4 +18,5 @@ pytest_plugins = ['fuel_ccp_tests.fixtures.common_fixtures',
                   'fuel_ccp_tests.fixtures.k8s_fixtures',
                   'fuel_ccp_tests.fixtures.rally_fixtures',
                   'fuel_ccp_tests.fixtures.ccp_fixtures',
-                  'fuel_ccp_tests.fixtures.influxdb_fixtures']
+                  'fuel_ccp_tests.fixtures.influxdb_fixtures',
+                  'fuel_ccp_tests.fixtures.os_fixtures']

fuel_ccp_tests/tests/system/conftest.py

@@ -17,4 +17,5 @@ pytest_plugins = ['fuel_ccp_tests.fixtures.common_fixtures',
                   'fuel_ccp_tests.fixtures.underlay_fixtures',
                   'fuel_ccp_tests.fixtures.k8s_fixtures',
                   'fuel_ccp_tests.fixtures.rally_fixtures',
-                  'fuel_ccp_tests.fixtures.ccp_fixtures']
+                  'fuel_ccp_tests.fixtures.ccp_fixtures',
+                  'fuel_ccp_tests.fixtures.os_fixtures']