Test logging aggregation
- to run this test, the following env variable should be exported: export STACKLIGHT_ENABLE=true - os manager - os deployed fixture - os deployed with stacklight fixture - assertion messages added - moved to component folder - list of files switched from ls to find - elastic client return None if not items found - show_step applied to all scenarios Change-Id: I4ded419730889761ffa102a23e97d655a07d20fa
This commit is contained in:
Jenkins
ydehtiarov
parent
7a46e34d90
commit
52ddfb73e3
|
|
@ -0,0 +1,52 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import pytest
|
||||
|
||||
from fuel_ccp_tests import logger
|
||||
from fuel_ccp_tests.helpers import ext
|
||||
from fuel_ccp_tests.managers.osmanager import OSManager
|
||||
|
||||
LOG = logger.logger
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def os_deployed(ccpcluster,
|
||||
hardware,
|
||||
underlay,
|
||||
revert_snapshot,
|
||||
config,
|
||||
k8s_actions):
|
||||
"""Deploy openstack
|
||||
"""
|
||||
# If no snapshot was reverted, then try to revert the snapshot
|
||||
# that belongs to the fixture.
|
||||
# Note: keep fixtures in strict dependences from each other!
|
||||
if not revert_snapshot:
|
||||
if hardware.has_snapshot(ext.SNAPSHOT.os_deployed) and \
|
||||
hardware.has_snapshot_config(ext.SNAPSHOT.os_deployed):
|
||||
hardware.revert_snapshot(ext.SNAPSHOT.os_deployed)
|
||||
|
||||
osmanager = OSManager(config, underlay, k8s_actions, ccpcluster)
|
||||
if not config.os.running:
|
||||
LOG.info("Preparing openstack log collector fixture...")
|
||||
topology = None
|
||||
if config.os_deploy.stacklight_enable:
|
||||
topology = ('/fuel_ccp_tests/templates/k8s_templates/'
|
||||
'stacklight_topology.yaml')
|
||||
osmanager.install_os(topology=topology)
|
||||
hardware.create_snapshot(ext.SNAPSHOT.os_deployed)
|
||||
else:
|
||||
LOG.info("Openstack allready installed and running...")
|
||||
osmanager.check_os_ready()
|
||||
|
|
@ -14,6 +14,8 @@
|
|||
|
||||
import collections
|
||||
|
||||
from enum import IntEnum
|
||||
|
||||
|
||||
def enum(*values, **kwargs):
|
||||
names = kwargs.get('names')
|
||||
|
|
@ -38,4 +40,47 @@ SNAPSHOT = enum(
|
|||
'underlay',
|
||||
'k8s_deployed',
|
||||
'ccp_deployed',
|
||||
'os_deployed',
|
||||
'os_deployed_stacklight'
|
||||
)
|
||||
|
||||
LOG_LEVELS = enum(
|
||||
'INFO',
|
||||
'WARNING',
|
||||
'ERROR',
|
||||
'CRITICAL',
|
||||
'DEBUG',
|
||||
'NOTE'
|
||||
)
|
||||
|
||||
|
||||
class ExitCodes(IntEnum):
|
||||
EX_OK = 0 # successful termination
|
||||
EX_INVALID = 0xDEADBEEF # uint32 debug value. Impossible for POSIX
|
||||
EX_ERROR = 1 # general failure
|
||||
EX_BUILTIN = 2 # Misuse of shell builtins (according to Bash)
|
||||
EX_USAGE = 64 # command line usage error
|
||||
EX_DATAERR = 65 # data format error
|
||||
EX_NOINPUT = 66 # cannot open input
|
||||
EX_NOUSER = 67 # addressee unknown
|
||||
EX_NOHOST = 68 # host name unknown
|
||||
EX_UNAVAILABLE = 69 # service unavailable
|
||||
EX_SOFTWARE = 70 # internal software error
|
||||
EX_OSERR = 71 # system error (e.g., can't fork)
|
||||
EX_OSFILE = 72 # critical OS file missing
|
||||
EX_CANTCREAT = 73 # can't create (user) output file
|
||||
EX_IOERR = 74 # input/output error
|
||||
EX_TEMPFAIL = 75 # temp failure; user is invited to retry
|
||||
EX_PROTOCOL = 76 # remote error in protocol
|
||||
EX_NOPERM = 77 # permission denied
|
||||
EX_CONFIG = 78 # configuration error
|
||||
EX_NOEXEC = 126 # If a command is found but is not executable
|
||||
EX_NOCMD = 127 # If a command is not found
|
||||
|
||||
|
||||
class HttpCodes(enumerate):
|
||||
OK = '200'
|
||||
|
||||
|
||||
class Namespace(enumerate):
|
||||
BASE_NAMESPACE = 'ccp'
|
||||
|
|
|
|||
|
|
@ -14,16 +14,19 @@
|
|||
|
||||
|
||||
import os
|
||||
import time
|
||||
import paramiko
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
|
||||
import paramiko
|
||||
import yaml
|
||||
from devops.helpers import helpers
|
||||
from elasticsearch import Elasticsearch
|
||||
|
||||
from fuel_ccp_tests import logger
|
||||
from fuel_ccp_tests import settings
|
||||
from fuel_ccp_tests.helpers import ext
|
||||
|
||||
LOG = logger.logger
|
||||
|
||||
|
|
@ -179,3 +182,98 @@ def retry(tries_number=3, exception=Exception):
|
|||
iter_number += 1
|
||||
return wrapper
|
||||
return _retry
|
||||
|
||||
|
||||
class ElasticClient(object):
|
||||
def __init__(self, host='localhost', port=9200):
|
||||
self.es = Elasticsearch([{'host': '{}'.format(host),
|
||||
'port': port}])
|
||||
self.host = host
|
||||
self.port = port
|
||||
|
||||
def find(self, key, value):
|
||||
LOG.info('Search for {} for {}'.format(key, value))
|
||||
search_request_body = '{' +\
|
||||
' "query": {' +\
|
||||
' "simple_query_string": {' +\
|
||||
' "query": "{}",'.format(value) +\
|
||||
' "analyze_wildcard" : "true",' +\
|
||||
' "fields" : ["{}"],'.format(key) +\
|
||||
' "default_operator": "AND"' +\
|
||||
' }' +\
|
||||
' },' +\
|
||||
' "size": 1' +\
|
||||
'}'
|
||||
LOG.info('Search by {}'.format(search_request_body))
|
||||
|
||||
def is_found():
|
||||
def temporary_status():
|
||||
res = self.es.search(index='_all', body=search_request_body)
|
||||
return res['hits']['total'] != 0
|
||||
return temporary_status
|
||||
|
||||
predicate = is_found()
|
||||
helpers.wait(predicate, timeout=300,
|
||||
timeout_msg='Timeout waiting, result from elastic')
|
||||
|
||||
es_raw = self.es.search(index='_all', body=search_request_body)
|
||||
if es_raw['timed_out']:
|
||||
raise RuntimeError('Elastic search timeout exception')
|
||||
|
||||
return ElasticSearchResult(key, value, es_raw['hits']['total'], es_raw)
|
||||
|
||||
|
||||
class ElasticSearchResult(object):
|
||||
def __init__(self, key, value, count, raw):
|
||||
self.key = key
|
||||
self.value = value
|
||||
self.count = count
|
||||
self.raw = raw
|
||||
if self.count != 0:
|
||||
self.items = raw['hits']['hits']
|
||||
|
||||
def get(self, index):
|
||||
if self.count != 0:
|
||||
return self.items[index]['_source']
|
||||
else:
|
||||
None
|
||||
|
||||
|
||||
def create_file(node, pod, path, size,
|
||||
namespace=ext.Namespace.BASE_NAMESPACE):
|
||||
node.check_call(
|
||||
'kubectl exec {} --namespace={} {}'.format(
|
||||
pod.name,
|
||||
namespace,
|
||||
'dd -- if=/dev/zero -- of={} bs=1MB count={}'.format(path, size)),
|
||||
expected=[ext.ExitCodes.EX_OK])
|
||||
|
||||
|
||||
def run_daily_cron(node, pod, task,
|
||||
namespace=ext.Namespace.BASE_NAMESPACE):
|
||||
node.check_call(
|
||||
'kubectl exec {} --namespace={} {}'.format(
|
||||
pod.name,
|
||||
namespace,
|
||||
'/etc/cron.daily/{}'.format(task)),
|
||||
expected=[ext.ExitCodes.EX_OK])
|
||||
|
||||
|
||||
def list_files(node, pod, path, mask,
|
||||
namespace=ext.Namespace.BASE_NAMESPACE):
|
||||
return "".join(node.check_call(
|
||||
'kubectl exec {} --namespace={} {}'.format(
|
||||
pod.name,
|
||||
namespace,
|
||||
'find {} -- -iname {}'.format(path, mask)),
|
||||
expected=[ext.ExitCodes.EX_OK])['stdout']) \
|
||||
.replace('\n', ' ').strip().split(" ")
|
||||
|
||||
|
||||
def rm_files(node, pod, path,
|
||||
namespace=ext.Namespace.BASE_NAMESPACE):
|
||||
node.execute(
|
||||
'kubectl exec {} --namespace={} {}'.format(
|
||||
pod.name,
|
||||
namespace,
|
||||
'rm -- {}'.format(path)))
|
||||
|
|
|
|||
|
|
@ -0,0 +1,84 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import os
|
||||
|
||||
from fuel_ccp_tests import logger
|
||||
from fuel_ccp_tests import settings
|
||||
from fuel_ccp_tests.helpers import post_os_deploy_checks
|
||||
|
||||
LOG = logger.logger
|
||||
|
||||
|
||||
class OSManager(object):
|
||||
"""docstring for K8SManager"""
|
||||
|
||||
__config = None
|
||||
__underlay = None
|
||||
|
||||
def __init__(self, config, underlay, k8s_actions, ccpcluster):
|
||||
self.__config = config
|
||||
self.__underlay = underlay
|
||||
self.__k8s_actions = k8s_actions
|
||||
self.__ccpcluster = ccpcluster
|
||||
|
||||
def install_os(self, topology=None,
|
||||
check_os_ready=True):
|
||||
"""Action to deploy openstack by ccp tool
|
||||
|
||||
Additional steps:
|
||||
TODO
|
||||
|
||||
:param env: EnvManager
|
||||
:param custom_yaml: False if deploy with kargo default, None if deploy
|
||||
with environment settings, or put you own
|
||||
:rtype: None
|
||||
"""
|
||||
LOG.info("Trying to install k8s")
|
||||
|
||||
"""
|
||||
Deploy openstack with stacklight topology
|
||||
"""
|
||||
LOG.info("Preparing openstack log collector fixture...")
|
||||
if settings.BUILD_IMAGES:
|
||||
LOG.info("Creating registry...")
|
||||
self.__k8s_actions.create_registry()
|
||||
LOG.info("Building images...")
|
||||
self.__ccpcluster.build()
|
||||
if topology:
|
||||
LOG.info("Pushing topology yaml...")
|
||||
LOG.warn(
|
||||
"Patched topology used, workaround until kube 1.4 released")
|
||||
topology_path = \
|
||||
os.getcwd() + topology
|
||||
self.__underlay.remote(
|
||||
host=self.__config.k8s.kube_host).upload(
|
||||
topology_path,
|
||||
settings.DEPLOY_CONFIG)
|
||||
LOG.info("Deploy openstack")
|
||||
self.__ccpcluster.deploy()
|
||||
if check_os_ready:
|
||||
self.check_os_ready()
|
||||
self.__config.os.running = True
|
||||
|
||||
def check_os_ready(self,
|
||||
check_jobs_ready=True,
|
||||
check_pods_ready=True):
|
||||
if check_jobs_ready:
|
||||
LOG.info("Checking openstack jobs statuses...")
|
||||
post_os_deploy_checks.check_jobs_status(self.__k8s_actions.api,
|
||||
timeout=3600)
|
||||
if check_pods_ready:
|
||||
LOG.info("Checking openstack pods statuses...")
|
||||
post_os_deploy_checks.check_pods_status(self.__k8s_actions.api,
|
||||
timeout=3600)
|
||||
|
|
@ -10,3 +10,4 @@ urllib3
|
|||
psycopg2
|
||||
python-k8sclient==0.3.0
|
||||
junit-xml
|
||||
elasticsearch>=2.0.0,<=3.0.0 # Apache-2.0
|
||||
|
|
|
|||
|
|
@ -113,6 +113,16 @@ ccp_opts = [
|
|||
help="", default='0.0.0.0'),
|
||||
]
|
||||
|
||||
os_deploy_opts = [
|
||||
ct.Cfg('stacklight_enable', ct.Boolean(),
|
||||
help="", default=False),
|
||||
]
|
||||
|
||||
os_opts = [
|
||||
ct.Cfg('running', ct.Boolean(),
|
||||
help="", default=False),
|
||||
]
|
||||
|
||||
|
||||
_group_opts = [
|
||||
('hardware', hardware_opts),
|
||||
|
|
@ -121,6 +131,8 @@ _group_opts = [
|
|||
('k8s', k8s_opts),
|
||||
('ccp_deploy', ccp_deploy_opts),
|
||||
('ccp', ccp_opts),
|
||||
('os_deploy', os_deploy_opts),
|
||||
('os', os_opts),
|
||||
]
|
||||
|
||||
|
||||
|
|
@ -148,6 +160,15 @@ def register_opts(config):
|
|||
config.register_group(cfg.OptGroup(name='ccp',
|
||||
title="CCP config and credentials", help=""))
|
||||
config.register_opts(group='ccp', opts=ccp_opts)
|
||||
|
||||
config.register_group(cfg.OptGroup(name='os',
|
||||
title="Openstack config and credentials", help=""))
|
||||
config.register_opts(group='os', opts=os_opts)
|
||||
config.register_group(
|
||||
cfg.OptGroup(name='os_deploy',
|
||||
title="Openstack deploy config and credentials",
|
||||
help=""))
|
||||
config.register_opts(group='os_deploy', opts=os_deploy_opts)
|
||||
return config
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,706 @@
|
|||
# Copyright 2016 Mirantis, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import json
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
|
||||
import pytest
|
||||
|
||||
from fuel_ccp_tests.helpers import ext
|
||||
from fuel_ccp_tests.helpers import utils
|
||||
from fuel_ccp_tests.logger import logger
|
||||
|
||||
|
||||
@pytest.yield_fixture(scope='function')
|
||||
def admin_node(config, underlay, ccpcluster):
|
||||
logger.info("Get SSH access to admin node")
|
||||
with underlay.remote(host=config.k8s.kube_host) as remote:
|
||||
yield remote
|
||||
|
||||
|
||||
@pytest.yield_fixture(scope='function')
|
||||
def elastic_client_public(os_deployed, k8s_actions, config):
|
||||
"""
|
||||
Discover elasticsearch on the cluster and return simple elastic client
|
||||
initialized with public endpoint
|
||||
:param os_deployed:
|
||||
:param k8s_actions:
|
||||
:param config:
|
||||
:return: utils.ElasticClient
|
||||
"""
|
||||
service_list = k8s_actions.api.services.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
service = [service for service in
|
||||
service_list if 'elasticsearch' in service.name][0]
|
||||
|
||||
elastic_search_public_port = service.spec.ports[0].node_port
|
||||
elastic_search_public_host = config.k8s.kube_host
|
||||
|
||||
yield utils.ElasticClient(elastic_search_public_host,
|
||||
elastic_search_public_port)
|
||||
|
||||
|
||||
@pytest.yield_fixture(scope='function')
|
||||
def elastic_client_private(os_deployed,
|
||||
k8s_actions,
|
||||
admin_node,
|
||||
config):
|
||||
"""
|
||||
Discover elasticsearch on the cluster and return simple elastic client
|
||||
initialized with pod ip endpoint
|
||||
:param os_deployed:
|
||||
:param k8s_actions:
|
||||
:param config:
|
||||
:return: utils.ElasticClient
|
||||
"""
|
||||
service_list = k8s_actions.api.services.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
service = [service for service in
|
||||
service_list if 'elasticsearch' in service.name][0]
|
||||
|
||||
elastic_search_service_host = service.spec.cluster_ip
|
||||
elastic_search_service_port = service.spec.ports[0].name
|
||||
|
||||
yield utils.ElasticClient(elastic_search_service_host,
|
||||
elastic_search_service_port)
|
||||
|
||||
|
||||
@pytest.fixture(scope='function')
|
||||
def kibana_public_endpoint(os_deployed,
|
||||
k8s_actions,
|
||||
admin_node,
|
||||
config):
|
||||
"""
|
||||
Discover kibana on the cluster and return kibana endpoint
|
||||
:param os_deployed:
|
||||
:param k8s_actions:
|
||||
:param admin_node:
|
||||
:param config:
|
||||
:return: host:port
|
||||
"""
|
||||
service_list = k8s_actions.api.services.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
service = [service for service in
|
||||
service_list if 'kibana' in service.name][0]
|
||||
|
||||
kibana_public_port = service.spec.ports[0].node_port
|
||||
kibana_public_address = config.k8s.kube_host
|
||||
|
||||
return '{}:{}'.format(kibana_public_address, kibana_public_port)
|
||||
|
||||
|
||||
@pytest.mark.ccp_logging
|
||||
@pytest.mark.revert_snapshot(ext.SNAPSHOT.os_deployed)
|
||||
class TestCppLogging(object):
|
||||
"""Check logging aggregation"""
|
||||
def test_logging_connection_to_elasticsearch_public(
|
||||
self, admin_node, elastic_client_public, show_step):
|
||||
"""Elasticsearch api test
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Test elasticsearch is accessible on public ip
|
||||
"""
|
||||
show_step(1)
|
||||
elastic_call = 'curl -s -o /dev/null -w "%{{http_code}}" http://{}:{}/'
|
||||
elastic_http_response = admin_node.execute(elastic_call.format(
|
||||
elastic_client_public.host,
|
||||
elastic_client_public.port))['stdout']
|
||||
|
||||
assert ext.HttpCodes.OK in elastic_http_response, \
|
||||
"Elastic respond with unexpected " \
|
||||
"HTTP_RESPONSE on public endpoint Expected {} Actual {}".format(
|
||||
ext.HttpCodes.OK, elastic_http_response)
|
||||
|
||||
def test_logging_connection_to_elasticsearch_private(
|
||||
self,
|
||||
admin_node,
|
||||
elastic_client_private,
|
||||
show_step):
|
||||
"""Elasticsearch api test
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Test elasticsearch is accessibile on private ip
|
||||
"""
|
||||
show_step(1)
|
||||
elastic_call = 'curl -s -o /dev/null -w "%{{http_code}}" http://{}:{}/'
|
||||
elastic_http_response = admin_node.execute(
|
||||
elastic_call.format(
|
||||
elastic_client_private.host,
|
||||
elastic_client_private.port))['stdout']
|
||||
assert ext.HttpCodes.OK in elastic_http_response, \
|
||||
"Elastic respond with unexpected " \
|
||||
"HTTP_RESPONSE on private endpoint Expected {} Actual {}".format(
|
||||
ext.HttpCodes.OK, elastic_http_response)
|
||||
|
||||
def test_logging_search_for_logs_from_all_running_heka_instances(
|
||||
self, admin_node, k8scluster, elastic_client_public, show_step):
|
||||
"""Heka connection test
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Find logs from all heka nodes
|
||||
2. Test that logs from each heka node exist
|
||||
"""
|
||||
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
k8sclient = k8scluster.api
|
||||
# get all nodes
|
||||
nodes = k8sclient.nodes.list()
|
||||
# get all heka instances
|
||||
hekas = [pod for pod
|
||||
in k8sclient.pods.list(namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
if 'heka' in pod.name]
|
||||
# ensure heka is running on each node
|
||||
assert len(nodes) == len(hekas)
|
||||
|
||||
show_step(2)
|
||||
for heka_job in hekas:
|
||||
logger.info('Checking presense in aggregated log messages from {}'
|
||||
.format(heka_job.name))
|
||||
assert ec.find('Hostname', heka_job.name).count > 0, \
|
||||
"Log message from heka node {} not found on elastic".format(
|
||||
heka_job.name)
|
||||
|
||||
def test_logging_trigger_event_into_mysql(self,
|
||||
admin_node,
|
||||
k8scluster,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Trigger event in mysql container
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Get mysql pod
|
||||
2. Trigger mysql log event inside the container
|
||||
3. Test that triggered event pushed to elasticsearch
|
||||
"""
|
||||
ec = elastic_client_public
|
||||
k8sclient = k8scluster.api
|
||||
|
||||
show_step(1)
|
||||
mysql_pod = [pod for pod in
|
||||
k8sclient.pods.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
if 'mariadb' in pod.name][0]
|
||||
|
||||
show_step(2)
|
||||
mysql_id = str(uuid.uuid4()).replace('-', '')
|
||||
mysql_template = \
|
||||
'{} 140115909998528 ' \
|
||||
'[Note] mysqld: ready for connections. {}\n'.format(
|
||||
datetime.today().strftime('%Y-%m-%d %H:%M:%S'),
|
||||
mysql_id)
|
||||
|
||||
admin_node.check_call(
|
||||
'kubectl exec {} --namespace={} -- {}'.format(
|
||||
mysql_pod.name,
|
||||
ext.Namespace.BASE_NAMESPACE,
|
||||
'\'/bin/bash\' -xc \'(echo \"{}\" >> '
|
||||
'/var/log/ccp/mysql/mysql.log)\''.format(
|
||||
mysql_template)
|
||||
),
|
||||
expected=[ext.ExitCodes.EX_OK])
|
||||
|
||||
show_step(3)
|
||||
injected = ec.find('Payload', mysql_id)
|
||||
assert injected.count == 1, \
|
||||
"New log message from mysql from {} not picked by heka".format(
|
||||
mysql_pod)
|
||||
|
||||
def test_logging_trigger_event_into_rabbitmq(self,
|
||||
admin_node,
|
||||
k8scluster,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Trigger event in rabbitmq container
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Get rabbitmq pod
|
||||
2. Trigger rabbitmq log event inside the container
|
||||
3. Test that triggered event pushed to elasticsearch
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
k8sclient = k8scluster.api
|
||||
rabbitmq_pod = [pod for pod in
|
||||
k8sclient.pods.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
if 'rabbitmq' in pod.name][0]
|
||||
|
||||
show_step(2)
|
||||
rabbitmq_id = str(uuid.uuid4()).replace('-', '')
|
||||
rabbitmq_template = "=INFO REPORT==== {} ===\n" \
|
||||
"accepting AMQP connection <0.580.0> " \
|
||||
"(10.233.83.7 -> 10.233.83.89:5672):\n" \
|
||||
"{}".format(datetime.today()
|
||||
.strftime("%d-%b-%Y::%H:%M:%S"),
|
||||
rabbitmq_id)
|
||||
admin_node.check_call(
|
||||
'kubectl exec {} --namespace={} -- {}'.format(
|
||||
rabbitmq_pod.name,
|
||||
ext.Namespace.BASE_NAMESPACE,
|
||||
'\'/bin/bash\' -xc \'(echo -e \"{}\n\" >> '
|
||||
'/var/log/ccp/rabbitmq/rabbitmq.log)\''.format(
|
||||
rabbitmq_template)),
|
||||
expected=[ext.ExitCodes.EX_OK])
|
||||
|
||||
show_step(3)
|
||||
injected = ec.find('Payload', rabbitmq_id)
|
||||
assert injected.count == 1,\
|
||||
"New log message from mysql from {} not picked by heka".format(
|
||||
rabbitmq_pod)
|
||||
|
||||
def test_logging_attributes_for_mysql_message(self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for mysql
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search mysql log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'mysql').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'mysql')
|
||||
|
||||
def test_logging_attributes_for_rabbitmq_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for rabbitmq
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search rabbitmq log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'rabbitmq').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'rabbitmq')
|
||||
|
||||
def test_logging_attributes_for_openstack_horizon_apache_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
k8s_actions,
|
||||
admin_node,
|
||||
show_step):
|
||||
"""Test attibute population consistency for horizon-apache
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Trigger horizon to produce logs
|
||||
2. Search horizon-apache log event in the elasticsearch
|
||||
3. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
service_list = k8s_actions.api.services.list(
|
||||
ext.Namespace.BASE_NAMESPACE)
|
||||
service = [service for service in
|
||||
service_list if 'horizon' in service.name][0]
|
||||
horizon_service_host = service.spec.cluster_ip
|
||||
horizon_service_port = service.spec.ports[0].name
|
||||
admin_node.check_call(
|
||||
'curl http://{}:{}'.format(horizon_service_host,
|
||||
horizon_service_port),
|
||||
expected=[ext.ExitCodes.EX_OK])
|
||||
|
||||
show_step(2)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openstack.horizon-apache').get(0)
|
||||
|
||||
show_step(3)
|
||||
self.check_message_format(event, 'openstack.horizon-apache')
|
||||
|
||||
def test_logging_attributes_for_openstack_keystone_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for keystone
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search keystone log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openstack.keystone').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'openstack.keystone')
|
||||
|
||||
def test_logging_attributes_for_openstack_keystone_apache_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for keystone-apache
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search keystone-apache log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openstack.keystone-apache').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'openstack.keystone-apache')
|
||||
|
||||
def test_logging_attributes_for_openstack_nova_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for nova
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search nova log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openstack.nova').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'openstack.nova')
|
||||
|
||||
def test_logging_attributes_for_openstack_neutron_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for neutron
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search neutron log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openstack.neutron').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'openstack.neutron')
|
||||
|
||||
def test_logging_attributes_for_openstack_glance_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for glance
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search glance log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openstack.glance').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'openstack.glance')
|
||||
|
||||
def test_logging_attributes_for_openstack_heat_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for heat
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search heat log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openstack.heat').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'openstack.heat')
|
||||
|
||||
def test_logging_attributes_for_openvswitch_message(
|
||||
self,
|
||||
elastic_client_public,
|
||||
show_step):
|
||||
"""Test attibute population consistency for openvswitch
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Search openvswitch log event in the elasticsearch
|
||||
2. Test logged message consistency
|
||||
"""
|
||||
show_step(1)
|
||||
ec = elastic_client_public
|
||||
event = ec.find('Logger', 'openvswitch').get(0)
|
||||
|
||||
show_step(2)
|
||||
self.check_message_format(event, 'openvswitch')
|
||||
|
||||
def test_logging_explore_indexes_with_kibana(
|
||||
self,
|
||||
admin_node,
|
||||
kibana_public_endpoint,
|
||||
show_step):
|
||||
"""Test index availability from kibana
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Get kibana service status
|
||||
2. Test overal status should be green
|
||||
"""
|
||||
show_step(1)
|
||||
status_full = json.loads(
|
||||
"".join(admin_node.execute('curl http://{}/api/status'.format(
|
||||
kibana_public_endpoint))['stdout']))
|
||||
status_overall = status_full['status']['overall']['state']
|
||||
|
||||
show_step(2)
|
||||
assert status_overall == 'green', "Kibaba service have issues. " \
|
||||
"Status {}".format(status_overall)
|
||||
|
||||
def test_logging_kibana_running_single_node(
|
||||
self,
|
||||
admin_node,
|
||||
os_deployed,
|
||||
k8s_actions,
|
||||
show_step):
|
||||
"""Test kibana running single node
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Check kibana running single node
|
||||
"""
|
||||
show_step(1)
|
||||
pods_list = k8s_actions.api.pods.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
kibana_count = len(
|
||||
[pod for pod in pods_list
|
||||
if 'kibana' in pod.name])
|
||||
assert kibana_count == 1, "Unexpected count of kibana instances"
|
||||
|
||||
def test_logging_elastic_running_single_node(
|
||||
self,
|
||||
admin_node,
|
||||
os_deployed,
|
||||
k8s_actions,
|
||||
show_step):
|
||||
"""Test elasticsearch running single node
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Check elasticsearch running single node
|
||||
"""
|
||||
show_step(1)
|
||||
pods_list = k8s_actions.api.pods.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
elastic_count = len(
|
||||
[pod for pod in pods_list
|
||||
if 'elasticsearch' in pod.name])
|
||||
assert elastic_count == 1, \
|
||||
"Unexpected count of elasticsearch instances"
|
||||
|
||||
def test_logging_heka_running_all_nodes(
|
||||
self,
|
||||
admin_node,
|
||||
os_deployed,
|
||||
k8s_actions,
|
||||
show_step):
|
||||
"""Test heka running all nodes
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Check heka running on each node
|
||||
"""
|
||||
show_step(1)
|
||||
pods_list = k8s_actions.api.pods.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
nodes_list = [node.name for node in k8s_actions.api.nodes.list()]
|
||||
heka_nodes = [
|
||||
pod.spec.node_name for pod in pods_list if 'heka' in pod.name]
|
||||
|
||||
assert sorted(nodes_list) == sorted(heka_nodes),\
|
||||
"Unexpected count of heka running on nodes instances"
|
||||
|
||||
def test_logging_log_rotate_for_mysql(
|
||||
self,
|
||||
admin_node,
|
||||
k8s_actions,
|
||||
show_step,
|
||||
os_deployed):
|
||||
"""Test log rotate for mysql
|
||||
Precondition:
|
||||
1. Install k8s
|
||||
2. Install microservices
|
||||
3. Fetch all repos
|
||||
4. Build images or use external registry
|
||||
5. Deploy openstack
|
||||
|
||||
Scenario:
|
||||
1. Clean mysql log on cron pod
|
||||
2. Simulate 8 days log rotation
|
||||
3. Ensure that count of log files is equal to 7(week rotation)
|
||||
"""
|
||||
logger.info('Log rotate for mysql')
|
||||
log_path = '/var/log/ccp/mysql/'
|
||||
log_file = 'mysql.log'
|
||||
|
||||
show_step(1)
|
||||
# get cron pod
|
||||
cron_pod = [pod for pod in k8s_actions.api.pods.list(
|
||||
namespace=ext.Namespace.BASE_NAMESPACE)
|
||||
if 'cron-' in pod.name][0]
|
||||
# clean files
|
||||
utils.rm_files(admin_node, cron_pod, log_path + log_file + '*')
|
||||
|
||||
show_step(2)
|
||||
for day in range(0, 8):
|
||||
utils.create_file(admin_node, cron_pod, log_path + log_file, 110)
|
||||
utils.run_daily_cron(admin_node, cron_pod, 'logrotate')
|
||||
sleep(5)
|
||||
|
||||
show_step(3)
|
||||
log_files = utils.list_files(
|
||||
admin_node, cron_pod, log_path, log_file + '*')
|
||||
assert len(log_files) == 7,\
|
||||
"Count of log files after rotation is wrong. " \
|
||||
"Expected {} Actual {}".format(log_files, 7)
|
||||
|
||||
@staticmethod
|
||||
def check_message_format(event, logger):
|
||||
"""Event consistency validator"""
|
||||
assert event is not None, "No result found for {}".format(logger)
|
||||
assert len(event['Timestamp']) > 0, \
|
||||
"Logger {} have wrong [timestamp] field".format(logger)
|
||||
assert event['Type'] == 'log', \
|
||||
"Logger {} have wrong [Type] field".format(logger)
|
||||
assert len(event['Payload']) > 0, \
|
||||
"Logger {} have wrong [Payload] field".format(logger)
|
||||
assert isinstance(event['Pid'], int), \
|
||||
"Logger {} have wrong [Pid] field".format(logger)
|
||||
assert len(event['Hostname']) > 0, \
|
||||
"Logger {} have wrong [Hostname] field".format(logger)
|
||||
assert event['severity_label'] in ext.LOG_LEVELS, \
|
||||
"Logger {} have wrong [severity_label] field".format(logger)
|
||||
assert len(event['programname']) > 0, \
|
||||
"Logger {} have wrong [programname] field".format(logger)
|
||||
assert isinstance(event['Severity'], int), \
|
||||
"Logger {} have wrong [Severity] field".format(logger)
|
||||
|
|
@ -18,4 +18,5 @@ pytest_plugins = ['fuel_ccp_tests.fixtures.common_fixtures',
|
|||
'fuel_ccp_tests.fixtures.k8s_fixtures',
|
||||
'fuel_ccp_tests.fixtures.rally_fixtures',
|
||||
'fuel_ccp_tests.fixtures.ccp_fixtures',
|
||||
'fuel_ccp_tests.fixtures.influxdb_fixtures']
|
||||
'fuel_ccp_tests.fixtures.influxdb_fixtures',
|
||||
'fuel_ccp_tests.fixtures.os_fixtures']
|
||||
|
|
|
|||
|
|
@ -17,4 +17,5 @@ pytest_plugins = ['fuel_ccp_tests.fixtures.common_fixtures',
|
|||
'fuel_ccp_tests.fixtures.underlay_fixtures',
|
||||
'fuel_ccp_tests.fixtures.k8s_fixtures',
|
||||
'fuel_ccp_tests.fixtures.rally_fixtures',
|
||||
'fuel_ccp_tests.fixtures.ccp_fixtures']
|
||||
'fuel_ccp_tests.fixtures.ccp_fixtures',
|
||||
'fuel_ccp_tests.fixtures.os_fixtures']
|
||||
|
|
|
|||
Loading…
Reference in New Issue