From 8cff9c154b3bdfc688b1e54b412283c8d4ed2dc8 Mon Sep 17 00:00:00 2001 From: Ashwin Agate Date: Mon, 11 Dec 2017 23:13:54 -0800 Subject: [PATCH] Remove oslo config global conf dependency Ceilometer in Ocata made several changes to move away from oslo config's global conf. Instead ceilometer/service.py now creates a oslo config object that gets passed to the publisher. To keep up with the changes had to make the following changes in monasca-ceilometer 1.) Monasca Publisher now extends ConfigPublisherBase, with conf as argument 2.) monasca_client now takes conf as an argument during intialization. 3.) Added monasca_ceilometer_opts to intialize all ConfigOpts. This will have to be regiested in ceilometer/opts.py. (will need a additional line in opts.py) 4.) Introduced three new properties service_username, service_password and service_auth_url since username, password and auth_url were removed from [service_credentials] section. 5.) Added api/health.py, service.py and opts.py from ceilometer code base which were modified to read monasca ceilometer options via monasca_ceilometer_opts.py 6.) Added ostestr to test requirements and updated tox.ini to run tests in serial (to avoid problems with singleton mapping classes, stepping on each other test values when run in parallel) List of Ceilometer changes to remove global conf [1] https://review.openstack.org/#/c/384834 [2] https://review.openstack.org/#/c/384624 [3] https://review.openstack.org/#/c/386025 This change also modifies test-requirements.txt to bring this master branch up to the stable/pike version of ceilometer, and python-monascaclient up to 1.7.1 (pike). Change-Id: Ieb7962a50fdb5d12ea6238c96ed116e8b2c83c48 --- ceilosca/ceilometer/api/health.py | 81 +++++++++ .../ceilometer_static_info_mapping.py | 32 ++-- .../ceilosca_mapping/ceilosca_mapping.py | 39 ++--- .../ceilometer/monasca_ceilometer_opts.py | 114 +++++++++++++ ceilosca/ceilometer/monasca_client.py | 125 +++++++------- ceilosca/ceilometer/opts.py | 157 ++++++++++++++++++ .../publisher/monasca_data_filter.py | 38 +---- ceilosca/ceilometer/publisher/monclient.py | 135 ++++++--------- ceilosca/ceilometer/service.py | 72 ++++++++ ceilosca/ceilometer/storage/impl_monasca.py | 38 +++-- .../tests/functional/api/__init__.py | 16 +- .../api/v2/test_api_with_monasca_driver.py | 32 ++-- .../ceilosca_mapping/test_ceilosca_mapping.py | 60 ++++--- .../test_static_ceilometer_mapping.py | 38 +++-- .../publisher/test_monasca_data_filter.py | 26 +-- .../unit/publisher/test_monasca_publisher.py | 46 +++-- .../tests/unit/storage/test_impl_monasca.py | 85 +++++----- .../tests/unit/test_monascaclient.py | 127 +++++++------- monasca_test_setup.py | 1 + setup.py | 2 +- test-requirements.txt | 15 +- tox.ini | 2 +- 22 files changed, 828 insertions(+), 453 deletions(-) create mode 100644 ceilosca/ceilometer/api/health.py create mode 100644 ceilosca/ceilometer/monasca_ceilometer_opts.py create mode 100644 ceilosca/ceilometer/opts.py create mode 100644 ceilosca/ceilometer/service.py diff --git a/ceilosca/ceilometer/api/health.py b/ceilosca/ceilometer/api/health.py new file mode 100644 index 0000000..8e8c56d --- /dev/null +++ b/ceilosca/ceilometer/api/health.py @@ -0,0 +1,81 @@ +# +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log +LOG = log.getLogger(__name__) + + +def filter_factory(global_config, **local_conf): + """Returns a WSGI Filter.""" + conf = global_config.copy() + conf.update(local_conf) + LOG.debug('Filter Factory') + + def health_filter(app): + return HealthFilterApi(app, conf) + return health_filter + + +class HealthFilterApi(object): + + def __init__(self, app, conf): + self.conf = conf + self.app = app + self.db = None + + def __call__(self, environ, start_response): + """Handle the incoming request and filters it. + + Interjects the request and acts on it only if + it is related to the filter functionality, otherwise + it just let the request pass through to the app. + + """ + + LOG.debug('Health Check Filter') + + if environ['PATH_INFO'].startswith('/v2/health'): + response_code = '204 No Content' + if environ['REQUEST_METHOD'] != 'HEAD': + response_code = '200 OK' +# Commenting out healthcheck behavior when request method +# is not of type HEAD.# As it creates load on monasca vertica +# FIXME: fix this when monasca creates get versions api in +# in Monascaclient. USe that instead +# try: +# if not self.db: +# self.db = storage.get_connection_from_config(cfg.CONF) +# meters = self.db.get_meters(unique=True, limit=1) +# if not meters: +# response_code = '503 Backend Unavailable' +# except Exception as e: +# response_code = '503 Backend Unavailable' +# LOG.warning('DB health check connection failed: %s', e) +# self.db = None + resp = MiniResp(response_code, environ) + start_response(response_code, resp.headers) + return resp.body + else: + return self.app(environ, start_response) + + +class MiniResp(object): + + def __init__(self, message, env, headers=[]): + if env['REQUEST_METHOD'] == 'HEAD': + self.body = [''] + else: + self.body = [message] + self.headers = list(headers) diff --git a/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py b/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py index ab9c620..c66d5f6 100644 --- a/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py +++ b/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,23 +21,12 @@ import os import pkg_resources import yaml -from oslo_config import cfg from oslo_log import log - from ceilometer import sample LOG = log.getLogger(__name__) -OPTS = [ - cfg.StrOpt('ceilometer_static_info_mapping', - default='ceilometer_static_info_mapping.yaml', - help='Configuration mapping file to map ceilometer meters to ' - 'their units an type informaiton'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - class CeilometerStaticMappingDefinitionException(Exception): def __init__(self, message, definition_cfg): @@ -67,19 +57,19 @@ class CeilometerStaticMappingDefinition(object): "Invalid type %s specified" % self.cfg['type'], self.cfg) -def get_config_file(): - config_file = cfg.CONF.monasca.ceilometer_static_info_mapping +def get_config_file(conf): + config_file = conf.monasca.ceilometer_static_info_mapping if not os.path.exists(config_file): - config_file = cfg.CONF.find_file(config_file) + config_file = conf.find_file(config_file) if not config_file: config_file = pkg_resources.resource_filename( __name__, "data/ceilometer_static_info_mapping.yaml") return config_file -def setup_ceilometer_static_mapping_config(): +def setup_ceilometer_static_mapping_config(conf): """Setup the meters definitions from yaml config file.""" - config_file = get_config_file() + config_file = get_config_file(conf) if config_file is not None: LOG.debug("Static Ceilometer mapping file to map static info: %s", config_file) @@ -160,11 +150,12 @@ class ProcessMappedCeilometerStaticInfo(object): __new__(cls, *args, **kwargs) return cls._instance - def __init__(self): + def __init__(self, conf): if not (self._instance and self._inited): + self.conf = conf self._inited = True self.__definitions = load_definitions( - setup_ceilometer_static_mapping_config()) + setup_ceilometer_static_mapping_config(self.conf)) self.__mapped_meter_info_map = dict() for d in self.__definitions: self.__mapped_meter_info_map[d.cfg['name']] = d @@ -178,9 +169,10 @@ class ProcessMappedCeilometerStaticInfo(object): def get_meter_static_info_key_val(self, meter_name, key): return self.__mapped_meter_info_map.get(meter_name).cfg[key] - def reinitialize(self): + def reinitialize(self, conf): + self.conf = conf self.__definitions = load_definitions( - setup_ceilometer_static_mapping_config()) + setup_ceilometer_static_mapping_config(self.conf)) self.__mapped_meter_info_map = dict() for d in self.__definitions: self.__mapped_meter_info_map[d.cfg['name']] = d diff --git a/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py b/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py index fb81230..2fc4cca 100644 --- a/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py +++ b/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -23,24 +24,13 @@ import six import yaml from jsonpath_rw_ext import parser -from oslo_config import cfg from oslo_log import log - from ceilometer import pipeline from ceilometer import sample LOG = log.getLogger(__name__) -OPTS = [ - cfg.StrOpt('ceilometer_monasca_metrics_mapping', - default='ceilosca_mapping.yaml', - help='Configuration mapping file to map monasca metrics to ' - 'ceilometer meters'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - class CeiloscaMappingDefinitionException(Exception): def __init__(self, message, definition_cfg): @@ -132,19 +122,19 @@ class CeiloscaMappingDefinition(object): return values -def get_config_file(): - config_file = cfg.CONF.monasca.ceilometer_monasca_metrics_mapping +def get_config_file(conf): + config_file = conf.monasca.ceilometer_monasca_metrics_mapping if not os.path.exists(config_file): - config_file = cfg.CONF.find_file(config_file) + config_file = conf.find_file(config_file) if not config_file: config_file = pkg_resources.resource_filename( __name__, "data/ceilosca_mapping.yaml") return config_file -def setup_ceilosca_mapping_config(): +def setup_ceilosca_mapping_config(conf): """Setup the meters definitions from yaml config file.""" - config_file = get_config_file() + config_file = get_config_file(conf) if config_file is not None: LOG.debug("Ceilometer Monasca Mapping Definitions file: %s", config_file) @@ -228,11 +218,12 @@ class ProcessMappedCeiloscaMetric(object): cls, *args, **kwargs) return cls._instance - def __init__(self): + def __init__(self, conf): if not (self._instance and self._inited): + self.conf = conf self._inited = True self.__definitions = load_definitions( - setup_ceilosca_mapping_config()) + setup_ceilosca_mapping_config(self.conf)) self.__mapped_metric_map = dict() self.__mon_metric_to_cm_meter_map = dict() for d in self.__definitions: @@ -252,9 +243,10 @@ class ProcessMappedCeiloscaMetric(object): def get_ceilosca_mapped_definition_key_val(self, monasca_metric_name, key): return self.__mapped_metric_map.get(monasca_metric_name).cfg[key] - def reinitialize(self): + def reinitialize(self, conf): + self.conf = conf self.__definitions = load_definitions( - setup_ceilosca_mapping_config()) + setup_ceilosca_mapping_config(self.conf)) self.__mapped_metric_map = dict() self.__mon_metric_to_cm_meter_map = dict() for d in self.__definitions: @@ -281,10 +273,11 @@ class PipelineReader(object): cls, *args, **kwargs) return cls._instance - def __init__(self): + def __init__(self, conf): if not (self._instance and self._inited): self._inited = True - self.__pipeline_manager = pipeline.setup_pipeline() + self.conf = conf + self.__pipeline_manager = pipeline.setup_pipeline(self.conf) self.__meters_from_pipeline = set() for pipe in self.__pipeline_manager.pipelines: if not isinstance(pipe, pipeline.EventPipeline): @@ -296,7 +289,7 @@ class PipelineReader(object): return self.__meters_from_pipeline def reinitialize(self): - self.__pipeline_manager = pipeline.setup_pipeline() + self.__pipeline_manager = pipeline.setup_pipeline(self.conf) self.__meters_from_pipeline = set() for pipe in self.__pipeline_manager.pipelines: if not isinstance(pipe, pipeline.EventPipeline): diff --git a/ceilosca/ceilometer/monasca_ceilometer_opts.py b/ceilosca/ceilometer/monasca_ceilometer_opts.py new file mode 100644 index 0000000..b79b0a0 --- /dev/null +++ b/ceilosca/ceilometer/monasca_ceilometer_opts.py @@ -0,0 +1,114 @@ +# +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" All monasca ceilometer config opts""" + +from oslo_config import cfg + +OPTS = [ + + # from ceilometer_static_info_mapping + cfg.StrOpt('ceilometer_static_info_mapping', + default='ceilometer_static_info_mapping.yaml', + help='Configuration mapping file to map ceilometer meters to ' + 'their units an type information'), + + # from ceilosca_mapping + cfg.StrOpt('ceilometer_monasca_metrics_mapping', + default='ceilosca_mapping.yaml', + help='Configuration mapping file to map monasca metrics to ' + 'ceilometer meters'), + + # from monasca_client + cfg.StrOpt('clientapi_version', + default='2_0', + help='Version of Monasca client to use while publishing.'), + cfg.BoolOpt('enable_api_pagination', + default=False, + help='Enable paging through monasca api resultset.'), + + cfg.StrOpt('service_auth_url', help='auth url connecting to service'), + cfg.StrOpt('service_password', help='password connecting to service'), + cfg.StrOpt('service_username', help='username connecting to service'), + cfg.StrOpt('service_project_id', help='username connecting to service'), + cfg.StrOpt('service_domain_name', help='domain connecting to service'), + cfg.StrOpt('service_region_name', help='region connecting to service'), + cfg.StrOpt('service_project_name', + help='project name connecting to service'), + cfg.StrOpt('service_verify', + help='path to ssl cert to verify connecting to service'), + + # from monasca_data_filter + cfg.StrOpt('monasca_mappings', + default='/etc/ceilometer/monasca_field_definitions.yaml', + help='Monasca static and dynamic field mappings'), + + # from multi region opts + cfg.StrOpt('control_plane', + default='None', + help='The name of control plane'), + cfg.StrOpt('cluster', + default='None', + help='The name of cluster'), + cfg.StrOpt('cloud_name', + default='None', + help='The name of cloud'), + + # from publisher monclient + cfg.BoolOpt('batch_mode', + default=True, + help='Indicates whether samples are' + ' published in a batch.'), + cfg.IntOpt('batch_count', + default=1000, + help='Maximum number of samples in a batch.'), + cfg.IntOpt('batch_timeout', + default=15, + help='Maximum time interval(seconds) after which ' + 'samples are published in a batch.'), + cfg.IntOpt('batch_polling_interval', + default=5, + help='Frequency of checking if batch criteria is met.'), + cfg.BoolOpt('retry_on_failure', + default=False, + help='Indicates whether publisher retries publishing' + 'sample in case of failure. Only a few error cases' + 'are queued for a retry.'), + cfg.IntOpt('retry_interval', + default=60, + help='Frequency of attempting a retry.'), + cfg.IntOpt('max_retries', + default=3, + help='Maximum number of retry attempts on a publishing ' + 'failure.'), + cfg.BoolOpt('archive_on_failure', + default=False, + help='When turned on, archives metrics in file system when' + 'publish to Monasca fails or metric publish maxes out' + 'retry attempts.'), + cfg.StrOpt('archive_path', + default='mon_pub_failures.txt', + help='File of metrics that failed to publish to ' + 'Monasca. These include metrics that failed to ' + 'publish on first attempt and failed metrics that' + ' maxed out their retries.'), + + # from impl_monasca + cfg.IntOpt('default_stats_period', + default=300, + help='Default period (in seconds) to use for querying stats ' + 'in case no period specified in the stats API call.'), + +] diff --git a/ceilosca/ceilometer/monasca_client.py b/ceilosca/ceilometer/monasca_client.py index 81f3e26..6fa0271 100644 --- a/ceilosca/ceilometer/monasca_client.py +++ b/ceilosca/ceilometer/monasca_client.py @@ -1,4 +1,5 @@ # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -16,27 +17,10 @@ import copy from monascaclient import client from monascaclient import exc -from monascaclient import ksclient -from oslo_config import cfg from oslo_log import log -import retrying +import tenacity from ceilometer.i18n import _ -from ceilometer import keystone_client - - -monclient_opts = [ - cfg.StrOpt('clientapi_version', - default='2_0', - help='Version of Monasca client to use while publishing.'), - cfg.BoolOpt('enable_api_pagination', - default=False, - help='Enable paging through monasca api resultset.'), -] - -cfg.CONF.register_opts(monclient_opts, group='monasca') -keystone_client.register_keystoneauth_opts(cfg.CONF) -cfg.CONF.import_group('service_credentials', 'ceilometer.service') LOG = log.getLogger(__name__) @@ -68,20 +52,23 @@ class MonascaInvalidParametersException(Exception): class Client(object): """A client which gets information via python-monascaclient.""" - _ksclient = None - - def __init__(self, parsed_url): - self._retry_interval = cfg.CONF.database.retry_interval * 1000 - self._max_retries = cfg.CONF.database.max_retries or 1 + def __init__(self, conf, parsed_url): + self.conf = conf + self._retry_interval = self.conf.database.retry_interval + self._max_retries = self.conf.database.max_retries or 1 # enable monasca api pagination - self._enable_api_pagination = cfg.CONF.monasca.enable_api_pagination + self._enable_api_pagination = self.conf.monasca.enable_api_pagination # NOTE(zqfan): There are many concurrency requests while using # Ceilosca, to save system resource, we don't retry too many times. if self._max_retries < 0 or self._max_retries > 10: LOG.warning('Reduce max retries from %s to 10', self._max_retries) self._max_retries = 10 - conf = cfg.CONF.service_credentials + + # self.conf.log_opt_values(LOG, logging.INFO) + + conf = self.conf.service_credentials + monasca_conf = self.conf.monasca # because our ansible script are in another repo, the old setting # of auth_type is password-ceilometer-legacy which doesn't register # os_xxx options, so here we need to provide a compatible way to @@ -90,14 +77,18 @@ class Client(object): username = conf.os_username password = conf.os_password auth_url = conf.os_auth_url - project_id = conf.os_tenant_id + # project_id = conf.os_tenant_id project_name = conf.os_tenant_name else: - username = conf.username - password = conf.password - auth_url = conf.auth_url - project_id = conf.project_id - project_name = conf.project_name + username = monasca_conf.service_username + password = monasca_conf.service_password + auth_url = monasca_conf.service_auth_url + # project_id = monasca_conf.service_project_id + project_name = monasca_conf.service_project_name + default_domain_name = monasca_conf.service_domain_name + region_name = monasca_conf.service_region_name + service_verify = monasca_conf.service_verify + if not username or not password or not auth_url: err_msg = _("No user name or password or auth_url " "found in service_credentials") @@ -108,13 +99,17 @@ class Client(object): 'username': username, 'password': password, 'auth_url': auth_url.replace("v2.0", "v3"), - 'project_id': project_id, + # 'project_id': project_id, 'project_name': project_name, - 'region_name': conf.region_name, - 'read_timeout': cfg.CONF.http_timeout, - 'write_timeout': cfg.CONF.http_timeout, + 'region_name': region_name, + 'default_domain_name': default_domain_name, + 'project_domain_name': default_domain_name, + 'user_domain_name': default_domain_name, + 'read_timeout': self.conf.http_timeout, + 'write_timeout': self.conf.http_timeout, + 'keystone_timeout': self.conf.http_timeout, + 'verify': service_verify } - self._kwargs = kwargs self._endpoint = parsed_url.netloc + parsed_url.path LOG.info(_("monasca_client: using %s as monasca end point") % @@ -122,35 +117,29 @@ class Client(object): self._refresh_client() def _refresh_client(self): - if not Client._ksclient: - Client._ksclient = ksclient.KSClient(**self._kwargs) - self._kwargs['token'] = Client._ksclient.token - self._mon_client = client.Client(cfg.CONF.monasca.clientapi_version, + self._mon_client = client.Client(self.conf.monasca.clientapi_version, self._endpoint, **self._kwargs) - @staticmethod - def _retry_on_exception(e): - return not isinstance(e, MonascaInvalidParametersException) - def call_func(self, func, **kwargs): - @retrying.retry(wait_fixed=self._retry_interval, - stop_max_attempt_number=self._max_retries, - retry_on_exception=self._retry_on_exception) + @tenacity.retry( + wait=tenacity.wait_fixed(self._retry_interval), + stop=tenacity.stop_after_attempt(self._max_retries), + retry=(tenacity.retry_if_exception_type(MonascaServiceException) | + tenacity.retry_if_exception_type(MonascaException))) def _inner(): try: return func(**kwargs) - except (exc.HTTPInternalServerError, - exc.HTTPServiceUnavailable, - exc.HTTPBadGateway, - exc.CommunicationError) as e: + except (exc.http.InternalServerError, + exc.http.ServiceUnavailable, + exc.http.BadGateway, + exc.connection.ConnectionError) as e: LOG.exception(e) msg = '%s: %s' % (e.__class__.__name__, e) raise MonascaServiceException(msg) - except exc.HTTPException as e: + except exc.http.HttpError as e: LOG.exception(e) msg = '%s: %s' % (e.__class__.__name__, e) - status_code = e.code - # exc.HTTPException has string code 'N/A' + status_code = e.http_status if not isinstance(status_code, int): status_code = 500 if 400 <= status_code < 500: @@ -206,13 +195,17 @@ class Client(object): **search_args) # check if api pagination is enabled if self._enable_api_pagination: - while measurements: + while measurements and len(measurements[0]["measurements"]) > 0: for measurement in measurements: + if measurement["measurements"] is not None and \ + len(measurement["measurements"]) > 0: + # offset for measurements is measurement id composited + # with the last measurement's timestamp + last_good_offset = '%s_%s' % ( + measurement['id'], + measurement['measurements'][-1][0]) yield measurement - # offset for measurements is measurement id composited with - # the last measurement's timestamp - search_args['offset'] = '%s_%s' % ( - measurement['id'], measurement['measurements'][-1][0]) + search_args['offset'] = last_good_offset measurements = self.call_func( self._mon_client.metrics.list_measurements, **search_args) @@ -231,17 +224,21 @@ class Client(object): **search_args) # check if api pagination is enabled if self._enable_api_pagination: - while statistics: + while statistics and len(statistics[0]["statistics"]) > 0: for statistic in statistics: + if statistic["statistics"] is not None and \ + len(statistic["statistics"]) > 0: + # offset for statistics is statistic id composited with + # the last statistic's timestamp + last_good_offset = '%s_%s' % ( + statistic['id'], statistic['statistics'][-1][0]) yield statistic + # with groupby, the offset is unpredictable to me, we don't # support pagination for it now. if kwargs.get('group_by'): break - # offset for statistics is statistic id composited with - # the last statistic's timestamp - search_args['offset'] = '%s_%s' % ( - statistic['id'], statistic['statistics'][-1][0]) + search_args['offset'] = last_good_offset statistics = self.call_func( self._mon_client.metrics.list_statistics, **search_args) diff --git a/ceilosca/ceilometer/opts.py b/ceilosca/ceilometer/opts.py new file mode 100644 index 0000000..f3dd2ae --- /dev/null +++ b/ceilosca/ceilometer/opts.py @@ -0,0 +1,157 @@ +# Copyright 2014 eNovance +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import itertools +import socket + +from keystoneauth1 import loading +from oslo_config import cfg + +import ceilometer.agent.manager +import ceilometer.api.app +import ceilometer.api.controllers.v2.root +import ceilometer.collector +import ceilometer.compute.discovery +import ceilometer.compute.virt.inspector +import ceilometer.compute.virt.libvirt.utils +import ceilometer.compute.virt.vmware.inspector +import ceilometer.compute.virt.xenapi.inspector +import ceilometer.dispatcher +import ceilometer.dispatcher.file +import ceilometer.dispatcher.gnocchi_opts +import ceilometer.dispatcher.http +import ceilometer.event.converter +import ceilometer.exchange_control +import ceilometer.hardware.discovery +import ceilometer.hardware.pollsters.generic +import ceilometer.image.discovery +import ceilometer.ipmi.notifications.ironic +import ceilometer.ipmi.platform.intel_node_manager +import ceilometer.ipmi.pollsters +import ceilometer.keystone_client +import ceilometer.meter.notifications +import ceilometer.middleware +import ceilometer.monasca_ceilometer_opts +import ceilometer.neutron_client +import ceilometer.notification +import ceilometer.nova_client +import ceilometer.objectstore.rgw +import ceilometer.objectstore.swift +import ceilometer.pipeline +import ceilometer.publisher.messaging +import ceilometer.publisher.utils +import ceilometer.sample +import ceilometer.storage +import ceilometer.utils +import ceilometer.volume.discovery + + +OPTS = [ + cfg.HostAddressOpt('host', + default=socket.gethostname(), + sample_default='', + help='Name of this node, which must be valid in an ' + 'AMQP key. Can be an opaque identifier. For ZeroMQ ' + 'only, must be a valid host name, FQDN, or IP ' + 'address.'), + cfg.IntOpt('http_timeout', + default=600, + help='Timeout seconds for HTTP requests. Set it to None to ' + 'disable timeout.'), + cfg.IntOpt('max_parallel_requests', + default=64, + min=1, + help='Maximum number of parallel requests for ' + 'services to handle at the same time.'), +] + + +def list_opts(): + # FIXME(sileht): readd pollster namespaces in the generated configfile + # This have been removed due to a recursive import issue + return [ + ('DEFAULT', + itertools.chain(ceilometer.agent.manager.OPTS, + ceilometer.api.app.OPTS, + ceilometer.compute.virt.inspector.OPTS, + ceilometer.compute.virt.libvirt.utils.OPTS, + ceilometer.dispatcher.OPTS, + ceilometer.ipmi.notifications.ironic.OPTS, + ceilometer.nova_client.OPTS, + ceilometer.objectstore.swift.OPTS, + ceilometer.pipeline.OPTS, + ceilometer.sample.OPTS, + ceilometer.utils.OPTS, + ceilometer.exchange_control.EXCHANGE_OPTS, + OPTS)), + ('api', itertools.chain(ceilometer.api.app.API_OPTS, + ceilometer.api.controllers.v2.root.API_OPTS)), + ('collector', ceilometer.collector.OPTS), + ('compute', ceilometer.compute.discovery.OPTS), + ('coordination', [ + cfg.StrOpt( + 'backend_url', + help='The backend URL to use for distributed coordination. If ' + 'left empty, per-deployment central agent and per-host ' + 'compute agent won\'t do workload ' + 'partitioning and will only function correctly if a ' + 'single instance of that service is running.'), + cfg.FloatOpt( + 'check_watchers', + default=10.0, + help='Number of seconds between checks to see if group ' + 'membership has changed'), + ]), + ('database', ceilometer.storage.OPTS), + ('dispatcher_file', ceilometer.dispatcher.file.OPTS), + ('dispatcher_http', ceilometer.dispatcher.http.http_dispatcher_opts), + ('dispatcher_gnocchi', + ceilometer.dispatcher.gnocchi_opts.dispatcher_opts), + ('event', ceilometer.event.converter.OPTS), + ('hardware', itertools.chain( + ceilometer.hardware.discovery.OPTS, + ceilometer.hardware.pollsters.generic.OPTS)), + ('ipmi', + itertools.chain(ceilometer.ipmi.platform.intel_node_manager.OPTS, + ceilometer.ipmi.pollsters.OPTS)), + ('meter', ceilometer.meter.notifications.OPTS), + ('monasca', ceilometer.monasca_ceilometer_opts.OPTS), + ('notification', + itertools.chain(ceilometer.notification.OPTS, + ceilometer.notification.EXCHANGES_OPTS)), + ('polling', ceilometer.agent.manager.POLLING_OPTS), + ('publisher', ceilometer.publisher.utils.OPTS), + ('publisher_notifier', ceilometer.publisher.messaging.NOTIFIER_OPTS), + ('rgw_admin_credentials', ceilometer.objectstore.rgw.CREDENTIAL_OPTS), + ('service_types', + itertools.chain(ceilometer.image.discovery.SERVICE_OPTS, + ceilometer.neutron_client.SERVICE_OPTS, + ceilometer.nova_client.SERVICE_OPTS, + ceilometer.objectstore.rgw.SERVICE_OPTS, + ceilometer.objectstore.swift.SERVICE_OPTS, + ceilometer.volume.discovery.SERVICE_OPTS,)), + ('vmware', ceilometer.compute.virt.vmware.inspector.OPTS), + ('xenapi', ceilometer.compute.virt.xenapi.inspector.OPTS), + ] + + +def list_keystoneauth_opts(): + # NOTE(sileht): the configuration file contains only the options + # for the password plugin that handles keystone v2 and v3 API + # with discovery. But other options are possible. + return [('service_credentials', itertools.chain( + loading.get_auth_common_conf_options(), + loading.get_auth_plugin_conf_options('password'), + ceilometer.keystone_client.CLI_OPTS + ))] diff --git a/ceilosca/ceilometer/publisher/monasca_data_filter.py b/ceilosca/ceilometer/publisher/monasca_data_filter.py index d68635f..70557ad 100644 --- a/ceilosca/ceilometer/publisher/monasca_data_filter.py +++ b/ceilosca/ceilometer/publisher/monasca_data_filter.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -16,7 +17,6 @@ import datetime from jsonpath_rw_ext import parser -from oslo_config import cfg from oslo_log import log from oslo_utils import timeutils import yaml @@ -25,27 +25,6 @@ from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinitionException) from ceilometer import sample as sample_util -OPTS = [ - cfg.StrOpt('monasca_mappings', - default='/etc/ceilometer/monasca_field_definitions.yaml', - help='Monasca static and dynamic field mappings'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - -MULTI_REGION_OPTS = [ - cfg.StrOpt('control_plane', - default='None', - help='The name of control plane'), - cfg.StrOpt('cluster', - default='None', - help='The name of cluster'), - cfg.StrOpt('cloud_name', - default='None', - help='The name of cloud') -] -cfg.CONF.register_opts(MULTI_REGION_OPTS) - LOG = log.getLogger(__name__) @@ -60,12 +39,13 @@ class NoMappingsFound(Exception): class MonascaDataFilter(object): JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser() - def __init__(self): + def __init__(self, conf): + self.conf = conf self._mapping = {} self._mapping = self._get_mapping() def _get_mapping(self): - with open(cfg.CONF.monasca.monasca_mappings, 'r') as f: + with open(self.conf.monasca.monasca_mappings, 'r') as f: try: return yaml.safe_load(f) except yaml.YAMLError as err: @@ -74,13 +54,13 @@ class MonascaDataFilter(object): errmsg = ("Invalid YAML syntax in Monasca Data " "Filter file %(file)s at line: " "%(line)s, column: %(column)s." - % dict(file=cfg.CONF.monasca.monasca_mappings, + % dict(file=self.conf.monasca.monasca_mappings, line=mark.line + 1, column=mark.column + 1)) else: errmsg = ("YAML error reading Monasca Data Filter " "file %(file)s" % - dict(file=cfg.CONF.monasca.monasca_mappings)) + dict(file=self.conf.monasca.monasca_mappings)) LOG.error(errmsg) raise UnableToLoadMappings(err.message) @@ -181,9 +161,9 @@ class MonascaDataFilter(object): dimensions['datasource'] = 'ceilometer' # control_plane, cluster and cloud_name can be None, but we use # literal 'None' for such case - dimensions['control_plane'] = cfg.CONF.control_plane or 'None' - dimensions['cluster'] = cfg.CONF.cluster or 'None' - dimensions['cloud_name'] = cfg.CONF.cloud_name or 'None' + dimensions['control_plane'] = self.conf.monasca.control_plane or 'None' + dimensions['cluster'] = self.conf.monasca.cluster or 'None' + dimensions['cloud_name'] = self.conf.monasca.cloud_name or 'None' if isinstance(sample_obj, sample_util.Sample): sample = sample_obj.as_dict() elif isinstance(sample_obj, dict): diff --git a/ceilosca/ceilometer/publisher/monclient.py b/ceilosca/ceilometer/publisher/monclient.py index 8d25fe3..840dac8 100755 --- a/ceilosca/ceilometer/publisher/monclient.py +++ b/ceilosca/ceilometer/publisher/monclient.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -19,79 +20,37 @@ import os import threading import time -from oslo_config import cfg from oslo_log import log import ceilometer -from ceilometer.i18n import _ from ceilometer import monasca_client as mon_client from ceilometer import publisher from ceilometer.publisher.monasca_data_filter import MonascaDataFilter from monascaclient import exc +import traceback - -monpub_opts = [ - cfg.BoolOpt('batch_mode', - default=True, - help='Indicates whether samples are' - ' published in a batch.'), - cfg.IntOpt('batch_count', - default=1000, - help='Maximum number of samples in a batch.'), - cfg.IntOpt('batch_timeout', - default=15, - help='Maximum time interval(seconds) after which ' - 'samples are published in a batch.'), - cfg.IntOpt('batch_polling_interval', - default=5, - help='Frequency of checking if batch criteria is met.'), - cfg.BoolOpt('retry_on_failure', - default=False, - help='Indicates whether publisher retries publishing' - 'sample in case of failure. Only a few error cases' - 'are queued for a retry.'), - cfg.IntOpt('retry_interval', - default=60, - help='Frequency of attempting a retry.'), - cfg.IntOpt('max_retries', - default=3, - help='Maximum number of retry attempts on a publishing ' - 'failure.'), - cfg.BoolOpt('archive_on_failure', - default=False, - help='When turned on, archives metrics in file system when' - 'publish to Monasca fails or metric publish maxes out' - 'retry attempts.'), - cfg.StrOpt('archive_path', - default='mon_pub_failures.txt', - help='File of metrics that failed to publish to ' - 'Monasca. These include metrics that failed to ' - 'publish on first attempt and failed metrics that' - ' maxed out their retries.'), -] - -cfg.CONF.register_opts(monpub_opts, group='monasca') -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +batch_polling_interval = 5 +retry_interval = 60 LOG = log.getLogger(__name__) -class MonascaPublisher(publisher.PublisherBase): +class MonascaPublisher(publisher.ConfigPublisherBase): """Publisher to publish samples to monasca using monasca-client. Example URL to place in pipeline.yaml: - monclient://http://192.168.10.4:8070/v2.0 """ - def __init__(self, parsed_url): - super(MonascaPublisher, self).__init__(parsed_url) + def __init__(self, conf, parsed_url): + super(MonascaPublisher, self).__init__(conf, parsed_url) # list to hold metrics to be published in batch (behaves like queue) self.metric_queue = [] self.time_of_last_batch_run = time.time() - self.mon_client = mon_client.Client(parsed_url) - self.mon_filter = MonascaDataFilter() + self.mon_client = mon_client.Client(self.conf, parsed_url) + self.mon_filter = MonascaDataFilter(self.conf) # add flush_batch function to periodic callables periodic_callables = [ @@ -101,7 +60,7 @@ class MonascaPublisher(publisher.PublisherBase): (self.flush_batch, (), {}), ] - if cfg.CONF.monasca.retry_on_failure: + if self.conf.monasca.retry_on_failure: # list to hold metrics to be re-tried (behaves like queue) self.retry_queue = [] # list to store retry attempts for metrics in retry_queue @@ -110,13 +69,16 @@ class MonascaPublisher(publisher.PublisherBase): # add retry_batch function to periodic callables periodic_callables.append((self.retry_batch, (), {})) - if cfg.CONF.monasca.archive_on_failure: - archive_path = cfg.CONF.monasca.archive_path + if self.conf.monasca.archive_on_failure: + archive_path = self.conf.monasca.archive_path if not os.path.exists(archive_path): - archive_path = cfg.CONF.find_file(archive_path) + archive_path = self.conf.find_file(archive_path) - self.archive_handler = publisher.get_publisher('file://' + - str(archive_path)) + self.archive_handler = publisher.get_publisher( + self.conf, + 'file://' + + str(archive_path), + 'ceilometer.sample.publisher') # start periodic worker self.periodic_worker = periodics.PeriodicWorker(periodic_callables) @@ -134,22 +96,22 @@ class MonascaPublisher(publisher.PublisherBase): func(**{'jsonbody': metrics}) else: func(**metrics[0]) - LOG.debug(_('Successfully published %d metric(s)') % metric_count) + LOG.info('Successfully published %d metric(s)' % metric_count) except mon_client.MonascaServiceException: # Assuming atomicity of create or failure - meaning # either all succeed or all fail in a batch - LOG.error(_('Metric create failed for %(count)d metric(s) with' - ' name(s) %(names)s ') % + LOG.error('Metric create failed for %(count)d metric(s) with' + ' name(s) %(names)s ' % ({'count': len(metrics), 'names': ','.join([metric['name'] for metric in metrics])})) - if cfg.CONF.monasca.retry_on_failure: + if self.conf.monasca.retry_on_failure: # retry payload in case of internal server error(500), # service unavailable error(503),bad gateway (502) or # Communication Error # append failed metrics to retry_queue - LOG.debug(_('Adding metrics to retry queue.')) + LOG.debug('Adding metrics to retry queue.') self.retry_queue.extend(metrics) # initialize the retry_attempt for the each failed # metric in retry_counter @@ -159,6 +121,7 @@ class MonascaPublisher(publisher.PublisherBase): if hasattr(self, 'archive_handler'): self.archive_handler.publish_samples(None, metrics) except Exception: + LOG.info(traceback.format_exc()) if hasattr(self, 'archive_handler'): self.archive_handler.publish_samples(None, metrics) @@ -169,14 +132,14 @@ class MonascaPublisher(publisher.PublisherBase): metric = self.mon_filter.process_sample_for_monasca(sample) # In batch mode, push metric to queue, # else publish the metric - if cfg.CONF.monasca.batch_mode: - LOG.debug(_('Adding metric to queue.')) + if self.conf.monasca.batch_mode: + LOG.debug('Adding metric to queue.') self.metric_queue.append(metric) else: - LOG.debug(_('Publishing metric with name %(name)s and' - ' timestamp %(ts)s to endpoint.') % - ({'name': metric['name'], - 'ts': metric['timestamp']})) + LOG.info('Publishing metric with name %(name)s and' + ' timestamp %(ts)s to endpoint.' % + ({'name': metric['name'], + 'ts': metric['timestamp']})) self._publish_handler(self.mon_client.metrics_create, [metric]) @@ -187,18 +150,18 @@ class MonascaPublisher(publisher.PublisherBase): current_time = time.time() elapsed_time = current_time - previous_time - if elapsed_time >= cfg.CONF.monasca.batch_timeout and len(self. + if elapsed_time >= self.conf.monasca.batch_timeout and len(self. metric_queue) > 0: - LOG.debug(_('Batch timeout exceeded, triggering batch publish.')) + LOG.info('Batch timeout exceeded, triggering batch publish.') return True else: - if len(self.metric_queue) >= cfg.CONF.monasca.batch_count: - LOG.debug(_('Batch queue full, triggering batch publish.')) + if len(self.metric_queue) >= self.conf.monasca.batch_count: + LOG.info('Batch queue full, triggering batch publish.') return True else: return False - @periodics.periodic(cfg.CONF.monasca.batch_polling_interval) + @periodics.periodic(batch_polling_interval) def flush_batch(self): """Method to flush the queued metrics.""" # print "flush batch... %s" % str(time.time()) @@ -206,6 +169,8 @@ class MonascaPublisher(publisher.PublisherBase): # publish all metrics in queue at this point batch_count = len(self.metric_queue) + LOG.info("batch is ready: batch_count %s" % str(batch_count)) + self._publish_handler(self.mon_client.metrics_create, self.metric_queue[:batch_count], batch=True) @@ -220,12 +185,12 @@ class MonascaPublisher(publisher.PublisherBase): """Method to check if retry batch is ready to trigger.""" if len(self.retry_queue) > 0: - LOG.debug(_('Retry queue has items, triggering retry.')) + LOG.info('Retry queue has items, triggering retry.') return True else: return False - @periodics.periodic(cfg.CONF.monasca.retry_interval) + @periodics.periodic(retry_interval) def retry_batch(self): """Method to retry the failed metrics.""" # print "retry batch...%s" % str(time.time()) @@ -235,14 +200,14 @@ class MonascaPublisher(publisher.PublisherBase): # Iterate over the retry_queue to eliminate # metrics that have maxed out their retry attempts for ctr in xrange(retry_count): - if self.retry_counter[ctr] > cfg.CONF.monasca.max_retries: + if self.retry_counter[ctr] > self.conf.monasca.max_retries: if hasattr(self, 'archive_handler'): self.archive_handler.publish_samples( None, [self.retry_queue[ctr]]) - LOG.debug(_('Removing metric %s from retry queue.' - ' Metric retry maxed out retry attempts') % - self.retry_queue[ctr]['name']) + LOG.info('Removing metric %s from retry queue.' + ' Metric retry maxed out retry attempts' % + self.retry_queue[ctr]['name']) del self.retry_queue[ctr] del self.retry_counter[ctr] @@ -255,17 +220,17 @@ class MonascaPublisher(publisher.PublisherBase): ctr = 0 while ctr < len(self.retry_queue): try: - LOG.debug(_('Retrying metric publish from retry queue.')) + LOG.info('Retrying metric publish from retry queue.') self.mon_client.metrics_create(**self.retry_queue[ctr]) # remove from retry queue if publish was success - LOG.debug(_('Retrying metric %s successful,' - ' removing metric from retry queue.') % - self.retry_queue[ctr]['name']) + LOG.info('Retrying metric %s successful,' + ' removing metric from retry queue.' % + self.retry_queue[ctr]['name']) del self.retry_queue[ctr] del self.retry_counter[ctr] - except exc.BaseException: - LOG.error(_('Exception encountered in retry. ' - 'Batch will be retried in next attempt.')) + except exc.ClientException: + LOG.error('Exception encountered in retry. ' + 'Batch will be retried in next attempt.') # if retry failed, increment the retry counter self.retry_counter[ctr] += 1 ctr += 1 diff --git a/ceilosca/ceilometer/service.py b/ceilosca/ceilometer/service.py new file mode 100644 index 0000000..57f7b49 --- /dev/null +++ b/ceilosca/ceilometer/service.py @@ -0,0 +1,72 @@ +# Copyright 2012-2014 eNovance +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from oslo_config import cfg +from oslo_db import options as db_options +import oslo_i18n +from oslo_log import log +from oslo_policy import opts as policy_opts +# from oslo_reports import guru_meditation_report as gmr + +from ceilometer.conf import defaults +from ceilometer import keystone_client +from ceilometer import messaging +from ceilometer import opts +from ceilometer import sample +from ceilometer import utils +from ceilometer import version + + +def prepare_service(argv=None, config_files=None, conf=None): + if argv is None: + argv = sys.argv + + if conf is None: + conf = cfg.ConfigOpts() + + oslo_i18n.enable_lazy() + for group, options in opts.list_opts(): + conf.register_opts(list(options), + group=None if group == "DEFAULT" else group) + keystone_client.register_keystoneauth_opts(conf) + log.register_options(conf) + log_levels = (conf.default_log_levels + + ['futurist=INFO', 'neutronclient=INFO', + 'keystoneclient=INFO']) + log.set_defaults(default_log_levels=log_levels) + defaults.set_cors_middleware_defaults() + policy_opts.set_defaults(conf) + db_options.set_defaults(conf) + + conf(argv[1:], project='ceilometer', validate_default_values=True, + version=version.version_info.version_string(), + default_config_files=config_files) + + keystone_client.post_register_keystoneauth_opts(conf) + + log.setup(conf, 'ceilometer') + utils.setup_root_helper(conf) + sample.setup(conf) + + # NOTE(liusheng): guru cannot run with service under apache daemon, so when + # ceilometer-api running with mod_wsgi, the argv is [], we don't start + # guru. + if argv: + # gmr.TextGuruMeditation.setup_autorun(version) + pass + messaging.setup() + return conf diff --git a/ceilosca/ceilometer/storage/impl_monasca.py b/ceilosca/ceilometer/storage/impl_monasca.py index 2bfb133..2bf5755 100644 --- a/ceilosca/ceilometer/storage/impl_monasca.py +++ b/ceilosca/ceilometer/storage/impl_monasca.py @@ -1,5 +1,6 @@ # # (C) Copyright 2015-2017 Hewlett Packard Enterprise Development LP +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -15,6 +16,7 @@ """Simple monasca storage backend. """ +import traceback from collections import defaultdict import copy @@ -23,7 +25,6 @@ import itertools import operator from monascaclient import exc as monasca_exc -from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils from oslo_utils import netutils @@ -43,15 +44,6 @@ from ceilometer.storage import base from ceilometer.storage import models as api_models from ceilometer import utils -OPTS = [ - cfg.IntOpt('default_stats_period', - default=300, - help='Default period (in seconds) to use for querying stats ' - 'in case no period specified in the stats API call.'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - LOG = log.getLogger(__name__) AVAILABLE_CAPABILITIES = { @@ -96,13 +88,23 @@ class Connection(base.Connection): AVAILABLE_STORAGE_CAPABILITIES, ) - def __init__(self, url): - self.mc = monasca_client.Client(netutils.urlsplit(url)) - self.mon_filter = MonascaDataFilter() - self.ceilosca_mapper = ProcessMappedCeiloscaMetric() - self.pipeline_reader = PipelineReader() - self.meter_static_info = ProcessMappedCeilometerStaticInfo() - self.meters_from_pipeline = self.pipeline_reader.get_pipeline_meters() + def __init__(self, conf, url): + try: + super(Connection, self).__init__(conf, url) + self.conf = conf + self.mc = monasca_client.Client(self.conf, netutils.urlsplit(url)) + self.mon_filter = MonascaDataFilter(self.conf) + self.pipeline_reader = PipelineReader(conf) + self.ceilosca_mapper = ProcessMappedCeiloscaMetric(conf) + self.meter_static_info = ProcessMappedCeilometerStaticInfo(conf) + self.meters_from_pipeline = self.pipeline_reader\ + .get_pipeline_meters() + except Exception as ex: + LOG.info("ERROR: creating monasca connection: " + str(ex.message)) + LOG.info(ex) + LOG.info("ERROR: creating monasca connection: done") + LOG.info(traceback.format_ex()) + traceback.print_ex() @staticmethod def _convert_to_dict(stats, cols): @@ -878,7 +880,7 @@ class Connection(base.Connection): dims_filter = {k: v for k, v in dims_filter.items() if v is not None} period = period if period \ - else cfg.CONF.monasca.default_stats_period + else self.conf.monasca.default_stats_period _search_args = dict( name=filter.meter, diff --git a/ceilosca/ceilometer/tests/functional/api/__init__.py b/ceilosca/ceilometer/tests/functional/api/__init__.py index 520009c..a0359dc 100644 --- a/ceilosca/ceilometer/tests/functional/api/__init__.py +++ b/ceilosca/ceilometer/tests/functional/api/__init__.py @@ -1,5 +1,6 @@ # # Copyright 2012 New Dream Network, LLC (DreamHost) +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,18 +15,15 @@ # under the License. """Base classes for API tests. """ - -from oslo_config import cfg -from oslo_config import fixture as fixture_config from oslo_policy import opts import pecan import pecan.testing from ceilometer.api import rbac +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer.tests import db as db_test_base -cfg.CONF.import_group('api', 'ceilometer.api.controllers.v2.root') - class FunctionalTest(db_test_base.TestBase): """Used for functional tests of Pecan controllers. @@ -38,7 +36,9 @@ class FunctionalTest(db_test_base.TestBase): def setUp(self): super(FunctionalTest, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') self.setup_messaging(self.CONF) opts.set_defaults(self.CONF) @@ -48,6 +48,7 @@ class FunctionalTest(db_test_base.TestBase): self.CONF.set_override('gnocchi_is_enabled', False, group='api') self.CONF.set_override('aodh_is_enabled', False, group='api') + self.CONF.set_override('panko_is_enabled', False, group='api') self.app = self._make_app() @@ -63,7 +64,8 @@ class FunctionalTest(db_test_base.TestBase): }, } - return pecan.testing.load_test_app(self.config) + return pecan.testing.load_test_app(self.config, + conf=self.CONF) def tearDown(self): super(FunctionalTest, self).tearDown() diff --git a/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py b/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py index a862b87..3c113ef 100644 --- a/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py +++ b/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,10 +21,11 @@ import mock import pkg_resources from oslo_config import cfg -from oslo_config import fixture as fixture_config from stevedore import driver from stevedore import extension +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer import storage from ceilometer.tests import base as test_base from oslo_policy import opts @@ -52,31 +54,28 @@ class TestApi(test_base.BaseTestCase): mgr._init_plugins([a_driver]) return mgr - def get_connection_with_mock_driver_manager(self, url, namespace): + def get_connection_with_mock_driver_manager(self, conf, url, namespace): mgr = self._get_driver_from_entry_point( entry_point='monasca = ceilometer.storage.impl_monasca:Connection', namespace='ceilometer.metering.storage') - return mgr.driver(url) + return mgr.driver(conf, url) - def get_publisher_with_mock_driver_manager(self, url, namespace): + def get_publisher_with_mock_driver_manager(self, conf, url, namespace): mgr = self._get_driver_from_entry_point( entry_point='monasca = ceilometer.publisher.monclient:' 'MonascaPublisher', - namespace='ceilometer.publisher') - return mgr.driver(url) + namespace='ceilometer.sample.publisher') + return mgr.driver(conf, url) def setUp(self): super(TestApi, self).setUp() self.PATH_PREFIX = '/v2' - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) - + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') self.setup_messaging(self.CONF) opts.set_defaults(self.CONF) - - self.CONF.set_override("auth_version", "v2.0", - group=OPT_GROUP_NAME) self.CONF.set_override("policy_file", self.path_get('etc/ceilometer/policy.json'), group='oslo_policy') @@ -106,8 +105,10 @@ class TestApi(test_base.BaseTestCase): self.get_connection_with_mock_driver_manager) get_pub.side_effect = self.get_publisher_with_mock_driver_manager self.mock_mon_client = mock_client - self.conn = storage.get_connection('monasca://127.0.0.1:8080', - 'ceilometer.metering.storage') + self.conn = storage.get_connection( + self.CONF, + 'monasca://127.0.0.1:8080', + 'ceilometer.metering.storage') self.useFixture(fixtures.MockPatch( 'ceilometer.storage.get_connection', @@ -127,7 +128,8 @@ class TestApi(test_base.BaseTestCase): }, } - return pecan.testing.load_test_app(self.config) + return pecan.testing.load_test_app(self.config, + conf=self.CONF) def get_json(self, path, expect_errors=False, headers=None, extra_environ=None, q=None, groupby=None, status=None, diff --git a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py index 3907fc5..5ff857a 100644 --- a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py +++ b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -18,7 +19,6 @@ import os import fixtures import mock -from oslo_config import fixture as fixture_config from oslo_utils import fileutils from oslo_utils import timeutils from oslotest import base @@ -31,6 +31,8 @@ from ceilometer.ceilosca_mapping.ceilosca_mapping import ( from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinitionException) from ceilometer.ceilosca_mapping.ceilosca_mapping import PipelineReader +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer import storage from ceilometer.storage import impl_monasca from ceilometer.storage import models as storage_models @@ -150,15 +152,16 @@ class TestGetPipelineReader(TestCeiloscaMapping): def setUp(self): super(TestGetPipelineReader, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') def test_pipeline_reader(self): pipeline_cfg_file = self.setup_pipeline_file( self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) - test_pipeline_reader = PipelineReader() + test_pipeline_reader = PipelineReader(self.CONF) self.assertEqual(set(['testbatch', 'testbatch2']), test_pipeline_reader.get_pipeline_meters() @@ -223,14 +226,16 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): def setUp(self): super(TestMappedCeiloscaMetricProcessing, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') def test_fallback_mapping_file_path(self): self.useFixture(fixtures.MockPatchObject(self.CONF, 'find_file', return_value=None)) - fall_bak_path = ceilosca_mapping.get_config_file() + fall_bak_path = ceilosca_mapping.get_config_file(self.CONF) self.assertIn("ceilosca_mapping/data/ceilosca_mapping.yaml", fall_bak_path) @@ -264,7 +269,7 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - data = ceilosca_mapping.setup_ceilosca_mapping_config() + data = ceilosca_mapping.setup_ceilosca_mapping_config(self.CONF) meter_loaded = ceilosca_mapping.load_definitions(data) self.assertEqual(1, len(meter_loaded)) LOG.error.assert_called_with( @@ -275,8 +280,9 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) self.assertItemsEqual(['fake_metric', 'fake_metric2', 'fake_metric3'], ceilosca_mapper.get_list_monasca_metrics().keys() ) @@ -299,8 +305,9 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) self.assertEqual('fake_metric', ceilosca_mapper.get_monasca_metric_name('fake_meter') ) @@ -317,20 +324,27 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): Aggregate = collections.namedtuple("Aggregate", ['func', 'param']) def setUp(self): + super(TestMoanscaDriverForMappedMetrics, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + pipeline_cfg_file = self.setup_pipeline_file(self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) + ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) def test_get_samples_for_mapped_meters(self, mdf_mock): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") ml_mock = mock_client().measurements_list # TODO(this test case needs more work) ml_mock.return_value = ([MONASCA_MEASUREMENT]) @@ -386,7 +400,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): u'id': u'2015-04-16T18:42:31Z', u'name': u'testbatch'}]) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.side_effect = [data1, data2] kwargs = dict(limit=4) @@ -407,7 +421,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", "name": "subnet.update"}]) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metric_names_list_mock = mock_client().metric_names_list metric_names_list_mock.return_value = ( dummy_metric_names_mocked_return_value) @@ -419,7 +433,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): def test_stats_list_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -455,7 +469,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): def test_get_resources_for_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") dummy_metric_names_mocked_return_value = ( [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", "name": "fake_metric"}, @@ -489,7 +503,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): def test_stats_list_with_groupby_for_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -567,7 +581,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): return samples.pop() with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") with mock.patch.object(conn, 'get_samples') as gsm: gsm.side_effect = _get_samples diff --git a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py index 5925956..3f7c998 100644 --- a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py +++ b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -17,7 +18,7 @@ import os import fixtures import mock -from oslo_config import fixture as fixture_config +# from oslo_config import fixture as fixture_config from oslo_utils import fileutils from oslotest import base import six @@ -28,6 +29,8 @@ from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( CeilometerStaticMappingDefinition) from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( CeilometerStaticMappingDefinitionException) +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer.storage import impl_monasca @@ -152,13 +155,16 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): def setUp(self): super(TestMappedCeilometerStaticInfoProcessing, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf + # self.CONF = self.useFixture(fixture_config.Config()).conf + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') static_info_mapping_file = self.setup_static_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_static_info_mapping', static_info_mapping_file, group='monasca') self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo() - self.CONF([], project='ceilometer', validate_default_values=True) + .ProcessMappedCeilometerStaticInfo(self.CONF) + # self.CONF([], project='ceilometer', validate_default_values=True) def test_fallback_mapping_file_path(self): self.useFixture(fixtures.MockPatchObject(self.CONF, @@ -166,8 +172,9 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): return_value=None)) self.CONF.set_override('ceilometer_static_info_mapping', ' ', group='monasca') - self.static_info_mapper.reinitialize() - fall_bak_path = ceilometer_static_info_mapping.get_config_file() + self.static_info_mapper.reinitialize(self.CONF) + fall_bak_path = ceilometer_static_info_mapping.get_config_file( + self.CONF) self.assertIn( "ceilosca_mapping/data/ceilometer_static_info_mapping.yaml", fall_bak_path) @@ -198,7 +205,7 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): self.CONF.set_override('ceilometer_static_info_mapping', static_info_mapping_file, group='monasca') data = ceilometer_static_info_mapping.\ - setup_ceilometer_static_mapping_config() + setup_ceilometer_static_mapping_config(self.CONF) meter_loaded = ceilometer_static_info_mapping.load_definitions(data) self.assertEqual(3, len(meter_loaded)) LOG.error.assert_called_with( @@ -206,7 +213,7 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): "Invalid type foo specified") def test_list_of_meters_returned(self): - self.static_info_mapper.reinitialize() + self.static_info_mapper.reinitialize(self.CONF) self.assertItemsEqual(['disk.ephemeral.size', 'disk.root.size', 'image', 'image.delete'], self.static_info_mapper. @@ -225,7 +232,7 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): static_info_mapping_file = self.setup_static_mapping_def_file(cfg) self.CONF.set_override('ceilometer_static_info_mapping', static_info_mapping_file, group='monasca') - self.static_info_mapper.reinitialize() + self.static_info_mapper.reinitialize(self.CONF) self.assertEqual('gauge', self.static_info_mapper.get_meter_static_info_key_val( 'disk.ephemeral.size', 'type') @@ -239,8 +246,11 @@ class TestMoanscaDriverForMappedStaticInfo(TestStaticInfoBase): def setUp(self): super(TestMoanscaDriverForMappedStaticInfo, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + # self.CONF = self.useFixture(fixture_config.Config()).conf + # self.CONF([], project='ceilometer', validate_default_values=True) + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') pipeline_cfg_file = self.setup_pipeline_file(self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) static_info_mapping_file = self.setup_static_mapping_def_file(self.cfg) @@ -251,8 +261,8 @@ class TestMoanscaDriverForMappedStaticInfo(TestStaticInfoBase): self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo() - self.static_info_mapper.reinitialize() + .ProcessMappedCeilometerStaticInfo(self.CONF) + self.static_info_mapper.reinitialize(self.CONF) def test_get_statc_info_for_mapped_meters_uniq(self, mdf_mock): dummy_metric_names_mocked_return_value = ( @@ -262,7 +272,7 @@ class TestMoanscaDriverForMappedStaticInfo(TestStaticInfoBase): "name": "fake_metric"}]) with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') metric_names_list_mock = mock_client().metric_names_list metric_names_list_mock.return_value = ( dummy_metric_names_mocked_return_value diff --git a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py index af320d8..2c87230 100644 --- a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py +++ b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,13 +21,18 @@ from oslotest import base from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinitionException) +from ceilometer import monasca_ceilometer_opts from ceilometer.publisher import monasca_data_filter as mdf from ceilometer import sample +from ceilometer import service class TestMonUtils(base.BaseTestCase): def setUp(self): super(TestMonUtils, self).setUp() + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') self._field_mappings = { 'dimensions': ['resource_id', 'project_id', @@ -124,7 +130,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -160,7 +166,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[field_map]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertIsNone(r['dimensions'].get('project_id')) @@ -199,7 +205,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) self.assertIsNotNone(r.get('value_meta')) @@ -229,7 +235,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -261,7 +267,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -312,7 +318,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -362,7 +368,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -408,7 +414,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) try: # Don't assign to a variable, this should raise data_filter.process_sample_for_monasca(s) @@ -445,7 +451,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -493,7 +499,7 @@ class TestMonUtils(base.BaseTestCase): # use the bad mapping with mock.patch(to_patch, side_effect=[self._field_mappings_bad_format]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) try: # Don't assign to a variable as this should raise data_filter.process_sample_for_monasca(s) diff --git a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py index 6030505..42c9bf0 100755 --- a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py +++ b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -17,20 +18,16 @@ import datetime import fixtures -import os import time -from keystoneauth1 import loading as ka_loading import mock -from oslo_config import cfg -from oslo_config import fixture as fixture_config -from oslo_utils import fileutils from oslotest import base +from ceilometer import monasca_ceilometer_opts from ceilometer import monasca_client as mon_client from ceilometer.publisher import monclient from ceilometer import sample -from monascaclient import ksclient +from ceilometer import service class FakeResponse(object): @@ -112,21 +109,17 @@ class TestMonascaPublisher(base.BaseTestCase): def setUp(self): super(TestMonascaPublisher, self).setUp() - content = ("[service_credentials]\n" - "auth_type = password\n" - "username = ceilometer\n" - "password = admin\n" - "auth_url = http://localhost:5000/v2.0\n") - tempfile = fileutils.write_to_tempfile(content=content, - prefix='ceilometer', - suffix='.conf') - self.addCleanup(os.remove, tempfile) - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], default_config_files=[tempfile]) - ka_loading.load_auth_from_conf_options(self.CONF, - "service_credentials") + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + self.CONF.set_override('service_username', 'ceilometer', 'monasca') + self.CONF.set_override('service_password', 'admin', 'monasca') + self.CONF.set_override('service_auth_url', + 'http://localhost:5000/v2.0', + 'monasca') + self.parsed_url = mock.MagicMock() - ksclient.KSClient = mock.MagicMock() def tearDown(self): # For some reason, cfg.CONF is registered a required option named @@ -134,8 +127,8 @@ class TestMonascaPublisher(base.BaseTestCase): # case test_event_pipeline_endpoint_requeue_on_failure, so we # unregister it here. self.CONF.reset() - self.CONF.unregister_opt(cfg.StrOpt('auth_url'), - group='service_credentials') + # self.CONF.unregister_opt(cfg.StrOpt('service_auth_url'), + # group='monasca') super(TestMonascaPublisher, self).tearDown() @mock.patch("ceilometer.publisher.monasca_data_filter." @@ -143,7 +136,7 @@ class TestMonascaPublisher(base.BaseTestCase): side_effect=[field_mappings]) def test_publisher_publish(self, mapping_patch): self.CONF.set_override('batch_mode', False, group='monasca') - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, @@ -161,7 +154,7 @@ class TestMonascaPublisher(base.BaseTestCase): self.CONF.set_override('batch_count', 3, group='monasca') self.CONF.set_override('batch_polling_interval', 1, group='monasca') - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, 'metrics_create') as mock_create: @@ -182,7 +175,7 @@ class TestMonascaPublisher(base.BaseTestCase): self.CONF.set_override('retry_interval', 2, group='monasca') self.CONF.set_override('max_retries', 1, group='monasca') - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, 'metrics_create') as mock_create: @@ -207,7 +200,8 @@ class TestMonascaPublisher(base.BaseTestCase): 'ceilometer.publisher.file.FilePublisher', return_value=self.fake_publisher)) - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, + self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, diff --git a/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py b/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py index 93ecd59..ce59c7f 100644 --- a/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py +++ b/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -19,7 +20,6 @@ import os import dateutil.parser import mock -from oslo_config import fixture as fixture_config from oslo_utils import fileutils from oslo_utils import timeutils from oslotest import base @@ -30,6 +30,8 @@ import ceilometer from ceilometer.api.controllers.v2.meters import Aggregate from ceilometer.ceilosca_mapping import ceilometer_static_info_mapping from ceilometer.ceilosca_mapping import ceilosca_mapping +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer import storage from ceilometer.storage import impl_monasca from ceilometer.storage import models as storage_models @@ -39,18 +41,14 @@ class _BaseTestCase(base.BaseTestCase): def setUp(self): super(_BaseTestCase, self).setUp() - content = ("[service_credentials]\n" - "auth_type = password\n" - "username = ceilometer\n" - "password = admin\n" - "auth_url = http://localhost:5000/v2.0\n") - tempfile = fileutils.write_to_tempfile(content=content, - prefix='ceilometer', - suffix='.conf') - self.addCleanup(os.remove, tempfile) - conf = self.useFixture(fixture_config.Config()).conf - conf([], default_config_files=[tempfile]) - self.CONF = conf + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + self.CONF.set_override('service_username', 'ceilometer', 'monasca') + self.CONF.set_override('service_password', 'admin', 'monasca') + self.CONF.set_override('service_auth_url', + 'http://localhost:5000/v2.0', + 'monasca') mdf = mock.patch.object(impl_monasca, 'MonascaDataFilter') mdf.start() self.addCleanup(mdf.stop) @@ -58,8 +56,8 @@ class _BaseTestCase(base.BaseTestCase): spl.start() self.addCleanup(spl.stop) self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo() - self.static_info_mapper.reinitialize() + .ProcessMappedCeilometerStaticInfo(self.CONF) + self.static_info_mapper.reinitialize(self.CONF) def assertRaisesWithMessage(self, msg, exc_class, func, *args, **kwargs): try: @@ -133,12 +131,13 @@ class TestGetResources(_BaseTestCase): TestGetResources.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) def test_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") kwargs = dict(start_timestamp_op='le') self.assertRaises(ceilometer.NotImplementedError, @@ -149,7 +148,7 @@ class TestGetResources(_BaseTestCase): def test_dims_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ { @@ -178,7 +177,7 @@ class TestGetResources(_BaseTestCase): def test_get_resources(self, mock_utcnow): mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 20) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ { @@ -225,7 +224,7 @@ class TestGetResources(_BaseTestCase): def test_get_resources_limit(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ @@ -263,7 +262,7 @@ class TestGetResources(_BaseTestCase): def test_get_resources_simple_metaquery(self, mock_utcnow): mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 28) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ { @@ -325,7 +324,7 @@ class MeterTest(_BaseTestCase): def test_not_implemented_params(self): with mock.patch('ceilometer.monasca_client.Client'): - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') kwargs = dict(metaquery=True) self.assertRaises(ceilometer.NotImplementedError, @@ -333,7 +332,7 @@ class MeterTest(_BaseTestCase): def test_metrics_list_call(self): with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') metrics_list_mock = mock_client().metrics_list kwargs = dict(user='user-1', @@ -381,7 +380,7 @@ class MeterTest(_BaseTestCase): {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", "name": "subnet.update"}]) with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') metric_names_list_mock = mock_client().metric_names_list metric_names_list_mock.return_value = ( dummy_metric_names_mocked_return_value @@ -429,7 +428,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sample_filter = storage.SampleFilter(meter='specific meter', message_id='specific message') @@ -438,7 +437,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_name(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( TestGetSamples.dummy_metrics_mocked_return_value @@ -460,7 +459,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_start_timestamp_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( @@ -482,7 +481,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_timestamp_filter_exclusive_range(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( @@ -515,7 +514,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_limit(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list @@ -544,7 +543,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_project_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': dict(project_id='specific project'), @@ -564,7 +563,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_resource_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': dict(resource_id='specific resource'), @@ -583,7 +582,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_source_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': dict(source='specific source'), @@ -602,7 +601,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_simple_metaquery(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( TestGetSamples.dummy_metrics_mocked_return_value @@ -620,7 +619,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_simple_metaquery_with_extended_key(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( TestGetSamples.dummy_metrics_mocked_return_value @@ -639,7 +638,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_results(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': { @@ -718,7 +717,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") self.assertRaisesWithMessage("Query without filter " "not implemented", @@ -781,7 +780,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_stats_list_called_with(self, mock_utcnow): mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 31) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sf = storage.SampleFilter() @@ -813,7 +812,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_stats_list(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -851,7 +850,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_stats_list_with_groupby(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -911,7 +910,7 @@ class TestQuerySamples(_BaseTestCase): def test_query_samples_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") query = {'and': [{'=': {'counter_name': 'instance'}}, {'or': [{'=': {"project_id": "123"}}, {'=': {"user_id": "456"}}]}]} @@ -956,7 +955,7 @@ class TestQuerySamples(_BaseTestCase): return samples.pop() with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") with mock.patch.object(conn, 'get_samples') as gsm: gsm.side_effect = _get_samples @@ -998,7 +997,7 @@ class TestQuerySamples(_BaseTestCase): return samples.pop() with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") with mock.patch.object(conn, 'get_samples') as gsm: gsm.side_effect = _get_samples diff --git a/ceilosca/ceilometer/tests/unit/test_monascaclient.py b/ceilosca/ceilometer/tests/unit/test_monascaclient.py index 6603ef5..50cc929 100644 --- a/ceilosca/ceilometer/tests/unit/test_monascaclient.py +++ b/ceilosca/ceilometer/tests/unit/test_monascaclient.py @@ -1,4 +1,5 @@ # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -11,40 +12,33 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -import os - -from keystoneauth1 import loading as ka_loading import mock -from oslo_config import cfg -from oslo_config import fixture as fixture_config -from oslo_utils import fileutils + from oslo_utils import netutils from oslotest import base +from ceilometer import monasca_ceilometer_opts from ceilometer import monasca_client -from monascaclient import exc +from ceilometer import service -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +from monascaclient import exc +import tenacity class TestMonascaClient(base.BaseTestCase): def setUp(self): super(TestMonascaClient, self).setUp() - content = ("[service_credentials]\n" - "auth_type = password\n" - "username = ceilometer\n" - "password = admin\n" - "auth_url = http://localhost:5000/v2.0\n") - tempfile = fileutils.write_to_tempfile(content=content, - prefix='ceilometer', - suffix='.conf') - self.addCleanup(os.remove, tempfile) - self.conf = self.useFixture(fixture_config.Config()).conf - self.conf([], default_config_files=[tempfile]) - ka_loading.load_auth_from_conf_options(self.conf, - "service_credentials") - self.conf.set_override('max_retries', 0, 'database') + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + self.CONF.set_override('service_username', 'ceilometer', 'monasca') + self.CONF.set_override('service_password', 'admin', 'monasca') + self.CONF.set_override('service_auth_url', + 'http://localhost:5000/v2.0', + 'monasca') + + self.CONF.set_override('max_retries', 0, 'database') self.mc = self._get_client() def tearDown(self): @@ -52,25 +46,21 @@ class TestMonascaClient(base.BaseTestCase): # auth_url after these tests run, which occasionally blocks test # case test_event_pipeline_endpoint_requeue_on_failure, so we # unregister it here. - self.conf.reset() - self.conf.unregister_opt(cfg.StrOpt('auth_url'), - group='service_credentials') + self.CONF.reset() + # self.CONF.unregister_opt(cfg.StrOpt('service_auth_url'), + # group='monasca') super(TestMonascaClient, self).tearDown() @mock.patch('monascaclient.client.Client') - @mock.patch('monascaclient.ksclient.KSClient') - def _get_client(self, ksclass_mock, monclient_mock): - ksclient_mock = ksclass_mock.return_value - ksclient_mock.token.return_value = "token123" + def _get_client(self, monclient_mock): return monasca_client.Client( + self.CONF, netutils.urlsplit("http://127.0.0.1:8080")) @mock.patch('monascaclient.client.Client') - @mock.patch('monascaclient.ksclient.KSClient') - def test_client_url_correctness(self, ksclass_mock, monclient_mock): - ksclient_mock = ksclass_mock.return_value - ksclient_mock.token.return_value = "token123" + def test_client_url_correctness(self, monclient_mock): mon_client = monasca_client.Client( + self.CONF, netutils.urlsplit("monasca://https://127.0.0.1:8080")) self.assertEqual("https://127.0.0.1:8080", mon_client._endpoint) @@ -84,44 +74,46 @@ class TestMonascaClient(base.BaseTestCase): def test_metrics_create_exception(self): with mock.patch.object( self.mc._mon_client.metrics, 'create', - side_effect=[exc.HTTPInternalServerError, True])\ + side_effect=[exc.http.InternalServerError, True])\ as create_patch: - self.assertRaises(monasca_client.MonascaServiceException, - self.mc.metrics_create) + e = self.assertRaises(tenacity.RetryError, + self.mc.metrics_create) + (original_ex, traceobj) = e.last_attempt.exception_info() + self.assertIsInstance(original_ex, + monasca_client.MonascaServiceException) self.assertEqual(1, create_patch.call_count) def test_metrics_create_unprocessable_exception(self): with mock.patch.object( self.mc._mon_client.metrics, 'create', - side_effect=[exc.HTTPUnProcessable, True])\ + side_effect=[exc.http.UnprocessableEntity, True])\ as create_patch: self.assertRaises(monasca_client.MonascaInvalidParametersException, self.mc.metrics_create) self.assertEqual(1, create_patch.call_count) def test_invalid_service_creds(self): - conf = cfg.CONF.service_credentials + conf = self.CONF.monasca class SetOpt(object): def __enter__(self): - self.username = conf.username - conf.username = "" + self.username = conf.service_username + conf.service_username = "" def __exit__(self, exc_type, exc_val, exc_tb): - conf.username = self.username + conf.service_username = self.username with SetOpt(): self.assertRaises( monasca_client.MonascaInvalidServiceCredentialsException, self._get_client) - self.assertIsNotNone(True, conf.username) + self.assertIsNotNone(True, conf.service_username) def test_retry_on_key_error(self): - self.conf.set_override('max_retries', 2, 'database') - self.conf.set_override('retry_interval', 1, 'database') + self.CONF.set_override('max_retries', 2, 'database') + self.CONF.set_override('retry_interval', 1, 'database') self.mc = self._get_client() - with mock.patch.object( self.mc._mon_client.metrics, 'list', side_effect=[KeyError, []]) as mocked_metrics_list: @@ -129,8 +121,8 @@ class TestMonascaClient(base.BaseTestCase): self.assertEqual(2, mocked_metrics_list.call_count) def test_no_retry_on_invalid_parameter(self): - self.conf.set_override('max_retries', 2, 'database') - self.conf.set_override('retry_interval', 1, 'database') + self.CONF.set_override('max_retries', 2, 'database') + self.CONF.set_override('retry_interval', 1, 'database') self.mc = self._get_client() def _check(exception): @@ -142,12 +134,12 @@ class TestMonascaClient(base.BaseTestCase): self.assertRaises(expected_exc, list, self.mc.metrics_list()) self.assertEqual(1, mocked_metrics_list.call_count) - _check(exc.HTTPUnProcessable) - _check(exc.HTTPBadRequest) + _check(exc.http.UnprocessableEntity) + _check(exc.http.BadRequest) def test_max_retris_not_too_much(self): def _check(configured, expected): - self.conf.set_override('max_retries', configured, 'database') + self.CONF.set_override('max_retries', configured, 'database') self.mc = self._get_client() self.assertEqual(expected, self.mc._max_retries) @@ -159,27 +151,32 @@ class TestMonascaClient(base.BaseTestCase): def test_meaningful_exception_message(self): with mock.patch.object( self.mc._mon_client.metrics, 'list', - side_effect=[exc.HTTPInternalServerError, - exc.HTTPUnProcessable, + side_effect=[exc.http.InternalServerError, + exc.http.UnprocessableEntity, KeyError]): e = self.assertRaises( - monasca_client.MonascaServiceException, + tenacity.RetryError, list, self.mc.metrics_list()) - self.assertIn('Monasca service is unavailable', str(e)) + (original_ex, traceobj) = e.last_attempt.exception_info() + self.assertIn('Monasca service is unavailable', + str(original_ex)) e = self.assertRaises( monasca_client.MonascaInvalidParametersException, list, self.mc.metrics_list()) - self.assertIn('Request cannot be handled by Monasca', str(e)) + self.assertIn('Request cannot be handled by Monasca', + str(e)) e = self.assertRaises( - monasca_client.MonascaException, + tenacity.RetryError, list, self.mc.metrics_list()) - self.assertIn('An exception is raised from Monasca', str(e)) + (original_ex, traceobj) = e.last_attempt.exception_info() + self.assertIn('An exception is raised from Monasca', + str(original_ex)) @mock.patch.object(monasca_client.Client, '_refresh_client') def test_metrics_create_with_401(self, rc_patch): with mock.patch.object( self.mc._mon_client.metrics, 'create', - side_effect=[exc.HTTPUnauthorized, True]): + side_effect=[exc.http.Unauthorized, True]): self.assertRaises( monasca_client.MonascaInvalidParametersException, self.mc.metrics_create) @@ -206,7 +203,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = len(metric_list_pages) expected_metric_names = ["test1", "test2"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', True, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -245,7 +242,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = 1 expected_metric_names = ["test1"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', False, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -283,7 +280,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = len(measurement_list_pages) expected_metric_names = ["test1", "test2"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', True, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -322,7 +319,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = 1 expected_metric_names = ["test1"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', False, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -364,7 +361,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = len(statistics_list_pages) expected_metric_names = ["test1", "test2"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', True, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -403,7 +400,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = 1 expected_metric_names = ["test1"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', False, group='monasca') # get a new ceilosca mc mc = self._get_client() diff --git a/monasca_test_setup.py b/monasca_test_setup.py index 22e8d87..7696b1a 100644 --- a/monasca_test_setup.py +++ b/monasca_test_setup.py @@ -23,6 +23,7 @@ ceilosca_files = { for file in [ 'monasca_client.py', + 'monasca_ceilometer_opts.py', 'publisher/monasca_data_filter.py', 'publisher/monclient.py', 'storage/impl_monasca.py' diff --git a/setup.py b/setup.py index e2be832..2c6a1d9 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ # (c) Copyright 2017 Hewlett Packard Enterprise Development LP -# (c) Copyright 2017 SUSE LLC +# (c) Copyright 2018 SUSE LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/test-requirements.txt b/test-requirements.txt index b954a82..425353a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,16 +1,13 @@ -hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0 -git+https://github.com/openstack/ceilometer.git@newton-eol#egg=ceilometer +git+https://github.com/openstack/ceilometer.git@stable/pike#egg=ceilometer mock>=1.2 testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 -oslosphinx>=2.5.0 # Apache-2.0 -# update oslotest version after ceilometer newton -oslotest>=1.10.0,<2.0.0 # Apache-2.0 +oslotest>=2.15.0 # Apache-2.0 oslo.vmware>=1.16.0,<2.17.0 # Apache-2.0 # Use lower versions of config and utils since # Keystone client depends on it -oslo.config>=2.3.0 # Apache-2.0 -oslo.utils!=2.6.0,>=2.0.0 # Apache-2.0 -oslo.log>=1.8.0 # Apache-2.0 -python-monascaclient<=1.2.0 +oslo.config>=3.22.0 # Apache-2.0 +oslo.log>=1.14.0 # Apache-2.0 +os-testr>=1.0.0 # Apache-2.0 +python-monascaclient<=1.7.1 diff --git a/tox.ini b/tox.ini index 54ba240..564558e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ install_command = pip install -U {opts} {packages} deps = -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete python monasca_test_setup.py - python setup.py testr --slowest --testr-args="{posargs}" + ostestr --serial {posargs} whitelist_externals = bash find