diff --git a/ceilosca/ceilometer/api/health.py b/ceilosca/ceilometer/api/health.py new file mode 100644 index 0000000..8e8c56d --- /dev/null +++ b/ceilosca/ceilometer/api/health.py @@ -0,0 +1,81 @@ +# +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log +LOG = log.getLogger(__name__) + + +def filter_factory(global_config, **local_conf): + """Returns a WSGI Filter.""" + conf = global_config.copy() + conf.update(local_conf) + LOG.debug('Filter Factory') + + def health_filter(app): + return HealthFilterApi(app, conf) + return health_filter + + +class HealthFilterApi(object): + + def __init__(self, app, conf): + self.conf = conf + self.app = app + self.db = None + + def __call__(self, environ, start_response): + """Handle the incoming request and filters it. + + Interjects the request and acts on it only if + it is related to the filter functionality, otherwise + it just let the request pass through to the app. + + """ + + LOG.debug('Health Check Filter') + + if environ['PATH_INFO'].startswith('/v2/health'): + response_code = '204 No Content' + if environ['REQUEST_METHOD'] != 'HEAD': + response_code = '200 OK' +# Commenting out healthcheck behavior when request method +# is not of type HEAD.# As it creates load on monasca vertica +# FIXME: fix this when monasca creates get versions api in +# in Monascaclient. USe that instead +# try: +# if not self.db: +# self.db = storage.get_connection_from_config(cfg.CONF) +# meters = self.db.get_meters(unique=True, limit=1) +# if not meters: +# response_code = '503 Backend Unavailable' +# except Exception as e: +# response_code = '503 Backend Unavailable' +# LOG.warning('DB health check connection failed: %s', e) +# self.db = None + resp = MiniResp(response_code, environ) + start_response(response_code, resp.headers) + return resp.body + else: + return self.app(environ, start_response) + + +class MiniResp(object): + + def __init__(self, message, env, headers=[]): + if env['REQUEST_METHOD'] == 'HEAD': + self.body = [''] + else: + self.body = [message] + self.headers = list(headers) diff --git a/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py b/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py index ab9c620..c66d5f6 100644 --- a/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py +++ b/ceilosca/ceilometer/ceilosca_mapping/ceilometer_static_info_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,23 +21,12 @@ import os import pkg_resources import yaml -from oslo_config import cfg from oslo_log import log - from ceilometer import sample LOG = log.getLogger(__name__) -OPTS = [ - cfg.StrOpt('ceilometer_static_info_mapping', - default='ceilometer_static_info_mapping.yaml', - help='Configuration mapping file to map ceilometer meters to ' - 'their units an type informaiton'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - class CeilometerStaticMappingDefinitionException(Exception): def __init__(self, message, definition_cfg): @@ -67,19 +57,19 @@ class CeilometerStaticMappingDefinition(object): "Invalid type %s specified" % self.cfg['type'], self.cfg) -def get_config_file(): - config_file = cfg.CONF.monasca.ceilometer_static_info_mapping +def get_config_file(conf): + config_file = conf.monasca.ceilometer_static_info_mapping if not os.path.exists(config_file): - config_file = cfg.CONF.find_file(config_file) + config_file = conf.find_file(config_file) if not config_file: config_file = pkg_resources.resource_filename( __name__, "data/ceilometer_static_info_mapping.yaml") return config_file -def setup_ceilometer_static_mapping_config(): +def setup_ceilometer_static_mapping_config(conf): """Setup the meters definitions from yaml config file.""" - config_file = get_config_file() + config_file = get_config_file(conf) if config_file is not None: LOG.debug("Static Ceilometer mapping file to map static info: %s", config_file) @@ -160,11 +150,12 @@ class ProcessMappedCeilometerStaticInfo(object): __new__(cls, *args, **kwargs) return cls._instance - def __init__(self): + def __init__(self, conf): if not (self._instance and self._inited): + self.conf = conf self._inited = True self.__definitions = load_definitions( - setup_ceilometer_static_mapping_config()) + setup_ceilometer_static_mapping_config(self.conf)) self.__mapped_meter_info_map = dict() for d in self.__definitions: self.__mapped_meter_info_map[d.cfg['name']] = d @@ -178,9 +169,10 @@ class ProcessMappedCeilometerStaticInfo(object): def get_meter_static_info_key_val(self, meter_name, key): return self.__mapped_meter_info_map.get(meter_name).cfg[key] - def reinitialize(self): + def reinitialize(self, conf): + self.conf = conf self.__definitions = load_definitions( - setup_ceilometer_static_mapping_config()) + setup_ceilometer_static_mapping_config(self.conf)) self.__mapped_meter_info_map = dict() for d in self.__definitions: self.__mapped_meter_info_map[d.cfg['name']] = d diff --git a/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py b/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py index fb81230..2fc4cca 100644 --- a/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py +++ b/ceilosca/ceilometer/ceilosca_mapping/ceilosca_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -23,24 +24,13 @@ import six import yaml from jsonpath_rw_ext import parser -from oslo_config import cfg from oslo_log import log - from ceilometer import pipeline from ceilometer import sample LOG = log.getLogger(__name__) -OPTS = [ - cfg.StrOpt('ceilometer_monasca_metrics_mapping', - default='ceilosca_mapping.yaml', - help='Configuration mapping file to map monasca metrics to ' - 'ceilometer meters'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - class CeiloscaMappingDefinitionException(Exception): def __init__(self, message, definition_cfg): @@ -132,19 +122,19 @@ class CeiloscaMappingDefinition(object): return values -def get_config_file(): - config_file = cfg.CONF.monasca.ceilometer_monasca_metrics_mapping +def get_config_file(conf): + config_file = conf.monasca.ceilometer_monasca_metrics_mapping if not os.path.exists(config_file): - config_file = cfg.CONF.find_file(config_file) + config_file = conf.find_file(config_file) if not config_file: config_file = pkg_resources.resource_filename( __name__, "data/ceilosca_mapping.yaml") return config_file -def setup_ceilosca_mapping_config(): +def setup_ceilosca_mapping_config(conf): """Setup the meters definitions from yaml config file.""" - config_file = get_config_file() + config_file = get_config_file(conf) if config_file is not None: LOG.debug("Ceilometer Monasca Mapping Definitions file: %s", config_file) @@ -228,11 +218,12 @@ class ProcessMappedCeiloscaMetric(object): cls, *args, **kwargs) return cls._instance - def __init__(self): + def __init__(self, conf): if not (self._instance and self._inited): + self.conf = conf self._inited = True self.__definitions = load_definitions( - setup_ceilosca_mapping_config()) + setup_ceilosca_mapping_config(self.conf)) self.__mapped_metric_map = dict() self.__mon_metric_to_cm_meter_map = dict() for d in self.__definitions: @@ -252,9 +243,10 @@ class ProcessMappedCeiloscaMetric(object): def get_ceilosca_mapped_definition_key_val(self, monasca_metric_name, key): return self.__mapped_metric_map.get(monasca_metric_name).cfg[key] - def reinitialize(self): + def reinitialize(self, conf): + self.conf = conf self.__definitions = load_definitions( - setup_ceilosca_mapping_config()) + setup_ceilosca_mapping_config(self.conf)) self.__mapped_metric_map = dict() self.__mon_metric_to_cm_meter_map = dict() for d in self.__definitions: @@ -281,10 +273,11 @@ class PipelineReader(object): cls, *args, **kwargs) return cls._instance - def __init__(self): + def __init__(self, conf): if not (self._instance and self._inited): self._inited = True - self.__pipeline_manager = pipeline.setup_pipeline() + self.conf = conf + self.__pipeline_manager = pipeline.setup_pipeline(self.conf) self.__meters_from_pipeline = set() for pipe in self.__pipeline_manager.pipelines: if not isinstance(pipe, pipeline.EventPipeline): @@ -296,7 +289,7 @@ class PipelineReader(object): return self.__meters_from_pipeline def reinitialize(self): - self.__pipeline_manager = pipeline.setup_pipeline() + self.__pipeline_manager = pipeline.setup_pipeline(self.conf) self.__meters_from_pipeline = set() for pipe in self.__pipeline_manager.pipelines: if not isinstance(pipe, pipeline.EventPipeline): diff --git a/ceilosca/ceilometer/monasca_ceilometer_opts.py b/ceilosca/ceilometer/monasca_ceilometer_opts.py new file mode 100644 index 0000000..b79b0a0 --- /dev/null +++ b/ceilosca/ceilometer/monasca_ceilometer_opts.py @@ -0,0 +1,114 @@ +# +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" All monasca ceilometer config opts""" + +from oslo_config import cfg + +OPTS = [ + + # from ceilometer_static_info_mapping + cfg.StrOpt('ceilometer_static_info_mapping', + default='ceilometer_static_info_mapping.yaml', + help='Configuration mapping file to map ceilometer meters to ' + 'their units an type information'), + + # from ceilosca_mapping + cfg.StrOpt('ceilometer_monasca_metrics_mapping', + default='ceilosca_mapping.yaml', + help='Configuration mapping file to map monasca metrics to ' + 'ceilometer meters'), + + # from monasca_client + cfg.StrOpt('clientapi_version', + default='2_0', + help='Version of Monasca client to use while publishing.'), + cfg.BoolOpt('enable_api_pagination', + default=False, + help='Enable paging through monasca api resultset.'), + + cfg.StrOpt('service_auth_url', help='auth url connecting to service'), + cfg.StrOpt('service_password', help='password connecting to service'), + cfg.StrOpt('service_username', help='username connecting to service'), + cfg.StrOpt('service_project_id', help='username connecting to service'), + cfg.StrOpt('service_domain_name', help='domain connecting to service'), + cfg.StrOpt('service_region_name', help='region connecting to service'), + cfg.StrOpt('service_project_name', + help='project name connecting to service'), + cfg.StrOpt('service_verify', + help='path to ssl cert to verify connecting to service'), + + # from monasca_data_filter + cfg.StrOpt('monasca_mappings', + default='/etc/ceilometer/monasca_field_definitions.yaml', + help='Monasca static and dynamic field mappings'), + + # from multi region opts + cfg.StrOpt('control_plane', + default='None', + help='The name of control plane'), + cfg.StrOpt('cluster', + default='None', + help='The name of cluster'), + cfg.StrOpt('cloud_name', + default='None', + help='The name of cloud'), + + # from publisher monclient + cfg.BoolOpt('batch_mode', + default=True, + help='Indicates whether samples are' + ' published in a batch.'), + cfg.IntOpt('batch_count', + default=1000, + help='Maximum number of samples in a batch.'), + cfg.IntOpt('batch_timeout', + default=15, + help='Maximum time interval(seconds) after which ' + 'samples are published in a batch.'), + cfg.IntOpt('batch_polling_interval', + default=5, + help='Frequency of checking if batch criteria is met.'), + cfg.BoolOpt('retry_on_failure', + default=False, + help='Indicates whether publisher retries publishing' + 'sample in case of failure. Only a few error cases' + 'are queued for a retry.'), + cfg.IntOpt('retry_interval', + default=60, + help='Frequency of attempting a retry.'), + cfg.IntOpt('max_retries', + default=3, + help='Maximum number of retry attempts on a publishing ' + 'failure.'), + cfg.BoolOpt('archive_on_failure', + default=False, + help='When turned on, archives metrics in file system when' + 'publish to Monasca fails or metric publish maxes out' + 'retry attempts.'), + cfg.StrOpt('archive_path', + default='mon_pub_failures.txt', + help='File of metrics that failed to publish to ' + 'Monasca. These include metrics that failed to ' + 'publish on first attempt and failed metrics that' + ' maxed out their retries.'), + + # from impl_monasca + cfg.IntOpt('default_stats_period', + default=300, + help='Default period (in seconds) to use for querying stats ' + 'in case no period specified in the stats API call.'), + +] diff --git a/ceilosca/ceilometer/monasca_client.py b/ceilosca/ceilometer/monasca_client.py index 81f3e26..6fa0271 100644 --- a/ceilosca/ceilometer/monasca_client.py +++ b/ceilosca/ceilometer/monasca_client.py @@ -1,4 +1,5 @@ # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -16,27 +17,10 @@ import copy from monascaclient import client from monascaclient import exc -from monascaclient import ksclient -from oslo_config import cfg from oslo_log import log -import retrying +import tenacity from ceilometer.i18n import _ -from ceilometer import keystone_client - - -monclient_opts = [ - cfg.StrOpt('clientapi_version', - default='2_0', - help='Version of Monasca client to use while publishing.'), - cfg.BoolOpt('enable_api_pagination', - default=False, - help='Enable paging through monasca api resultset.'), -] - -cfg.CONF.register_opts(monclient_opts, group='monasca') -keystone_client.register_keystoneauth_opts(cfg.CONF) -cfg.CONF.import_group('service_credentials', 'ceilometer.service') LOG = log.getLogger(__name__) @@ -68,20 +52,23 @@ class MonascaInvalidParametersException(Exception): class Client(object): """A client which gets information via python-monascaclient.""" - _ksclient = None - - def __init__(self, parsed_url): - self._retry_interval = cfg.CONF.database.retry_interval * 1000 - self._max_retries = cfg.CONF.database.max_retries or 1 + def __init__(self, conf, parsed_url): + self.conf = conf + self._retry_interval = self.conf.database.retry_interval + self._max_retries = self.conf.database.max_retries or 1 # enable monasca api pagination - self._enable_api_pagination = cfg.CONF.monasca.enable_api_pagination + self._enable_api_pagination = self.conf.monasca.enable_api_pagination # NOTE(zqfan): There are many concurrency requests while using # Ceilosca, to save system resource, we don't retry too many times. if self._max_retries < 0 or self._max_retries > 10: LOG.warning('Reduce max retries from %s to 10', self._max_retries) self._max_retries = 10 - conf = cfg.CONF.service_credentials + + # self.conf.log_opt_values(LOG, logging.INFO) + + conf = self.conf.service_credentials + monasca_conf = self.conf.monasca # because our ansible script are in another repo, the old setting # of auth_type is password-ceilometer-legacy which doesn't register # os_xxx options, so here we need to provide a compatible way to @@ -90,14 +77,18 @@ class Client(object): username = conf.os_username password = conf.os_password auth_url = conf.os_auth_url - project_id = conf.os_tenant_id + # project_id = conf.os_tenant_id project_name = conf.os_tenant_name else: - username = conf.username - password = conf.password - auth_url = conf.auth_url - project_id = conf.project_id - project_name = conf.project_name + username = monasca_conf.service_username + password = monasca_conf.service_password + auth_url = monasca_conf.service_auth_url + # project_id = monasca_conf.service_project_id + project_name = monasca_conf.service_project_name + default_domain_name = monasca_conf.service_domain_name + region_name = monasca_conf.service_region_name + service_verify = monasca_conf.service_verify + if not username or not password or not auth_url: err_msg = _("No user name or password or auth_url " "found in service_credentials") @@ -108,13 +99,17 @@ class Client(object): 'username': username, 'password': password, 'auth_url': auth_url.replace("v2.0", "v3"), - 'project_id': project_id, + # 'project_id': project_id, 'project_name': project_name, - 'region_name': conf.region_name, - 'read_timeout': cfg.CONF.http_timeout, - 'write_timeout': cfg.CONF.http_timeout, + 'region_name': region_name, + 'default_domain_name': default_domain_name, + 'project_domain_name': default_domain_name, + 'user_domain_name': default_domain_name, + 'read_timeout': self.conf.http_timeout, + 'write_timeout': self.conf.http_timeout, + 'keystone_timeout': self.conf.http_timeout, + 'verify': service_verify } - self._kwargs = kwargs self._endpoint = parsed_url.netloc + parsed_url.path LOG.info(_("monasca_client: using %s as monasca end point") % @@ -122,35 +117,29 @@ class Client(object): self._refresh_client() def _refresh_client(self): - if not Client._ksclient: - Client._ksclient = ksclient.KSClient(**self._kwargs) - self._kwargs['token'] = Client._ksclient.token - self._mon_client = client.Client(cfg.CONF.monasca.clientapi_version, + self._mon_client = client.Client(self.conf.monasca.clientapi_version, self._endpoint, **self._kwargs) - @staticmethod - def _retry_on_exception(e): - return not isinstance(e, MonascaInvalidParametersException) - def call_func(self, func, **kwargs): - @retrying.retry(wait_fixed=self._retry_interval, - stop_max_attempt_number=self._max_retries, - retry_on_exception=self._retry_on_exception) + @tenacity.retry( + wait=tenacity.wait_fixed(self._retry_interval), + stop=tenacity.stop_after_attempt(self._max_retries), + retry=(tenacity.retry_if_exception_type(MonascaServiceException) | + tenacity.retry_if_exception_type(MonascaException))) def _inner(): try: return func(**kwargs) - except (exc.HTTPInternalServerError, - exc.HTTPServiceUnavailable, - exc.HTTPBadGateway, - exc.CommunicationError) as e: + except (exc.http.InternalServerError, + exc.http.ServiceUnavailable, + exc.http.BadGateway, + exc.connection.ConnectionError) as e: LOG.exception(e) msg = '%s: %s' % (e.__class__.__name__, e) raise MonascaServiceException(msg) - except exc.HTTPException as e: + except exc.http.HttpError as e: LOG.exception(e) msg = '%s: %s' % (e.__class__.__name__, e) - status_code = e.code - # exc.HTTPException has string code 'N/A' + status_code = e.http_status if not isinstance(status_code, int): status_code = 500 if 400 <= status_code < 500: @@ -206,13 +195,17 @@ class Client(object): **search_args) # check if api pagination is enabled if self._enable_api_pagination: - while measurements: + while measurements and len(measurements[0]["measurements"]) > 0: for measurement in measurements: + if measurement["measurements"] is not None and \ + len(measurement["measurements"]) > 0: + # offset for measurements is measurement id composited + # with the last measurement's timestamp + last_good_offset = '%s_%s' % ( + measurement['id'], + measurement['measurements'][-1][0]) yield measurement - # offset for measurements is measurement id composited with - # the last measurement's timestamp - search_args['offset'] = '%s_%s' % ( - measurement['id'], measurement['measurements'][-1][0]) + search_args['offset'] = last_good_offset measurements = self.call_func( self._mon_client.metrics.list_measurements, **search_args) @@ -231,17 +224,21 @@ class Client(object): **search_args) # check if api pagination is enabled if self._enable_api_pagination: - while statistics: + while statistics and len(statistics[0]["statistics"]) > 0: for statistic in statistics: + if statistic["statistics"] is not None and \ + len(statistic["statistics"]) > 0: + # offset for statistics is statistic id composited with + # the last statistic's timestamp + last_good_offset = '%s_%s' % ( + statistic['id'], statistic['statistics'][-1][0]) yield statistic + # with groupby, the offset is unpredictable to me, we don't # support pagination for it now. if kwargs.get('group_by'): break - # offset for statistics is statistic id composited with - # the last statistic's timestamp - search_args['offset'] = '%s_%s' % ( - statistic['id'], statistic['statistics'][-1][0]) + search_args['offset'] = last_good_offset statistics = self.call_func( self._mon_client.metrics.list_statistics, **search_args) diff --git a/ceilosca/ceilometer/opts.py b/ceilosca/ceilometer/opts.py new file mode 100644 index 0000000..f3dd2ae --- /dev/null +++ b/ceilosca/ceilometer/opts.py @@ -0,0 +1,157 @@ +# Copyright 2014 eNovance +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +import itertools +import socket + +from keystoneauth1 import loading +from oslo_config import cfg + +import ceilometer.agent.manager +import ceilometer.api.app +import ceilometer.api.controllers.v2.root +import ceilometer.collector +import ceilometer.compute.discovery +import ceilometer.compute.virt.inspector +import ceilometer.compute.virt.libvirt.utils +import ceilometer.compute.virt.vmware.inspector +import ceilometer.compute.virt.xenapi.inspector +import ceilometer.dispatcher +import ceilometer.dispatcher.file +import ceilometer.dispatcher.gnocchi_opts +import ceilometer.dispatcher.http +import ceilometer.event.converter +import ceilometer.exchange_control +import ceilometer.hardware.discovery +import ceilometer.hardware.pollsters.generic +import ceilometer.image.discovery +import ceilometer.ipmi.notifications.ironic +import ceilometer.ipmi.platform.intel_node_manager +import ceilometer.ipmi.pollsters +import ceilometer.keystone_client +import ceilometer.meter.notifications +import ceilometer.middleware +import ceilometer.monasca_ceilometer_opts +import ceilometer.neutron_client +import ceilometer.notification +import ceilometer.nova_client +import ceilometer.objectstore.rgw +import ceilometer.objectstore.swift +import ceilometer.pipeline +import ceilometer.publisher.messaging +import ceilometer.publisher.utils +import ceilometer.sample +import ceilometer.storage +import ceilometer.utils +import ceilometer.volume.discovery + + +OPTS = [ + cfg.HostAddressOpt('host', + default=socket.gethostname(), + sample_default='', + help='Name of this node, which must be valid in an ' + 'AMQP key. Can be an opaque identifier. For ZeroMQ ' + 'only, must be a valid host name, FQDN, or IP ' + 'address.'), + cfg.IntOpt('http_timeout', + default=600, + help='Timeout seconds for HTTP requests. Set it to None to ' + 'disable timeout.'), + cfg.IntOpt('max_parallel_requests', + default=64, + min=1, + help='Maximum number of parallel requests for ' + 'services to handle at the same time.'), +] + + +def list_opts(): + # FIXME(sileht): readd pollster namespaces in the generated configfile + # This have been removed due to a recursive import issue + return [ + ('DEFAULT', + itertools.chain(ceilometer.agent.manager.OPTS, + ceilometer.api.app.OPTS, + ceilometer.compute.virt.inspector.OPTS, + ceilometer.compute.virt.libvirt.utils.OPTS, + ceilometer.dispatcher.OPTS, + ceilometer.ipmi.notifications.ironic.OPTS, + ceilometer.nova_client.OPTS, + ceilometer.objectstore.swift.OPTS, + ceilometer.pipeline.OPTS, + ceilometer.sample.OPTS, + ceilometer.utils.OPTS, + ceilometer.exchange_control.EXCHANGE_OPTS, + OPTS)), + ('api', itertools.chain(ceilometer.api.app.API_OPTS, + ceilometer.api.controllers.v2.root.API_OPTS)), + ('collector', ceilometer.collector.OPTS), + ('compute', ceilometer.compute.discovery.OPTS), + ('coordination', [ + cfg.StrOpt( + 'backend_url', + help='The backend URL to use for distributed coordination. If ' + 'left empty, per-deployment central agent and per-host ' + 'compute agent won\'t do workload ' + 'partitioning and will only function correctly if a ' + 'single instance of that service is running.'), + cfg.FloatOpt( + 'check_watchers', + default=10.0, + help='Number of seconds between checks to see if group ' + 'membership has changed'), + ]), + ('database', ceilometer.storage.OPTS), + ('dispatcher_file', ceilometer.dispatcher.file.OPTS), + ('dispatcher_http', ceilometer.dispatcher.http.http_dispatcher_opts), + ('dispatcher_gnocchi', + ceilometer.dispatcher.gnocchi_opts.dispatcher_opts), + ('event', ceilometer.event.converter.OPTS), + ('hardware', itertools.chain( + ceilometer.hardware.discovery.OPTS, + ceilometer.hardware.pollsters.generic.OPTS)), + ('ipmi', + itertools.chain(ceilometer.ipmi.platform.intel_node_manager.OPTS, + ceilometer.ipmi.pollsters.OPTS)), + ('meter', ceilometer.meter.notifications.OPTS), + ('monasca', ceilometer.monasca_ceilometer_opts.OPTS), + ('notification', + itertools.chain(ceilometer.notification.OPTS, + ceilometer.notification.EXCHANGES_OPTS)), + ('polling', ceilometer.agent.manager.POLLING_OPTS), + ('publisher', ceilometer.publisher.utils.OPTS), + ('publisher_notifier', ceilometer.publisher.messaging.NOTIFIER_OPTS), + ('rgw_admin_credentials', ceilometer.objectstore.rgw.CREDENTIAL_OPTS), + ('service_types', + itertools.chain(ceilometer.image.discovery.SERVICE_OPTS, + ceilometer.neutron_client.SERVICE_OPTS, + ceilometer.nova_client.SERVICE_OPTS, + ceilometer.objectstore.rgw.SERVICE_OPTS, + ceilometer.objectstore.swift.SERVICE_OPTS, + ceilometer.volume.discovery.SERVICE_OPTS,)), + ('vmware', ceilometer.compute.virt.vmware.inspector.OPTS), + ('xenapi', ceilometer.compute.virt.xenapi.inspector.OPTS), + ] + + +def list_keystoneauth_opts(): + # NOTE(sileht): the configuration file contains only the options + # for the password plugin that handles keystone v2 and v3 API + # with discovery. But other options are possible. + return [('service_credentials', itertools.chain( + loading.get_auth_common_conf_options(), + loading.get_auth_plugin_conf_options('password'), + ceilometer.keystone_client.CLI_OPTS + ))] diff --git a/ceilosca/ceilometer/publisher/monasca_data_filter.py b/ceilosca/ceilometer/publisher/monasca_data_filter.py index d68635f..70557ad 100644 --- a/ceilosca/ceilometer/publisher/monasca_data_filter.py +++ b/ceilosca/ceilometer/publisher/monasca_data_filter.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -16,7 +17,6 @@ import datetime from jsonpath_rw_ext import parser -from oslo_config import cfg from oslo_log import log from oslo_utils import timeutils import yaml @@ -25,27 +25,6 @@ from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinitionException) from ceilometer import sample as sample_util -OPTS = [ - cfg.StrOpt('monasca_mappings', - default='/etc/ceilometer/monasca_field_definitions.yaml', - help='Monasca static and dynamic field mappings'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - -MULTI_REGION_OPTS = [ - cfg.StrOpt('control_plane', - default='None', - help='The name of control plane'), - cfg.StrOpt('cluster', - default='None', - help='The name of cluster'), - cfg.StrOpt('cloud_name', - default='None', - help='The name of cloud') -] -cfg.CONF.register_opts(MULTI_REGION_OPTS) - LOG = log.getLogger(__name__) @@ -60,12 +39,13 @@ class NoMappingsFound(Exception): class MonascaDataFilter(object): JSONPATH_RW_PARSER = parser.ExtentedJsonPathParser() - def __init__(self): + def __init__(self, conf): + self.conf = conf self._mapping = {} self._mapping = self._get_mapping() def _get_mapping(self): - with open(cfg.CONF.monasca.monasca_mappings, 'r') as f: + with open(self.conf.monasca.monasca_mappings, 'r') as f: try: return yaml.safe_load(f) except yaml.YAMLError as err: @@ -74,13 +54,13 @@ class MonascaDataFilter(object): errmsg = ("Invalid YAML syntax in Monasca Data " "Filter file %(file)s at line: " "%(line)s, column: %(column)s." - % dict(file=cfg.CONF.monasca.monasca_mappings, + % dict(file=self.conf.monasca.monasca_mappings, line=mark.line + 1, column=mark.column + 1)) else: errmsg = ("YAML error reading Monasca Data Filter " "file %(file)s" % - dict(file=cfg.CONF.monasca.monasca_mappings)) + dict(file=self.conf.monasca.monasca_mappings)) LOG.error(errmsg) raise UnableToLoadMappings(err.message) @@ -181,9 +161,9 @@ class MonascaDataFilter(object): dimensions['datasource'] = 'ceilometer' # control_plane, cluster and cloud_name can be None, but we use # literal 'None' for such case - dimensions['control_plane'] = cfg.CONF.control_plane or 'None' - dimensions['cluster'] = cfg.CONF.cluster or 'None' - dimensions['cloud_name'] = cfg.CONF.cloud_name or 'None' + dimensions['control_plane'] = self.conf.monasca.control_plane or 'None' + dimensions['cluster'] = self.conf.monasca.cluster or 'None' + dimensions['cloud_name'] = self.conf.monasca.cloud_name or 'None' if isinstance(sample_obj, sample_util.Sample): sample = sample_obj.as_dict() elif isinstance(sample_obj, dict): diff --git a/ceilosca/ceilometer/publisher/monclient.py b/ceilosca/ceilometer/publisher/monclient.py index 8d25fe3..840dac8 100755 --- a/ceilosca/ceilometer/publisher/monclient.py +++ b/ceilosca/ceilometer/publisher/monclient.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -19,79 +20,37 @@ import os import threading import time -from oslo_config import cfg from oslo_log import log import ceilometer -from ceilometer.i18n import _ from ceilometer import monasca_client as mon_client from ceilometer import publisher from ceilometer.publisher.monasca_data_filter import MonascaDataFilter from monascaclient import exc +import traceback - -monpub_opts = [ - cfg.BoolOpt('batch_mode', - default=True, - help='Indicates whether samples are' - ' published in a batch.'), - cfg.IntOpt('batch_count', - default=1000, - help='Maximum number of samples in a batch.'), - cfg.IntOpt('batch_timeout', - default=15, - help='Maximum time interval(seconds) after which ' - 'samples are published in a batch.'), - cfg.IntOpt('batch_polling_interval', - default=5, - help='Frequency of checking if batch criteria is met.'), - cfg.BoolOpt('retry_on_failure', - default=False, - help='Indicates whether publisher retries publishing' - 'sample in case of failure. Only a few error cases' - 'are queued for a retry.'), - cfg.IntOpt('retry_interval', - default=60, - help='Frequency of attempting a retry.'), - cfg.IntOpt('max_retries', - default=3, - help='Maximum number of retry attempts on a publishing ' - 'failure.'), - cfg.BoolOpt('archive_on_failure', - default=False, - help='When turned on, archives metrics in file system when' - 'publish to Monasca fails or metric publish maxes out' - 'retry attempts.'), - cfg.StrOpt('archive_path', - default='mon_pub_failures.txt', - help='File of metrics that failed to publish to ' - 'Monasca. These include metrics that failed to ' - 'publish on first attempt and failed metrics that' - ' maxed out their retries.'), -] - -cfg.CONF.register_opts(monpub_opts, group='monasca') -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +batch_polling_interval = 5 +retry_interval = 60 LOG = log.getLogger(__name__) -class MonascaPublisher(publisher.PublisherBase): +class MonascaPublisher(publisher.ConfigPublisherBase): """Publisher to publish samples to monasca using monasca-client. Example URL to place in pipeline.yaml: - monclient://http://192.168.10.4:8070/v2.0 """ - def __init__(self, parsed_url): - super(MonascaPublisher, self).__init__(parsed_url) + def __init__(self, conf, parsed_url): + super(MonascaPublisher, self).__init__(conf, parsed_url) # list to hold metrics to be published in batch (behaves like queue) self.metric_queue = [] self.time_of_last_batch_run = time.time() - self.mon_client = mon_client.Client(parsed_url) - self.mon_filter = MonascaDataFilter() + self.mon_client = mon_client.Client(self.conf, parsed_url) + self.mon_filter = MonascaDataFilter(self.conf) # add flush_batch function to periodic callables periodic_callables = [ @@ -101,7 +60,7 @@ class MonascaPublisher(publisher.PublisherBase): (self.flush_batch, (), {}), ] - if cfg.CONF.monasca.retry_on_failure: + if self.conf.monasca.retry_on_failure: # list to hold metrics to be re-tried (behaves like queue) self.retry_queue = [] # list to store retry attempts for metrics in retry_queue @@ -110,13 +69,16 @@ class MonascaPublisher(publisher.PublisherBase): # add retry_batch function to periodic callables periodic_callables.append((self.retry_batch, (), {})) - if cfg.CONF.monasca.archive_on_failure: - archive_path = cfg.CONF.monasca.archive_path + if self.conf.monasca.archive_on_failure: + archive_path = self.conf.monasca.archive_path if not os.path.exists(archive_path): - archive_path = cfg.CONF.find_file(archive_path) + archive_path = self.conf.find_file(archive_path) - self.archive_handler = publisher.get_publisher('file://' + - str(archive_path)) + self.archive_handler = publisher.get_publisher( + self.conf, + 'file://' + + str(archive_path), + 'ceilometer.sample.publisher') # start periodic worker self.periodic_worker = periodics.PeriodicWorker(periodic_callables) @@ -134,22 +96,22 @@ class MonascaPublisher(publisher.PublisherBase): func(**{'jsonbody': metrics}) else: func(**metrics[0]) - LOG.debug(_('Successfully published %d metric(s)') % metric_count) + LOG.info('Successfully published %d metric(s)' % metric_count) except mon_client.MonascaServiceException: # Assuming atomicity of create or failure - meaning # either all succeed or all fail in a batch - LOG.error(_('Metric create failed for %(count)d metric(s) with' - ' name(s) %(names)s ') % + LOG.error('Metric create failed for %(count)d metric(s) with' + ' name(s) %(names)s ' % ({'count': len(metrics), 'names': ','.join([metric['name'] for metric in metrics])})) - if cfg.CONF.monasca.retry_on_failure: + if self.conf.monasca.retry_on_failure: # retry payload in case of internal server error(500), # service unavailable error(503),bad gateway (502) or # Communication Error # append failed metrics to retry_queue - LOG.debug(_('Adding metrics to retry queue.')) + LOG.debug('Adding metrics to retry queue.') self.retry_queue.extend(metrics) # initialize the retry_attempt for the each failed # metric in retry_counter @@ -159,6 +121,7 @@ class MonascaPublisher(publisher.PublisherBase): if hasattr(self, 'archive_handler'): self.archive_handler.publish_samples(None, metrics) except Exception: + LOG.info(traceback.format_exc()) if hasattr(self, 'archive_handler'): self.archive_handler.publish_samples(None, metrics) @@ -169,14 +132,14 @@ class MonascaPublisher(publisher.PublisherBase): metric = self.mon_filter.process_sample_for_monasca(sample) # In batch mode, push metric to queue, # else publish the metric - if cfg.CONF.monasca.batch_mode: - LOG.debug(_('Adding metric to queue.')) + if self.conf.monasca.batch_mode: + LOG.debug('Adding metric to queue.') self.metric_queue.append(metric) else: - LOG.debug(_('Publishing metric with name %(name)s and' - ' timestamp %(ts)s to endpoint.') % - ({'name': metric['name'], - 'ts': metric['timestamp']})) + LOG.info('Publishing metric with name %(name)s and' + ' timestamp %(ts)s to endpoint.' % + ({'name': metric['name'], + 'ts': metric['timestamp']})) self._publish_handler(self.mon_client.metrics_create, [metric]) @@ -187,18 +150,18 @@ class MonascaPublisher(publisher.PublisherBase): current_time = time.time() elapsed_time = current_time - previous_time - if elapsed_time >= cfg.CONF.monasca.batch_timeout and len(self. + if elapsed_time >= self.conf.monasca.batch_timeout and len(self. metric_queue) > 0: - LOG.debug(_('Batch timeout exceeded, triggering batch publish.')) + LOG.info('Batch timeout exceeded, triggering batch publish.') return True else: - if len(self.metric_queue) >= cfg.CONF.monasca.batch_count: - LOG.debug(_('Batch queue full, triggering batch publish.')) + if len(self.metric_queue) >= self.conf.monasca.batch_count: + LOG.info('Batch queue full, triggering batch publish.') return True else: return False - @periodics.periodic(cfg.CONF.monasca.batch_polling_interval) + @periodics.periodic(batch_polling_interval) def flush_batch(self): """Method to flush the queued metrics.""" # print "flush batch... %s" % str(time.time()) @@ -206,6 +169,8 @@ class MonascaPublisher(publisher.PublisherBase): # publish all metrics in queue at this point batch_count = len(self.metric_queue) + LOG.info("batch is ready: batch_count %s" % str(batch_count)) + self._publish_handler(self.mon_client.metrics_create, self.metric_queue[:batch_count], batch=True) @@ -220,12 +185,12 @@ class MonascaPublisher(publisher.PublisherBase): """Method to check if retry batch is ready to trigger.""" if len(self.retry_queue) > 0: - LOG.debug(_('Retry queue has items, triggering retry.')) + LOG.info('Retry queue has items, triggering retry.') return True else: return False - @periodics.periodic(cfg.CONF.monasca.retry_interval) + @periodics.periodic(retry_interval) def retry_batch(self): """Method to retry the failed metrics.""" # print "retry batch...%s" % str(time.time()) @@ -235,14 +200,14 @@ class MonascaPublisher(publisher.PublisherBase): # Iterate over the retry_queue to eliminate # metrics that have maxed out their retry attempts for ctr in xrange(retry_count): - if self.retry_counter[ctr] > cfg.CONF.monasca.max_retries: + if self.retry_counter[ctr] > self.conf.monasca.max_retries: if hasattr(self, 'archive_handler'): self.archive_handler.publish_samples( None, [self.retry_queue[ctr]]) - LOG.debug(_('Removing metric %s from retry queue.' - ' Metric retry maxed out retry attempts') % - self.retry_queue[ctr]['name']) + LOG.info('Removing metric %s from retry queue.' + ' Metric retry maxed out retry attempts' % + self.retry_queue[ctr]['name']) del self.retry_queue[ctr] del self.retry_counter[ctr] @@ -255,17 +220,17 @@ class MonascaPublisher(publisher.PublisherBase): ctr = 0 while ctr < len(self.retry_queue): try: - LOG.debug(_('Retrying metric publish from retry queue.')) + LOG.info('Retrying metric publish from retry queue.') self.mon_client.metrics_create(**self.retry_queue[ctr]) # remove from retry queue if publish was success - LOG.debug(_('Retrying metric %s successful,' - ' removing metric from retry queue.') % - self.retry_queue[ctr]['name']) + LOG.info('Retrying metric %s successful,' + ' removing metric from retry queue.' % + self.retry_queue[ctr]['name']) del self.retry_queue[ctr] del self.retry_counter[ctr] - except exc.BaseException: - LOG.error(_('Exception encountered in retry. ' - 'Batch will be retried in next attempt.')) + except exc.ClientException: + LOG.error('Exception encountered in retry. ' + 'Batch will be retried in next attempt.') # if retry failed, increment the retry counter self.retry_counter[ctr] += 1 ctr += 1 diff --git a/ceilosca/ceilometer/service.py b/ceilosca/ceilometer/service.py new file mode 100644 index 0000000..57f7b49 --- /dev/null +++ b/ceilosca/ceilometer/service.py @@ -0,0 +1,72 @@ +# Copyright 2012-2014 eNovance +# (c) Copyright 2018 SUSE LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +from oslo_config import cfg +from oslo_db import options as db_options +import oslo_i18n +from oslo_log import log +from oslo_policy import opts as policy_opts +# from oslo_reports import guru_meditation_report as gmr + +from ceilometer.conf import defaults +from ceilometer import keystone_client +from ceilometer import messaging +from ceilometer import opts +from ceilometer import sample +from ceilometer import utils +from ceilometer import version + + +def prepare_service(argv=None, config_files=None, conf=None): + if argv is None: + argv = sys.argv + + if conf is None: + conf = cfg.ConfigOpts() + + oslo_i18n.enable_lazy() + for group, options in opts.list_opts(): + conf.register_opts(list(options), + group=None if group == "DEFAULT" else group) + keystone_client.register_keystoneauth_opts(conf) + log.register_options(conf) + log_levels = (conf.default_log_levels + + ['futurist=INFO', 'neutronclient=INFO', + 'keystoneclient=INFO']) + log.set_defaults(default_log_levels=log_levels) + defaults.set_cors_middleware_defaults() + policy_opts.set_defaults(conf) + db_options.set_defaults(conf) + + conf(argv[1:], project='ceilometer', validate_default_values=True, + version=version.version_info.version_string(), + default_config_files=config_files) + + keystone_client.post_register_keystoneauth_opts(conf) + + log.setup(conf, 'ceilometer') + utils.setup_root_helper(conf) + sample.setup(conf) + + # NOTE(liusheng): guru cannot run with service under apache daemon, so when + # ceilometer-api running with mod_wsgi, the argv is [], we don't start + # guru. + if argv: + # gmr.TextGuruMeditation.setup_autorun(version) + pass + messaging.setup() + return conf diff --git a/ceilosca/ceilometer/storage/impl_monasca.py b/ceilosca/ceilometer/storage/impl_monasca.py index 2bfb133..2bf5755 100644 --- a/ceilosca/ceilometer/storage/impl_monasca.py +++ b/ceilosca/ceilometer/storage/impl_monasca.py @@ -1,5 +1,6 @@ # # (C) Copyright 2015-2017 Hewlett Packard Enterprise Development LP +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -15,6 +16,7 @@ """Simple monasca storage backend. """ +import traceback from collections import defaultdict import copy @@ -23,7 +25,6 @@ import itertools import operator from monascaclient import exc as monasca_exc -from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils from oslo_utils import netutils @@ -43,15 +44,6 @@ from ceilometer.storage import base from ceilometer.storage import models as api_models from ceilometer import utils -OPTS = [ - cfg.IntOpt('default_stats_period', - default=300, - help='Default period (in seconds) to use for querying stats ' - 'in case no period specified in the stats API call.'), -] - -cfg.CONF.register_opts(OPTS, group='monasca') - LOG = log.getLogger(__name__) AVAILABLE_CAPABILITIES = { @@ -96,13 +88,23 @@ class Connection(base.Connection): AVAILABLE_STORAGE_CAPABILITIES, ) - def __init__(self, url): - self.mc = monasca_client.Client(netutils.urlsplit(url)) - self.mon_filter = MonascaDataFilter() - self.ceilosca_mapper = ProcessMappedCeiloscaMetric() - self.pipeline_reader = PipelineReader() - self.meter_static_info = ProcessMappedCeilometerStaticInfo() - self.meters_from_pipeline = self.pipeline_reader.get_pipeline_meters() + def __init__(self, conf, url): + try: + super(Connection, self).__init__(conf, url) + self.conf = conf + self.mc = monasca_client.Client(self.conf, netutils.urlsplit(url)) + self.mon_filter = MonascaDataFilter(self.conf) + self.pipeline_reader = PipelineReader(conf) + self.ceilosca_mapper = ProcessMappedCeiloscaMetric(conf) + self.meter_static_info = ProcessMappedCeilometerStaticInfo(conf) + self.meters_from_pipeline = self.pipeline_reader\ + .get_pipeline_meters() + except Exception as ex: + LOG.info("ERROR: creating monasca connection: " + str(ex.message)) + LOG.info(ex) + LOG.info("ERROR: creating monasca connection: done") + LOG.info(traceback.format_ex()) + traceback.print_ex() @staticmethod def _convert_to_dict(stats, cols): @@ -878,7 +880,7 @@ class Connection(base.Connection): dims_filter = {k: v for k, v in dims_filter.items() if v is not None} period = period if period \ - else cfg.CONF.monasca.default_stats_period + else self.conf.monasca.default_stats_period _search_args = dict( name=filter.meter, diff --git a/ceilosca/ceilometer/tests/functional/api/__init__.py b/ceilosca/ceilometer/tests/functional/api/__init__.py index 520009c..a0359dc 100644 --- a/ceilosca/ceilometer/tests/functional/api/__init__.py +++ b/ceilosca/ceilometer/tests/functional/api/__init__.py @@ -1,5 +1,6 @@ # # Copyright 2012 New Dream Network, LLC (DreamHost) +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -14,18 +15,15 @@ # under the License. """Base classes for API tests. """ - -from oslo_config import cfg -from oslo_config import fixture as fixture_config from oslo_policy import opts import pecan import pecan.testing from ceilometer.api import rbac +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer.tests import db as db_test_base -cfg.CONF.import_group('api', 'ceilometer.api.controllers.v2.root') - class FunctionalTest(db_test_base.TestBase): """Used for functional tests of Pecan controllers. @@ -38,7 +36,9 @@ class FunctionalTest(db_test_base.TestBase): def setUp(self): super(FunctionalTest, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') self.setup_messaging(self.CONF) opts.set_defaults(self.CONF) @@ -48,6 +48,7 @@ class FunctionalTest(db_test_base.TestBase): self.CONF.set_override('gnocchi_is_enabled', False, group='api') self.CONF.set_override('aodh_is_enabled', False, group='api') + self.CONF.set_override('panko_is_enabled', False, group='api') self.app = self._make_app() @@ -63,7 +64,8 @@ class FunctionalTest(db_test_base.TestBase): }, } - return pecan.testing.load_test_app(self.config) + return pecan.testing.load_test_app(self.config, + conf=self.CONF) def tearDown(self): super(FunctionalTest, self).tearDown() diff --git a/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py b/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py index a862b87..3c113ef 100644 --- a/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py +++ b/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,10 +21,11 @@ import mock import pkg_resources from oslo_config import cfg -from oslo_config import fixture as fixture_config from stevedore import driver from stevedore import extension +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer import storage from ceilometer.tests import base as test_base from oslo_policy import opts @@ -52,31 +54,28 @@ class TestApi(test_base.BaseTestCase): mgr._init_plugins([a_driver]) return mgr - def get_connection_with_mock_driver_manager(self, url, namespace): + def get_connection_with_mock_driver_manager(self, conf, url, namespace): mgr = self._get_driver_from_entry_point( entry_point='monasca = ceilometer.storage.impl_monasca:Connection', namespace='ceilometer.metering.storage') - return mgr.driver(url) + return mgr.driver(conf, url) - def get_publisher_with_mock_driver_manager(self, url, namespace): + def get_publisher_with_mock_driver_manager(self, conf, url, namespace): mgr = self._get_driver_from_entry_point( entry_point='monasca = ceilometer.publisher.monclient:' 'MonascaPublisher', - namespace='ceilometer.publisher') - return mgr.driver(url) + namespace='ceilometer.sample.publisher') + return mgr.driver(conf, url) def setUp(self): super(TestApi, self).setUp() self.PATH_PREFIX = '/v2' - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) - + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') self.setup_messaging(self.CONF) opts.set_defaults(self.CONF) - - self.CONF.set_override("auth_version", "v2.0", - group=OPT_GROUP_NAME) self.CONF.set_override("policy_file", self.path_get('etc/ceilometer/policy.json'), group='oslo_policy') @@ -106,8 +105,10 @@ class TestApi(test_base.BaseTestCase): self.get_connection_with_mock_driver_manager) get_pub.side_effect = self.get_publisher_with_mock_driver_manager self.mock_mon_client = mock_client - self.conn = storage.get_connection('monasca://127.0.0.1:8080', - 'ceilometer.metering.storage') + self.conn = storage.get_connection( + self.CONF, + 'monasca://127.0.0.1:8080', + 'ceilometer.metering.storage') self.useFixture(fixtures.MockPatch( 'ceilometer.storage.get_connection', @@ -127,7 +128,8 @@ class TestApi(test_base.BaseTestCase): }, } - return pecan.testing.load_test_app(self.config) + return pecan.testing.load_test_app(self.config, + conf=self.CONF) def get_json(self, path, expect_errors=False, headers=None, extra_environ=None, q=None, groupby=None, status=None, diff --git a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py index 3907fc5..5ff857a 100644 --- a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py +++ b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -18,7 +19,6 @@ import os import fixtures import mock -from oslo_config import fixture as fixture_config from oslo_utils import fileutils from oslo_utils import timeutils from oslotest import base @@ -31,6 +31,8 @@ from ceilometer.ceilosca_mapping.ceilosca_mapping import ( from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinitionException) from ceilometer.ceilosca_mapping.ceilosca_mapping import PipelineReader +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer import storage from ceilometer.storage import impl_monasca from ceilometer.storage import models as storage_models @@ -150,15 +152,16 @@ class TestGetPipelineReader(TestCeiloscaMapping): def setUp(self): super(TestGetPipelineReader, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') def test_pipeline_reader(self): pipeline_cfg_file = self.setup_pipeline_file( self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) - test_pipeline_reader = PipelineReader() + test_pipeline_reader = PipelineReader(self.CONF) self.assertEqual(set(['testbatch', 'testbatch2']), test_pipeline_reader.get_pipeline_meters() @@ -223,14 +226,16 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): def setUp(self): super(TestMappedCeiloscaMetricProcessing, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') def test_fallback_mapping_file_path(self): self.useFixture(fixtures.MockPatchObject(self.CONF, 'find_file', return_value=None)) - fall_bak_path = ceilosca_mapping.get_config_file() + fall_bak_path = ceilosca_mapping.get_config_file(self.CONF) self.assertIn("ceilosca_mapping/data/ceilosca_mapping.yaml", fall_bak_path) @@ -264,7 +269,7 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - data = ceilosca_mapping.setup_ceilosca_mapping_config() + data = ceilosca_mapping.setup_ceilosca_mapping_config(self.CONF) meter_loaded = ceilosca_mapping.load_definitions(data) self.assertEqual(1, len(meter_loaded)) LOG.error.assert_called_with( @@ -275,8 +280,9 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) self.assertItemsEqual(['fake_metric', 'fake_metric2', 'fake_metric3'], ceilosca_mapper.get_list_monasca_metrics().keys() ) @@ -299,8 +305,9 @@ class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) self.assertEqual('fake_metric', ceilosca_mapper.get_monasca_metric_name('fake_meter') ) @@ -317,20 +324,27 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): Aggregate = collections.namedtuple("Aggregate", ['func', 'param']) def setUp(self): + super(TestMoanscaDriverForMappedMetrics, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + pipeline_cfg_file = self.setup_pipeline_file(self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) + ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) def test_get_samples_for_mapped_meters(self, mdf_mock): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") ml_mock = mock_client().measurements_list # TODO(this test case needs more work) ml_mock.return_value = ([MONASCA_MEASUREMENT]) @@ -386,7 +400,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): u'id': u'2015-04-16T18:42:31Z', u'name': u'testbatch'}]) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.side_effect = [data1, data2] kwargs = dict(limit=4) @@ -407,7 +421,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", "name": "subnet.update"}]) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metric_names_list_mock = mock_client().metric_names_list metric_names_list_mock.return_value = ( dummy_metric_names_mocked_return_value) @@ -419,7 +433,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): def test_stats_list_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -455,7 +469,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): def test_get_resources_for_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") dummy_metric_names_mocked_return_value = ( [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", "name": "fake_metric"}, @@ -489,7 +503,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): def test_stats_list_with_groupby_for_mapped_meters(self, mock_mdf): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -567,7 +581,7 @@ class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): return samples.pop() with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") with mock.patch.object(conn, 'get_samples') as gsm: gsm.side_effect = _get_samples diff --git a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py index 5925956..3f7c998 100644 --- a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py +++ b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py @@ -1,5 +1,6 @@ # # Copyright 2016 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -17,7 +18,7 @@ import os import fixtures import mock -from oslo_config import fixture as fixture_config +# from oslo_config import fixture as fixture_config from oslo_utils import fileutils from oslotest import base import six @@ -28,6 +29,8 @@ from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( CeilometerStaticMappingDefinition) from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( CeilometerStaticMappingDefinitionException) +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer.storage import impl_monasca @@ -152,13 +155,16 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): def setUp(self): super(TestMappedCeilometerStaticInfoProcessing, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf + # self.CONF = self.useFixture(fixture_config.Config()).conf + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') static_info_mapping_file = self.setup_static_mapping_def_file(self.cfg) self.CONF.set_override('ceilometer_static_info_mapping', static_info_mapping_file, group='monasca') self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo() - self.CONF([], project='ceilometer', validate_default_values=True) + .ProcessMappedCeilometerStaticInfo(self.CONF) + # self.CONF([], project='ceilometer', validate_default_values=True) def test_fallback_mapping_file_path(self): self.useFixture(fixtures.MockPatchObject(self.CONF, @@ -166,8 +172,9 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): return_value=None)) self.CONF.set_override('ceilometer_static_info_mapping', ' ', group='monasca') - self.static_info_mapper.reinitialize() - fall_bak_path = ceilometer_static_info_mapping.get_config_file() + self.static_info_mapper.reinitialize(self.CONF) + fall_bak_path = ceilometer_static_info_mapping.get_config_file( + self.CONF) self.assertIn( "ceilosca_mapping/data/ceilometer_static_info_mapping.yaml", fall_bak_path) @@ -198,7 +205,7 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): self.CONF.set_override('ceilometer_static_info_mapping', static_info_mapping_file, group='monasca') data = ceilometer_static_info_mapping.\ - setup_ceilometer_static_mapping_config() + setup_ceilometer_static_mapping_config(self.CONF) meter_loaded = ceilometer_static_info_mapping.load_definitions(data) self.assertEqual(3, len(meter_loaded)) LOG.error.assert_called_with( @@ -206,7 +213,7 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): "Invalid type foo specified") def test_list_of_meters_returned(self): - self.static_info_mapper.reinitialize() + self.static_info_mapper.reinitialize(self.CONF) self.assertItemsEqual(['disk.ephemeral.size', 'disk.root.size', 'image', 'image.delete'], self.static_info_mapper. @@ -225,7 +232,7 @@ class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): static_info_mapping_file = self.setup_static_mapping_def_file(cfg) self.CONF.set_override('ceilometer_static_info_mapping', static_info_mapping_file, group='monasca') - self.static_info_mapper.reinitialize() + self.static_info_mapper.reinitialize(self.CONF) self.assertEqual('gauge', self.static_info_mapper.get_meter_static_info_key_val( 'disk.ephemeral.size', 'type') @@ -239,8 +246,11 @@ class TestMoanscaDriverForMappedStaticInfo(TestStaticInfoBase): def setUp(self): super(TestMoanscaDriverForMappedStaticInfo, self).setUp() - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], project='ceilometer', validate_default_values=True) + # self.CONF = self.useFixture(fixture_config.Config()).conf + # self.CONF([], project='ceilometer', validate_default_values=True) + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') pipeline_cfg_file = self.setup_pipeline_file(self.pipeline_data) self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) static_info_mapping_file = self.setup_static_mapping_def_file(self.cfg) @@ -251,8 +261,8 @@ class TestMoanscaDriverForMappedStaticInfo(TestStaticInfoBase): self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo() - self.static_info_mapper.reinitialize() + .ProcessMappedCeilometerStaticInfo(self.CONF) + self.static_info_mapper.reinitialize(self.CONF) def test_get_statc_info_for_mapped_meters_uniq(self, mdf_mock): dummy_metric_names_mocked_return_value = ( @@ -262,7 +272,7 @@ class TestMoanscaDriverForMappedStaticInfo(TestStaticInfoBase): "name": "fake_metric"}]) with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') metric_names_list_mock = mock_client().metric_names_list metric_names_list_mock.return_value = ( dummy_metric_names_mocked_return_value diff --git a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py index af320d8..2c87230 100644 --- a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py +++ b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_data_filter.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,13 +21,18 @@ from oslotest import base from ceilometer.ceilosca_mapping.ceilosca_mapping import ( CeiloscaMappingDefinitionException) +from ceilometer import monasca_ceilometer_opts from ceilometer.publisher import monasca_data_filter as mdf from ceilometer import sample +from ceilometer import service class TestMonUtils(base.BaseTestCase): def setUp(self): super(TestMonUtils, self).setUp() + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') self._field_mappings = { 'dimensions': ['resource_id', 'project_id', @@ -124,7 +130,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -160,7 +166,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[field_map]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertIsNone(r['dimensions'].get('project_id')) @@ -199,7 +205,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) self.assertIsNotNone(r.get('value_meta')) @@ -229,7 +235,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -261,7 +267,7 @@ class TestMonUtils(base.BaseTestCase): to_patch = ("ceilometer.publisher.monasca_data_filter." "MonascaDataFilter._get_mapping") with mock.patch(to_patch, side_effect=[self._field_mappings]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -312,7 +318,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -362,7 +368,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -408,7 +414,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) try: # Don't assign to a variable, this should raise data_filter.process_sample_for_monasca(s) @@ -445,7 +451,7 @@ class TestMonUtils(base.BaseTestCase): "MonascaDataFilter._get_mapping") # use the cinder specific mapping with mock.patch(to_patch, side_effect=[self._field_mappings_cinder]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) r = data_filter.process_sample_for_monasca(s) self.assertEqual(s.name, r['name']) @@ -493,7 +499,7 @@ class TestMonUtils(base.BaseTestCase): # use the bad mapping with mock.patch(to_patch, side_effect=[self._field_mappings_bad_format]): - data_filter = mdf.MonascaDataFilter() + data_filter = mdf.MonascaDataFilter(self.CONF) try: # Don't assign to a variable as this should raise data_filter.process_sample_for_monasca(s) diff --git a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py index 6030505..42c9bf0 100755 --- a/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py +++ b/ceilosca/ceilometer/tests/unit/publisher/test_monasca_publisher.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -17,20 +18,16 @@ import datetime import fixtures -import os import time -from keystoneauth1 import loading as ka_loading import mock -from oslo_config import cfg -from oslo_config import fixture as fixture_config -from oslo_utils import fileutils from oslotest import base +from ceilometer import monasca_ceilometer_opts from ceilometer import monasca_client as mon_client from ceilometer.publisher import monclient from ceilometer import sample -from monascaclient import ksclient +from ceilometer import service class FakeResponse(object): @@ -112,21 +109,17 @@ class TestMonascaPublisher(base.BaseTestCase): def setUp(self): super(TestMonascaPublisher, self).setUp() - content = ("[service_credentials]\n" - "auth_type = password\n" - "username = ceilometer\n" - "password = admin\n" - "auth_url = http://localhost:5000/v2.0\n") - tempfile = fileutils.write_to_tempfile(content=content, - prefix='ceilometer', - suffix='.conf') - self.addCleanup(os.remove, tempfile) - self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF([], default_config_files=[tempfile]) - ka_loading.load_auth_from_conf_options(self.CONF, - "service_credentials") + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + self.CONF.set_override('service_username', 'ceilometer', 'monasca') + self.CONF.set_override('service_password', 'admin', 'monasca') + self.CONF.set_override('service_auth_url', + 'http://localhost:5000/v2.0', + 'monasca') + self.parsed_url = mock.MagicMock() - ksclient.KSClient = mock.MagicMock() def tearDown(self): # For some reason, cfg.CONF is registered a required option named @@ -134,8 +127,8 @@ class TestMonascaPublisher(base.BaseTestCase): # case test_event_pipeline_endpoint_requeue_on_failure, so we # unregister it here. self.CONF.reset() - self.CONF.unregister_opt(cfg.StrOpt('auth_url'), - group='service_credentials') + # self.CONF.unregister_opt(cfg.StrOpt('service_auth_url'), + # group='monasca') super(TestMonascaPublisher, self).tearDown() @mock.patch("ceilometer.publisher.monasca_data_filter." @@ -143,7 +136,7 @@ class TestMonascaPublisher(base.BaseTestCase): side_effect=[field_mappings]) def test_publisher_publish(self, mapping_patch): self.CONF.set_override('batch_mode', False, group='monasca') - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, @@ -161,7 +154,7 @@ class TestMonascaPublisher(base.BaseTestCase): self.CONF.set_override('batch_count', 3, group='monasca') self.CONF.set_override('batch_polling_interval', 1, group='monasca') - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, 'metrics_create') as mock_create: @@ -182,7 +175,7 @@ class TestMonascaPublisher(base.BaseTestCase): self.CONF.set_override('retry_interval', 2, group='monasca') self.CONF.set_override('max_retries', 1, group='monasca') - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, 'metrics_create') as mock_create: @@ -207,7 +200,8 @@ class TestMonascaPublisher(base.BaseTestCase): 'ceilometer.publisher.file.FilePublisher', return_value=self.fake_publisher)) - publisher = monclient.MonascaPublisher(self.parsed_url) + publisher = monclient.MonascaPublisher(self.CONF, + self.parsed_url) publisher.mon_client = mock.MagicMock() with mock.patch.object(publisher.mon_client, diff --git a/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py b/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py index 93ecd59..ce59c7f 100644 --- a/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py +++ b/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py @@ -1,5 +1,6 @@ # # Copyright 2015 Hewlett Packard +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -19,7 +20,6 @@ import os import dateutil.parser import mock -from oslo_config import fixture as fixture_config from oslo_utils import fileutils from oslo_utils import timeutils from oslotest import base @@ -30,6 +30,8 @@ import ceilometer from ceilometer.api.controllers.v2.meters import Aggregate from ceilometer.ceilosca_mapping import ceilometer_static_info_mapping from ceilometer.ceilosca_mapping import ceilosca_mapping +from ceilometer import monasca_ceilometer_opts +from ceilometer import service from ceilometer import storage from ceilometer.storage import impl_monasca from ceilometer.storage import models as storage_models @@ -39,18 +41,14 @@ class _BaseTestCase(base.BaseTestCase): def setUp(self): super(_BaseTestCase, self).setUp() - content = ("[service_credentials]\n" - "auth_type = password\n" - "username = ceilometer\n" - "password = admin\n" - "auth_url = http://localhost:5000/v2.0\n") - tempfile = fileutils.write_to_tempfile(content=content, - prefix='ceilometer', - suffix='.conf') - self.addCleanup(os.remove, tempfile) - conf = self.useFixture(fixture_config.Config()).conf - conf([], default_config_files=[tempfile]) - self.CONF = conf + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + self.CONF.set_override('service_username', 'ceilometer', 'monasca') + self.CONF.set_override('service_password', 'admin', 'monasca') + self.CONF.set_override('service_auth_url', + 'http://localhost:5000/v2.0', + 'monasca') mdf = mock.patch.object(impl_monasca, 'MonascaDataFilter') mdf.start() self.addCleanup(mdf.stop) @@ -58,8 +56,8 @@ class _BaseTestCase(base.BaseTestCase): spl.start() self.addCleanup(spl.stop) self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo() - self.static_info_mapper.reinitialize() + .ProcessMappedCeilometerStaticInfo(self.CONF) + self.static_info_mapper.reinitialize(self.CONF) def assertRaisesWithMessage(self, msg, exc_class, func, *args, **kwargs): try: @@ -133,12 +131,13 @@ class TestGetResources(_BaseTestCase): TestGetResources.cfg) self.CONF.set_override('ceilometer_monasca_metrics_mapping', ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping.ProcessMappedCeiloscaMetric() - ceilosca_mapper.reinitialize() + ceilosca_mapper = ceilosca_mapping\ + .ProcessMappedCeiloscaMetric(self.CONF) + ceilosca_mapper.reinitialize(self.CONF) def test_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") kwargs = dict(start_timestamp_op='le') self.assertRaises(ceilometer.NotImplementedError, @@ -149,7 +148,7 @@ class TestGetResources(_BaseTestCase): def test_dims_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ { @@ -178,7 +177,7 @@ class TestGetResources(_BaseTestCase): def test_get_resources(self, mock_utcnow): mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 20) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ { @@ -225,7 +224,7 @@ class TestGetResources(_BaseTestCase): def test_get_resources_limit(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ @@ -263,7 +262,7 @@ class TestGetResources(_BaseTestCase): def test_get_resources_simple_metaquery(self, mock_utcnow): mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 28) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") mnl_mock = mock_client().metric_names_list mnl_mock.return_value = [ { @@ -325,7 +324,7 @@ class MeterTest(_BaseTestCase): def test_not_implemented_params(self): with mock.patch('ceilometer.monasca_client.Client'): - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') kwargs = dict(metaquery=True) self.assertRaises(ceilometer.NotImplementedError, @@ -333,7 +332,7 @@ class MeterTest(_BaseTestCase): def test_metrics_list_call(self): with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') metrics_list_mock = mock_client().metrics_list kwargs = dict(user='user-1', @@ -381,7 +380,7 @@ class MeterTest(_BaseTestCase): {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", "name": "subnet.update"}]) with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection('127.0.0.1:8080') + conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') metric_names_list_mock = mock_client().metric_names_list metric_names_list_mock.return_value = ( dummy_metric_names_mocked_return_value @@ -429,7 +428,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sample_filter = storage.SampleFilter(meter='specific meter', message_id='specific message') @@ -438,7 +437,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_name(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( TestGetSamples.dummy_metrics_mocked_return_value @@ -460,7 +459,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_start_timestamp_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( @@ -482,7 +481,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_timestamp_filter_exclusive_range(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( @@ -515,7 +514,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_limit(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list @@ -544,7 +543,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_project_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': dict(project_id='specific project'), @@ -564,7 +563,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_resource_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': dict(resource_id='specific resource'), @@ -583,7 +582,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_source_filter(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': dict(source='specific source'), @@ -602,7 +601,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_simple_metaquery(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( TestGetSamples.dummy_metrics_mocked_return_value @@ -620,7 +619,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_simple_metaquery_with_extended_key(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( TestGetSamples.dummy_metrics_mocked_return_value @@ -639,7 +638,7 @@ class TestGetSamples(_BaseTestCase): def test_get_samples_results(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") metrics_list_mock = mock_client().metrics_list metrics_list_mock.return_value = ( [{u'dimensions': { @@ -718,7 +717,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") self.assertRaisesWithMessage("Query without filter " "not implemented", @@ -781,7 +780,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_stats_list_called_with(self, mock_utcnow): mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 31) with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sf = storage.SampleFilter() @@ -813,7 +812,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_stats_list(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -851,7 +850,7 @@ class MeterStatisticsTest(_BaseTestCase): def test_stats_list_with_groupby(self): with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") sl_mock = mock_client().statistics_list sl_mock.return_value = [ { @@ -911,7 +910,7 @@ class TestQuerySamples(_BaseTestCase): def test_query_samples_not_implemented_params(self): with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") query = {'and': [{'=': {'counter_name': 'instance'}}, {'or': [{'=': {"project_id": "123"}}, {'=': {"user_id": "456"}}]}]} @@ -956,7 +955,7 @@ class TestQuerySamples(_BaseTestCase): return samples.pop() with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") with mock.patch.object(conn, 'get_samples') as gsm: gsm.side_effect = _get_samples @@ -998,7 +997,7 @@ class TestQuerySamples(_BaseTestCase): return samples.pop() with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection("127.0.0.1:8080") + conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") with mock.patch.object(conn, 'get_samples') as gsm: gsm.side_effect = _get_samples diff --git a/ceilosca/ceilometer/tests/unit/test_monascaclient.py b/ceilosca/ceilometer/tests/unit/test_monascaclient.py index 6603ef5..50cc929 100644 --- a/ceilosca/ceilometer/tests/unit/test_monascaclient.py +++ b/ceilosca/ceilometer/tests/unit/test_monascaclient.py @@ -1,4 +1,5 @@ # Copyright 2015 Hewlett-Packard Company +# (c) Copyright 2018 SUSE LLC # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -11,40 +12,33 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. - -import os - -from keystoneauth1 import loading as ka_loading import mock -from oslo_config import cfg -from oslo_config import fixture as fixture_config -from oslo_utils import fileutils + from oslo_utils import netutils from oslotest import base +from ceilometer import monasca_ceilometer_opts from ceilometer import monasca_client -from monascaclient import exc +from ceilometer import service -cfg.CONF.import_group('service_credentials', 'ceilometer.service') +from monascaclient import exc +import tenacity class TestMonascaClient(base.BaseTestCase): def setUp(self): super(TestMonascaClient, self).setUp() - content = ("[service_credentials]\n" - "auth_type = password\n" - "username = ceilometer\n" - "password = admin\n" - "auth_url = http://localhost:5000/v2.0\n") - tempfile = fileutils.write_to_tempfile(content=content, - prefix='ceilometer', - suffix='.conf') - self.addCleanup(os.remove, tempfile) - self.conf = self.useFixture(fixture_config.Config()).conf - self.conf([], default_config_files=[tempfile]) - ka_loading.load_auth_from_conf_options(self.conf, - "service_credentials") - self.conf.set_override('max_retries', 0, 'database') + + self.CONF = service.prepare_service([], []) + self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), + 'monasca') + self.CONF.set_override('service_username', 'ceilometer', 'monasca') + self.CONF.set_override('service_password', 'admin', 'monasca') + self.CONF.set_override('service_auth_url', + 'http://localhost:5000/v2.0', + 'monasca') + + self.CONF.set_override('max_retries', 0, 'database') self.mc = self._get_client() def tearDown(self): @@ -52,25 +46,21 @@ class TestMonascaClient(base.BaseTestCase): # auth_url after these tests run, which occasionally blocks test # case test_event_pipeline_endpoint_requeue_on_failure, so we # unregister it here. - self.conf.reset() - self.conf.unregister_opt(cfg.StrOpt('auth_url'), - group='service_credentials') + self.CONF.reset() + # self.CONF.unregister_opt(cfg.StrOpt('service_auth_url'), + # group='monasca') super(TestMonascaClient, self).tearDown() @mock.patch('monascaclient.client.Client') - @mock.patch('monascaclient.ksclient.KSClient') - def _get_client(self, ksclass_mock, monclient_mock): - ksclient_mock = ksclass_mock.return_value - ksclient_mock.token.return_value = "token123" + def _get_client(self, monclient_mock): return monasca_client.Client( + self.CONF, netutils.urlsplit("http://127.0.0.1:8080")) @mock.patch('monascaclient.client.Client') - @mock.patch('monascaclient.ksclient.KSClient') - def test_client_url_correctness(self, ksclass_mock, monclient_mock): - ksclient_mock = ksclass_mock.return_value - ksclient_mock.token.return_value = "token123" + def test_client_url_correctness(self, monclient_mock): mon_client = monasca_client.Client( + self.CONF, netutils.urlsplit("monasca://https://127.0.0.1:8080")) self.assertEqual("https://127.0.0.1:8080", mon_client._endpoint) @@ -84,44 +74,46 @@ class TestMonascaClient(base.BaseTestCase): def test_metrics_create_exception(self): with mock.patch.object( self.mc._mon_client.metrics, 'create', - side_effect=[exc.HTTPInternalServerError, True])\ + side_effect=[exc.http.InternalServerError, True])\ as create_patch: - self.assertRaises(monasca_client.MonascaServiceException, - self.mc.metrics_create) + e = self.assertRaises(tenacity.RetryError, + self.mc.metrics_create) + (original_ex, traceobj) = e.last_attempt.exception_info() + self.assertIsInstance(original_ex, + monasca_client.MonascaServiceException) self.assertEqual(1, create_patch.call_count) def test_metrics_create_unprocessable_exception(self): with mock.patch.object( self.mc._mon_client.metrics, 'create', - side_effect=[exc.HTTPUnProcessable, True])\ + side_effect=[exc.http.UnprocessableEntity, True])\ as create_patch: self.assertRaises(monasca_client.MonascaInvalidParametersException, self.mc.metrics_create) self.assertEqual(1, create_patch.call_count) def test_invalid_service_creds(self): - conf = cfg.CONF.service_credentials + conf = self.CONF.monasca class SetOpt(object): def __enter__(self): - self.username = conf.username - conf.username = "" + self.username = conf.service_username + conf.service_username = "" def __exit__(self, exc_type, exc_val, exc_tb): - conf.username = self.username + conf.service_username = self.username with SetOpt(): self.assertRaises( monasca_client.MonascaInvalidServiceCredentialsException, self._get_client) - self.assertIsNotNone(True, conf.username) + self.assertIsNotNone(True, conf.service_username) def test_retry_on_key_error(self): - self.conf.set_override('max_retries', 2, 'database') - self.conf.set_override('retry_interval', 1, 'database') + self.CONF.set_override('max_retries', 2, 'database') + self.CONF.set_override('retry_interval', 1, 'database') self.mc = self._get_client() - with mock.patch.object( self.mc._mon_client.metrics, 'list', side_effect=[KeyError, []]) as mocked_metrics_list: @@ -129,8 +121,8 @@ class TestMonascaClient(base.BaseTestCase): self.assertEqual(2, mocked_metrics_list.call_count) def test_no_retry_on_invalid_parameter(self): - self.conf.set_override('max_retries', 2, 'database') - self.conf.set_override('retry_interval', 1, 'database') + self.CONF.set_override('max_retries', 2, 'database') + self.CONF.set_override('retry_interval', 1, 'database') self.mc = self._get_client() def _check(exception): @@ -142,12 +134,12 @@ class TestMonascaClient(base.BaseTestCase): self.assertRaises(expected_exc, list, self.mc.metrics_list()) self.assertEqual(1, mocked_metrics_list.call_count) - _check(exc.HTTPUnProcessable) - _check(exc.HTTPBadRequest) + _check(exc.http.UnprocessableEntity) + _check(exc.http.BadRequest) def test_max_retris_not_too_much(self): def _check(configured, expected): - self.conf.set_override('max_retries', configured, 'database') + self.CONF.set_override('max_retries', configured, 'database') self.mc = self._get_client() self.assertEqual(expected, self.mc._max_retries) @@ -159,27 +151,32 @@ class TestMonascaClient(base.BaseTestCase): def test_meaningful_exception_message(self): with mock.patch.object( self.mc._mon_client.metrics, 'list', - side_effect=[exc.HTTPInternalServerError, - exc.HTTPUnProcessable, + side_effect=[exc.http.InternalServerError, + exc.http.UnprocessableEntity, KeyError]): e = self.assertRaises( - monasca_client.MonascaServiceException, + tenacity.RetryError, list, self.mc.metrics_list()) - self.assertIn('Monasca service is unavailable', str(e)) + (original_ex, traceobj) = e.last_attempt.exception_info() + self.assertIn('Monasca service is unavailable', + str(original_ex)) e = self.assertRaises( monasca_client.MonascaInvalidParametersException, list, self.mc.metrics_list()) - self.assertIn('Request cannot be handled by Monasca', str(e)) + self.assertIn('Request cannot be handled by Monasca', + str(e)) e = self.assertRaises( - monasca_client.MonascaException, + tenacity.RetryError, list, self.mc.metrics_list()) - self.assertIn('An exception is raised from Monasca', str(e)) + (original_ex, traceobj) = e.last_attempt.exception_info() + self.assertIn('An exception is raised from Monasca', + str(original_ex)) @mock.patch.object(monasca_client.Client, '_refresh_client') def test_metrics_create_with_401(self, rc_patch): with mock.patch.object( self.mc._mon_client.metrics, 'create', - side_effect=[exc.HTTPUnauthorized, True]): + side_effect=[exc.http.Unauthorized, True]): self.assertRaises( monasca_client.MonascaInvalidParametersException, self.mc.metrics_create) @@ -206,7 +203,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = len(metric_list_pages) expected_metric_names = ["test1", "test2"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', True, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -245,7 +242,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = 1 expected_metric_names = ["test1"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', False, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -283,7 +280,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = len(measurement_list_pages) expected_metric_names = ["test1", "test2"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', True, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -322,7 +319,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = 1 expected_metric_names = ["test1"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', False, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -364,7 +361,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = len(statistics_list_pages) expected_metric_names = ["test1", "test2"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', True, group='monasca') # get a new ceilosca mc mc = self._get_client() @@ -403,7 +400,7 @@ class TestMonascaClient(base.BaseTestCase): expected_page_count = 1 expected_metric_names = ["test1"] - self.conf.set_override('enable_api_pagination', + self.CONF.set_override('enable_api_pagination', False, group='monasca') # get a new ceilosca mc mc = self._get_client() diff --git a/monasca_test_setup.py b/monasca_test_setup.py index 22e8d87..7696b1a 100644 --- a/monasca_test_setup.py +++ b/monasca_test_setup.py @@ -23,6 +23,7 @@ ceilosca_files = { for file in [ 'monasca_client.py', + 'monasca_ceilometer_opts.py', 'publisher/monasca_data_filter.py', 'publisher/monclient.py', 'storage/impl_monasca.py' diff --git a/setup.py b/setup.py index e2be832..2c6a1d9 100755 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ # (c) Copyright 2017 Hewlett Packard Enterprise Development LP -# (c) Copyright 2017 SUSE LLC +# (c) Copyright 2018 SUSE LLC # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at diff --git a/test-requirements.txt b/test-requirements.txt index b954a82..425353a 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,16 +1,13 @@ -hacking>=0.12.0,!=0.13.0,<0.14 # Apache-2.0 -git+https://github.com/openstack/ceilometer.git@newton-eol#egg=ceilometer +git+https://github.com/openstack/ceilometer.git@stable/pike#egg=ceilometer mock>=1.2 testrepository>=0.0.18 testscenarios>=0.4 testtools>=1.4.0 -oslosphinx>=2.5.0 # Apache-2.0 -# update oslotest version after ceilometer newton -oslotest>=1.10.0,<2.0.0 # Apache-2.0 +oslotest>=2.15.0 # Apache-2.0 oslo.vmware>=1.16.0,<2.17.0 # Apache-2.0 # Use lower versions of config and utils since # Keystone client depends on it -oslo.config>=2.3.0 # Apache-2.0 -oslo.utils!=2.6.0,>=2.0.0 # Apache-2.0 -oslo.log>=1.8.0 # Apache-2.0 -python-monascaclient<=1.2.0 +oslo.config>=3.22.0 # Apache-2.0 +oslo.log>=1.14.0 # Apache-2.0 +os-testr>=1.0.0 # Apache-2.0 +python-monascaclient<=1.7.1 diff --git a/tox.ini b/tox.ini index 54ba240..564558e 100644 --- a/tox.ini +++ b/tox.ini @@ -11,7 +11,7 @@ install_command = pip install -U {opts} {packages} deps = -r{toxinidir}/test-requirements.txt commands = find . -type f -name "*.pyc" -delete python monasca_test_setup.py - python setup.py testr --slowest --testr-args="{posargs}" + ostestr --serial {posargs} whitelist_externals = bash find