diff --git a/ceilosca/ceilometer/api/health.py b/ceilosca/ceilometer/api/health.py deleted file mode 100644 index 8e8c56d..0000000 --- a/ceilosca/ceilometer/api/health.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# (c) Copyright 2018 SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from oslo_log import log -LOG = log.getLogger(__name__) - - -def filter_factory(global_config, **local_conf): - """Returns a WSGI Filter.""" - conf = global_config.copy() - conf.update(local_conf) - LOG.debug('Filter Factory') - - def health_filter(app): - return HealthFilterApi(app, conf) - return health_filter - - -class HealthFilterApi(object): - - def __init__(self, app, conf): - self.conf = conf - self.app = app - self.db = None - - def __call__(self, environ, start_response): - """Handle the incoming request and filters it. - - Interjects the request and acts on it only if - it is related to the filter functionality, otherwise - it just let the request pass through to the app. - - """ - - LOG.debug('Health Check Filter') - - if environ['PATH_INFO'].startswith('/v2/health'): - response_code = '204 No Content' - if environ['REQUEST_METHOD'] != 'HEAD': - response_code = '200 OK' -# Commenting out healthcheck behavior when request method -# is not of type HEAD.# As it creates load on monasca vertica -# FIXME: fix this when monasca creates get versions api in -# in Monascaclient. USe that instead -# try: -# if not self.db: -# self.db = storage.get_connection_from_config(cfg.CONF) -# meters = self.db.get_meters(unique=True, limit=1) -# if not meters: -# response_code = '503 Backend Unavailable' -# except Exception as e: -# response_code = '503 Backend Unavailable' -# LOG.warning('DB health check connection failed: %s', e) -# self.db = None - resp = MiniResp(response_code, environ) - start_response(response_code, resp.headers) - return resp.body - else: - return self.app(environ, start_response) - - -class MiniResp(object): - - def __init__(self, message, env, headers=[]): - if env['REQUEST_METHOD'] == 'HEAD': - self.body = [''] - else: - self.body = [message] - self.headers = list(headers) diff --git a/ceilosca/ceilometer/monasca_ceilometer_opts.py b/ceilosca/ceilometer/monasca_ceilometer_opts.py index b79b0a0..790c158 100644 --- a/ceilosca/ceilometer/monasca_ceilometer_opts.py +++ b/ceilosca/ceilometer/monasca_ceilometer_opts.py @@ -104,11 +104,17 @@ OPTS = [ 'Monasca. These include metrics that failed to ' 'publish on first attempt and failed metrics that' ' maxed out their retries.'), - # from impl_monasca cfg.IntOpt('default_stats_period', default=300, help='Default period (in seconds) to use for querying stats ' 'in case no period specified in the stats API call.'), + cfg.IntOpt('database_max_retries', + default=3, + help='Maximum number of retry attempts of connecting to ' + 'database failure.'), + cfg.IntOpt('database_retry_interval', + default=60, + help='Frequency of attempting a retry connecting to database'), ] diff --git a/ceilosca/ceilometer/monasca_client.py b/ceilosca/ceilometer/monasca_client.py index 6fa0271..4771458 100644 --- a/ceilosca/ceilometer/monasca_client.py +++ b/ceilosca/ceilometer/monasca_client.py @@ -54,8 +54,8 @@ class Client(object): def __init__(self, conf, parsed_url): self.conf = conf - self._retry_interval = self.conf.database.retry_interval - self._max_retries = self.conf.database.max_retries or 1 + self._retry_interval = self.conf.monasca.database_retry_interval + self._max_retries = self.conf.monasca.database_max_retries or 1 # enable monasca api pagination self._enable_api_pagination = self.conf.monasca.enable_api_pagination # NOTE(zqfan): There are many concurrency requests while using diff --git a/ceilosca/ceilometer/opts.py b/ceilosca/ceilometer/opts.py index f3dd2ae..5946cc6 100644 --- a/ceilosca/ceilometer/opts.py +++ b/ceilosca/ceilometer/opts.py @@ -18,25 +18,15 @@ import socket from keystoneauth1 import loading from oslo_config import cfg -import ceilometer.agent.manager -import ceilometer.api.app -import ceilometer.api.controllers.v2.root -import ceilometer.collector import ceilometer.compute.discovery import ceilometer.compute.virt.inspector import ceilometer.compute.virt.libvirt.utils import ceilometer.compute.virt.vmware.inspector import ceilometer.compute.virt.xenapi.inspector -import ceilometer.dispatcher -import ceilometer.dispatcher.file -import ceilometer.dispatcher.gnocchi_opts -import ceilometer.dispatcher.http import ceilometer.event.converter -import ceilometer.exchange_control import ceilometer.hardware.discovery import ceilometer.hardware.pollsters.generic import ceilometer.image.discovery -import ceilometer.ipmi.notifications.ironic import ceilometer.ipmi.platform.intel_node_manager import ceilometer.ipmi.pollsters import ceilometer.keystone_client @@ -48,11 +38,11 @@ import ceilometer.notification import ceilometer.nova_client import ceilometer.objectstore.rgw import ceilometer.objectstore.swift -import ceilometer.pipeline +import ceilometer.pipeline.base +import ceilometer.polling.manager import ceilometer.publisher.messaging import ceilometer.publisher.utils import ceilometer.sample -import ceilometer.storage import ceilometer.utils import ceilometer.volume.discovery @@ -82,22 +72,14 @@ def list_opts(): # This have been removed due to a recursive import issue return [ ('DEFAULT', - itertools.chain(ceilometer.agent.manager.OPTS, - ceilometer.api.app.OPTS, + itertools.chain(ceilometer.polling.manager.OPTS, ceilometer.compute.virt.inspector.OPTS, ceilometer.compute.virt.libvirt.utils.OPTS, - ceilometer.dispatcher.OPTS, - ceilometer.ipmi.notifications.ironic.OPTS, - ceilometer.nova_client.OPTS, ceilometer.objectstore.swift.OPTS, - ceilometer.pipeline.OPTS, + ceilometer.pipeline.base.OPTS, ceilometer.sample.OPTS, ceilometer.utils.OPTS, - ceilometer.exchange_control.EXCHANGE_OPTS, OPTS)), - ('api', itertools.chain(ceilometer.api.app.API_OPTS, - ceilometer.api.controllers.v2.root.API_OPTS)), - ('collector', ceilometer.collector.OPTS), ('compute', ceilometer.compute.discovery.OPTS), ('coordination', [ cfg.StrOpt( @@ -113,11 +95,28 @@ def list_opts(): help='Number of seconds between checks to see if group ' 'membership has changed'), ]), - ('database', ceilometer.storage.OPTS), - ('dispatcher_file', ceilometer.dispatcher.file.OPTS), - ('dispatcher_http', ceilometer.dispatcher.http.http_dispatcher_opts), - ('dispatcher_gnocchi', - ceilometer.dispatcher.gnocchi_opts.dispatcher_opts), + ('dispatcher_gnocchi', ( + cfg.StrOpt( + 'filter_project', + deprecated_for_removal=True, + default='gnocchi', + help='Gnocchi project used to filter out samples ' + 'generated by Gnocchi service activity'), + cfg.StrOpt( + 'archive_policy', + deprecated_for_removal=True, + help='The archive policy to use when the dispatcher ' + 'create a new metric.'), + cfg.StrOpt( + 'resources_definition_file', + deprecated_for_removal=True, + default='gnocchi_resources.yaml', + help=('The Yaml file that defines mapping between samples ' + 'and gnocchi resources/metrics')), + cfg.FloatOpt( + 'request_timeout', default=6.05, min=0.0, + deprecated_for_removal=True, + help='Number of seconds before request to gnocchi times out'))), ('event', ceilometer.event.converter.OPTS), ('hardware', itertools.chain( ceilometer.hardware.discovery.OPTS, @@ -130,7 +129,7 @@ def list_opts(): ('notification', itertools.chain(ceilometer.notification.OPTS, ceilometer.notification.EXCHANGES_OPTS)), - ('polling', ceilometer.agent.manager.POLLING_OPTS), + ('polling', ceilometer.polling.manager.POLLING_OPTS), ('publisher', ceilometer.publisher.utils.OPTS), ('publisher_notifier', ceilometer.publisher.messaging.NOTIFIER_OPTS), ('rgw_admin_credentials', ceilometer.objectstore.rgw.CREDENTIAL_OPTS), diff --git a/ceilosca/ceilometer/service.py b/ceilosca/ceilometer/service.py index 57f7b49..c38991b 100644 --- a/ceilosca/ceilometer/service.py +++ b/ceilosca/ceilometer/service.py @@ -16,13 +16,10 @@ import sys from oslo_config import cfg -from oslo_db import options as db_options import oslo_i18n from oslo_log import log -from oslo_policy import opts as policy_opts # from oslo_reports import guru_meditation_report as gmr -from ceilometer.conf import defaults from ceilometer import keystone_client from ceilometer import messaging from ceilometer import opts @@ -48,9 +45,6 @@ def prepare_service(argv=None, config_files=None, conf=None): ['futurist=INFO', 'neutronclient=INFO', 'keystoneclient=INFO']) log.set_defaults(default_log_levels=log_levels) - defaults.set_cors_middleware_defaults() - policy_opts.set_defaults(conf) - db_options.set_defaults(conf) conf(argv[1:], project='ceilometer', validate_default_values=True, version=version.version_info.version_string(), @@ -62,11 +56,6 @@ def prepare_service(argv=None, config_files=None, conf=None): utils.setup_root_helper(conf) sample.setup(conf) - # NOTE(liusheng): guru cannot run with service under apache daemon, so when - # ceilometer-api running with mod_wsgi, the argv is [], we don't start - # guru. - if argv: - # gmr.TextGuruMeditation.setup_autorun(version) - pass + # gmr.TextGuruMeditation.setup_autorun(version) messaging.setup() return conf diff --git a/ceilosca/ceilometer/storage/impl_monasca.py b/ceilosca/ceilometer/storage/impl_monasca.py deleted file mode 100644 index 2bf5755..0000000 --- a/ceilosca/ceilometer/storage/impl_monasca.py +++ /dev/null @@ -1,1295 +0,0 @@ -# -# (C) Copyright 2015-2017 Hewlett Packard Enterprise Development LP -# (c) Copyright 2018 SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -"""Simple monasca storage backend. -""" -import traceback - -from collections import defaultdict -import copy -import datetime -import itertools -import operator - -from monascaclient import exc as monasca_exc -from oslo_log import log -from oslo_serialization import jsonutils -from oslo_utils import netutils -from oslo_utils import timeutils - -import ceilometer -from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( - ProcessMappedCeilometerStaticInfo) -from ceilometer.ceilosca_mapping.ceilosca_mapping import ( - ProcessMappedCeiloscaMetric) -from ceilometer.ceilosca_mapping.ceilosca_mapping import PipelineReader -from ceilometer.i18n import _ -from ceilometer import monasca_client -from ceilometer.publisher.monasca_data_filter import MonascaDataFilter -from ceilometer import storage -from ceilometer.storage import base -from ceilometer.storage import models as api_models -from ceilometer import utils - -LOG = log.getLogger(__name__) - -AVAILABLE_CAPABILITIES = { - 'meters': {'query': {'simple': True, - 'metadata': False}}, - 'resources': {'query': {'simple': True, - 'metadata': True}}, - 'samples': {'query': {'simple': True, - 'metadata': True, - 'complex': True}}, - 'statistics': {'groupby': False, - 'query': {'simple': True, - 'metadata': False}, - 'aggregation': {'standard': True, - 'selectable': { - 'max': True, - 'min': True, - 'sum': True, - 'avg': True, - 'count': True, - 'stddev': False, - 'cardinality': False}} - }, - 'events': {'query': {'simple': False}} -} - - -AVAILABLE_STORAGE_CAPABILITIES = { - 'storage': {'production_ready': True} -} - - -class InvalidInputException(Exception): - code = 400 - - -class Connection(base.Connection): - CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES, - AVAILABLE_CAPABILITIES) - STORAGE_CAPABILITIES = utils.update_nested( - base.Connection.STORAGE_CAPABILITIES, - AVAILABLE_STORAGE_CAPABILITIES, - ) - - def __init__(self, conf, url): - try: - super(Connection, self).__init__(conf, url) - self.conf = conf - self.mc = monasca_client.Client(self.conf, netutils.urlsplit(url)) - self.mon_filter = MonascaDataFilter(self.conf) - self.pipeline_reader = PipelineReader(conf) - self.ceilosca_mapper = ProcessMappedCeiloscaMetric(conf) - self.meter_static_info = ProcessMappedCeilometerStaticInfo(conf) - self.meters_from_pipeline = self.pipeline_reader\ - .get_pipeline_meters() - except Exception as ex: - LOG.info("ERROR: creating monasca connection: " + str(ex.message)) - LOG.info(ex) - LOG.info("ERROR: creating monasca connection: done") - LOG.info(traceback.format_ex()) - traceback.print_ex() - - @staticmethod - def _convert_to_dict(stats, cols): - return {c: stats[i] for i, c in enumerate(cols)} - - def _convert_metaquery(self, metaquery): - """Strip "metadata." from key and convert value to string - - :param metaquery: { 'metadata.KEY': VALUE, ... } - :returns: converted metaquery - """ - query = {} - for k, v in metaquery.items(): - key = '.'.join(k.split('.')[1:]) - if isinstance(v, basestring): - query[key] = v - else: - query[key] = str(int(v)) - return query - - def _match_metaquery_to_value_meta(self, query, value_meta): - """Check if metaquery matches value_meta - - :param query: metaquery with converted format - :param value_meta: metadata from monasca - :returns: True for matched, False for not matched - """ - if (query and (len(value_meta) == 0 or - not set(query.items()).issubset(set(value_meta.items())))): - return False - else: - return True - - def _incr_date_by_millisecond(self, date): - # Monasca only supports millisecond for now - epoch = datetime.datetime(1970, 1, 1) - seconds_since_epoch = timeutils.delta_seconds(epoch, date) - millis_since_epoch = seconds_since_epoch * 1000 - millis_since_epoch += 1 - return (timeutils.iso8601_from_timestamp( - millis_since_epoch / 1000, True)) - - def _decr_date_by_millisecond(self, date): - epoch = datetime.datetime(1970, 1, 1) - seconds_since_epoch = timeutils.delta_seconds(epoch, date) - - millis_since_epoch = seconds_since_epoch * 1000 - millis_since_epoch -= 1 - return (timeutils.iso8601_from_timestamp( - millis_since_epoch / 1000, True)) - - def upgrade(self): - pass - - def clear(self): - pass - - def record_metering_data(self, data): - """Write the data to the backend storage system. - - :param data: a dictionary such as returned by - ceilometer.meter.meter_message_from_counter. - """ - LOG.info(_('metering data %(counter_name)s for %(resource_id)s: ' - '%(counter_volume)s') - % ({'counter_name': data['counter_name'], - 'resource_id': data['resource_id'], - 'counter_volume': data['counter_volume']})) - - metric = self.mon_filter.process_sample_for_monasca(data) - self.mc.metrics_create(**metric) - - def clear_expired_metering_data(self, ttl): - """Clear expired data from the backend storage system. - - Clearing occurs according to the time-to-live. - :param ttl: Number of seconds to keep records for. - """ - LOG.info(_("Dropping data with TTL %d"), ttl) - - def get_metrics_with_mapped_dimensions(self, source_dimension, - mapped_dimension, search_args=None): - # Return metric list with the results of mapped dimensions. - if source_dimension in search_args['dimensions']: - search_args2 = copy.deepcopy(search_args) - filter_val = search_args2['dimensions'].pop(source_dimension, None) - search_args2['dimensions'][mapped_dimension] = filter_val - metric_list = self.mc.metrics_list(**search_args2) - if metric_list is not None: - return metric_list - return [] - - def get_metric_names_with_mapped_dimensions( - self, source_dimension, mapped_dimension, search_args=None): - # Return metric list with the results of mapped dimensions. - if source_dimension in search_args['dimensions']: - search_args2 = copy.deepcopy(search_args) - filter_val = search_args2['dimensions'].pop(source_dimension, None) - search_args2['dimensions'][mapped_dimension] = filter_val - metric_names_list = self.mc.metric_names_list(**search_args2) - if metric_names_list is not None: - return metric_names_list - return [] - - def get_resources(self, user=None, project=None, source=None, - start_timestamp=None, start_timestamp_op=None, - end_timestamp=None, end_timestamp_op=None, - metaquery=None, resource=None, limit=None): - """Return an iterable of dictionaries containing resource information. - - { 'resource_id': UUID of the resource, - 'project_id': UUID of project owning the resource, - 'user_id': UUID of user owning the resource, - 'timestamp': UTC datetime of last update to the resource, - 'metadata': most current metadata for the resource, - 'meter': list of the meters reporting data for the resource, - } - - :param user: Optional ID for user that owns the resource. - :param project: Optional ID for project that owns the resource. - :param source: Optional source filter. - :param start_timestamp: Optional modified timestamp start range. - :param start_timestamp_op: Optional start time operator, like gt, ge. - :param end_timestamp: Optional modified timestamp end range. - :param end_timestamp_op: Optional end time operator, like lt, le. - :param metaquery: Optional dict with metadata to match on. - :param resource: Optional resource filter. - :param limit: Maximum number of results to return. - """ - if limit == 0: - return - - q = {} - if metaquery: - q = self._convert_metaquery(metaquery) - - if start_timestamp_op and start_timestamp_op not in ['ge', 'gt']: - raise ceilometer.NotImplementedError(('Start time op %s ' - 'not implemented') % - start_timestamp_op) - - if end_timestamp_op and end_timestamp_op not in ['le', 'lt']: - raise ceilometer.NotImplementedError(('End time op %s ' - 'not implemented') % - end_timestamp_op) - - if not start_timestamp: - start_timestamp = datetime.datetime(1970, 1, 1) - - if not end_timestamp: - end_timestamp = timeutils.utcnow() - - self._ensure_start_time_le_end_time(start_timestamp, - end_timestamp) - - # Equivalent of doing a start_timestamp_op = 'ge' - if (start_timestamp_op and - start_timestamp_op == 'gt'): - start_timestamp = self._incr_date_by_millisecond( - start_timestamp) - start_timestamp_op = 'ge' - else: - start_timestamp = timeutils.isotime(start_timestamp, - subsecond=True) - - # Equivalent of doing a end_timestamp_op = 'le' - if (end_timestamp_op and - end_timestamp_op == 'lt'): - end_timestamp = self._decr_date_by_millisecond( - end_timestamp) - end_timestamp_op = 'le' - else: - end_timestamp = timeutils.isotime(end_timestamp, subsecond=True) - - dims_filter = dict(user_id=user, - project_id=project, - source=source, - resource_id=resource - ) - dims_filter = {k: v for k, v in dims_filter.items() if v is not None} - - _search_args = dict( - start_time=start_timestamp, - end_time=end_timestamp, - start_timestamp_op=start_timestamp_op, - end_timestamp_op=end_timestamp_op) - - _search_args = {k: v for k, v in _search_args.items() - if v is not None} - - result_count = 0 - _search_kwargs = {'dimensions': dims_filter} - meter_names = list() - meter_names_list = itertools.chain( - # Accumulate a list from monascaclient starting with no filter - self.mc.metric_names_list(**_search_kwargs), - # query monasca with hostname = resource_id filter - self.get_metric_names_with_mapped_dimensions( - "resource_id", "hostname", _search_kwargs), - # query monasca with tenant_id = project_id filter - self.get_metric_names_with_mapped_dimensions( - "project_id", "tenant_id", _search_kwargs) - ) - - for metric in meter_names_list: - if metric['name'] in meter_names: - continue - elif (metric['name'] in - self.meter_static_info.get_list_supported_meters()): - meter_names.insert(0, metric['name']) - elif (metric['name'] in - self.ceilosca_mapper.get_list_monasca_metrics()): - meter_names.append(metric['name']) - - for meter_name in meter_names: - _search_args['name'] = meter_name - _search_args['group_by'] = '*' - _search_args.pop('dimensions', None) - _search_args['dimensions'] = dims_filter - - # if meter is a Ceilometer meter... - if (meter_name not in - self.ceilosca_mapper.get_list_monasca_metrics()): - try: - if meter_name not in self.meters_from_pipeline: - _search_args['dimensions']['datasource'] = 'ceilometer' - for sample in (self.mc.measurements_list(**_search_args)): - d = sample['dimensions'] - for meas in sample['measurements']: - m = self._convert_to_dict(meas, sample['columns']) - vm = m['value_meta'] - if not self._match_metaquery_to_value_meta(q, vm): - continue - if d.get('resource_id'): - result_count += 1 - - yield api_models.Resource( - resource_id=d.get('resource_id'), - first_sample_timestamp=( - timeutils.parse_isotime( - m['timestamp'])), - last_sample_timestamp=timeutils.utcnow(), - project_id=d.get('project_id'), - source=d.get('source'), - user_id=d.get('user_id'), - metadata=m['value_meta'] - ) - - if result_count == limit: - return - - except monasca_exc.HTTPConflict: - pass - - # else if meter is a Monasca meter... - else: - try: - meter_def = self.ceilosca_mapper.\ - get_ceilosca_mapped_metric_definition(meter_name) - - # if for a meter name being queried, project exists in - # ceilometer-monasca mapping file, query by - # mapped_field instead - if not (project is None): - mapped_field = self.ceilosca_mapper.\ - get_ceilosca_mapped_definition_key_val( - _search_args['name'], 'project_id') - if 'dimensions' in mapped_field: - _search_args['dimensions'].pop('project_id', None) - _search_args['dimensions'][mapped_field.split(".")[-1]] = \ - project - - # if for a meter name being queried, resource_id exists in - # ceilometer-monasca mapping file, query by - # mapped_field instead - if not (resource is None): - mapped_field = self.ceilosca_mapper.\ - get_ceilosca_mapped_definition_key_val( - _search_args['name'], 'resource_id') - if 'dimensions' in mapped_field: - _search_args['dimensions'].pop('resource_id', None) - _search_args['dimensions'][mapped_field.split(".")[-1]] = \ - resource - - for sample in (self.mc.measurements_list(**_search_args)): - d = sample['dimensions'] - for meas in sample['measurements']: - m = self._convert_to_dict(meas, sample['columns']) - vm = m['value_meta'] - if not self._match_metaquery_to_value_meta(q, vm): - continue - if meter_def.parse_fields('resource_id', sample): - result_count += 1 - - yield api_models.Resource( - resource_id=meter_def.parse_fields( - 'resource_id', sample), - first_sample_timestamp=( - timeutils.parse_isotime( - m['timestamp'])), - last_sample_timestamp=(timeutils.utcnow()), - project_id=meter_def.parse_fields( - 'project_id', sample), - source=meter_def.parse_fields('source', - sample), - user_id=meter_def.parse_fields( - 'user_id', sample), - metadata=meter_def.parse_fields( - 'resource_metadata', sample) - ) - - if result_count == limit: - return - - except monasca_exc.HTTPConflict: - pass - - def get_meters(self, user=None, project=None, resource=None, source=None, - metaquery=None, limit=None, unique=False): - """Return an iterable of dictionaries containing meter information. - - { 'name': name of the meter, - 'type': type of the meter (gauge, delta, cumulative), - 'resource_id': UUID of the resource, - 'project_id': UUID of project owning the resource, - 'user_id': UUID of user owning the resource, - } - - :param user: Optional ID for user that owns the resource. - :param project: Optional ID for project that owns the resource. - :param resource: Optional resource filter. - :param source: Optional source filter. - :param metaquery: Optional dict with metadata to match on. - :param limit: Maximum number of results to return. - :param unique: If set to true, return only unique meter information. - """ - if limit == 0: - return - - if metaquery: - raise ceilometer.NotImplementedError('Metaquery not implemented') - - _dimensions = dict( - user_id=user, - project_id=project, - resource_id=resource, - source=source - ) - _dimensions = {k: v for k, v in _dimensions.items() if v is not None} - _search_kwargs = {'dimensions': _dimensions} - if unique: - meter_names = set() - for metric in self.mc.metric_names_list(**_search_kwargs): - if metric['name'] in meter_names: - continue - elif (metric['name'] in - self.meter_static_info.get_list_supported_meters()): - if limit and len(meter_names) >= limit: - return - meter_names.add(metric['name']) - yield api_models.Meter( - name=metric['name'], - type=self.meter_static_info - .get_meter_static_info_key_val(metric['name'], - 'type'), - unit=self.meter_static_info - .get_meter_static_info_key_val(metric['name'], - 'unit'), - resource_id=None, - project_id=None, - source=None, - user_id=None) - - elif (metric['name'] not in - self.ceilosca_mapper.get_list_monasca_metrics()): - continue - else: - if limit and len(meter_names) >= limit: - return - meter_names.add(metric['name']) - meter = (self.ceilosca_mapper. - get_ceilosca_mapped_metric_definition - (metric['name'])) - yield api_models.Meter( - name=meter.parse_fields('name', metric), - type=meter.parse_fields('type', metric), - unit=meter.parse_fields('unit', metric), - resource_id=None, - project_id=None, - source=None, - user_id=None) - - else: - result_count = 0 - # Search for ceilometer published data first - _search_kwargs_tmp = copy.deepcopy(_search_kwargs) - _search_kwargs_tmp['dimensions']['datasource'] = 'ceilometer' - metrics_list = self.mc.metrics_list(**_search_kwargs_tmp) - for metric in metrics_list: - if result_count == limit: - return - result_count += 1 - yield api_models.Meter( - name=metric['name'], - type=metric['dimensions'].get('type') or 'cumulative', - unit=metric['dimensions'].get('unit'), - resource_id=metric['dimensions'].get('resource_id'), - project_id=metric['dimensions'].get('project_id'), - source=metric['dimensions'].get('source'), - user_id=metric['dimensions'].get('user_id')) - - # because we enable monasca pagination, so we should use iterator - # instead of a list, to reduce unnecessary requests to monasca-api - metrics_list = itertools.chain( - self.mc.metrics_list(**_search_kwargs), - # for vm performance metrics collected by monasca-agent, the - # project_id is mapped to tenant_id - self.get_metrics_with_mapped_dimensions("project_id", - "tenant_id", - _search_kwargs), - # for compute.node metrics collected by monasca-agent, the - # resource_id is mapped to hostname - self.get_metrics_with_mapped_dimensions("resource_id", - "hostname", - _search_kwargs), - ) - - for metric in metrics_list: - if result_count == limit: - return - if (metric['dimensions'].get('datasource') != 'ceilometer' and - metric['name'] in self.meters_from_pipeline): - result_count += 1 - yield api_models.Meter( - name=metric['name'], - type=metric['dimensions'].get('type') or 'cumulative', - unit=metric['dimensions'].get('unit'), - resource_id=metric['dimensions'].get('resource_id'), - project_id=metric['dimensions'].get('project_id'), - source=metric['dimensions'].get('source'), - user_id=metric['dimensions'].get('user_id')) - - elif (metric['name'] in - self.ceilosca_mapper.get_list_monasca_metrics()): - meter = (self.ceilosca_mapper. - get_ceilosca_mapped_metric_definition - (metric['name'])) - result_count += 1 - yield api_models.Meter( - name=meter.parse_fields('name', metric), - type=meter.parse_fields('type', metric), - unit=meter.parse_fields('unit', metric), - resource_id=meter.parse_fields('resource_id', metric), - project_id=meter.parse_fields('project_id', metric), - source=meter.parse_fields('source', metric), - user_id=meter.parse_fields('user_id', metric)) - - def get_samples(self, sample_filter, limit=None): - """Return an iterable of dictionaries containing sample information. - - { - 'source': source of the resource, - 'counter_name': name of the resource,if groupby: - raise ceilometer.NotImplementedError('Groupby not implemented') - 'counter_type': type of the sample (gauge, delta, cumulative), - 'counter_unit': unit of the sample, - 'counter_volume': volume of the sample, - 'user_id': UUID of user owning the resource, - 'project_id': UUID of project owning the resource, - 'resource_id': UUID of the resource, - 'timestamp': timestamp of the sample, - 'resource_metadata': metadata of the sample, - 'message_id': message ID of the sample, - 'message_signature': message signature of the sample, - 'recorded_at': time the sample was recorded - } - - :param sample_filter: constraints for the sample search. - :param limit: Maximum number of results to return. - """ - - if limit == 0: - return - - if not sample_filter or not sample_filter.meter: - raise ceilometer.NotImplementedError( - "Supply meter name at the least") - - if (sample_filter.start_timestamp_op and - sample_filter.start_timestamp_op not in ['ge', 'gt']): - raise ceilometer.NotImplementedError(('Start time op %s ' - 'not implemented') % - sample_filter. - start_timestamp_op) - - if (sample_filter.end_timestamp_op and - sample_filter.end_timestamp_op not in ['le', 'lt']): - raise ceilometer.NotImplementedError(('End time op %s ' - 'not implemented') % - sample_filter. - end_timestamp_op) - - q = {} - if sample_filter.metaquery: - q = self._convert_metaquery(sample_filter.metaquery) - - if sample_filter.message_id: - raise ceilometer.NotImplementedError('message_id not ' - 'implemented ' - 'in get_samples') - - if not sample_filter.start_timestamp: - sample_filter.start_timestamp = datetime.datetime(1970, 1, 1) - - if not sample_filter.end_timestamp: - sample_filter.end_timestamp = timeutils.utcnow() - - self._ensure_start_time_le_end_time(sample_filter.start_timestamp, - sample_filter.end_timestamp) - - if (sample_filter.start_timestamp_op and sample_filter. - start_timestamp_op == 'gt'): - sample_filter.start_timestamp = self._incr_date_by_millisecond( - sample_filter.start_timestamp) - sample_filter.start_timestamp_op = 'ge' - else: - sample_filter.start_timestamp = timeutils.isotime( - sample_filter.start_timestamp, subsecond=True) - - if (sample_filter.end_timestamp_op and sample_filter. - end_timestamp_op == 'lt'): - sample_filter.end_timestamp = self._decr_date_by_millisecond( - sample_filter.end_timestamp) - sample_filter.end_timestamp_op = 'le' - else: - sample_filter.end_timestamp = timeutils.isotime( - sample_filter.end_timestamp, subsecond=True) - - _dimensions = dict( - user_id=sample_filter.user, - project_id=sample_filter.project, - resource_id=sample_filter.resource, - source=sample_filter.source, - # Dynamic sample filter attributes, these fields are useful for - # filtering result. - unit=getattr(sample_filter, 'unit', None), - type=getattr(sample_filter, 'type', None), - ) - - _dimensions = {k: v for k, v in _dimensions.items() if v is not None} - - _search_args = dict( - start_time=sample_filter.start_timestamp, - start_timestamp_op=sample_filter.start_timestamp_op, - end_time=sample_filter.end_timestamp, - end_timestamp_op=sample_filter.end_timestamp_op, - dimensions=_dimensions, - name=sample_filter.meter, - group_by='*' - ) - - result_count = 0 - _search_args = {k: v for k, v in _search_args.items() - if v is not None} - - if self.ceilosca_mapper.get_monasca_metric_name(sample_filter.meter): - _search_args['name'] = ( - self.ceilosca_mapper.get_monasca_metric_name( - sample_filter.meter)) - meter_def = ( - self.ceilosca_mapper.get_ceilosca_mapped_metric_definition( - _search_args['name'])) - - # if for a meter name being queried, project exists in - # ceilometer-monasca mapping file, query by mapped_field instead - if not (sample_filter.project is None): - mapped_field = self.ceilosca_mapper.\ - get_ceilosca_mapped_definition_key_val( - _search_args['name'], 'project_id') - if 'dimensions' in mapped_field: - _search_args['dimensions'].pop('project_id', None) - _search_args['dimensions'][mapped_field.split(".")[-1]] = \ - sample_filter.project - - # if for a meter name being queried, resource_id exists in - # ceilometer-monasca mapping file, query by mapped_field instead - if not (sample_filter.resource is None): - mapped_field = self.ceilosca_mapper.\ - get_ceilosca_mapped_definition_key_val( - _search_args['name'], 'resource_id') - if 'dimensions' in mapped_field: - _search_args['dimensions'].pop('resource_id', None) - _search_args['dimensions'][mapped_field.split(".")[-1]] = \ - sample_filter.resource - - for sample in self.mc.measurements_list(**_search_args): - d = sample['dimensions'] - for meas in sample['measurements']: - m = self._convert_to_dict( - meas, sample['columns']) - vm = m['value_meta'] - if not self._match_metaquery_to_value_meta(q, vm): - continue - result_count += 1 - - yield api_models.Sample( - source=meter_def.parse_fields('source', sample), - counter_name=sample_filter.meter, - counter_type=meter_def.parse_fields('type', sample), - counter_unit=meter_def.parse_fields('unit', sample), - counter_volume=m['value'], - user_id=meter_def.parse_fields('user_id', sample), - project_id=meter_def.parse_fields('project_id', - sample), - resource_id=meter_def.parse_fields('resource_id', - sample), - timestamp=timeutils.parse_isotime(m['timestamp']), - resource_metadata=meter_def.parse_fields( - 'resource_metadata', sample), - message_id=sample['id'], - message_signature='', - recorded_at=(timeutils.parse_isotime(m['timestamp']))) - - if result_count == limit: - return - - return - - # This if statement is putting the dimension information for 2 cases - # 1. To safeguard against querying an existing monasca metric which is - # not mapped in ceilometer, unless it is published in monasca by - # ceilometer itself - # 2. Also, this will allow you to query a metric which was being - # collected historically but not being collected any more as far as - # those metrics exists in monasca - if sample_filter.meter not in self.meters_from_pipeline: - _search_args['dimensions']['datasource'] = 'ceilometer' - - for sample in self.mc.measurements_list(**_search_args): - d = sample['dimensions'] - for meas in sample['measurements']: - m = self._convert_to_dict( - meas, sample['columns']) - vm = m['value_meta'] - if not self._match_metaquery_to_value_meta(q, vm): - continue - result_count += 1 - - yield api_models.Sample( - source=d.get('source'), - counter_name=sample_filter.meter, - counter_type=d.get('type'), - counter_unit=d.get('unit'), - counter_volume=m['value'], - user_id=d.get('user_id'), - project_id=d.get('project_id'), - resource_id=d.get('resource_id'), - timestamp=timeutils.parse_isotime(m['timestamp']), - resource_metadata=m['value_meta'], - message_id=sample['id'], - message_signature='', - recorded_at=(timeutils.parse_isotime(m['timestamp']))) - - if result_count == limit: - return - - def get_meter_statistics(self, filter, period=None, groupby=None, - aggregate=None): - """Return a dictionary containing meter statistics. - - Meter statistics is described by the query parameters. - The filter must have a meter value set. - - { 'min': - 'max': - 'avg': - 'sum': - 'count': - 'period': - 'period_start': - 'period_end': - 'duration': - 'duration_start': - 'duration_end': - } - """ - if filter: - if not filter.meter: - raise ceilometer.NotImplementedError('Query without meter ' - 'not implemented') - else: - raise ceilometer.NotImplementedError('Query without filter ' - 'not implemented') - - allowed_groupby = ['user_id', 'project_id', 'resource_id', 'source'] - - if groupby: - if len(groupby) > 1: - raise ceilometer.NotImplementedError('Only one groupby ' - 'supported') - - groupby = groupby[0] - if groupby not in allowed_groupby: - raise ceilometer.NotImplementedError('Groupby %s not' - ' implemented' % groupby) - - if filter.metaquery: - raise ceilometer.NotImplementedError('Metaquery not implemented') - - if filter.message_id: - raise ceilometer.NotImplementedError('Message_id query ' - 'not implemented') - - if filter.start_timestamp_op and ( - filter.start_timestamp_op not in ['ge', 'gt']): - raise ceilometer.NotImplementedError(('Start time op %s ' - 'not implemented') % - filter.start_timestamp_op) - - if filter.end_timestamp_op and ( - filter.end_timestamp_op not in ['le', 'lt']): - raise ceilometer.NotImplementedError(('End time op %s ' - 'not implemented') % - filter.end_timestamp_op) - - if not filter.start_timestamp: - filter.start_timestamp = datetime.datetime(1970, 1, 1) - - if not filter.end_timestamp: - filter.end_timestamp = timeutils.utcnow() - - self._ensure_start_time_le_end_time(filter.start_timestamp, - filter.end_timestamp) - - if (filter.start_timestamp_op and filter. - start_timestamp_op == 'gt'): - filter.start_timestamp = self._incr_date_by_millisecond( - filter.start_timestamp) - filter.start_timestamp_op = 'ge' - else: - filter.start_timestamp = timeutils.isotime(filter.start_timestamp, - subsecond=True) - - if (filter.end_timestamp_op and filter. - end_timestamp_op == 'lt'): - filter.end_timestamp = self._decr_date_by_millisecond( - filter.end_timestamp) - filter.end_timestamp_op = 'le' - else: - filter.end_timestamp = timeutils.isotime(filter.end_timestamp, - subsecond=True) - - # TODO(monasca): Add this a config parameter - allowed_stats = ['avg', 'min', 'max', 'sum', 'count'] - if aggregate: - not_allowed_stats = [a.func for a in aggregate - if a.func not in allowed_stats] - if not_allowed_stats: - raise ceilometer.NotImplementedError(('Aggregate function(s) ' - '%s not implemented') % - not_allowed_stats) - - statistics = [a.func for a in aggregate - if a.func in allowed_stats] - else: - statistics = allowed_stats - - dims_filter = dict(user_id=filter.user, - project_id=filter.project, - source=filter.source, - resource_id=filter.resource - ) - dims_filter = {k: v for k, v in dims_filter.items() if v is not None} - - period = period if period \ - else self.conf.monasca.default_stats_period - - _search_args = dict( - name=filter.meter, - dimensions=dims_filter, - start_time=filter.start_timestamp, - end_time=filter.end_timestamp, - period=period, - statistics=','.join(statistics), - ) - - _search_args = {k: v for k, v in _search_args.items() - if v is not None} - - is_mapped_metric = False - if self.ceilosca_mapper.get_monasca_metric_name(filter.meter): - _search_args['name'] = self.ceilosca_mapper.\ - get_monasca_metric_name(filter.meter) - meter_def = ( - self.ceilosca_mapper.get_ceilosca_mapped_metric_definition( - _search_args['name'])) - is_mapped_metric = True - # if for a meter name being queried, project exists in - # ceilometer-monasca mapping file, query by mapped_field instead - if not (filter.project is None): - mapped_field = self.ceilosca_mapper.\ - get_ceilosca_mapped_definition_key_val( - _search_args['name'], 'project_id') - if 'dimensions' in mapped_field: - _search_args['dimensions'].pop('project_id', None) - _search_args['dimensions'][mapped_field.split(".")[-1]] = \ - filter.project - - # if for a meter name being queried, resource_id exists in - # ceilometer-monasca mapping file, query by mapped_field instead - if not (filter.resource is None): - mapped_field = self.ceilosca_mapper.\ - get_ceilosca_mapped_definition_key_val( - _search_args['name'], 'resource_id') - if 'dimensions' in mapped_field: - _search_args['dimensions'].pop('resource_id', None) - _search_args['dimensions'][mapped_field.split(".")[-1]] = \ - filter.resource - - elif filter.meter not in self.meters_from_pipeline: - _search_args['dimensions']['datasource'] = 'ceilometer' - - if groupby: - _search_args['group_by'] = '*' - stats_list = self.mc.statistics_list(**_search_args) - group_stats_dict = defaultdict(list) - - for stats in stats_list: - if is_mapped_metric: - groupby_val = meter_def.parse_fields(groupby, stats) - else: - groupby_val = stats['dimensions'].get(groupby) - group_stats_dict[groupby_val].append(stats) - - def get_max(items): - return max(items) - - def get_min(items): - return min(items) - - def get_avg(items): - return sum(items)/len(items) - - def get_sum(items): - return sum(items) - - def get_count(items): - count = 0 - for item in items: - count = count + item - return count - - for group_key, stats_group in group_stats_dict.iteritems(): - max_list = [] - min_list = [] - avg_list = [] - sum_list = [] - count_list = [] - ts_list = [] - group_statistics = {} - for stats in stats_group: - for s in stats['statistics']: - stats_dict = self._convert_to_dict(s, stats['columns']) - - if 'max' in stats['columns']: - max_list.append(stats_dict['max']) - if 'min' in stats['columns']: - min_list.append(stats_dict['min']) - if 'avg' in stats['columns']: - avg_list.append(stats_dict['avg']) - if 'sum' in stats['columns']: - sum_list.append(stats_dict['sum']) - if 'count' in stats['columns']: - count_list.append(stats_dict['count']) - - ts_list.append(stats_dict['timestamp']) - if is_mapped_metric: - group_statistics['unit'] = (meter_def.parse_fields( - 'unit', stats)) - else: - group_statistics['unit'] = (stats['dimensions']. - get('unit')) - - if len(max_list): - group_statistics['max'] = get_max(max_list) - if len(min_list): - group_statistics['min'] = get_min(min_list) - if len(avg_list): - group_statistics['avg'] = get_avg(avg_list) - if len(sum_list): - group_statistics['sum'] = get_sum(sum_list) - if len(count_list): - group_statistics['count'] = get_count(count_list) - - if 'count' in group_statistics: - group_statistics['count'] = int(group_statistics['count']) - - unit = group_statistics.get('unit') - - if 'unit' in group_statistics: - del group_statistics['unit'] - - if aggregate: - group_statistics['aggregate'] = {} - for a in aggregate: - key = '%s%s' % (a.func, '/%s' % a.param if a.param - else '') - group_statistics['aggregate'][key] = ( - group_statistics.get(key)) - - if group_statistics and len(ts_list): - group_statistics['end_timestamp'] = get_max(ts_list) - group_statistics['timestamp'] = get_min(ts_list) - - ts_start = timeutils.parse_isotime( - group_statistics['timestamp']).replace(tzinfo=None) - - ts_end = timeutils.parse_isotime( - group_statistics['end_timestamp']).replace(tzinfo=None) - - del group_statistics['end_timestamp'] - if ts_start and ts_end: - yield api_models.Statistics( - unit=unit, - period=period, - period_start=ts_start, - period_end=ts_end, - duration=period, - duration_start=ts_start, - duration_end=ts_end, - groupby={groupby: group_key}, - **group_statistics - ) - else: - _search_args['merge_metrics'] = True - stats_list = self.mc.statistics_list(**_search_args) - for stats in stats_list: - for s in stats['statistics']: - stats_dict = self._convert_to_dict(s, stats['columns']) - ts_start = timeutils.parse_isotime( - stats_dict['timestamp']).replace(tzinfo=None) - ts_end = (ts_start + datetime.timedelta( - 0, period)).replace(tzinfo=None) - del stats_dict['timestamp'] - if 'count' in stats_dict: - stats_dict['count'] = int(stats_dict['count']) - - if aggregate: - stats_dict['aggregate'] = {} - for a in aggregate: - key = '%s%s' % (a.func, '/%s' % a.param if a.param - else '') - stats_dict['aggregate'][key] = stats_dict.get(key) - if is_mapped_metric: - unit = meter_def.parse_fields('unit', stats) - else: - unit = stats['dimensions'].get('unit') - if ts_start and ts_end: - yield api_models.Statistics( - unit=unit, - period=period, - period_start=ts_start, - period_end=ts_end, - duration=period, - duration_start=ts_start, - duration_end=ts_end, - groupby={u'': u''}, - **stats_dict - ) - - def _parse_to_filter_list(self, filter_expr): - """Parse complex query expression to simple filter list. - - For i.e. parse: - {"or":[{"=":{"meter":"cpu"}},{"=":{"meter":"memory"}}]} - to - [[{"=":{"counter_name":"cpu"}}], - [{"=":{"counter_name":"memory"}}]] - """ - op, nodes = filter_expr.items()[0] - msg = "%s operator is not supported" % op - - if op == 'or': - filter_list = [] - for node in nodes: - filter_list.extend(self._parse_to_filter_list(node)) - return filter_list - elif op == 'and': - filter_list_subtree = [] - for node in nodes: - filter_list_subtree.append(self._parse_to_filter_list(node)) - filter_list = [[]] - for filters in filter_list_subtree: - tmp = [] - for filter in filters: - for f in filter_list: - tmp.append(f + filter) - filter_list = tmp - return filter_list - elif op == 'not': - raise ceilometer.NotImplementedError(msg) - elif op in ("<", "<=", "=", ">=", ">", '!='): - return [[filter_expr]] - else: - raise ceilometer.NotImplementedError(msg) - - def _ensure_start_time_le_end_time(self, start_time, end_time): - if start_time is None: - start_time = datetime.datetime(1970, 1, 1) - if end_time is None: - end_time = timeutils.utcnow() - # here we don't handle the corner case of start_time == end_time, - # while start_time_op & end_time_op can be gt and lt, let Monasca - # decides it valid or not. - if start_time > end_time: - msg = _('start time (%(start_time)s) should not after end_time ' - '(%(end_time)s). (start time defaults to ' - '1970-01-01T00:00:00Z, end time defaults to utc now)') % { - 'start_time': start_time, 'end_time': end_time} - raise InvalidInputException(msg) - - def _parse_to_sample_filter(self, simple_filters): - """Parse simple filters to sample filter. - - For i.e.: parse - [{"=":{"counter_name":"cpu"}},{"=":{"counter_volume": 1}}] - to - SampleFilter(counter_name="cpu", counter_volume=1) - """ - equal_only_fields = ( - 'counter_name', - 'counter_unit', - 'counter_type', - 'project_id', - 'user_id', - 'source', - 'resource_id', - # These fields are supported by Ceilometer but cannot supported - # by Monasca. - # 'message_id', - # 'message_signature', - # 'recorded_at', - ) - field_map = { - "project_id": "project", - "user_id": "user", - "resource_id": "resource", - "counter_name": "meter", - "counter_type": "type", - "counter_unit": "unit", - } - msg = "operator %s cannot be applied to field %s" - kwargs = {'metaquery': {}} - for sf in simple_filters: - op = sf.keys()[0] - field, value = sf.values()[0].items()[0] - if field in equal_only_fields: - if op != '=': - raise ceilometer.NotImplementedError(msg % (op, field)) - field = field_map.get(field, field) - kwargs[field] = value - elif field == 'timestamp': - if op == '>=': - kwargs['start_timestamp'] = value - kwargs['start_timestamp_op'] = 'ge' - elif op == '>': - kwargs['start_timestamp'] = value - kwargs['start_timestamp_op'] = 'gt' - elif op == '<=': - kwargs['end_timestamp'] = value - kwargs['end_timestamp_op'] = 'le' - elif op == '<': - kwargs['end_timestamp'] = value - kwargs['end_timestamp_op'] = 'lt' - else: - raise ceilometer.NotImplementedError(msg % (op, field)) - elif field == 'counter_volume': - kwargs['volume'] = value - kwargs['volume_op'] = op - elif (field.startswith('resource_metadata.') or - field.startswith('metadata.')): - kwargs['metaquery'][field] = value - else: - ra_msg = "field %s is not supported" % field - raise ceilometer.NotImplementedError(ra_msg) - self._ensure_start_time_le_end_time(kwargs.get('start_timestamp'), - kwargs.get('end_timestamp')) - sample_type = kwargs.pop('type', None) - sample_unit = kwargs.pop('unit', None) - sample_volume = kwargs.pop('volume', None) - sample_volume_op = kwargs.pop('volume_op', None) - sample_filter = storage.SampleFilter(**kwargs) - # Add some dynamic attributes, type and unit attributes can be used - # when query Monasca API, volume and volime_op attributes can - # be used for volume comparison. - sample_filter.type = sample_type - sample_filter.unit = sample_unit - sample_filter.volume = sample_volume - sample_filter.volume_op = sample_volume_op - return sample_filter - - def _is_meter_name_exist(self, filters): - for f in filters: - field = f.values()[0].keys()[0] - if field == 'counter_name': - return True - return False - - def _validate_filters(self, filters, msg=''): - """Validate filters to ensure fail at early stage.""" - if not self._is_meter_name_exist(filters): - msg = _('%(msg)s, meter name is not found in %(filters)s') % { - 'msg': msg, 'filters': jsonutils.dumps(filters)} - raise InvalidInputException(msg) - - def _parse_to_sample_filters(self, filter_expr): - """Parse complex query expression to sample filter list.""" - filter_list = self._parse_to_filter_list(filter_expr) - msg = _('complex query filter expression has been translated to ' - '%(count)d sub queries: %(filters)s') % { - 'count': len(filter_list), - 'filters': jsonutils.dumps(filter_list)} - sample_filters = [] - for filters in filter_list: - self._validate_filters(filters, msg) - sf = self._parse_to_sample_filter(filters) - if sf: - sample_filters.append(sf) - return sample_filters - - def _validate_samples_by_volume(self, samples, sf): - if not sf.volume: - return samples - - op_func_map = { - '<': operator.lt, - '<=': operator.le, - '=': operator.eq, - '>=': operator.ge, - '>': operator.gt, - '!=': operator.ne, - } - - ret = [] - for s in samples: - op_func = op_func_map[sf.volume_op] - volume = getattr(s, 'volume', getattr(s, 'counter_volume', None)) - if op_func(volume, sf.volume): - ret.append(s) - return ret - - def query_samples(self, filter_expr=None, orderby=None, limit=None): - if not filter_expr: - msg = _("filter must be specified") - raise ceilometer.NotImplementedError(msg) - if orderby: - msg = _("orderby is not supported") - raise ceilometer.NotImplementedError(msg) - - # limit won't be None because we have limit enforcement in API level - if limit == 0: - return [] - - LOG.debug("filter_expr = %s", filter_expr) - try: - sample_filters = self._parse_to_sample_filters(filter_expr) - # ValueError: year=1016 is before 1900; the datetime strftime() - # methods require year >= 1900 - except ValueError as e: - raise InvalidInputException(e) - LOG.debug("sample_filters = %s", sample_filters) - - ret = [] - for sf in sample_filters: - if not sf.volume: - samples = list(self.get_samples(sf, limit)) - else: - samples = self.get_samples(sf) - samples = list(self._validate_samples_by_volume(samples, sf)) - - if limit <= len(samples): - ret.extend(samples[0:limit]) - break - else: - ret.extend(samples) - limit -= len(samples) - - return ret diff --git a/ceilosca/ceilometer/tests/functional/__init__.py b/ceilosca/ceilometer/tests/functional/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ceilosca/ceilometer/tests/functional/api/__init__.py b/ceilosca/ceilometer/tests/functional/api/__init__.py deleted file mode 100644 index a0359dc..0000000 --- a/ceilosca/ceilometer/tests/functional/api/__init__.py +++ /dev/null @@ -1,185 +0,0 @@ -# -# Copyright 2012 New Dream Network, LLC (DreamHost) -# (c) Copyright 2018 SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Base classes for API tests. -""" -from oslo_policy import opts -import pecan -import pecan.testing - -from ceilometer.api import rbac -from ceilometer import monasca_ceilometer_opts -from ceilometer import service -from ceilometer.tests import db as db_test_base - - -class FunctionalTest(db_test_base.TestBase): - """Used for functional tests of Pecan controllers. - - Used in case when you need to test your literal application and its - integration with the framework. - """ - - PATH_PREFIX = '' - - def setUp(self): - super(FunctionalTest, self).setUp() - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - self.setup_messaging(self.CONF) - opts.set_defaults(self.CONF) - - self.CONF.set_override("policy_file", - self.path_get('etc/ceilometer/policy.json'), - group='oslo_policy') - - self.CONF.set_override('gnocchi_is_enabled', False, group='api') - self.CONF.set_override('aodh_is_enabled', False, group='api') - self.CONF.set_override('panko_is_enabled', False, group='api') - - self.app = self._make_app() - - def _make_app(self, enable_acl=False): - self.config = { - 'app': { - 'root': 'ceilometer.api.controllers.root.RootController', - 'modules': ['ceilometer.api'], - 'enable_acl': enable_acl, - }, - 'wsme': { - 'debug': True, - }, - } - - return pecan.testing.load_test_app(self.config, - conf=self.CONF) - - def tearDown(self): - super(FunctionalTest, self).tearDown() - rbac.reset() - pecan.set_config({}, overwrite=True) - - def put_json(self, path, params, expect_errors=False, headers=None, - extra_environ=None, status=None): - """Sends simulated HTTP PUT request to Pecan test app. - - :param path: url path of target service - :param params: content for wsgi.input of request - :param expect_errors: boolean value whether an error is expected based - on request - :param headers: A dictionary of headers to send along with the request - :param extra_environ: A dictionary of environ variables to send along - with the request - :param status: Expected status code of response - """ - return self.post_json(path=path, params=params, - expect_errors=expect_errors, - headers=headers, extra_environ=extra_environ, - status=status, method="put") - - def post_json(self, path, params, expect_errors=False, headers=None, - method="post", extra_environ=None, status=None): - """Sends simulated HTTP POST request to Pecan test app. - - :param path: url path of target service - :param params: content for wsgi.input of request - :param expect_errors: boolean value whether an error is expected based - on request - :param headers: A dictionary of headers to send along with the request - :param method: Request method type. Appropriate method function call - should be used rather than passing attribute in. - :param extra_environ: A dictionary of environ variables to send along - with the request - :param status: Expected status code of response - """ - full_path = self.PATH_PREFIX + path - response = getattr(self.app, "%s_json" % method)( - str(full_path), - params=params, - headers=headers, - status=status, - extra_environ=extra_environ, - expect_errors=expect_errors - ) - return response - - def delete(self, path, expect_errors=False, headers=None, - extra_environ=None, status=None): - """Sends simulated HTTP DELETE request to Pecan test app. - - :param path: url path of target service - :param expect_errors: boolean value whether an error is expected based - on request - :param headers: A dictionary of headers to send along with the request - :param extra_environ: A dictionary of environ variables to send along - with the request - :param status: Expected status code of response - """ - full_path = self.PATH_PREFIX + path - response = self.app.delete(str(full_path), - headers=headers, - status=status, - extra_environ=extra_environ, - expect_errors=expect_errors) - return response - - def get_json(self, path, expect_errors=False, headers=None, - extra_environ=None, q=None, groupby=None, status=None, - override_params=None, **params): - """Sends simulated HTTP GET request to Pecan test app. - - :param path: url path of target service - :param expect_errors: boolean value whether an error is expected based - on request - :param headers: A dictionary of headers to send along with the request - :param extra_environ: A dictionary of environ variables to send along - with the request - :param q: list of queries consisting of: field, value, op, and type - keys - :param groupby: list of fields to group by - :param status: Expected status code of response - :param override_params: literally encoded query param string - :param params: content for wsgi.input of request - """ - q = q or [] - groupby = groupby or [] - full_path = self.PATH_PREFIX + path - if override_params: - all_params = override_params - else: - query_params = {'q.field': [], - 'q.value': [], - 'q.op': [], - 'q.type': [], - } - for query in q: - for name in ['field', 'op', 'value', 'type']: - query_params['q.%s' % name].append(query.get(name, '')) - all_params = {} - all_params.update(params) - if q: - all_params.update(query_params) - if groupby: - all_params.update({'groupby': groupby}) - response = self.app.get(full_path, - params=all_params, - headers=headers, - extra_environ=extra_environ, - expect_errors=expect_errors, - status=status) - if not expect_errors: - response = response.json - return response diff --git a/ceilosca/ceilometer/tests/functional/api/v2/__init__.py b/ceilosca/ceilometer/tests/functional/api/v2/__init__.py deleted file mode 100644 index fc70f5e..0000000 --- a/ceilosca/ceilometer/tests/functional/api/v2/__init__.py +++ /dev/null @@ -1,20 +0,0 @@ -# -# Copyright 2012 New Dream Network, LLC (DreamHost) -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from ceilometer.tests.functional import api - - -class FunctionalTest(api.FunctionalTest): - PATH_PREFIX = '/v2' diff --git a/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py b/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py deleted file mode 100644 index 3c113ef..0000000 --- a/ceilosca/ceilometer/tests/functional/api/v2/test_api_with_monasca_driver.py +++ /dev/null @@ -1,270 +0,0 @@ -# -# Copyright 2015 Hewlett Packard -# (c) Copyright 2018 SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""Test api with Monasca driver -""" - -import fixtures -import mock -import pkg_resources - -from oslo_config import cfg -from stevedore import driver -from stevedore import extension - -from ceilometer import monasca_ceilometer_opts -from ceilometer import service -from ceilometer import storage -from ceilometer.tests import base as test_base -from oslo_policy import opts - -import pecan -import pecan.testing - -OPT_GROUP_NAME = 'keystone_authtoken' -cfg.CONF.import_group(OPT_GROUP_NAME, "keystonemiddleware.auth_token") - - -class TestApi(test_base.BaseTestCase): - - # TODO(Unresolved comment from git review: Can we include CM-api test - # cases for get_samples in - # ceilometer/tests/api/v2/test_api_with_monasca_driver.py?) - - def _get_driver_from_entry_point(self, entry_point, namespace): - ep = pkg_resources.EntryPoint.parse(entry_point) - a_driver = extension.Extension('con_driver', ep, - ep.load(require=False), None) - - mgr = driver.DriverManager.make_test_instance( - a_driver, namespace=namespace - ) - mgr._init_plugins([a_driver]) - return mgr - - def get_connection_with_mock_driver_manager(self, conf, url, namespace): - mgr = self._get_driver_from_entry_point( - entry_point='monasca = ceilometer.storage.impl_monasca:Connection', - namespace='ceilometer.metering.storage') - return mgr.driver(conf, url) - - def get_publisher_with_mock_driver_manager(self, conf, url, namespace): - mgr = self._get_driver_from_entry_point( - entry_point='monasca = ceilometer.publisher.monclient:' - 'MonascaPublisher', - namespace='ceilometer.sample.publisher') - return mgr.driver(conf, url) - - def setUp(self): - super(TestApi, self).setUp() - self.PATH_PREFIX = '/v2' - - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - self.setup_messaging(self.CONF) - opts.set_defaults(self.CONF) - self.CONF.set_override("policy_file", - self.path_get('etc/ceilometer/policy.json'), - group='oslo_policy') - - self.CONF.import_opt('pipeline_cfg_file', 'ceilometer.pipeline') - self.CONF.set_override( - 'pipeline_cfg_file', - self.path_get('etc/ceilometer/monasca_pipeline.yaml') - ) - - self.CONF.import_opt('monasca_mappings', - 'ceilometer.publisher.monasca_data_filter', - group='monasca') - - self.CONF.set_override( - 'monasca_mappings', - self.path_get('etc/ceilometer/monasca_field_definitions.yaml'), - group='monasca' - ) - - with mock.patch("ceilometer.monasca_client.Client") as mock_client,\ - mock.patch('ceilometer.storage.get_connection') as \ - get_storage_conn, \ - mock.patch('ceilometer.publisher.get_publisher') as get_pub: - - get_storage_conn.side_effect = ( - self.get_connection_with_mock_driver_manager) - get_pub.side_effect = self.get_publisher_with_mock_driver_manager - self.mock_mon_client = mock_client - self.conn = storage.get_connection( - self.CONF, - 'monasca://127.0.0.1:8080', - 'ceilometer.metering.storage') - - self.useFixture(fixtures.MockPatch( - 'ceilometer.storage.get_connection', - return_value=self.conn)) - - self.app = self._make_app() - - def _make_app(self, enable_acl=False): - self.config = { - 'app': { - 'root': 'ceilometer.api.controllers.root.RootController', - 'modules': ['ceilometer.api'], - 'enable_acl': enable_acl, - }, - 'wsme': { - 'debug': True, - }, - } - - return pecan.testing.load_test_app(self.config, - conf=self.CONF) - - def get_json(self, path, expect_errors=False, headers=None, - extra_environ=None, q=None, groupby=None, status=None, - override_params=None, **params): - """Sends simulated HTTP GET request to Pecan test app. - - :param path: url path of target service - :param expect_errors: boolean value whether an error is expected based - on request - :param headers: A dictionary of headers to send along with the request - :param extra_environ: A dictionary of environ variables to send along - with the request - :param q: list of queries consisting of: field, value, op, and type - keys - :param groupby: list of fields to group by - :param status: Expected status code of response - :param override_params: literally encoded query param string - :param params: content for wsgi.input of request - """ - - q = q or [] - groupby = groupby or [] - full_path = self.PATH_PREFIX + path - if override_params: - all_params = override_params - else: - query_params = {'q.field': [], - 'q.value': [], - 'q.op': [], - 'q.type': [], - } - for query in q: - for name in ['field', 'op', 'value', 'type']: - query_params['q.%s' % name].append(query.get(name, '')) - all_params = {} - all_params.update(params) - if q: - all_params.update(query_params) - if groupby: - all_params.update({'groupby': groupby}) - response = self.app.get(full_path, - params=all_params, - headers=headers, - extra_environ=extra_environ, - expect_errors=expect_errors, - status=status) - if not expect_errors: - response = response.json - return response - - -class TestListMeters(TestApi): - - def setUp(self): - super(TestListMeters, self).setUp() - - self.meter_payload = [{'name': 'm1', - 'dimensions': { - 'type': 'gauge', - 'unit': 'any', - 'resource_id': 'resource-1', - 'project_id': 'project-1', - 'user_id': 'user-1', - 'source': 'source'}}, - {'name': 'm2', - 'dimensions': { - 'type': 'delta', - 'unit': 'any', - 'resource_id': 'resource-1', - 'project_id': 'project-1', - 'user_id': 'user-1', - 'source': 'source'}}] - - def test_empty(self): - data = self.get_json('/meters') - self.assertEqual([], data) - - def test_get_meters(self): - - mnl_mock = self.mock_mon_client().metrics_list - mnl_mock.return_value = self.meter_payload - - data = self.get_json('/meters') - self.assertEqual(True, mnl_mock.called) - self.assertEqual(2, mnl_mock.call_count, - "impl_monasca.py calls the metrics_list api twice.") - self.assertEqual(2, len(data)) - - (self.assertIn(meter['name'], - [payload.get('name') for payload in - self.meter_payload]) for meter in data) - - def test_get_meters_query_with_project_resource(self): - """Test meter name conversion for project-id and resource-id. - - Previous versions of the monasca client did not do this conversion. - - Pre-Newton expected: - 'dimensions': {'project_id': u'project-1','resource_id': u'resource-1'} - - Newton expected: - 'dimensions': {'hostname': u'resource-1','project_id': u'project-1'} - """ - - mnl_mock = self.mock_mon_client().metrics_list - mnl_mock.return_value = self.meter_payload - - self.get_json('/meters', - q=[{'field': 'resource_id', - 'value': 'resource-1'}, - {'field': 'project_id', - 'value': 'project-1'}]) - self.assertEqual(True, mnl_mock.called) - self.assertEqual(4, mnl_mock.call_count, - "impl_monasca.py expected to make 4 calls to mock.") - # Note - previous versions of the api included a limit value - self.assertEqual(dict(dimensions=dict(hostname=u'resource-1', - project_id=u'project-1')), - mnl_mock.call_args[1]) - - def test_get_meters_query_with_user(self): - mnl_mock = self.mock_mon_client().metrics_list - mnl_mock.return_value = self.meter_payload - - self.get_json('/meters', - q=[{'field': 'user_id', - 'value': 'user-1'}]) - self.assertEqual(True, mnl_mock.called) - self.assertEqual(2, mnl_mock.call_count, - "impl_monasca.py calls the metrics_list api twice.") - # Note - previous versions of the api included a limit value - self.assertEqual(dict(dimensions=dict(user_id=u'user-1')), - mnl_mock.call_args[1]) - - # TODO(joadavis) Test a bad query parameter - # Like using 'hostname' instead of 'resource_id' - # Expected result with bad parameter: - # webtest.app.AppError: Bad response: 400 Bad Request diff --git a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/__init__.py b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py deleted file mode 100644 index 5ff857a..0000000 --- a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_ceilosca_mapping.py +++ /dev/null @@ -1,601 +0,0 @@ -# -# Copyright 2016 Hewlett Packard -# (c) Copyright 2018 SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections -import os - -import fixtures -import mock -from oslo_utils import fileutils -from oslo_utils import timeutils -from oslotest import base -import six -import yaml - -from ceilometer.ceilosca_mapping import ceilosca_mapping -from ceilometer.ceilosca_mapping.ceilosca_mapping import ( - CeiloscaMappingDefinition) -from ceilometer.ceilosca_mapping.ceilosca_mapping import ( - CeiloscaMappingDefinitionException) -from ceilometer.ceilosca_mapping.ceilosca_mapping import PipelineReader -from ceilometer import monasca_ceilometer_opts -from ceilometer import service -from ceilometer import storage -from ceilometer.storage import impl_monasca -from ceilometer.storage import models as storage_models - -MONASCA_MEASUREMENT = { - "id": "fef26f9d27f8027ea44b940cf3626fc398f7edfb", - "name": "fake_metric", - "dimensions": { - "resource_id": "2fe6e3a9-9bdf-4c98-882c-a826cf0107a1", - "cloud_name": "helion-poc-hlm-003", - "component": "vm", - "control_plane": "control-plane-1", - "service": "compute", - "device": "tap3356676e-a5", - "tenant_id": "50ce24dd577c43879cede72b77224e2f", - "hostname": "hlm003-cp1-comp0003-mgmt", - "cluster": "compute", - "zone": "nova" - }, - "columns": ["timestamp", "value", "value_meta"], - "measurements": [["2016-05-23T22:22:42.000Z", 54.0, { - "audit_period_ending": "None", - "audit_period_beginning": "None", - "host": "network.hlm003-cp1-c1-m2-mgmt", - "availability_zone": "None", - "event_type": "subnet.create.end", - "enable_dhcp": "true", - "gateway_ip": "10.43.0.1", - "ip_version": "4", - "cidr": "10.43.0.0/28"}]] -} - -MONASCA_VALUE_META = { - 'audit_period_beginning': 'None', - 'audit_period_ending': 'None', - 'availability_zone': 'None', - 'cidr': '10.43.0.0/28', - 'enable_dhcp': 'true', - 'event_type': 'subnet.create.end', - 'gateway_ip': '10.43.0.1', - 'host': 'network.hlm003-cp1-c1-m2-mgmt', - 'ip_version': '4' -} - - -class TestCeiloscaMapping(base.BaseTestCase): - pipeline_data = yaml.dump({ - 'sources': [{ - 'name': 'test_pipeline', - 'interval': 1, - 'meters': ['testbatch', 'testbatch2'], - 'resources': ['alpha', 'beta', 'gamma', 'delta'], - 'sinks': ['test_sink']}], - 'sinks': [{ - 'name': 'test_sink', - 'transformers': [], - 'publishers': ["test"]}] - }) - - cfg = yaml.dump({ - 'meter_metric_map': [{ - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric', - 'source': 'NA', - 'project_id': '$.dimensions.tenant_id', - 'type': 'gauge', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }, { - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter2', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric2', - 'source': 'NA', - 'project_id': '$.dimensions.project_id', - 'type': 'delta', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }, { - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter3', - 'resource_id': '$.dimensions.hostname', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric3', - 'source': 'NA', - 'project_id': '$.dimensions.project_id', - 'type': 'delta', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - } - ] - }) - - def setup_pipeline_file(self, pipeline_data): - if six.PY3: - pipeline_data = pipeline_data.encode('utf-8') - pipeline_cfg_file = fileutils.write_to_tempfile(content=pipeline_data, - prefix="pipeline", - suffix="yaml") - self.addCleanup(os.remove, pipeline_cfg_file) - return pipeline_cfg_file - - def setup_ceilosca_mapping_def_file(self, cfg): - if six.PY3: - cfg = cfg.encode('utf-8') - ceilosca_mapping_file = fileutils.write_to_tempfile( - content=cfg, prefix='ceilosca_mapping', suffix='yaml') - self.addCleanup(os.remove, ceilosca_mapping_file) - return ceilosca_mapping_file - - -class TestGetPipelineReader(TestCeiloscaMapping): - - def setUp(self): - super(TestGetPipelineReader, self).setUp() - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - - def test_pipeline_reader(self): - pipeline_cfg_file = self.setup_pipeline_file( - self.pipeline_data) - self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) - - test_pipeline_reader = PipelineReader(self.CONF) - - self.assertEqual(set(['testbatch', 'testbatch2']), - test_pipeline_reader.get_pipeline_meters() - ) - - -class TestMappingDefinition(base.BaseTestCase): - - def test_mapping_definition(self): - cfg = dict(name="network.outgoing.rate", - monasca_metric_name="vm.net.out_bytes_sec", - resource_id="$.dimensions.resource_id", - project_id="$.dimensions.tenant_id", - user_id="$.dimensions.user_id", - region="NA", - type="gauge", - unit="B/s", - source="NA", - resource_metadata="$.measurements[0][2]") - handler = CeiloscaMappingDefinition(cfg) - self.assertIsNone(handler.parse_fields("user_id", MONASCA_MEASUREMENT)) - self.assertEqual("2fe6e3a9-9bdf-4c98-882c-a826cf0107a1", - handler.parse_fields("resource_id", - MONASCA_MEASUREMENT)) - self.assertEqual("50ce24dd577c43879cede72b77224e2f", - handler.parse_fields("project_id", - MONASCA_MEASUREMENT)) - self.assertEqual(MONASCA_VALUE_META, - handler.parse_fields("resource_metadata", - MONASCA_MEASUREMENT)) - self.assertEqual("$.dimensions.tenant_id", handler.cfg["project_id"]) - - def test_config_required_missing_fields(self): - cfg = dict() - try: - CeiloscaMappingDefinition(cfg) - except CeiloscaMappingDefinitionException as e: - self.assertEqual("Required fields [" - "'name', 'monasca_metric_name', 'type', 'unit', " - "'source', 'resource_metadata', 'resource_id', " - "'project_id', 'user_id', 'region'] " - "not specified", e.message) - - def test_bad_type_cfg_definition(self): - cfg = dict(name="fake_meter", - monasca_metric_name="fake_metric", - resource_id="$.dimensions.resource_id", - project_id="$.dimensions.tenant_id", - user_id="$.dimensions.user_id", - region="NA", - type="foo", - unit="B/s", - source="NA", - resource_metadata="$.measurements[0][2]") - try: - CeiloscaMappingDefinition(cfg) - except CeiloscaMappingDefinitionException as e: - self.assertEqual("Invalid type foo specified", e.message) - - -class TestMappedCeiloscaMetricProcessing(TestCeiloscaMapping): - - def setUp(self): - super(TestMappedCeiloscaMetricProcessing, self).setUp() - - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - - def test_fallback_mapping_file_path(self): - self.useFixture(fixtures.MockPatchObject(self.CONF, - 'find_file', - return_value=None)) - fall_bak_path = ceilosca_mapping.get_config_file(self.CONF) - self.assertIn("ceilosca_mapping/data/ceilosca_mapping.yaml", - fall_bak_path) - - @mock.patch('ceilometer.ceilosca_mapping.ceilosca_mapping.LOG') - def test_bad_meter_definition_skip(self, LOG): - cfg = yaml.dump({ - 'meter_metric_map': [{ - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric', - 'source': 'NA', - 'project_id': '$.dimensions.tenant_id', - 'type': 'gauge', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }, { - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric', - 'source': 'NA', - 'project_id': '$.dimensions.tenant_id', - 'type': 'foo', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }] - }) - ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) - self.CONF.set_override('ceilometer_monasca_metrics_mapping', - ceilosca_mapping_file, group='monasca') - data = ceilosca_mapping.setup_ceilosca_mapping_config(self.CONF) - meter_loaded = ceilosca_mapping.load_definitions(data) - self.assertEqual(1, len(meter_loaded)) - LOG.error.assert_called_with( - "Error loading Ceilometer Monasca Mapping Definition : " - "Invalid type foo specified") - - def test_list_of_meters_returned(self): - ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) - self.CONF.set_override('ceilometer_monasca_metrics_mapping', - ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping\ - .ProcessMappedCeiloscaMetric(self.CONF) - ceilosca_mapper.reinitialize(self.CONF) - self.assertItemsEqual(['fake_metric', 'fake_metric2', 'fake_metric3'], - ceilosca_mapper.get_list_monasca_metrics().keys() - ) - - def test_monasca_metric_name_map_ceilometer_meter(self): - cfg = yaml.dump({ - 'meter_metric_map': [{ - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric', - 'source': 'NA', - 'project_id': '$.dimensions.tenant_id', - 'type': 'gauge', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }] - }) - ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(cfg) - self.CONF.set_override('ceilometer_monasca_metrics_mapping', - ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping\ - .ProcessMappedCeiloscaMetric(self.CONF) - ceilosca_mapper.reinitialize(self.CONF) - self.assertEqual('fake_metric', - ceilosca_mapper.get_monasca_metric_name('fake_meter') - ) - self.assertEqual('$.dimensions.tenant_id', - ceilosca_mapper. - get_ceilosca_mapped_definition_key_val('fake_metric', - 'project_id')) - - -# This Class will only test the driver for the mapped meteric -# Impl_Monasca Tests will be doing exhaustive tests for non mapped metrics -@mock.patch("ceilometer.storage.impl_monasca.MonascaDataFilter") -class TestMoanscaDriverForMappedMetrics(TestCeiloscaMapping): - Aggregate = collections.namedtuple("Aggregate", ['func', 'param']) - - def setUp(self): - - super(TestMoanscaDriverForMappedMetrics, self).setUp() - - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - - pipeline_cfg_file = self.setup_pipeline_file(self.pipeline_data) - self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) - - ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file(self.cfg) - self.CONF.set_override('ceilometer_monasca_metrics_mapping', - ceilosca_mapping_file, group='monasca') - - ceilosca_mapper = ceilosca_mapping\ - .ProcessMappedCeiloscaMetric(self.CONF) - ceilosca_mapper.reinitialize(self.CONF) - - def test_get_samples_for_mapped_meters(self, mdf_mock): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - ml_mock = mock_client().measurements_list - # TODO(this test case needs more work) - ml_mock.return_value = ([MONASCA_MEASUREMENT]) - - sample_filter = storage.SampleFilter( - meter='fake_meter', - start_timestamp='2015-03-20T00:00:00Z') - results = list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual('fake_meter', results[0].counter_name) - self.assertEqual(54.0, results[0].counter_volume) - self.assertEqual('gauge', results[0].counter_type) - self.assertEqual('2fe6e3a9-9bdf-4c98-882c-a826cf0107a1', - results[0].resource_id - ) - self.assertEqual(MONASCA_VALUE_META, results[0].resource_metadata) - self.assertEqual('50ce24dd577c43879cede72b77224e2f', - results[0].project_id, - ) - self.assertEqual('B/s', results[0].counter_unit) - self.assertIsNone(results[0].user_id) - - def test_get_meter_for_mapped_meters_non_uniq(self, mdf_mock): - data1 = ( - [{u'dimensions': {u'datasource': u'ceilometer'}, - u'id': u'2015-04-14T18:42:31Z', - u'name': u'meter-1'}, - {u'dimensions': {u'datasource': u'ceilometer'}, - u'id': u'2015-04-15T18:42:31Z', - u'name': u'meter-1'}]) - data2 = ( - [{u'dimensions': {u'datasource': u'ceilometer'}, - u'id': u'2015-04-14T18:42:31Z', - u'name': u'meter-1'}, - {u'dimensions': {u'datasource': u'ceilometer'}, - u'id': u'2015-04-15T18:42:31Z', - u'name': u'meter-1'}, - {u'id': u'fef26f9d27f8027ea44b940cf3626fc398f7edfb', - u'name': u'fake_metric', - u'dimensions': { - u'resource_id': u'2fe6e3a9-9bdf-4c98-882c-a826cf0107a1', - u'cloud_name': u'helion-poc-hlm-003', - u'component': u'vm', - u'control_plane': u'control-plane-1', - u'service': u'compute', - u'device': u'tap3356676e-a5', - u'tenant_id': u'50ce24dd577c43879cede72b77224e2f', - u'hostname': u'hlm003-cp1-comp0003-mgmt', - u'cluster': u'compute', - u'zone': u'nova'} - }, - {u'dimensions': {}, - u'id': u'2015-04-16T18:42:31Z', - u'name': u'testbatch'}]) - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.side_effect = [data1, data2] - kwargs = dict(limit=4) - - results = list(conn.get_meters(**kwargs)) - # result contains 2 records from data 1 since datasource - # = ceilometer, 2 records from data 2, 1 for pipeline - # meter but no datasource set to ceilometer and one for - # mapped meter - self.assertEqual(4, len(results)) - self.assertEqual(True, metrics_list_mock.called) - self.assertEqual(2, metrics_list_mock.call_count) - - def test_get_meter_for_mapped_meters_uniq(self, mdf_mock): - dummy_metric_names_mocked_return_value = ( - [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", - "name": "network.delete"}, - {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "subnet.update"}]) - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metric_names_list_mock = mock_client().metric_names_list - metric_names_list_mock.return_value = ( - dummy_metric_names_mocked_return_value) - kwargs = dict(limit=4, unique=True) - results = list(conn.get_meters(**kwargs)) - self.assertEqual(2, len(results)) - self.assertEqual(True, metric_names_list_mock.called) - self.assertEqual(1, metric_names_list_mock.call_count) - - def test_stats_list_mapped_meters(self, mock_mdf): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - sl_mock = mock_client().statistics_list - sl_mock.return_value = [ - { - 'statistics': - [ - ['2014-10-24T12:12:12Z', 0.008], - ['2014-10-24T12:52:12Z', 0.018] - ], - 'dimensions': {'unit': 'gb'}, - 'columns': ['timestamp', 'min'] - } - ] - - sf = storage.SampleFilter() - sf.meter = "fake_meter" - aggregate = self.Aggregate(func="min", param=None) - sf.start_timestamp = timeutils.parse_isotime( - '2014-10-24T12:12:42').replace(tzinfo=None) - stats = list(conn.get_meter_statistics(sf, aggregate=[aggregate], - period=30)) - self.assertEqual(2, len(stats)) - self.assertEqual('B/s', stats[0].unit) - self.assertEqual('B/s', stats[1].unit) - self.assertEqual(0.008, stats[0].min) - self.assertEqual(0.018, stats[1].min) - self.assertEqual(30, stats[0].period) - self.assertEqual('2014-10-24T12:12:42', - stats[0].period_end.isoformat()) - self.assertEqual('2014-10-24T12:52:42', - stats[1].period_end.isoformat()) - self.assertIsNotNone(stats[0].as_dict().get('aggregate')) - self.assertEqual({u'min': 0.008}, stats[0].as_dict()['aggregate']) - - def test_get_resources_for_mapped_meters(self, mock_mdf): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - dummy_metric_names_mocked_return_value = ( - [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", - "name": "fake_metric"}, - {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "metric1"}]) - mnl_mock = mock_client().metric_names_list - mnl_mock.return_value = ( - dummy_metric_names_mocked_return_value) - - dummy_get_resources_mocked_return_value = ( - [{u'dimensions': {u'resource_id': u'abcd'}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], - [u'2015-04-15T17:52:31Z', 2.0, {}], - [u'2015-04-16T17:52:31Z', 3.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'fake_metric'}]) - - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - dummy_get_resources_mocked_return_value) - - sample_filter = storage.SampleFilter( - meter='fake_meter', end_timestamp='2015-04-20T00:00:00Z') - resources = list(conn.get_resources(sample_filter, limit=2)) - self.assertEqual(2, len(resources)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - resources_without_limit = list(conn.get_resources(sample_filter)) - self.assertEqual(3, len(resources_without_limit)) - - def test_stats_list_with_groupby_for_mapped_meters(self, mock_mdf): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - sl_mock = mock_client().statistics_list - sl_mock.return_value = [ - { - 'statistics': - [ - ['2014-10-24T12:12:12Z', 0.008, 1.3, 3, 0.34], - ['2014-10-24T12:20:12Z', 0.078, 1.25, 2, 0.21], - ['2014-10-24T12:52:12Z', 0.018, 0.9, 4, 0.14] - ], - 'dimensions': {'hostname': '1234', 'unit': 'gb'}, - 'columns': ['timestamp', 'min', 'max', 'count', 'avg'] - }, - { - 'statistics': - [ - ['2014-10-24T12:14:12Z', 0.45, 2.5, 2, 2.1], - ['2014-10-24T12:20:12Z', 0.58, 3.2, 3, 3.4], - ['2014-10-24T13:52:42Z', 1.67, 3.5, 1, 5.3] - ], - 'dimensions': {'hostname': '5678', 'unit': 'gb'}, - 'columns': ['timestamp', 'min', 'max', 'count', 'avg'] - }] - - sf = storage.SampleFilter() - sf.meter = "fake_meter3" - sf.start_timestamp = timeutils.parse_isotime( - '2014-10-24T12:12:42').replace(tzinfo=None) - groupby = ['resource_id'] - stats = list(conn.get_meter_statistics(sf, period=30, - groupby=groupby)) - - self.assertEqual(2, len(stats)) - - for stat in stats: - self.assertIsNotNone(stat.groupby) - resource_id = stat.groupby.get('resource_id') - self.assertIn(resource_id, ['1234', '5678']) - if resource_id == '1234': - self.assertEqual(0.008, stat.min) - self.assertEqual(1.3, stat.max) - self.assertEqual(0.23, stat.avg) - self.assertEqual(9, stat.count) - self.assertEqual(30, stat.period) - self.assertEqual('2014-10-24T12:12:12', - stat.period_start.isoformat()) - if resource_id == '5678': - self.assertEqual(0.45, stat.min) - self.assertEqual(3.5, stat.max) - self.assertEqual(3.6, stat.avg) - self.assertEqual(6, stat.count) - self.assertEqual(30, stat.period) - self.assertEqual('2014-10-24T13:52:42', - stat.period_end.isoformat()) - - def test_query_samples_for_mapped_meter(self, mock_mdf): - SAMPLES = [[ - storage_models.Sample( - counter_name="fake_meter", - counter_type="gauge", - counter_unit="instance", - counter_volume=1, - project_id="123", - user_id="456", - resource_id="789", - resource_metadata={}, - source="openstack", - recorded_at=timeutils.utcnow(), - timestamp=timeutils.utcnow(), - message_id="0", - message_signature='', ) - ]] * 2 - samples = SAMPLES[:] - - def _get_samples(*args, **kwargs): - return samples.pop() - - with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - with mock.patch.object(conn, 'get_samples') as gsm: - gsm.side_effect = _get_samples - - query = {'and': [{'=': {'counter_name': 'fake_meter'}}, - {'or': [{'=': {"project_id": "123"}}, - {'=': {"user_id": "456"}}]}]} - samples = conn.query_samples(query, None, 100) - self.assertEqual(2, len(samples)) - self.assertEqual(2, gsm.call_count) - - samples = SAMPLES[:] - query = {'and': [{'=': {'counter_name': 'fake_meter'}}, - {'or': [{'=': {"project_id": "123"}}, - {'>': {"counter_volume": 2}}]}]} - samples = conn.query_samples(query, None, 100) - self.assertEqual(1, len(samples)) - self.assertEqual(4, gsm.call_count) diff --git a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py b/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py deleted file mode 100644 index 3f7c998..0000000 --- a/ceilosca/ceilometer/tests/unit/ceilosca_mapping/test_static_ceilometer_mapping.py +++ /dev/null @@ -1,286 +0,0 @@ -# -# Copyright 2016 Hewlett Packard -# (c) Copyright 2018 SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import os - -import fixtures -import mock -# from oslo_config import fixture as fixture_config -from oslo_utils import fileutils -from oslotest import base -import six -import yaml - -from ceilometer.ceilosca_mapping import ceilometer_static_info_mapping -from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( - CeilometerStaticMappingDefinition) -from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( - CeilometerStaticMappingDefinitionException) -from ceilometer import monasca_ceilometer_opts -from ceilometer import service -from ceilometer.storage import impl_monasca - - -class TestStaticInfoBase(base.BaseTestCase): - pipeline_data = yaml.dump({ - 'sources': [{ - 'name': 'test_pipeline', - 'interval': 1, - 'meters': ['testbatch', 'testbatch2'], - 'resources': ['alpha', 'beta', 'gamma', 'delta'], - 'sinks': ['test_sink']}], - 'sinks': [{ - 'name': 'test_sink', - 'transformers': [], - 'publishers': ["test"]}] - }) - - cfg = yaml.dump({ - 'meter_info_static_map': [{ - 'name': "disk.ephemeral.size", - 'type': "gauge", - 'unit': "GB" - }, { - 'name': "image.delete", - 'type': "delta", - 'unit': "image" - }, { - 'name': "image", - 'type': "gauge", - 'unit': "image" - }, { - 'name': "disk.root.size", - 'type': "gauge", - 'unit': "GB" - } - ] - }) - ceilosca_cfg = yaml.dump({ - 'meter_metric_map': [{ - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric', - 'source': 'NA', - 'project_id': '$.dimensions.tenant_id', - 'type': 'gauge', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }, { - 'user_id': '$.dimensions.user_id', - 'name': 'fake_meter2', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'fake_metric2', - 'source': 'NA', - 'project_id': '$.dimensions.project_id', - 'type': 'delta', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }] - }) - - def setup_static_mapping_def_file(self, cfg): - if six.PY3: - cfg = cfg.encode('utf-8') - ceilometer_static_info_mapping = fileutils.write_to_tempfile( - content=cfg, prefix='ceilometer_static_info_mapping', suffix='yaml' - ) - self.addCleanup(os.remove, ceilometer_static_info_mapping) - return ceilometer_static_info_mapping - - def setup_ceilosca_mapping_def_file(self, ceilosca_cfg): - if six.PY3: - ceilosca_cfg = ceilosca_cfg.encode('utf-8') - ceilosca_mapping_file = fileutils.write_to_tempfile( - content=ceilosca_cfg, prefix='ceilosca_mapping', suffix='yaml') - self.addCleanup(os.remove, ceilosca_mapping_file) - return ceilosca_mapping_file - - def setup_pipeline_file(self, pipeline_data): - if six.PY3: - pipeline_data = pipeline_data.encode('utf-8') - pipeline_cfg_file = fileutils.write_to_tempfile(content=pipeline_data, - prefix="pipeline", - suffix="yaml") - self.addCleanup(os.remove, pipeline_cfg_file) - return pipeline_cfg_file - - -class TestStaticInfoDefinition(base.BaseTestCase): - - def test_static_info_definition(self): - cfg = dict(name="image.delete", - type="delta", - unit="image") - handler = CeilometerStaticMappingDefinition(cfg) - self.assertEqual("delta", handler.cfg['type']) - self.assertEqual("image.delete", handler.cfg['name']) - self.assertEqual("image", handler.cfg['unit']) - - def test_config_required_missing_fields(self): - cfg = dict() - try: - CeilometerStaticMappingDefinition(cfg) - except CeilometerStaticMappingDefinitionException as e: - self.assertEqual("Required fields [" - "'name', 'type', 'unit'] " - "not specified", e.message) - - def test_bad_type_cfg_definition(self): - cfg = dict(name="fake_meter", - type="foo", - unit="B/s") - try: - CeilometerStaticMappingDefinition(cfg) - except CeilometerStaticMappingDefinitionException as e: - self.assertEqual("Invalid type foo specified", e.message) - - -class TestMappedCeilometerStaticInfoProcessing(TestStaticInfoBase): - - def setUp(self): - super(TestMappedCeilometerStaticInfoProcessing, self).setUp() - # self.CONF = self.useFixture(fixture_config.Config()).conf - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - static_info_mapping_file = self.setup_static_mapping_def_file(self.cfg) - self.CONF.set_override('ceilometer_static_info_mapping', - static_info_mapping_file, group='monasca') - self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo(self.CONF) - # self.CONF([], project='ceilometer', validate_default_values=True) - - def test_fallback_mapping_file_path(self): - self.useFixture(fixtures.MockPatchObject(self.CONF, - 'find_file', - return_value=None)) - self.CONF.set_override('ceilometer_static_info_mapping', - ' ', group='monasca') - self.static_info_mapper.reinitialize(self.CONF) - fall_bak_path = ceilometer_static_info_mapping.get_config_file( - self.CONF) - self.assertIn( - "ceilosca_mapping/data/ceilometer_static_info_mapping.yaml", - fall_bak_path) - - @mock.patch( - 'ceilometer.ceilosca_mapping.ceilometer_static_info_mapping.LOG') - def test_bad_mapping_definition_skip(self, LOG): - cfg = yaml.dump({ - 'meter_info_static_map': [{ - 'name': "disk.ephemeral.size", - 'type': "gauge", - 'unit': "GB" - }, { - 'name': "image.delete", - 'type': "delta", - 'unit': "image" - }, { - 'name': "image", - 'type': "gauge", - 'unit': "image" - }, { - 'name': "disk.root.size", - 'type': "foo", - 'unit': "GB" - }] - }) - static_info_mapping_file = self.setup_static_mapping_def_file(cfg) - self.CONF.set_override('ceilometer_static_info_mapping', - static_info_mapping_file, group='monasca') - data = ceilometer_static_info_mapping.\ - setup_ceilometer_static_mapping_config(self.CONF) - meter_loaded = ceilometer_static_info_mapping.load_definitions(data) - self.assertEqual(3, len(meter_loaded)) - LOG.error.assert_called_with( - "Error loading Ceilometer Static Mapping Definition : " - "Invalid type foo specified") - - def test_list_of_meters_returned(self): - self.static_info_mapper.reinitialize(self.CONF) - self.assertItemsEqual(['disk.ephemeral.size', 'disk.root.size', - 'image', 'image.delete'], - self.static_info_mapper. - get_list_supported_meters(). - keys() - ) - - def test_static_info_of_ceilometer_meter(self): - cfg = yaml.dump({ - 'meter_info_static_map': [{ - 'name': "disk.ephemeral.size", - 'type': "gauge", - 'unit': "GB" - }] - }) - static_info_mapping_file = self.setup_static_mapping_def_file(cfg) - self.CONF.set_override('ceilometer_static_info_mapping', - static_info_mapping_file, group='monasca') - self.static_info_mapper.reinitialize(self.CONF) - self.assertEqual('gauge', - self.static_info_mapper.get_meter_static_info_key_val( - 'disk.ephemeral.size', 'type') - ) - - -# This Class will only test the driver for the mapped static info -# Impl_Monasca Tests will be doing exhaustive tests for other test cases -@mock.patch("ceilometer.storage.impl_monasca.MonascaDataFilter") -class TestMoanscaDriverForMappedStaticInfo(TestStaticInfoBase): - - def setUp(self): - super(TestMoanscaDriverForMappedStaticInfo, self).setUp() - # self.CONF = self.useFixture(fixture_config.Config()).conf - # self.CONF([], project='ceilometer', validate_default_values=True) - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - pipeline_cfg_file = self.setup_pipeline_file(self.pipeline_data) - self.CONF.set_override("pipeline_cfg_file", pipeline_cfg_file) - static_info_mapping_file = self.setup_static_mapping_def_file(self.cfg) - self.CONF.set_override('ceilometer_static_info_mapping', - static_info_mapping_file, group='monasca') - ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file( - self.ceilosca_cfg) - self.CONF.set_override('ceilometer_monasca_metrics_mapping', - ceilosca_mapping_file, group='monasca') - self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo(self.CONF) - self.static_info_mapper.reinitialize(self.CONF) - - def test_get_statc_info_for_mapped_meters_uniq(self, mdf_mock): - dummy_metric_names_mocked_return_value = ( - [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", - "name": "image"}, - {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "fake_metric"}]) - - with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') - metric_names_list_mock = mock_client().metric_names_list - metric_names_list_mock.return_value = ( - dummy_metric_names_mocked_return_value - ) - - kwargs = dict(limit=4, - unique=True) - results = list(conn.get_meters(**kwargs)) - self.assertEqual(2, len(results)) - self.assertEqual(True, metric_names_list_mock.called) - self.assertEqual(1, metric_names_list_mock.call_count) diff --git a/ceilosca/ceilometer/tests/unit/storage/__init__.py b/ceilosca/ceilometer/tests/unit/storage/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py b/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py deleted file mode 100644 index ce59c7f..0000000 --- a/ceilosca/ceilometer/tests/unit/storage/test_impl_monasca.py +++ /dev/null @@ -1,1077 +0,0 @@ -# -# Copyright 2015 Hewlett Packard -# (c) Copyright 2018 SUSE LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import collections -import datetime -import os - -import dateutil.parser -import mock -from oslo_utils import fileutils -from oslo_utils import timeutils -from oslotest import base -import six -import yaml - -import ceilometer -from ceilometer.api.controllers.v2.meters import Aggregate -from ceilometer.ceilosca_mapping import ceilometer_static_info_mapping -from ceilometer.ceilosca_mapping import ceilosca_mapping -from ceilometer import monasca_ceilometer_opts -from ceilometer import service -from ceilometer import storage -from ceilometer.storage import impl_monasca -from ceilometer.storage import models as storage_models - - -class _BaseTestCase(base.BaseTestCase): - - def setUp(self): - super(_BaseTestCase, self).setUp() - self.CONF = service.prepare_service([], []) - self.CONF.register_opts(list(monasca_ceilometer_opts.OPTS), - 'monasca') - self.CONF.set_override('service_username', 'ceilometer', 'monasca') - self.CONF.set_override('service_password', 'admin', 'monasca') - self.CONF.set_override('service_auth_url', - 'http://localhost:5000/v2.0', - 'monasca') - mdf = mock.patch.object(impl_monasca, 'MonascaDataFilter') - mdf.start() - self.addCleanup(mdf.stop) - spl = mock.patch('ceilometer.pipeline.setup_pipeline') - spl.start() - self.addCleanup(spl.stop) - self.static_info_mapper = ceilometer_static_info_mapping\ - .ProcessMappedCeilometerStaticInfo(self.CONF) - self.static_info_mapper.reinitialize(self.CONF) - - def assertRaisesWithMessage(self, msg, exc_class, func, *args, **kwargs): - try: - func(*args, **kwargs) - self.fail('Expecting %s exception, none raised' % - exc_class.__name__) - except AssertionError: - raise - # Only catch specific exception so we can get stack trace when fail - except exc_class as e: - self.assertEqual(msg, e.message) - - def assert_raise_within_message(self, msg, e_cls, func, *args, **kwargs): - try: - func(*args, **kwargs) - self.fail('Expecting %s exception, none raised' % - e_cls.__name__) - except AssertionError: - raise - # Only catch specific exception so we can get stack trace when fail - except e_cls as e: - self.assertIn(msg, '%s' % e) - - -class TestGetResources(_BaseTestCase): - - dummy_get_resources_mocked_return_value = ( - [{u'dimensions': {}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'image'}]) - - cfg = yaml.dump({ - 'meter_metric_map': [{ - 'user_id': '$.dimensions.user_id', - 'name': 'network.incoming.rate', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'vm.net.in_rate', - 'source': 'NA', - 'project_id': '$.dimensions.tenant_id', - 'type': 'gauge', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }, { - 'user_id': '$.dimensions.user_id', - 'name': 'network.outgoing.rate', - 'resource_id': '$.dimensions.resource_id', - 'region': 'NA', - 'monasca_metric_name': 'vm.net.out_rate', - 'source': 'NA', - 'project_id': '$.dimensions.project_id', - 'type': 'delta', - 'resource_metadata': '$.measurements[0][2]', - 'unit': 'B/s' - }] - }) - - def setup_ceilosca_mapping_def_file(self, cfg): - if six.PY3: - cfg = cfg.encode('utf-8') - ceilosca_mapping_file = fileutils.write_to_tempfile( - content=cfg, prefix='ceilosca_mapping', suffix='yaml') - self.addCleanup(os.remove, ceilosca_mapping_file) - return ceilosca_mapping_file - - def setUp(self): - super(TestGetResources, self).setUp() - ceilosca_mapping_file = self.setup_ceilosca_mapping_def_file( - TestGetResources.cfg) - self.CONF.set_override('ceilometer_monasca_metrics_mapping', - ceilosca_mapping_file, group='monasca') - ceilosca_mapper = ceilosca_mapping\ - .ProcessMappedCeiloscaMetric(self.CONF) - ceilosca_mapper.reinitialize(self.CONF) - - def test_not_implemented_params(self): - with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - - kwargs = dict(start_timestamp_op='le') - self.assertRaises(ceilometer.NotImplementedError, - lambda: list(conn.get_resources(**kwargs))) - kwargs = dict(end_timestamp_op='ge') - self.assertRaises(ceilometer.NotImplementedError, - lambda: list(conn.get_resources(**kwargs))) - - def test_dims_filter(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - mnl_mock = mock_client().metric_names_list - mnl_mock.return_value = [ - { - "id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "some" - } - ] - end_time = datetime.datetime(2015, 4, 1, 12, 00, 00) - kwargs = dict(project='proj1', - end_timestamp=end_time) - list(conn.get_resources(**kwargs)) - self.assertEqual(True, mnl_mock.called) - - expected = [ - mock.call( - dimensions={ - 'project_id': 'proj1'}), - mock.call( - dimensions={ - 'tenant_id': 'proj1'}) - ] - self.assertTrue(expected == mnl_mock.call_args_list) - self.assertEqual(2, mnl_mock.call_count) - - @mock.patch('oslo_utils.timeutils.utcnow') - def test_get_resources(self, mock_utcnow): - mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 20) - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - mnl_mock = mock_client().metric_names_list - mnl_mock.return_value = [ - { - "id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "storage.objects.size" - }, - { - "id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "vm.net.in_rate" - } - ] - - kwargs = dict(source='openstack') - ml_mock = mock_client().measurements_list - data1 = ( - [{u'dimensions': {u'resource_id': u'abcd', - u'datasource': u'ceilometer'}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], - [u'2015-04-15T17:52:31Z', 2.0, {}], - [u'2015-04-16T17:52:31Z', 3.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'storage.objects.size'}]) - - data2 = ( - [{u'dimensions': {u'resource_id': u'abcd', - u'datasource': u'ceilometer'}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], - [u'2015-04-15T17:52:31Z', 2.0, {}], - [u'2015-04-16T17:52:31Z', 3.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'vm.net.in_rate'}]) - ml_mock.side_effect = [data1, data2] - list(conn.get_resources(**kwargs)) - self.assertEqual(2, ml_mock.call_count) - self.assertEqual(dict(dimensions=dict(datasource='ceilometer', - source='openstack'), - name='storage.objects.size', - start_time='1970-01-01T00:00:00.000000Z', - group_by='*', - end_time='2016-04-07T18:20:00.000000Z'), - ml_mock.call_args_list[0][1]) - - def test_get_resources_limit(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - - mnl_mock = mock_client().metric_names_list - mnl_mock.return_value = [ - { - "id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "storage.objects.size" - }, - { - "id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "vm.net.in_rate" - } - ] - dummy_get_resources_mocked_return_value = ( - [{u'dimensions': {u'resource_id': u'abcd', - u'datasource': u'ceilometer'}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], - [u'2015-04-15T17:52:31Z', 2.0, {}], - [u'2015-04-16T17:52:31Z', 3.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'image'}]) - - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - dummy_get_resources_mocked_return_value) - - sample_filter = storage.SampleFilter( - meter='specific meter', end_timestamp='2015-04-20T00:00:00Z') - resources = list(conn.get_resources(sample_filter, limit=2)) - self.assertEqual(2, len(resources)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - @mock.patch('oslo_utils.timeutils.utcnow') - def test_get_resources_simple_metaquery(self, mock_utcnow): - mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 28) - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - mnl_mock = mock_client().metric_names_list - mnl_mock.return_value = [ - { - "id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "storage.objects.size" - }, - { - "id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "vm.net.in_rate" - } - ] - kwargs = dict(metaquery={'metadata.key': 'value1'}) - - ml_mock = mock_client().measurements_list - data1 = ( - [{u'dimensions': {u'resource_id': u'abcd', - u'datasource': u'ceilometer'}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], - [u'2015-04-15T17:52:31Z', 2.0, {}], - [u'2015-04-16T17:52:31Z', 3.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'storage.objects.size'}]) - - data2 = ( - [{u'dimensions': {u'resource_id': u'abcd', - u'datasource': u'ceilometer'}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], - [u'2015-04-15T17:52:31Z', 2.0, {}], - [u'2015-04-16T17:52:31Z', 3.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'vm.net.in_rate'}]) - - ml_mock.side_effect = [data1, data2] - list(conn.get_resources(**kwargs)) - - self.assertEqual(2, ml_mock.call_count) - self.assertEqual(dict(dimensions=dict(datasource='ceilometer'), - name="storage.objects.size", - start_time='1970-01-01T00:00:00.000000Z', - group_by='*', - end_time='2016-04-07T18:28:00.000000Z'), - ml_mock.call_args_list[0][1]) - - -class MeterTest(_BaseTestCase): - - dummy_metrics_mocked_return_value = ( - [{u'dimensions': {u'datasource': u'ceilometer'}, - u'id': u'2015-04-14T18:42:31Z', - u'name': u'meter-1'}, - {u'dimensions': {u'datasource': u'ceilometer'}, - u'id': u'2015-04-15T18:42:31Z', - u'name': u'meter-1'}, - {u'dimensions': {u'datasource': u'ceilometer'}, - u'id': u'2015-04-16T18:42:31Z', - u'name': u'meter-2'}]) - - def test_not_implemented_params(self): - with mock.patch('ceilometer.monasca_client.Client'): - conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') - - kwargs = dict(metaquery=True) - self.assertRaises(ceilometer.NotImplementedError, - lambda: list(conn.get_meters(**kwargs))) - - def test_metrics_list_call(self): - with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') - metrics_list_mock = mock_client().metrics_list - - kwargs = dict(user='user-1', - project='project-1', - resource='resource-1', - source='openstack', - limit=100) - list(conn.get_meters(**kwargs)) - - self.assertEqual(True, metrics_list_mock.called) - self.assertEqual(4, metrics_list_mock.call_count) - expected = [ - mock.call( - dimensions={ - 'source': 'openstack', - 'project_id': 'project-1', - 'user_id': 'user-1', - 'datasource': 'ceilometer', - 'resource_id': 'resource-1'}), - mock.call( - dimensions={ - 'source': 'openstack', - 'project_id': 'project-1', - 'user_id': 'user-1', - 'resource_id': 'resource-1'}), - mock.call( - dimensions={ - 'source': 'openstack', - 'tenant_id': 'project-1', - 'user_id': 'user-1', - 'resource_id': 'resource-1'}), - mock.call( - dimensions={ - 'source': 'openstack', - 'project_id': 'project-1', - 'user_id': 'user-1', - 'hostname': 'resource-1'}) - ] - self.assertTrue(expected == metrics_list_mock.call_args_list) - - def test_unique_metrics_list_call(self): - dummy_metric_names_mocked_return_value = ( - [{"id": "015c995b1a770147f4ef18f5841ef566ab33521d", - "name": "network.delete"}, - {"id": "335b5d569ad29dc61b3dc24609fad3619e947944", - "name": "subnet.update"}]) - with mock.patch('ceilometer.monasca_client.Client') as mock_client: - conn = impl_monasca.Connection(self.CONF, '127.0.0.1:8080') - metric_names_list_mock = mock_client().metric_names_list - metric_names_list_mock.return_value = ( - dummy_metric_names_mocked_return_value - ) - kwargs = dict(user='user-1', - project='project-1', - resource='resource-1', - source='openstack', - limit=2, - unique=True) - - self.assertEqual(2, len(list(conn.get_meters(**kwargs)))) - - self.assertEqual(True, metric_names_list_mock.called) - self.assertEqual(1, metric_names_list_mock.call_count) - self.assertEqual(dict(dimensions=dict(user_id='user-1', - project_id='project-1', - resource_id='resource-1', - source='openstack')), - metric_names_list_mock.call_args[1]) - - -class TestGetSamples(_BaseTestCase): - - dummy_get_samples_mocked_return_value = ( - [{u'dimensions': {}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'image'}]) - - dummy_metrics_mocked_return_value = ( - [{u'dimensions': {}, - u'id': u'2015-04-14T18:42:31Z', - u'name': u'specific meter'}]) - - dummy_get_samples_mocked_return_extendedkey_value = ( - [{u'dimensions': {}, - u'measurements': [[u'2015-04-14T17:52:31Z', - 1.0, - {'image_meta.base_url': 'base_url'}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'image'}]) - - def test_get_samples_not_implemented_params(self): - with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - - sample_filter = storage.SampleFilter(meter='specific meter', - message_id='specific message') - self.assertRaises(ceilometer.NotImplementedError, - lambda: list(conn.get_samples(sample_filter))) - - def test_get_samples_name(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - TestGetSamples.dummy_metrics_mocked_return_value - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples.dummy_get_samples_mocked_return_value) - sample_filter = storage.SampleFilter( - meter='specific meter', end_timestamp='2015-04-20T00:00:00Z') - list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(dict( - dimensions=dict(datasource='ceilometer'), - start_time='1970-01-01T00:00:00.000000Z', - group_by='*', name='specific meter', - end_time='2015-04-20T00:00:00.000000Z'), - ml_mock.call_args[1]) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_start_timestamp_filter(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - TestGetSamples.dummy_metrics_mocked_return_value - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples.dummy_get_samples_mocked_return_value) - - start_time = datetime.datetime(2015, 3, 20) - - sample_filter = storage.SampleFilter( - meter='specific meter', - start_timestamp=timeutils.isotime(start_time), - start_timestamp_op='ge') - list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_timestamp_filter_exclusive_range(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - TestGetSamples.dummy_metrics_mocked_return_value - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples.dummy_get_samples_mocked_return_value) - - start_time = datetime.datetime(2015, 3, 20) - end_time = datetime.datetime(2015, 4, 1, 12, 00, 00) - - sample_filter = storage.SampleFilter( - meter='specific meter', - start_timestamp=timeutils.isotime(start_time), - start_timestamp_op='gt', - end_timestamp=timeutils.isotime(end_time), - end_timestamp_op='lt') - list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - self.assertEqual(dict(dimensions=dict(datasource='ceilometer'), - name='specific meter', - start_time='2015-03-20T00:00:00.001000Z', - end_time='2015-04-01T11:59:59.999000Z', - start_timestamp_op='ge', - end_timestamp_op='le', - group_by='*'), - ml_mock.call_args_list[0][1]) - - def test_get_samples_limit(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - - metrics_list_mock = mock_client().metrics_list - - dummy_get_samples_mocked_return_value = ( - [{u'dimensions': {}, - u'measurements': [[u'2015-04-14T17:52:31Z', 1.0, {}], - [u'2015-04-15T17:52:31Z', 2.0, {}], - [u'2015-04-16T17:52:31Z', 3.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'image'}]) - - metrics_list_mock.return_value = ( - TestGetSamples.dummy_metrics_mocked_return_value - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - dummy_get_samples_mocked_return_value) - - sample_filter = storage.SampleFilter( - meter='specific meter', end_timestamp='2015-04-20T00:00:00Z') - samples = list(conn.get_samples(sample_filter, limit=2)) - self.assertEqual(2, len(samples)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_project_filter(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - [{u'dimensions': dict(project_id='specific project'), - u'id': u'2015-04-14T18:42:31Z', - u'name': u'specific meter'}] - ) - - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples.dummy_get_samples_mocked_return_value) - - sample_filter = storage.SampleFilter(meter='specific meter', - project='specific project') - list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_resource_filter(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - [{u'dimensions': dict(resource_id='specific resource'), - u'id': u'2015-04-14T18:42:31Z', - u'name': u'specific meter'}] - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples.dummy_get_samples_mocked_return_value) - - sample_filter = storage.SampleFilter(meter='specific meter', - resource='specific resource') - list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_source_filter(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - [{u'dimensions': dict(source='specific source'), - u'id': u'2015-04-14T18:42:31Z', - u'name': u'specific meter'}] - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples.dummy_get_samples_mocked_return_value) - - sample_filter = storage.SampleFilter(meter='specific meter', - source='specific source') - list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_simple_metaquery(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - TestGetSamples.dummy_metrics_mocked_return_value - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples.dummy_get_samples_mocked_return_value) - - sample_filter = storage.SampleFilter( - meter='specific meter', - metaquery={'metadata.key': u'value'}) - list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_simple_metaquery_with_extended_key(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - TestGetSamples.dummy_metrics_mocked_return_value - ) - ml_mock = mock_client().measurements_list - ml_mock.return_value = ( - TestGetSamples. - dummy_get_samples_mocked_return_extendedkey_value - ) - sample_filter = storage.SampleFilter( - meter='specific meter', - metaquery={'metadata.image_meta.base_url': u'base_url'}) - self.assertTrue(len(list(conn.get_samples(sample_filter))) > 0) - self.assertEqual(True, ml_mock.called) - self.assertEqual(1, ml_mock.call_count) - - def test_get_samples_results(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - metrics_list_mock = mock_client().metrics_list - metrics_list_mock.return_value = ( - [{u'dimensions': { - 'source': 'some source', - 'datasource': 'ceilometer', - 'project_id': 'some project ID', - 'resource_id': 'some resource ID', - 'type': 'some type', - 'unit': 'some unit'}, - u'id': u'2015-04-14T18:42:31Z', - u'name': u'image'}] - ) - ml_mock = mock_client().measurements_list - # TODO(this test case needs more work) - ml_mock.return_value = ( - [{u'dimensions': { - 'source': 'some source', - 'datasource': 'ceilometer', - 'project_id': 'some project ID', - 'resource_id': 'some resource ID', - 'type': 'some type', - 'unit': 'some unit'}, - u'measurements': - [[u'2015-04-01T02:03:04Z', 1.0, {}], - [u'2015-04-11T22:33:44Z', 2.0, {}]], - u'id': u'2015-04-14T18:42:31Z', - u'columns': [u'timestamp', u'value', u'value_meta'], - u'name': u'image'}]) - - sample_filter = storage.SampleFilter( - meter='image', - start_timestamp='2015-03-20T00:00:00Z') - results = list(conn.get_samples(sample_filter)) - self.assertEqual(True, ml_mock.called) - - self.assertEqual(results[0].counter_name, - ml_mock.return_value[0].get('name')) - self.assertEqual(results[0].counter_type, - ml_mock.return_value[0].get('dimensions'). - get('type')) - self.assertEqual(results[0].counter_unit, - ml_mock.return_value[0].get('dimensions'). - get('unit')) - self.assertEqual(results[0].counter_volume, - ml_mock.return_value[0]. - get('measurements')[0][1]) - self.assertEqual(results[0].message_id, - ml_mock.return_value[0].get('id')) - self.assertEqual(results[0].message_signature, '') - self.assertEqual(results[0].project_id, - ml_mock.return_value[0].get('dimensions'). - get('project_id')) - self.assertEqual(results[0].recorded_at, - dateutil.parser.parse( - ml_mock.return_value[0]. - get('measurements')[0][0])) - self.assertEqual(results[0].resource_id, - ml_mock.return_value[0].get('dimensions'). - get('resource_id')) - self.assertEqual(results[0].resource_metadata, {}) - self.assertEqual(results[0].source, - ml_mock.return_value[0].get('dimensions'). - get('source')) - self.assertEqual(results[0].timestamp, - dateutil.parser.parse( - ml_mock.return_value[0]. - get('measurements')[0][0])) - self.assertEqual(results[0].user_id, None) - - self.assertEqual(1, ml_mock.call_count) - - -class MeterStatisticsTest(_BaseTestCase): - - Aggregate = collections.namedtuple("Aggregate", ['func', 'param']) - - def test_not_implemented_params(self): - with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - - self.assertRaisesWithMessage("Query without filter " - "not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics(None))) - - sf = storage.SampleFilter() - self.assertRaisesWithMessage("Query without meter " - "not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics(sf))) - - sf.meter = "image" - self.assertRaisesWithMessage("Groupby message_id not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics( - sf, - groupby=['message_id']))) - - sf.metaquery = "metaquery" - self.assertRaisesWithMessage("Metaquery not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics(sf))) - - sf.metaquery = None - sf.start_timestamp_op = 'le' - self.assertRaisesWithMessage("Start time op le not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics(sf))) - - sf.start_timestamp_op = None - sf.end_timestamp_op = 'ge' - self.assertRaisesWithMessage("End time op ge not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics(sf))) - - sf.end_timestamp_op = None - sf.message_id = "message_id" - self.assertRaisesWithMessage("Message_id query not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics(sf))) - - sf.message_id = None - aggregate = [self.Aggregate(func='stddev', param='test')] - self.assertRaisesWithMessage("Aggregate function(s) ['stddev']" - " not implemented", - ceilometer.NotImplementedError, - lambda: list( - conn.get_meter_statistics( - sf, aggregate=aggregate))) - - @mock.patch('oslo_utils.timeutils.utcnow') - def test_stats_list_called_with(self, mock_utcnow): - mock_utcnow.return_value = datetime.datetime(2016, 4, 7, 18, 31) - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - sl_mock = mock_client().statistics_list - - sf = storage.SampleFilter() - sf.meter = "image" - sf.project = "project_id" - sf.user = "user_id" - sf.resource = "resource_id" - sf.source = "source_id" - aggregate = [self.Aggregate(func="min", param="some")] - list(conn.get_meter_statistics(sf, period=10, aggregate=aggregate)) - - self.assertEqual(True, sl_mock.called) - self.assertEqual( - {'merge_metrics': True, - 'dimensions': {'source': 'source_id', - 'project_id': 'project_id', - 'user_id': 'user_id', - 'resource_id': 'resource_id', - 'datasource': 'ceilometer' - }, - 'end_time': '2016-04-07T18:31:00.000000Z', - 'start_time': '1970-01-01T00:00:00.000000Z', - 'period': 10, - 'statistics': 'min', - 'name': 'image' - }, - sl_mock.call_args[1] - ) - - def test_stats_list(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - sl_mock = mock_client().statistics_list - sl_mock.return_value = [ - { - 'statistics': - [ - ['2014-10-24T12:12:12Z', 0.008], - ['2014-10-24T12:52:12Z', 0.018] - ], - 'dimensions': {'unit': 'gb'}, - 'columns': ['timestamp', 'min'] - } - ] - - sf = storage.SampleFilter() - sf.meter = "image" - aggregate = Aggregate() - aggregate.func = 'min' - sf.start_timestamp = timeutils.parse_isotime( - '2014-10-24T12:12:42').replace(tzinfo=None) - stats = list(conn.get_meter_statistics(sf, aggregate=[aggregate], - period=30)) - - self.assertEqual(2, len(stats)) - self.assertEqual('gb', stats[0].unit) - self.assertEqual('gb', stats[1].unit) - self.assertEqual(0.008, stats[0].min) - self.assertEqual(0.018, stats[1].min) - self.assertEqual(30, stats[0].period) - self.assertEqual('2014-10-24T12:12:42', - stats[0].period_end.isoformat()) - self.assertEqual('2014-10-24T12:52:42', - stats[1].period_end.isoformat()) - self.assertIsNotNone(stats[0].as_dict().get('aggregate')) - self.assertEqual({u'min': 0.008}, stats[0].as_dict()['aggregate']) - - def test_stats_list_with_groupby(self): - with mock.patch("ceilometer.monasca_client.Client") as mock_client: - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - sl_mock = mock_client().statistics_list - sl_mock.return_value = [ - { - 'statistics': - [ - ['2014-10-24T12:12:12Z', 0.008, 1.3, 3, 0.34], - ['2014-10-24T12:20:12Z', 0.078, 1.25, 2, 0.21], - ['2014-10-24T12:52:12Z', 0.018, 0.9, 4, 0.14] - ], - 'dimensions': {'project_id': '1234', 'unit': 'gb'}, - 'columns': ['timestamp', 'min', 'max', 'count', 'avg'] - }, - { - 'statistics': - [ - ['2014-10-24T12:14:12Z', 0.45, 2.5, 2, 2.1], - ['2014-10-24T12:20:12Z', 0.58, 3.2, 3, 3.4], - ['2014-10-24T13:52:42Z', 1.67, 3.5, 1, 5.3] - ], - 'dimensions': {'project_id': '5678', 'unit': 'gb'}, - 'columns': ['timestamp', 'min', 'max', 'count', 'avg'] - }] - - sf = storage.SampleFilter() - sf.meter = "image" - sf.start_timestamp = timeutils.parse_isotime( - '2014-10-24T12:12:42').replace(tzinfo=None) - groupby = ['project_id'] - stats = list(conn.get_meter_statistics(sf, period=30, - groupby=groupby)) - - self.assertEqual(2, len(stats)) - - for stat in stats: - self.assertIsNotNone(stat.groupby) - project_id = stat.groupby.get('project_id') - self.assertIn(project_id, ['1234', '5678']) - if project_id == '1234': - self.assertEqual(0.008, stat.min) - self.assertEqual(1.3, stat.max) - self.assertEqual(0.23, stat.avg) - self.assertEqual(9, stat.count) - self.assertEqual(30, stat.period) - self.assertEqual('2014-10-24T12:12:12', - stat.period_start.isoformat()) - if project_id == '5678': - self.assertEqual(0.45, stat.min) - self.assertEqual(3.5, stat.max) - self.assertEqual(3.6, stat.avg) - self.assertEqual(6, stat.count) - self.assertEqual(30, stat.period) - self.assertEqual('2014-10-24T13:52:42', - stat.period_end.isoformat()) - - -class TestQuerySamples(_BaseTestCase): - - def test_query_samples_not_implemented_params(self): - with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - query = {'and': [{'=': {'counter_name': 'instance'}}, - {'or': [{'=': {"project_id": "123"}}, - {'=': {"user_id": "456"}}]}]} - - self.assertRaisesWithMessage( - 'filter must be specified', - ceilometer.NotImplementedError, - lambda: list(conn.query_samples())) - order_by = [{"timestamp": "desc"}] - self.assertRaisesWithMessage( - 'orderby is not supported', - ceilometer.NotImplementedError, - lambda: list(conn.query_samples(query, order_by))) - - query = {'or': [{'=': {"project_id": "123"}}, - {'=': {"user_id": "456"}}]} - self.assert_raise_within_message( - 'meter name is not found in', - impl_monasca.InvalidInputException, - lambda: list(conn.query_samples(query, None, 1))) - - def test_query_samples(self): - SAMPLES = [[ - storage_models.Sample( - counter_name="instance", - counter_type="gauge", - counter_unit="instance", - counter_volume=1, - project_id="123", - user_id="456", - resource_id="789", - resource_metadata={}, - source="openstack", - recorded_at=timeutils.utcnow(), - timestamp=timeutils.utcnow(), - message_id="0", - message_signature='',) - ]] * 2 - samples = SAMPLES[:] - - def _get_samples(*args, **kwargs): - return samples.pop() - - with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - with mock.patch.object(conn, 'get_samples') as gsm: - gsm.side_effect = _get_samples - - query = {'and': [{'=': {'counter_name': 'instance'}}, - {'or': [{'=': {"project_id": "123"}}, - {'=': {"user_id": "456"}}]}]} - samples = conn.query_samples(query, None, 100) - self.assertEqual(2, len(samples)) - self.assertEqual(2, gsm.call_count) - - samples = SAMPLES[:] - query = {'and': [{'=': {'counter_name': 'instance'}}, - {'or': [{'=': {"project_id": "123"}}, - {'>': {"counter_volume": 2}}]}]} - samples = conn.query_samples(query, None, 100) - self.assertEqual(1, len(samples)) - self.assertEqual(4, gsm.call_count) - - def test_query_samples_timestamp_gt_lt(self): - SAMPLES = [[ - storage_models.Sample( - counter_name="instance", - counter_type="gauge", - counter_unit="instance", - counter_volume=1, - project_id="123", - user_id="456", - resource_id="789", - resource_metadata={}, - source="openstack", - recorded_at=timeutils.utcnow(), - timestamp=timeutils.utcnow(), - message_id="0", - message_signature='',) - ]] * 2 - samples = SAMPLES[:] - - def _get_samples(*args, **kwargs): - return samples.pop() - - with mock.patch("ceilometer.monasca_client.Client"): - conn = impl_monasca.Connection(self.CONF, "127.0.0.1:8080") - with mock.patch.object(conn, 'get_samples') as gsm: - gsm.side_effect = _get_samples - - start = datetime.datetime(2014, 10, 24, 13, 52, 42) - end = datetime.datetime(2014, 10, 24, 14, 52, 42) - ts_query = { - 'or': [{'>': {"timestamp": start}}, - {'<': {"timestamp": end}}] - } - query = {'and': [{'=': {'counter_name': 'instance'}}, - ts_query]} - samples = conn.query_samples(query, None, 100) - self.assertEqual(2, len(samples)) - self.assertEqual(2, gsm.call_count) - - -class CapabilitiesTest(base.BaseTestCase): - - def test_capabilities(self): - expected_capabilities = { - 'meters': - { - 'query': - { - 'metadata': False, - 'simple': True - } - }, - 'resources': - { - 'query': - { - 'metadata': True, 'simple': True - } - }, - 'samples': - { - 'query': - { - 'complex': True, - 'metadata': True, - 'simple': True - } - }, - 'statistics': - { - 'aggregation': - { - 'selectable': - { - 'avg': True, - 'cardinality': False, - 'count': True, - 'max': True, - 'min': True, - 'stddev': False, - 'sum': True - }, - 'standard': True}, - 'groupby': False, - 'query': - { - 'metadata': False, - 'simple': True - } - }, - 'events': - { - 'query': - { - 'simple': False - } - } - } - - actual_capabilities = impl_monasca.Connection.get_capabilities() - self.assertEqual(expected_capabilities, actual_capabilities) diff --git a/ceilosca/ceilometer/tests/unit/test_monascaclient.py b/ceilosca/ceilometer/tests/unit/test_monascaclient.py index 50cc929..4fed2cf 100644 --- a/ceilosca/ceilometer/tests/unit/test_monascaclient.py +++ b/ceilosca/ceilometer/tests/unit/test_monascaclient.py @@ -38,7 +38,7 @@ class TestMonascaClient(base.BaseTestCase): 'http://localhost:5000/v2.0', 'monasca') - self.CONF.set_override('max_retries', 0, 'database') + self.CONF.set_override('database_max_retries', 0, 'monasca') self.mc = self._get_client() def tearDown(self): @@ -111,8 +111,8 @@ class TestMonascaClient(base.BaseTestCase): self.assertIsNotNone(True, conf.service_username) def test_retry_on_key_error(self): - self.CONF.set_override('max_retries', 2, 'database') - self.CONF.set_override('retry_interval', 1, 'database') + self.CONF.set_override('database_max_retries', 2, 'monasca') + self.CONF.set_override('database_retry_interval', 1, 'monasca') self.mc = self._get_client() with mock.patch.object( self.mc._mon_client.metrics, 'list', @@ -121,8 +121,8 @@ class TestMonascaClient(base.BaseTestCase): self.assertEqual(2, mocked_metrics_list.call_count) def test_no_retry_on_invalid_parameter(self): - self.CONF.set_override('max_retries', 2, 'database') - self.CONF.set_override('retry_interval', 1, 'database') + self.CONF.set_override('database_max_retries', 2, 'monasca') + self.CONF.set_override('database_retry_interval', 1, 'monasca') self.mc = self._get_client() def _check(exception): @@ -139,7 +139,8 @@ class TestMonascaClient(base.BaseTestCase): def test_max_retris_not_too_much(self): def _check(configured, expected): - self.CONF.set_override('max_retries', configured, 'database') + self.CONF.set_override('database_max_retries', configured, + 'monasca') self.mc = self._get_client() self.assertEqual(expected, self.mc._max_retries) diff --git a/etc/ceilometer/ceilometer.conf b/etc/ceilometer/ceilometer.conf index 562156b..1065dc7 100644 --- a/etc/ceilometer/ceilometer.conf +++ b/etc/ceilometer/ceilometer.conf @@ -14,7 +14,7 @@ rabbit_password = password rabbit_hosts = 16.78.179.83 [service_credentials] -auth_url = http://192.168.10.6:5000 +auth_url = http://192.168.10.6:5000/v3 region_name = RegionOne password = secretservice username = ceilometer @@ -22,18 +22,22 @@ project_name = service project_domain_id = default user_domain_id = default auth_type = password +interface = internalURL [keystone_authtoken] -signing_dir = /var/cache/ceilometer +auth_type = v3password cafile = /opt/stack/data/ca-bundle.pem auth_uri = http://16.78.179.83:5000 project_domain_id = default project_name = service user_domain_id = default +project_domain_name = Default +user_domain_name = Default password = password username = ceilometer auth_url = http://16.78.179.83:35357 auth_plugin = password +region_name = RegionOne [notification] store_events = True @@ -43,4 +47,18 @@ disable_non_metric_meters = False #metering_connection = mysql://root:password@127.0.0.1/ceilometer?charset=utf8 event_connection = mysql://root:password@127.0.0.1/ceilometer?charset=utf8 alarm_connection = mysql://root:password@127.0.0.1/ceilometer?charset=utf8 -metering_connection = monasca://http://127.0.0.1:8070/v2.0 +connection = monasca://http://127.0.0.1:8070/v2.0 + +[monasca] +enable_api_pagination = True +database_retry_interval = 5 +database_max_retries = 5 +service_auth_url = http://192.168.10.6:5000/v3 +service_password = secretservice +service_username = ceilometer +service_interface = internalURL +service_auth_type = password +service_domain_name = Default +service_verify = /opt/stack/data/ca-bundle.pem +service_region_name = RegionOne +monasca_mappings = /etc/ceilometer/monasca_field_definitions.yaml diff --git a/monasca_test_setup.py b/monasca_test_setup.py index 7696b1a..9495118 100644 --- a/monasca_test_setup.py +++ b/monasca_test_setup.py @@ -25,8 +25,7 @@ ceilosca_files = { 'monasca_client.py', 'monasca_ceilometer_opts.py', 'publisher/monasca_data_filter.py', - 'publisher/monclient.py', - 'storage/impl_monasca.py' + 'publisher/monclient.py' ] } diff --git a/test-requirements.txt b/test-requirements.txt index 425353a..9db6bea 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,4 +1,4 @@ -git+https://github.com/openstack/ceilometer.git@stable/pike#egg=ceilometer +git+https://github.com/openstack/ceilometer.git@master#egg=ceilometer mock>=1.2 testrepository>=0.0.18 testscenarios>=0.4