# # (C) Copyright 2015-2017 Hewlett Packard Enterprise Development LP # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Simple monasca storage backend. """ from collections import defaultdict import copy import datetime import itertools import operator from monascaclient import exc as monasca_exc from oslo_config import cfg from oslo_log import log from oslo_serialization import jsonutils from oslo_utils import netutils from oslo_utils import timeutils import ceilometer from ceilometer.ceilosca_mapping.ceilometer_static_info_mapping import ( ProcessMappedCeilometerStaticInfo) from ceilometer.ceilosca_mapping.ceilosca_mapping import ( ProcessMappedCeiloscaMetric) from ceilometer.ceilosca_mapping.ceilosca_mapping import PipelineReader from ceilometer.i18n import _ from ceilometer import monasca_client from ceilometer.publisher.monasca_data_filter import MonascaDataFilter from ceilometer import storage from ceilometer.storage import base from ceilometer.storage import models as api_models from ceilometer import utils OPTS = [ cfg.IntOpt('default_stats_period', default=300, help='Default period (in seconds) to use for querying stats ' 'in case no period specified in the stats API call.'), ] cfg.CONF.register_opts(OPTS, group='monasca') LOG = log.getLogger(__name__) AVAILABLE_CAPABILITIES = { 'meters': {'query': {'simple': True, 'metadata': False}}, 'resources': {'query': {'simple': True, 'metadata': True}}, 'samples': {'query': {'simple': True, 'metadata': True, 'complex': True}}, 'statistics': {'groupby': False, 'query': {'simple': True, 'metadata': False}, 'aggregation': {'standard': True, 'selectable': { 'max': True, 'min': True, 'sum': True, 'avg': True, 'count': True, 'stddev': False, 'cardinality': False}} }, 'events': {'query': {'simple': False}} } AVAILABLE_STORAGE_CAPABILITIES = { 'storage': {'production_ready': True} } class InvalidInputException(Exception): code = 400 class Connection(base.Connection): CAPABILITIES = utils.update_nested(base.Connection.CAPABILITIES, AVAILABLE_CAPABILITIES) STORAGE_CAPABILITIES = utils.update_nested( base.Connection.STORAGE_CAPABILITIES, AVAILABLE_STORAGE_CAPABILITIES, ) def __init__(self, url): self.mc = monasca_client.Client(netutils.urlsplit(url)) self.mon_filter = MonascaDataFilter() self.ceilosca_mapper = ProcessMappedCeiloscaMetric() self.pipeline_reader = PipelineReader() self.meter_static_info = ProcessMappedCeilometerStaticInfo() self.meters_from_pipeline = self.pipeline_reader.get_pipeline_meters() @staticmethod def _convert_to_dict(stats, cols): return {c: stats[i] for i, c in enumerate(cols)} def _convert_metaquery(self, metaquery): """Strip "metadata." from key and convert value to string :param metaquery: { 'metadata.KEY': VALUE, ... } :returns: converted metaquery """ query = {} for k, v in metaquery.items(): key = '.'.join(k.split('.')[1:]) if isinstance(v, basestring): query[key] = v else: query[key] = str(int(v)) return query def _match_metaquery_to_value_meta(self, query, value_meta): """Check if metaquery matches value_meta :param query: metaquery with converted format :param value_meta: metadata from monasca :returns: True for matched, False for not matched """ if (query and (len(value_meta) == 0 or not set(query.items()).issubset(set(value_meta.items())))): return False else: return True def _incr_date_by_millisecond(self, date): # Monasca only supports millisecond for now epoch = datetime.datetime(1970, 1, 1) seconds_since_epoch = timeutils.delta_seconds(epoch, date) millis_since_epoch = seconds_since_epoch * 1000 millis_since_epoch += 1 return (timeutils.iso8601_from_timestamp( millis_since_epoch / 1000, True)) def _decr_date_by_millisecond(self, date): epoch = datetime.datetime(1970, 1, 1) seconds_since_epoch = timeutils.delta_seconds(epoch, date) millis_since_epoch = seconds_since_epoch * 1000 millis_since_epoch -= 1 return (timeutils.iso8601_from_timestamp( millis_since_epoch / 1000, True)) def upgrade(self): pass def clear(self): pass def record_metering_data(self, data): """Write the data to the backend storage system. :param data: a dictionary such as returned by ceilometer.meter.meter_message_from_counter. """ LOG.info(_('metering data %(counter_name)s for %(resource_id)s: ' '%(counter_volume)s') % ({'counter_name': data['counter_name'], 'resource_id': data['resource_id'], 'counter_volume': data['counter_volume']})) metric = self.mon_filter.process_sample_for_monasca(data) self.mc.metrics_create(**metric) def clear_expired_metering_data(self, ttl): """Clear expired data from the backend storage system. Clearing occurs according to the time-to-live. :param ttl: Number of seconds to keep records for. """ LOG.info(_("Dropping data with TTL %d"), ttl) def get_metrics_with_mapped_dimensions(self, source_dimension, mapped_dimension, search_args=None): # Return metric list with the results of mapped dimensions. if source_dimension in search_args['dimensions']: search_args2 = copy.deepcopy(search_args) filter_val = search_args2['dimensions'].pop(source_dimension, None) search_args2['dimensions'][mapped_dimension] = filter_val metric_list = self.mc.metrics_list(**search_args2) if metric_list is not None: return metric_list return [] def get_metric_names_with_mapped_dimensions( self, source_dimension, mapped_dimension, search_args=None): # Return metric list with the results of mapped dimensions. if source_dimension in search_args['dimensions']: search_args2 = copy.deepcopy(search_args) filter_val = search_args2['dimensions'].pop(source_dimension, None) search_args2['dimensions'][mapped_dimension] = filter_val metric_names_list = self.mc.metric_names_list(**search_args2) if metric_names_list is not None: return metric_names_list return [] def get_resources(self, user=None, project=None, source=None, start_timestamp=None, start_timestamp_op=None, end_timestamp=None, end_timestamp_op=None, metaquery=None, resource=None, limit=None): """Return an iterable of dictionaries containing resource information. { 'resource_id': UUID of the resource, 'project_id': UUID of project owning the resource, 'user_id': UUID of user owning the resource, 'timestamp': UTC datetime of last update to the resource, 'metadata': most current metadata for the resource, 'meter': list of the meters reporting data for the resource, } :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param source: Optional source filter. :param start_timestamp: Optional modified timestamp start range. :param start_timestamp_op: Optional start time operator, like gt, ge. :param end_timestamp: Optional modified timestamp end range. :param end_timestamp_op: Optional end time operator, like lt, le. :param metaquery: Optional dict with metadata to match on. :param resource: Optional resource filter. :param limit: Maximum number of results to return. """ if limit == 0: return q = {} if metaquery: q = self._convert_metaquery(metaquery) if start_timestamp_op and start_timestamp_op not in ['ge', 'gt']: raise ceilometer.NotImplementedError(('Start time op %s ' 'not implemented') % start_timestamp_op) if end_timestamp_op and end_timestamp_op not in ['le', 'lt']: raise ceilometer.NotImplementedError(('End time op %s ' 'not implemented') % end_timestamp_op) if not start_timestamp: start_timestamp = datetime.datetime(1970, 1, 1) if not end_timestamp: end_timestamp = timeutils.utcnow() self._ensure_start_time_le_end_time(start_timestamp, end_timestamp) # Equivalent of doing a start_timestamp_op = 'ge' if (start_timestamp_op and start_timestamp_op == 'gt'): start_timestamp = self._incr_date_by_millisecond( start_timestamp) start_timestamp_op = 'ge' else: start_timestamp = timeutils.isotime(start_timestamp, subsecond=True) # Equivalent of doing a end_timestamp_op = 'le' if (end_timestamp_op and end_timestamp_op == 'lt'): end_timestamp = self._decr_date_by_millisecond( end_timestamp) end_timestamp_op = 'le' else: end_timestamp = timeutils.isotime(end_timestamp, subsecond=True) dims_filter = dict(user_id=user, project_id=project, source=source, resource_id=resource ) dims_filter = {k: v for k, v in dims_filter.items() if v is not None} _search_args = dict( start_time=start_timestamp, end_time=end_timestamp, start_timestamp_op=start_timestamp_op, end_timestamp_op=end_timestamp_op) _search_args = {k: v for k, v in _search_args.items() if v is not None} result_count = 0 _search_kwargs = {'dimensions': dims_filter} meter_names = list() meter_names_list = itertools.chain( # Accumulate a list from monascaclient starting with no filter self.mc.metric_names_list(**_search_kwargs), # query monasca with hostname = resource_id filter self.get_metric_names_with_mapped_dimensions( "resource_id", "hostname", _search_kwargs), # query monasca with tenant_id = project_id filter self.get_metric_names_with_mapped_dimensions( "project_id", "tenant_id", _search_kwargs) ) for metric in meter_names_list: if metric['name'] in meter_names: continue elif (metric['name'] in self.meter_static_info.get_list_supported_meters()): meter_names.insert(0, metric['name']) elif (metric['name'] in self.ceilosca_mapper.get_list_monasca_metrics()): meter_names.append(metric['name']) for meter_name in meter_names: _search_args['name'] = meter_name _search_args['group_by'] = '*' _search_args.pop('dimensions', None) _search_args['dimensions'] = dims_filter # if meter is a Ceilometer meter... if (meter_name not in self.ceilosca_mapper.get_list_monasca_metrics()): try: if meter_name not in self.meters_from_pipeline: _search_args['dimensions']['datasource'] = 'ceilometer' for sample in (self.mc.measurements_list(**_search_args)): d = sample['dimensions'] for meas in sample['measurements']: m = self._convert_to_dict(meas, sample['columns']) vm = m['value_meta'] if not self._match_metaquery_to_value_meta(q, vm): continue if d.get('resource_id'): result_count += 1 yield api_models.Resource( resource_id=d.get('resource_id'), first_sample_timestamp=( timeutils.parse_isotime( m['timestamp'])), last_sample_timestamp=timeutils.utcnow(), project_id=d.get('project_id'), source=d.get('source'), user_id=d.get('user_id'), metadata=m['value_meta'] ) if result_count == limit: return except monasca_exc.HTTPConflict: pass # else if meter is a Monasca meter... else: try: meter_def = self.ceilosca_mapper.\ get_ceilosca_mapped_metric_definition(meter_name) # if for a meter name being queried, project exists in # ceilometer-monasca mapping file, query by # mapped_field instead if not (project is None): mapped_field = self.ceilosca_mapper.\ get_ceilosca_mapped_definition_key_val( _search_args['name'], 'project_id') if 'dimensions' in mapped_field: _search_args['dimensions'].pop('project_id', None) _search_args['dimensions'][mapped_field.split(".")[-1]] = \ project # if for a meter name being queried, resource_id exists in # ceilometer-monasca mapping file, query by # mapped_field instead if not (resource is None): mapped_field = self.ceilosca_mapper.\ get_ceilosca_mapped_definition_key_val( _search_args['name'], 'resource_id') if 'dimensions' in mapped_field: _search_args['dimensions'].pop('resource_id', None) _search_args['dimensions'][mapped_field.split(".")[-1]] = \ resource for sample in (self.mc.measurements_list(**_search_args)): d = sample['dimensions'] for meas in sample['measurements']: m = self._convert_to_dict(meas, sample['columns']) vm = m['value_meta'] if not self._match_metaquery_to_value_meta(q, vm): continue if meter_def.parse_fields('resource_id', sample): result_count += 1 yield api_models.Resource( resource_id=meter_def.parse_fields( 'resource_id', sample), first_sample_timestamp=( timeutils.parse_isotime( m['timestamp'])), last_sample_timestamp=(timeutils.utcnow()), project_id=meter_def.parse_fields( 'project_id', sample), source=meter_def.parse_fields('source', sample), user_id=meter_def.parse_fields( 'user_id', sample), metadata=meter_def.parse_fields( 'resource_metadata', sample) ) if result_count == limit: return except monasca_exc.HTTPConflict: pass def get_meters(self, user=None, project=None, resource=None, source=None, metaquery=None, limit=None, unique=False): """Return an iterable of dictionaries containing meter information. { 'name': name of the meter, 'type': type of the meter (gauge, delta, cumulative), 'resource_id': UUID of the resource, 'project_id': UUID of project owning the resource, 'user_id': UUID of user owning the resource, } :param user: Optional ID for user that owns the resource. :param project: Optional ID for project that owns the resource. :param resource: Optional resource filter. :param source: Optional source filter. :param metaquery: Optional dict with metadata to match on. :param limit: Maximum number of results to return. :param unique: If set to true, return only unique meter information. """ if limit == 0: return if metaquery: raise ceilometer.NotImplementedError('Metaquery not implemented') _dimensions = dict( user_id=user, project_id=project, resource_id=resource, source=source ) _dimensions = {k: v for k, v in _dimensions.items() if v is not None} _search_kwargs = {'dimensions': _dimensions} if unique: meter_names = set() for metric in self.mc.metric_names_list(**_search_kwargs): if metric['name'] in meter_names: continue elif (metric['name'] in self.meter_static_info.get_list_supported_meters()): if limit and len(meter_names) >= limit: return meter_names.add(metric['name']) yield api_models.Meter( name=metric['name'], type=self.meter_static_info .get_meter_static_info_key_val(metric['name'], 'type'), unit=self.meter_static_info .get_meter_static_info_key_val(metric['name'], 'unit'), resource_id=None, project_id=None, source=None, user_id=None) elif (metric['name'] not in self.ceilosca_mapper.get_list_monasca_metrics()): continue else: if limit and len(meter_names) >= limit: return meter_names.add(metric['name']) meter = (self.ceilosca_mapper. get_ceilosca_mapped_metric_definition (metric['name'])) yield api_models.Meter( name=meter.parse_fields('name', metric), type=meter.parse_fields('type', metric), unit=meter.parse_fields('unit', metric), resource_id=None, project_id=None, source=None, user_id=None) else: result_count = 0 # Search for ceilometer published data first _search_kwargs_tmp = copy.deepcopy(_search_kwargs) _search_kwargs_tmp['dimensions']['datasource'] = 'ceilometer' metrics_list = self.mc.metrics_list(**_search_kwargs_tmp) for metric in metrics_list: if result_count == limit: return result_count += 1 yield api_models.Meter( name=metric['name'], type=metric['dimensions'].get('type') or 'cumulative', unit=metric['dimensions'].get('unit'), resource_id=metric['dimensions'].get('resource_id'), project_id=metric['dimensions'].get('project_id'), source=metric['dimensions'].get('source'), user_id=metric['dimensions'].get('user_id')) # because we enable monasca pagination, so we should use iterator # instead of a list, to reduce unnecessary requests to monasca-api metrics_list = itertools.chain( self.mc.metrics_list(**_search_kwargs), # for vm performance metrics collected by monasca-agent, the # project_id is mapped to tenant_id self.get_metrics_with_mapped_dimensions("project_id", "tenant_id", _search_kwargs), # for compute.node metrics collected by monasca-agent, the # resource_id is mapped to hostname self.get_metrics_with_mapped_dimensions("resource_id", "hostname", _search_kwargs), ) for metric in metrics_list: if result_count == limit: return if (metric['dimensions'].get('datasource') != 'ceilometer' and metric['name'] in self.meters_from_pipeline): result_count += 1 yield api_models.Meter( name=metric['name'], type=metric['dimensions'].get('type') or 'cumulative', unit=metric['dimensions'].get('unit'), resource_id=metric['dimensions'].get('resource_id'), project_id=metric['dimensions'].get('project_id'), source=metric['dimensions'].get('source'), user_id=metric['dimensions'].get('user_id')) elif (metric['name'] in self.ceilosca_mapper.get_list_monasca_metrics()): meter = (self.ceilosca_mapper. get_ceilosca_mapped_metric_definition (metric['name'])) result_count += 1 yield api_models.Meter( name=meter.parse_fields('name', metric), type=meter.parse_fields('type', metric), unit=meter.parse_fields('unit', metric), resource_id=meter.parse_fields('resource_id', metric), project_id=meter.parse_fields('project_id', metric), source=meter.parse_fields('source', metric), user_id=meter.parse_fields('user_id', metric)) def get_samples(self, sample_filter, limit=None): """Return an iterable of dictionaries containing sample information. { 'source': source of the resource, 'counter_name': name of the resource,if groupby: raise ceilometer.NotImplementedError('Groupby not implemented') 'counter_type': type of the sample (gauge, delta, cumulative), 'counter_unit': unit of the sample, 'counter_volume': volume of the sample, 'user_id': UUID of user owning the resource, 'project_id': UUID of project owning the resource, 'resource_id': UUID of the resource, 'timestamp': timestamp of the sample, 'resource_metadata': metadata of the sample, 'message_id': message ID of the sample, 'message_signature': message signature of the sample, 'recorded_at': time the sample was recorded } :param sample_filter: constraints for the sample search. :param limit: Maximum number of results to return. """ if limit == 0: return if not sample_filter or not sample_filter.meter: raise ceilometer.NotImplementedError( "Supply meter name at the least") if (sample_filter.start_timestamp_op and sample_filter.start_timestamp_op not in ['ge', 'gt']): raise ceilometer.NotImplementedError(('Start time op %s ' 'not implemented') % sample_filter. start_timestamp_op) if (sample_filter.end_timestamp_op and sample_filter.end_timestamp_op not in ['le', 'lt']): raise ceilometer.NotImplementedError(('End time op %s ' 'not implemented') % sample_filter. end_timestamp_op) q = {} if sample_filter.metaquery: q = self._convert_metaquery(sample_filter.metaquery) if sample_filter.message_id: raise ceilometer.NotImplementedError('message_id not ' 'implemented ' 'in get_samples') if not sample_filter.start_timestamp: sample_filter.start_timestamp = datetime.datetime(1970, 1, 1) if not sample_filter.end_timestamp: sample_filter.end_timestamp = timeutils.utcnow() self._ensure_start_time_le_end_time(sample_filter.start_timestamp, sample_filter.end_timestamp) if (sample_filter.start_timestamp_op and sample_filter. start_timestamp_op == 'gt'): sample_filter.start_timestamp = self._incr_date_by_millisecond( sample_filter.start_timestamp) sample_filter.start_timestamp_op = 'ge' else: sample_filter.start_timestamp = timeutils.isotime( sample_filter.start_timestamp, subsecond=True) if (sample_filter.end_timestamp_op and sample_filter. end_timestamp_op == 'lt'): sample_filter.end_timestamp = self._decr_date_by_millisecond( sample_filter.end_timestamp) sample_filter.end_timestamp_op = 'le' else: sample_filter.end_timestamp = timeutils.isotime( sample_filter.end_timestamp, subsecond=True) _dimensions = dict( user_id=sample_filter.user, project_id=sample_filter.project, resource_id=sample_filter.resource, source=sample_filter.source, # Dynamic sample filter attributes, these fields are useful for # filtering result. unit=getattr(sample_filter, 'unit', None), type=getattr(sample_filter, 'type', None), ) _dimensions = {k: v for k, v in _dimensions.items() if v is not None} _search_args = dict( start_time=sample_filter.start_timestamp, start_timestamp_op=sample_filter.start_timestamp_op, end_time=sample_filter.end_timestamp, end_timestamp_op=sample_filter.end_timestamp_op, dimensions=_dimensions, name=sample_filter.meter, group_by='*' ) result_count = 0 _search_args = {k: v for k, v in _search_args.items() if v is not None} if self.ceilosca_mapper.get_monasca_metric_name(sample_filter.meter): _search_args['name'] = ( self.ceilosca_mapper.get_monasca_metric_name( sample_filter.meter)) meter_def = ( self.ceilosca_mapper.get_ceilosca_mapped_metric_definition( _search_args['name'])) # if for a meter name being queried, project exists in # ceilometer-monasca mapping file, query by mapped_field instead if not (sample_filter.project is None): mapped_field = self.ceilosca_mapper.\ get_ceilosca_mapped_definition_key_val( _search_args['name'], 'project_id') if 'dimensions' in mapped_field: _search_args['dimensions'].pop('project_id', None) _search_args['dimensions'][mapped_field.split(".")[-1]] = \ sample_filter.project # if for a meter name being queried, resource_id exists in # ceilometer-monasca mapping file, query by mapped_field instead if not (sample_filter.resource is None): mapped_field = self.ceilosca_mapper.\ get_ceilosca_mapped_definition_key_val( _search_args['name'], 'resource_id') if 'dimensions' in mapped_field: _search_args['dimensions'].pop('resource_id', None) _search_args['dimensions'][mapped_field.split(".")[-1]] = \ sample_filter.resource for sample in self.mc.measurements_list(**_search_args): d = sample['dimensions'] for meas in sample['measurements']: m = self._convert_to_dict( meas, sample['columns']) vm = m['value_meta'] if not self._match_metaquery_to_value_meta(q, vm): continue result_count += 1 yield api_models.Sample( source=meter_def.parse_fields('source', sample), counter_name=sample_filter.meter, counter_type=meter_def.parse_fields('type', sample), counter_unit=meter_def.parse_fields('unit', sample), counter_volume=m['value'], user_id=meter_def.parse_fields('user_id', sample), project_id=meter_def.parse_fields('project_id', sample), resource_id=meter_def.parse_fields('resource_id', sample), timestamp=timeutils.parse_isotime(m['timestamp']), resource_metadata=meter_def.parse_fields( 'resource_metadata', sample), message_id=sample['id'], message_signature='', recorded_at=(timeutils.parse_isotime(m['timestamp']))) if result_count == limit: return return # This if statement is putting the dimension information for 2 cases # 1. To safeguard against querying an existing monasca metric which is # not mapped in ceilometer, unless it is published in monasca by # ceilometer itself # 2. Also, this will allow you to query a metric which was being # collected historically but not being collected any more as far as # those metrics exists in monasca if sample_filter.meter not in self.meters_from_pipeline: _search_args['dimensions']['datasource'] = 'ceilometer' for sample in self.mc.measurements_list(**_search_args): d = sample['dimensions'] for meas in sample['measurements']: m = self._convert_to_dict( meas, sample['columns']) vm = m['value_meta'] if not self._match_metaquery_to_value_meta(q, vm): continue result_count += 1 yield api_models.Sample( source=d.get('source'), counter_name=sample_filter.meter, counter_type=d.get('type'), counter_unit=d.get('unit'), counter_volume=m['value'], user_id=d.get('user_id'), project_id=d.get('project_id'), resource_id=d.get('resource_id'), timestamp=timeutils.parse_isotime(m['timestamp']), resource_metadata=m['value_meta'], message_id=sample['id'], message_signature='', recorded_at=(timeutils.parse_isotime(m['timestamp']))) if result_count == limit: return def get_meter_statistics(self, filter, period=None, groupby=None, aggregate=None): """Return a dictionary containing meter statistics. Meter statistics is described by the query parameters. The filter must have a meter value set. { 'min': 'max': 'avg': 'sum': 'count': 'period': 'period_start': 'period_end': 'duration': 'duration_start': 'duration_end': } """ if filter: if not filter.meter: raise ceilometer.NotImplementedError('Query without meter ' 'not implemented') else: raise ceilometer.NotImplementedError('Query without filter ' 'not implemented') allowed_groupby = ['user_id', 'project_id', 'resource_id', 'source'] if groupby: if len(groupby) > 1: raise ceilometer.NotImplementedError('Only one groupby ' 'supported') groupby = groupby[0] if groupby not in allowed_groupby: raise ceilometer.NotImplementedError('Groupby %s not' ' implemented' % groupby) if filter.metaquery: raise ceilometer.NotImplementedError('Metaquery not implemented') if filter.message_id: raise ceilometer.NotImplementedError('Message_id query ' 'not implemented') if filter.start_timestamp_op and ( filter.start_timestamp_op not in ['ge', 'gt']): raise ceilometer.NotImplementedError(('Start time op %s ' 'not implemented') % filter.start_timestamp_op) if filter.end_timestamp_op and ( filter.end_timestamp_op not in ['le', 'lt']): raise ceilometer.NotImplementedError(('End time op %s ' 'not implemented') % filter.end_timestamp_op) if not filter.start_timestamp: filter.start_timestamp = datetime.datetime(1970, 1, 1) if not filter.end_timestamp: filter.end_timestamp = timeutils.utcnow() self._ensure_start_time_le_end_time(filter.start_timestamp, filter.end_timestamp) if (filter.start_timestamp_op and filter. start_timestamp_op == 'gt'): filter.start_timestamp = self._incr_date_by_millisecond( filter.start_timestamp) filter.start_timestamp_op = 'ge' else: filter.start_timestamp = timeutils.isotime(filter.start_timestamp, subsecond=True) if (filter.end_timestamp_op and filter. end_timestamp_op == 'lt'): filter.end_timestamp = self._decr_date_by_millisecond( filter.end_timestamp) filter.end_timestamp_op = 'le' else: filter.end_timestamp = timeutils.isotime(filter.end_timestamp, subsecond=True) # TODO(monasca): Add this a config parameter allowed_stats = ['avg', 'min', 'max', 'sum', 'count'] if aggregate: not_allowed_stats = [a.func for a in aggregate if a.func not in allowed_stats] if not_allowed_stats: raise ceilometer.NotImplementedError(('Aggregate function(s) ' '%s not implemented') % not_allowed_stats) statistics = [a.func for a in aggregate if a.func in allowed_stats] else: statistics = allowed_stats dims_filter = dict(user_id=filter.user, project_id=filter.project, source=filter.source, resource_id=filter.resource ) dims_filter = {k: v for k, v in dims_filter.items() if v is not None} period = period if period \ else cfg.CONF.monasca.default_stats_period _search_args = dict( name=filter.meter, dimensions=dims_filter, start_time=filter.start_timestamp, end_time=filter.end_timestamp, period=period, statistics=','.join(statistics), ) _search_args = {k: v for k, v in _search_args.items() if v is not None} is_mapped_metric = False if self.ceilosca_mapper.get_monasca_metric_name(filter.meter): _search_args['name'] = self.ceilosca_mapper.\ get_monasca_metric_name(filter.meter) meter_def = ( self.ceilosca_mapper.get_ceilosca_mapped_metric_definition( _search_args['name'])) is_mapped_metric = True # if for a meter name being queried, project exists in # ceilometer-monasca mapping file, query by mapped_field instead if not (filter.project is None): mapped_field = self.ceilosca_mapper.\ get_ceilosca_mapped_definition_key_val( _search_args['name'], 'project_id') if 'dimensions' in mapped_field: _search_args['dimensions'].pop('project_id', None) _search_args['dimensions'][mapped_field.split(".")[-1]] = \ filter.project # if for a meter name being queried, resource_id exists in # ceilometer-monasca mapping file, query by mapped_field instead if not (filter.resource is None): mapped_field = self.ceilosca_mapper.\ get_ceilosca_mapped_definition_key_val( _search_args['name'], 'resource_id') if 'dimensions' in mapped_field: _search_args['dimensions'].pop('resource_id', None) _search_args['dimensions'][mapped_field.split(".")[-1]] = \ filter.resource elif filter.meter not in self.meters_from_pipeline: _search_args['dimensions']['datasource'] = 'ceilometer' if groupby: _search_args['group_by'] = '*' stats_list = self.mc.statistics_list(**_search_args) group_stats_dict = defaultdict(list) for stats in stats_list: if is_mapped_metric: groupby_val = meter_def.parse_fields(groupby, stats) else: groupby_val = stats['dimensions'].get(groupby) group_stats_dict[groupby_val].append(stats) def get_max(items): return max(items) def get_min(items): return min(items) def get_avg(items): return sum(items)/len(items) def get_sum(items): return sum(items) def get_count(items): count = 0 for item in items: count = count + item return count for group_key, stats_group in group_stats_dict.iteritems(): max_list = [] min_list = [] avg_list = [] sum_list = [] count_list = [] ts_list = [] group_statistics = {} for stats in stats_group: for s in stats['statistics']: stats_dict = self._convert_to_dict(s, stats['columns']) if 'max' in stats['columns']: max_list.append(stats_dict['max']) if 'min' in stats['columns']: min_list.append(stats_dict['min']) if 'avg' in stats['columns']: avg_list.append(stats_dict['avg']) if 'sum' in stats['columns']: sum_list.append(stats_dict['sum']) if 'count' in stats['columns']: count_list.append(stats_dict['count']) ts_list.append(stats_dict['timestamp']) if is_mapped_metric: group_statistics['unit'] = (meter_def.parse_fields( 'unit', stats)) else: group_statistics['unit'] = (stats['dimensions']. get('unit')) if len(max_list): group_statistics['max'] = get_max(max_list) if len(min_list): group_statistics['min'] = get_min(min_list) if len(avg_list): group_statistics['avg'] = get_avg(avg_list) if len(sum_list): group_statistics['sum'] = get_sum(sum_list) if len(count_list): group_statistics['count'] = get_count(count_list) if 'count' in group_statistics: group_statistics['count'] = int(group_statistics['count']) unit = group_statistics.get('unit') if 'unit' in group_statistics: del group_statistics['unit'] if aggregate: group_statistics['aggregate'] = {} for a in aggregate: key = '%s%s' % (a.func, '/%s' % a.param if a.param else '') group_statistics['aggregate'][key] = ( group_statistics.get(key)) if group_statistics and len(ts_list): group_statistics['end_timestamp'] = get_max(ts_list) group_statistics['timestamp'] = get_min(ts_list) ts_start = timeutils.parse_isotime( group_statistics['timestamp']).replace(tzinfo=None) ts_end = timeutils.parse_isotime( group_statistics['end_timestamp']).replace(tzinfo=None) del group_statistics['end_timestamp'] if ts_start and ts_end: yield api_models.Statistics( unit=unit, period=period, period_start=ts_start, period_end=ts_end, duration=period, duration_start=ts_start, duration_end=ts_end, groupby={groupby: group_key}, **group_statistics ) else: _search_args['merge_metrics'] = True stats_list = self.mc.statistics_list(**_search_args) for stats in stats_list: for s in stats['statistics']: stats_dict = self._convert_to_dict(s, stats['columns']) ts_start = timeutils.parse_isotime( stats_dict['timestamp']).replace(tzinfo=None) ts_end = (ts_start + datetime.timedelta( 0, period)).replace(tzinfo=None) del stats_dict['timestamp'] if 'count' in stats_dict: stats_dict['count'] = int(stats_dict['count']) if aggregate: stats_dict['aggregate'] = {} for a in aggregate: key = '%s%s' % (a.func, '/%s' % a.param if a.param else '') stats_dict['aggregate'][key] = stats_dict.get(key) if is_mapped_metric: unit = meter_def.parse_fields('unit', stats) else: unit = stats['dimensions'].get('unit') if ts_start and ts_end: yield api_models.Statistics( unit=unit, period=period, period_start=ts_start, period_end=ts_end, duration=period, duration_start=ts_start, duration_end=ts_end, groupby={u'': u''}, **stats_dict ) def _parse_to_filter_list(self, filter_expr): """Parse complex query expression to simple filter list. For i.e. parse: {"or":[{"=":{"meter":"cpu"}},{"=":{"meter":"memory"}}]} to [[{"=":{"counter_name":"cpu"}}], [{"=":{"counter_name":"memory"}}]] """ op, nodes = filter_expr.items()[0] msg = "%s operator is not supported" % op if op == 'or': filter_list = [] for node in nodes: filter_list.extend(self._parse_to_filter_list(node)) return filter_list elif op == 'and': filter_list_subtree = [] for node in nodes: filter_list_subtree.append(self._parse_to_filter_list(node)) filter_list = [[]] for filters in filter_list_subtree: tmp = [] for filter in filters: for f in filter_list: tmp.append(f + filter) filter_list = tmp return filter_list elif op == 'not': raise ceilometer.NotImplementedError(msg) elif op in ("<", "<=", "=", ">=", ">", '!='): return [[filter_expr]] else: raise ceilometer.NotImplementedError(msg) def _ensure_start_time_le_end_time(self, start_time, end_time): if start_time is None: start_time = datetime.datetime(1970, 1, 1) if end_time is None: end_time = timeutils.utcnow() # here we don't handle the corner case of start_time == end_time, # while start_time_op & end_time_op can be gt and lt, let Monasca # decides it valid or not. if start_time > end_time: msg = _('start time (%(start_time)s) should not after end_time ' '(%(end_time)s). (start time defaults to ' '1970-01-01T00:00:00Z, end time defaults to utc now)') % { 'start_time': start_time, 'end_time': end_time} raise InvalidInputException(msg) def _parse_to_sample_filter(self, simple_filters): """Parse simple filters to sample filter. For i.e.: parse [{"=":{"counter_name":"cpu"}},{"=":{"counter_volume": 1}}] to SampleFilter(counter_name="cpu", counter_volume=1) """ equal_only_fields = ( 'counter_name', 'counter_unit', 'counter_type', 'project_id', 'user_id', 'source', 'resource_id', # These fields are supported by Ceilometer but cannot supported # by Monasca. # 'message_id', # 'message_signature', # 'recorded_at', ) field_map = { "project_id": "project", "user_id": "user", "resource_id": "resource", "counter_name": "meter", "counter_type": "type", "counter_unit": "unit", } msg = "operator %s cannot be applied to field %s" kwargs = {'metaquery': {}} for sf in simple_filters: op = sf.keys()[0] field, value = sf.values()[0].items()[0] if field in equal_only_fields: if op != '=': raise ceilometer.NotImplementedError(msg % (op, field)) field = field_map.get(field, field) kwargs[field] = value elif field == 'timestamp': if op == '>=': kwargs['start_timestamp'] = value kwargs['start_timestamp_op'] = 'ge' elif op == '>': kwargs['start_timestamp'] = value kwargs['start_timestamp_op'] = 'gt' elif op == '<=': kwargs['end_timestamp'] = value kwargs['end_timestamp_op'] = 'le' elif op == '<': kwargs['end_timestamp'] = value kwargs['end_timestamp_op'] = 'lt' else: raise ceilometer.NotImplementedError(msg % (op, field)) elif field == 'counter_volume': kwargs['volume'] = value kwargs['volume_op'] = op elif (field.startswith('resource_metadata.') or field.startswith('metadata.')): kwargs['metaquery'][field] = value else: ra_msg = "field %s is not supported" % field raise ceilometer.NotImplementedError(ra_msg) self._ensure_start_time_le_end_time(kwargs.get('start_timestamp'), kwargs.get('end_timestamp')) sample_type = kwargs.pop('type', None) sample_unit = kwargs.pop('unit', None) sample_volume = kwargs.pop('volume', None) sample_volume_op = kwargs.pop('volume_op', None) sample_filter = storage.SampleFilter(**kwargs) # Add some dynamic attributes, type and unit attributes can be used # when query Monasca API, volume and volime_op attributes can # be used for volume comparison. sample_filter.type = sample_type sample_filter.unit = sample_unit sample_filter.volume = sample_volume sample_filter.volume_op = sample_volume_op return sample_filter def _is_meter_name_exist(self, filters): for f in filters: field = f.values()[0].keys()[0] if field == 'counter_name': return True return False def _validate_filters(self, filters, msg=''): """Validate filters to ensure fail at early stage.""" if not self._is_meter_name_exist(filters): msg = _('%(msg)s, meter name is not found in %(filters)s') % { 'msg': msg, 'filters': jsonutils.dumps(filters)} raise InvalidInputException(msg) def _parse_to_sample_filters(self, filter_expr): """Parse complex query expression to sample filter list.""" filter_list = self._parse_to_filter_list(filter_expr) msg = _('complex query filter expression has been translated to ' '%(count)d sub queries: %(filters)s') % { 'count': len(filter_list), 'filters': jsonutils.dumps(filter_list)} sample_filters = [] for filters in filter_list: self._validate_filters(filters, msg) sf = self._parse_to_sample_filter(filters) if sf: sample_filters.append(sf) return sample_filters def _validate_samples_by_volume(self, samples, sf): if not sf.volume: return samples op_func_map = { '<': operator.lt, '<=': operator.le, '=': operator.eq, '>=': operator.ge, '>': operator.gt, '!=': operator.ne, } ret = [] for s in samples: op_func = op_func_map[sf.volume_op] volume = getattr(s, 'volume', getattr(s, 'counter_volume', None)) if op_func(volume, sf.volume): ret.append(s) return ret def query_samples(self, filter_expr=None, orderby=None, limit=None): if not filter_expr: msg = _("fitler must be specified") raise ceilometer.NotImplementedError(msg) if orderby: msg = _("orderby is not supported") raise ceilometer.NotImplementedError(msg) # limit won't be None because we have limit enforcement in API level if limit == 0: return [] LOG.debug("filter_expr = %s", filter_expr) try: sample_filters = self._parse_to_sample_filters(filter_expr) # ValueError: year=1016 is before 1900; the datetime strftime() # methods require year >= 1900 except ValueError as e: raise InvalidInputException(e) LOG.debug("sample_filters = %s", sample_filters) ret = [] for sf in sample_filters: if not sf.volume: samples = list(self.get_samples(sf, limit)) else: samples = self.get_samples(sf) samples = list(self._validate_samples_by_volume(samples, sf)) if limit <= len(samples): ret.extend(samples[0:limit]) break else: ret.extend(samples) limit -= len(samples) return ret