diff --git a/cloudkitty/collector/monasca.py b/cloudkitty/collector/monasca.py new file mode 100644 index 00000000..e190891e --- /dev/null +++ b/cloudkitty/collector/monasca.py @@ -0,0 +1,273 @@ +# -*- coding: utf-8 -*- +# Copyright 2017 Objectif Libre +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# @author: Luka Peschke +# +import decimal + +from keystoneauth1 import loading as ks_loading +from keystoneclient.v3 import client as ks_client +from monascaclient import client as mclient +from oslo_config import cfg +from oslo_utils import units + +from cloudkitty import collector +from cloudkitty import transformer +from cloudkitty import utils as ck_utils + +MONASCA_API_VERSION = '2_0' +COLLECTOR_MONASCA_OPTS = 'collector_monasca' +collector_monasca_opts = ks_loading.get_auth_common_conf_options() + +cfg.CONF.register_opts(collector_monasca_opts, COLLECTOR_MONASCA_OPTS) +ks_loading.register_session_conf_options( + cfg.CONF, + COLLECTOR_MONASCA_OPTS) +ks_loading.register_auth_conf_options( + cfg.CONF, + COLLECTOR_MONASCA_OPTS) +CONF = cfg.CONF + + +class EndpointNotFound(Exception): + """Exception raised if the Monasca endpoint is not found""" + pass + + +class MonascaCollector(collector.BaseCollector): + collector_name = 'monasca' + dependencies = ['CloudKittyFormatTransformer'] + retrieve_mappings = { + 'compute': 'cpu', + 'image': 'image.size', + 'volume': 'volume.size', + 'network.floating': 'ip.floating', + 'network.bw.in': 'network.incoming.bytes', + 'network.bw.out': 'network.outgoing.bytes', + } + metric_mappings = { + 'compute': [ + ('cpu', 'max'), + ('vpcus', 'max'), + ('memory', 'max')], + 'image': [ + ('image.size', 'max'), + ('image.download', 'max'), + ('image.serve', 'max')], + 'volume': [ + ('volume.size', 'max')], + 'network.bw.in': [ + ('network.incoming.bytes', 'max')], + 'network.bw.out': [ + ('network.outgoing.bytes', 'max')], + 'network.floating': [ + ('ip.floating', 'max')], + } + # (qty, unit). qty must be either a metric name, an integer + # or a decimal.Decimal object + unit_mappings = { + 'compute': (1, 'instance'), + 'image': ('image.size', 'MB'), + 'volume': ('volume.size', 'GB'), + 'network.bw.out': ('network.outgoing.bytes', 'MB'), + 'network.bw.in': ('network.incoming.bytes', 'MB'), + 'network.floating': (1, 'ip'), + } + default_unit = (1, 'unknown') + + def __init__(self, transformers, **kwargs): + super(MonascaCollector, self).__init__(transformers, **kwargs) + + self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer'] + + self.auth = ks_loading.load_auth_from_conf_options( + CONF, + COLLECTOR_MONASCA_OPTS) + self.session = ks_loading.load_session_from_conf_options( + CONF, + COLLECTOR_MONASCA_OPTS, + auth=self.auth) + self.ks_client = ks_client.Client(session=self.session) + self.mon_endpoint = self._get_monasca_endpoint() + if not self.mon_endpoint: + raise EndpointNotFound() + # NOTE (lukapeschke) session authentication should be possible starting + # with OpenStack Q release. + self._conn = mclient.Client( + api_version=MONASCA_API_VERSION, + session=self.session, + endpoint=self.mon_endpoint) + + # NOTE(lukapeschke) This function should be removed as soon as the endpoint + # it no longer required by monascaclient + def _get_monasca_endpoint(self, service_name='monasca', + endpoint_interface_type='public'): + service_list = self.ks_client.services.list(name=service_name) + if not service_list: + return None + mon_service = service_list[0] + endpoints = self.ks_client.endpoints.list(mon_service.id) + for endpoint in endpoints: + if endpoint.interface == endpoint_interface_type: + return endpoint.url + return None + + def _get_metadata(self, resource_type, transformers): + info = {} + try: + info['unit'] = self.unit_mappings[resource_type][1] + except (KeyError, IndexError): + info['unit'] = self.default_unit[1] + start = ck_utils.dt2ts(ck_utils.get_month_start()) + end = ck_utils.dt2ts(ck_utils.get_month_end()) + try: + resource_id = self.active_resources(resource_type, start, + end, None)[0] + except IndexError: + resource_id = '' + metadata = self._get_resource_metadata(resource_type, start, + end, resource_id) + info['metadata'] = metadata.keys() + try: + for metric, statistics in self.metric_mappings[resource_type]: + info['metadata'].append(metric) + except (KeyError, IndexError): + pass + return info + + # NOTE(lukapeschke) if anyone sees a better way to do this, + # please make a patch + @classmethod + def get_metadata(cls, resource_type, transformers): + args = { + 'transformers': transformer.get_transformers(), + 'period': CONF.collect.period} + tmp = cls(**args) + return tmp._get_metadata(resource_type, transformers) + + def _get_resource_metadata(self, resource_type, start, end, resource_id): + meter = self.retrieve_mappings.get(resource_type) + if not meter: + return {} + measurements = self._conn.metrics.list_measurements( + name=meter, + start_time=ck_utils.ts2dt(start), + end_time=ck_utils.ts2dt(end), + merge_metrics=True, + dimensions={'resource_id': resource_id}, + ) + try: + # Getting the last measurement of given period + metadata = measurements[-1]['measurements'][-1][2] + except (KeyError, IndexError): + metadata = {} + return metadata + + def _get_resource_qty(self, meter, start, end, resource_id, statistics): + # NOTE(lukapeschke) the period trick is used to aggregate + # the measurements + period = end - start + statistics = self._conn.metrics.list_statistics( + name=meter, + start_time=ck_utils.ts2dt(start), + end_time=ck_utils.ts2dt(end), + dimensions={'resource_id': resource_id}, + statistics=statistics, + period=period, + merge_metrics=True, + ) + try: + # If several statistics are returned (should not happen), + # use the latest + qty = decimal.Decimal(statistics[-1]['statistics'][-1][1]) + except (KeyError, IndexError): + qty = decimal.Decimal(0) + return qty + + def _is_resource_active(self, meter, resource_id, start, end): + measurements = self._conn.metrics.list_measurements( + name=meter, + start_time=ck_utils.ts2dt(start), + end_time=ck_utils.ts2dt(end), + group_by='resource_id', + merge_metrics=True, + dimensions={'resource_id': resource_id}, + ) + return len(measurements) > 0 + + def active_resources(self, resource_type, start, + end, project_id, **kwargs): + meter = self.retrieve_mappings.get(resource_type) + if not meter: + return {} + dimensions = {} + dimensions.update(kwargs) + if project_id: + resources = self._conn.metrics.list(name=meter, + tenant_id=project_id, + **dimensions) + else: + resources = self._conn.metrics.list(name=meter, + **dimensions) + resource_ids = [] + for resource in resources: + try: + resource_id = resource['dimensions']['resource_id'] + if (resource_id not in resource_ids + and self._is_resource_active(meter, resource_id, + start, end)): + resource_ids.append(resource_id) + except KeyError: + continue + return resource_ids + + def _expand_metrics(self, resource, resource_id, mappings, start, end): + for name, statistics in mappings: + qty = self._get_resource_qty(name, start, + end, resource_id, statistics) + if name in ['network.outgoing.bytes', 'network.incoming.bytes']: + qty = qty / units.M + elif 'image.' in name: + qty = qty / units.Mi + resource[name] = qty + + def resource_info(self, resource_type, start, end, + project_id, q_filter=None): + qty, unit = self.unit_mappings.get(resource_type, self.default_unit) + active_resource_ids = self.active_resources( + resource_type, start, end, project_id + ) + resource_data = [] + for resource_id in active_resource_ids: + data = self._get_resource_metadata(resource_type, start, + end, resource_id) + mappings = self.metric_mappings[resource_type] + self._expand_metrics(data, resource_id, mappings, start, end) + resource_qty = qty + if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)): + resource_qty = data[self.retrieve_mappings[resource_type]] + resource = self.t_cloudkitty.format_item(data, unit, resource_qty) + resource['desc']['resource_id'] = resource_id + resource['resource_id'] = resource_id + resource_data.append(resource) + return resource_data + + def retrieve(self, resource_type, start, end, project_id, q_filter=None): + resources = self.resource_info(resource_type, start, end, + project_id=project_id, + q_filter=q_filter) + if not resources: + raise collector.NoDataCollected(self.collector_name, resource_type) + return self.t_cloudkitty.format_service(resource_type, resources) diff --git a/cloudkitty/common/config.py b/cloudkitty/common/config.py index 9f69828b..2a161200 100644 --- a/cloudkitty/common/config.py +++ b/cloudkitty/common/config.py @@ -20,6 +20,7 @@ import cloudkitty.api.app import cloudkitty.collector import cloudkitty.collector.ceilometer import cloudkitty.collector.gnocchi +import cloudkitty.collector.monasca import cloudkitty.config import cloudkitty.orchestrator import cloudkitty.service @@ -37,6 +38,8 @@ _opts = [ cloudkitty.collector.collect_opts))), ('ceilometer_collector', list(itertools.chain( cloudkitty.collector.ceilometer.ceilometer_collector_opts))), + ('collector_monasca', list(itertools.chain( + cloudkitty.collector.monasca.collector_monasca_opts))), ('gnocchi_collector', list(itertools.chain( cloudkitty.collector.gnocchi.gnocchi_collector_opts))), ('keystone_fetcher', list(itertools.chain( diff --git a/doc/source/configuration/index.rst b/doc/source/configuration/index.rst index 3cbab9a7..7fcf357c 100644 --- a/doc/source/configuration/index.rst +++ b/doc/source/configuration/index.rst @@ -136,8 +136,9 @@ If you want to use the pure gnocchi storage, add the following entry: [storage_gnocchi] auth_section = ks_auth -Two collectors are available: Ceilometer (deprecated, see the Telemetry -documentation), and Gnocchi. +Three collectors are available: Ceilometer (deprecated, see the Telemetry +documentation), Gnocchi and Monasca. The Monasca collector collects metrics +published by the Ceilometer agent to Monasca using Ceilosca_. .. code-block:: ini @@ -244,3 +245,6 @@ Choose and start the API server as:: $ cloudkitty-api -p 8889 + + +.. _Ceilosca: https://github.com/openstack/monasca-ceilometer \ No newline at end of file diff --git a/releasenotes/notes/collector-monasca-f0871406513ff22c.yaml b/releasenotes/notes/collector-monasca-f0871406513ff22c.yaml new file mode 100644 index 00000000..a1655cf9 --- /dev/null +++ b/releasenotes/notes/collector-monasca-f0871406513ff22c.yaml @@ -0,0 +1,5 @@ +--- +features: + - | + A collector for Monasca has been added. It works with telemetry metrics + published to Monasca by Ceilometer agent through Ceilosca. diff --git a/requirements.txt b/requirements.txt index c91fbd2b..6d5cc3b6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ eventlet!=0.18.3,>=0.18.2 # MIT keystonemiddleware!=4.1.0,>=4.0.0 # Apache-2.0 python-ceilometerclient>=2.2.1 # Apache-2.0 gnocchiclient>=2.5.0 # Apache-2.0 +python-monascaclient>=1.7.0 # Apache-2.0 python-keystoneclient!=1.8.0,!=2.1.0,>=1.6.0 # Apache-2.0 keystoneauth1>=2.1.0 # Apache-2.0 iso8601>=0.1.9 # MIT diff --git a/setup.cfg b/setup.cfg index ace5af08..355a617c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -42,6 +42,7 @@ cloudkitty.collector.backends = fake = cloudkitty.collector.fake:CSVCollector ceilometer = cloudkitty.collector.ceilometer:CeilometerCollector gnocchi = cloudkitty.collector.gnocchi:GnocchiCollector + monasca = cloudkitty.collector.monasca:MonascaCollector meta = cloudkitty.collector.meta:MetaCollector cloudkitty.tenant.fetchers =