372 lines
14 KiB
Python
372 lines
14 KiB
Python
# -*- coding: utf-8 -*-
|
|
# Copyright 2017 Objectif Libre
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
# @author: Luka Peschke
|
|
#
|
|
import decimal
|
|
|
|
from keystoneauth1 import loading as ks_loading
|
|
from keystoneclient.v3 import client as ks_client
|
|
from monascaclient import client as mclient
|
|
from oslo_config import cfg
|
|
from oslo_log import log as logging
|
|
from oslo_utils import units
|
|
|
|
from cloudkitty import collector
|
|
from cloudkitty import transformer
|
|
from cloudkitty import utils as ck_utils
|
|
|
|
|
|
LOG = logging.getLogger(__name__)
|
|
|
|
MONASCA_API_VERSION = '2_0'
|
|
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
|
|
collector_monasca_opts = ks_loading.get_auth_common_conf_options()
|
|
|
|
cfg.CONF.register_opts(collector_monasca_opts, COLLECTOR_MONASCA_OPTS)
|
|
ks_loading.register_session_conf_options(
|
|
cfg.CONF,
|
|
COLLECTOR_MONASCA_OPTS)
|
|
ks_loading.register_auth_conf_options(
|
|
cfg.CONF,
|
|
COLLECTOR_MONASCA_OPTS)
|
|
CONF = cfg.CONF
|
|
|
|
METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
|
|
|
|
|
|
class EndpointNotFound(Exception):
|
|
"""Exception raised if the Monasca endpoint is not found"""
|
|
pass
|
|
|
|
|
|
class MonascaCollector(collector.BaseCollector):
|
|
collector_name = 'monasca'
|
|
dependencies = ['CloudKittyFormatTransformer']
|
|
retrieve_mappings = {
|
|
'compute': 'cpu',
|
|
'image': 'image.size',
|
|
'volume': 'volume.size',
|
|
'network.floating': 'ip.floating',
|
|
'network.bw.in': 'network.incoming.bytes',
|
|
'network.bw.out': 'network.outgoing.bytes',
|
|
}
|
|
metrics_mappings = {
|
|
'compute': [
|
|
('cpu', 'max'),
|
|
('vpcus', 'max'),
|
|
('memory', 'max')],
|
|
'image': [
|
|
('image.size', 'max'),
|
|
('image.download', 'max'),
|
|
('image.serve', 'max')],
|
|
'volume': [
|
|
('volume.size', 'max')],
|
|
'network.bw.in': [
|
|
('network.incoming.bytes', 'max')],
|
|
'network.bw.out': [
|
|
('network.outgoing.bytes', 'max')],
|
|
'network.floating': [
|
|
('ip.floating', 'max')],
|
|
}
|
|
# (qty, unit). qty must be either a metric name, an integer
|
|
# or a decimal.Decimal object
|
|
units_mappings = {
|
|
'compute': (1, 'instance'),
|
|
'image': ('image.size', 'MiB'),
|
|
'volume': ('volume.size', 'GiB'),
|
|
'network.bw.out': ('network.outgoing.bytes', 'MB'),
|
|
'network.bw.in': ('network.incoming.bytes', 'MB'),
|
|
'network.floating': (1, 'ip'),
|
|
}
|
|
default_unit = (1, 'unknown')
|
|
|
|
def __init__(self, transformers, **kwargs):
|
|
super(MonascaCollector, self).__init__(transformers, **kwargs)
|
|
|
|
self.t_cloudkitty = self.transformers['CloudKittyFormatTransformer']
|
|
|
|
self.auth = ks_loading.load_auth_from_conf_options(
|
|
CONF,
|
|
COLLECTOR_MONASCA_OPTS)
|
|
self.session = ks_loading.load_session_from_conf_options(
|
|
CONF,
|
|
COLLECTOR_MONASCA_OPTS,
|
|
auth=self.auth)
|
|
self.ks_client = ks_client.Client(session=self.session)
|
|
self.mon_endpoint = self._get_monasca_endpoint()
|
|
if not self.mon_endpoint:
|
|
raise EndpointNotFound()
|
|
# NOTE (lukapeschke) session authentication should be possible starting
|
|
# with OpenStack Q release.
|
|
self._conn = mclient.Client(
|
|
api_version=MONASCA_API_VERSION,
|
|
session=self.session,
|
|
endpoint=self.mon_endpoint)
|
|
|
|
# NOTE(lukapeschke) This function should be removed as soon as the endpoint
|
|
# it no longer required by monascaclient
|
|
def _get_monasca_endpoint(self, service_name='monasca',
|
|
endpoint_interface_type='public'):
|
|
service_list = self.ks_client.services.list(name=service_name)
|
|
if not service_list:
|
|
return None
|
|
mon_service = service_list[0]
|
|
endpoints = self.ks_client.endpoints.list(mon_service.id)
|
|
for endpoint in endpoints:
|
|
if endpoint.interface == endpoint_interface_type:
|
|
return endpoint.url
|
|
return None
|
|
|
|
def _get_metadata(self, resource_type, transformers):
|
|
info = {}
|
|
try:
|
|
met = list(METRICS_CONF['metrics_units'][resource_type].values())
|
|
info['unit'] = met[0]['unit']
|
|
# NOTE(mc): deprecated second try kept for backward compatibility.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
|
try:
|
|
info['unit'] = self.units_mappings[resource_type][1]
|
|
except (KeyError, IndexError):
|
|
info['unit'] = self.default_unit[1]
|
|
|
|
start = ck_utils.dt2ts(ck_utils.get_month_start())
|
|
end = ck_utils.dt2ts(ck_utils.get_month_end())
|
|
try:
|
|
resource_id = self.active_resources(resource_type, start,
|
|
end, None)[0]
|
|
except IndexError:
|
|
resource_id = ''
|
|
metadata = self._get_resource_metadata(resource_type, start,
|
|
end, resource_id)
|
|
info['metadata'] = metadata.keys()
|
|
|
|
try:
|
|
for metric, statistics in METRICS_CONF['services_metrics']:
|
|
info['metadata'].append(metric)
|
|
# NOTE(mc): deprecated second try kept for backward compatibility.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
|
try:
|
|
for metric, statistics in self.metrics_mappings[resource_type]:
|
|
info['metadata'].append(metric)
|
|
except (KeyError, IndexError):
|
|
pass
|
|
return info
|
|
|
|
# NOTE(lukapeschke) if anyone sees a better way to do this,
|
|
# please make a patch
|
|
@classmethod
|
|
def get_metadata(cls, resource_type, transformers):
|
|
args = {
|
|
'transformers': transformer.get_transformers(),
|
|
'period': CONF.collect.period}
|
|
tmp = cls(**args)
|
|
return tmp._get_metadata(resource_type, transformers)
|
|
|
|
def _get_resource_metadata(self, resource_type, start, end, resource_id):
|
|
try:
|
|
meter = METRICS_CONF['services_objects'].get(resource_type)
|
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
|
meter = self.retrieve_mappings.get(resource_type)
|
|
|
|
if not meter:
|
|
return {}
|
|
measurements = self._conn.metrics.list_measurements(
|
|
name=meter,
|
|
start_time=ck_utils.ts2dt(start),
|
|
end_time=ck_utils.ts2dt(end),
|
|
merge_metrics=True,
|
|
dimensions={'resource_id': resource_id},
|
|
)
|
|
try:
|
|
# Getting the last measurement of given period
|
|
metadata = measurements[-1]['measurements'][-1][2]
|
|
except (KeyError, IndexError):
|
|
metadata = {}
|
|
return metadata
|
|
|
|
def _get_resource_qty(self, meter, start, end, resource_id, statistics):
|
|
# NOTE(lukapeschke) the period trick is used to aggregate
|
|
# the measurements
|
|
period = end - start
|
|
statistics = self._conn.metrics.list_statistics(
|
|
name=meter,
|
|
start_time=ck_utils.ts2dt(start),
|
|
end_time=ck_utils.ts2dt(end),
|
|
dimensions={'resource_id': resource_id},
|
|
statistics=statistics,
|
|
period=period,
|
|
merge_metrics=True,
|
|
)
|
|
try:
|
|
# If several statistics are returned (should not happen),
|
|
# use the latest
|
|
qty = decimal.Decimal(statistics[-1]['statistics'][-1][1])
|
|
except (KeyError, IndexError):
|
|
qty = decimal.Decimal(0)
|
|
return qty
|
|
|
|
def _is_resource_active(self, meter, resource_id, start, end):
|
|
measurements = self._conn.metrics.list_measurements(
|
|
name=meter,
|
|
start_time=ck_utils.ts2dt(start),
|
|
end_time=ck_utils.ts2dt(end),
|
|
group_by='resource_id',
|
|
merge_metrics=True,
|
|
dimensions={'resource_id': resource_id},
|
|
)
|
|
return len(measurements) > 0
|
|
|
|
def active_resources(self, resource_type, start,
|
|
end, project_id, **kwargs):
|
|
try:
|
|
meter = METRICS_CONF['services_objects'].get(resource_type)
|
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
|
meter = self.retrieve_mappings.get(resource_type)
|
|
|
|
if not meter:
|
|
return {}
|
|
dimensions = {}
|
|
if project_id:
|
|
dimensions['project_id'] = project_id
|
|
dimensions.update(kwargs)
|
|
resources = self._conn.metrics.list(name=meter,
|
|
**dimensions)
|
|
resource_ids = []
|
|
for resource in resources:
|
|
try:
|
|
resource_id = resource['dimensions']['resource_id']
|
|
if (resource_id not in resource_ids
|
|
and self._is_resource_active(meter, resource_id,
|
|
start, end)):
|
|
resource_ids.append(resource_id)
|
|
except KeyError:
|
|
continue
|
|
return resource_ids
|
|
|
|
def _expand_metrics(self, resource, resource_id,
|
|
mappings, start, end, resource_type):
|
|
try:
|
|
for name, statistics in mappings.items():
|
|
qty = self._get_resource_qty(
|
|
name,
|
|
start,
|
|
end,
|
|
resource_id,
|
|
statistics,
|
|
)
|
|
|
|
conv_data = METRICS_CONF['metrics_units'][resource_type][name]
|
|
resource[name] = ck_utils.convert_unit(
|
|
qty,
|
|
conv_data.get('factor', 1),
|
|
conv_data.get('offset', 0),
|
|
)
|
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
|
LOG.warning('Fallback on the deprecated hardcoded dict method.')
|
|
|
|
for name, statistics in mappings:
|
|
qty = self._get_resource_qty(
|
|
name,
|
|
start,
|
|
end,
|
|
resource_id,
|
|
statistics,
|
|
)
|
|
|
|
names = ['network.outgoing.bytes', 'network.incoming.bytes']
|
|
if name in names:
|
|
qty = qty / units.M
|
|
elif 'image.' in name:
|
|
qty = qty / units.Mi
|
|
resource[name] = qty
|
|
|
|
def resource_info(self, resource_type, start, end,
|
|
project_id, q_filter=None):
|
|
|
|
try:
|
|
tmp = METRICS_CONF['metrics_units'][resource_type]
|
|
qty = list(tmp.keys())[0]
|
|
unit = list(tmp.values())[0]['unit']
|
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
|
qty, unit = self.units_mappings.get(
|
|
resource_type,
|
|
self.default_unit
|
|
)
|
|
|
|
active_resource_ids = self.active_resources(
|
|
resource_type, start, end, project_id
|
|
)
|
|
resource_data = []
|
|
for resource_id in active_resource_ids:
|
|
data = self._get_resource_metadata(resource_type, start,
|
|
end, resource_id)
|
|
|
|
try:
|
|
mappings = METRICS_CONF['services_metrics'][resource_type]
|
|
# NOTE(mc): deprecated except part kept for backward compatibility.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf.')
|
|
LOG.warning('Fallback on the deprecated oslo config method.')
|
|
mappings = self.metrics_mappings[resource_type]
|
|
|
|
self._expand_metrics(
|
|
data,
|
|
resource_id,
|
|
mappings,
|
|
start,
|
|
end,
|
|
resource_type,
|
|
)
|
|
resource_qty = qty
|
|
if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
|
|
try:
|
|
resource_qty = METRICS_CONF['services_objects']
|
|
# NOTE(mc): deprecated except part kept for backward compat.
|
|
except KeyError:
|
|
LOG.warning('Error when trying to use yaml metrology conf')
|
|
msg = 'Fallback on the deprecated oslo config method'
|
|
LOG.warning(msg)
|
|
resource_qty = data[self.retrieve_mappings[resource_type]]
|
|
|
|
resource = self.t_cloudkitty.format_item(data, unit, resource_qty)
|
|
resource['desc']['resource_id'] = resource_id
|
|
resource['resource_id'] = resource_id
|
|
resource_data.append(resource)
|
|
return resource_data
|
|
|
|
def retrieve(self, resource_type, start, end, project_id, q_filter=None):
|
|
resources = self.resource_info(resource_type, start, end,
|
|
project_id=project_id,
|
|
q_filter=q_filter)
|
|
if not resources:
|
|
raise collector.NoDataCollected(self.collector_name, resource_type)
|
|
return self.t_cloudkitty.format_service(resource_type, resources)
|