openstack
/
cloudkitty
Code Issues Proposed changes

Merge "Removal of Monasca fetcher and collector"

This commit is contained in:
Zuul 2024-02-19 15:12:50 +00:00 committed by Gerrit Code Review
parent 9ca4a42189 b46dcd8983
commit 328307512b
14 changed files with 7 additions and 611 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

237
cloudkitty/collector/monasca.py
View File

@ -1,237 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2017 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from oslo_log import log as logging
from voluptuous import All
from voluptuous import In
from voluptuous import Length
from voluptuous import Required
from voluptuous import Schema
from cloudkitty import collector
from cloudkitty.common import monasca_client as mon_client_utils
from cloudkitty import utils as ck_utils
LOG = logging.getLogger(__name__)
MONASCA_API_VERSION = '2_0'
COLLECTOR_MONASCA_OPTS = 'collector_monasca'
collector_monasca_opts = [
cfg.StrOpt(
'interface',
default='internal',
help='Endpoint URL type (defaults to internal)',
),
cfg.StrOpt(
'monasca_service_name',
default='monasca',
help='Name of the Monasca service (defaults to monasca)',
),
]
CONF = cfg.CONF
CONF.register_opts(collector_monasca_opts, COLLECTOR_MONASCA_OPTS)
ks_loading.register_auth_conf_options(CONF, COLLECTOR_MONASCA_OPTS)
ks_loading.register_session_conf_options(CONF, COLLECTOR_MONASCA_OPTS)
MONASCA_EXTRA_SCHEMA = {
Required('extra_args', default={}): {
# Key corresponding to the resource id in a metric's dimensions
# Allows to adapt the resource identifier. Should not need to be
# modified in a standard OpenStack installation
Required('resource_key', default='resource_id'):
All(str, Length(min=1)),
Required('aggregation_method', default='max'):
In(['max', 'mean', 'min']),
# In case the metrics in Monasca do not belong to the project
# cloudkitty is identified in
Required('forced_project_id', default=''): str,
},
}
class MonascaCollector(collector.BaseCollector):
collector_name = 'monasca'
@staticmethod
def check_configuration(conf):
conf = collector.BaseCollector.check_configuration(conf)
metric_schema = Schema(collector.METRIC_BASE_SCHEMA).extend(
MONASCA_EXTRA_SCHEMA)
output = {}
for metric_name, metric in conf.items():
met = output[metric_name] = metric_schema(metric)
if met['extra_args']['resource_key'] not in met['groupby']:
met['groupby'].append(met['extra_args']['resource_key'])
return output
def __init__(self, **kwargs):
super(MonascaCollector, self).__init__(**kwargs)
self._conn = mon_client_utils.get_monasca_client(
CONF, COLLECTOR_MONASCA_OPTS)
def _get_metadata(self, metric_name, conf):
info = {}
info['unit'] = conf['metrics'][metric_name]['unit']
dimension_names = self._conn.metric.list_dimension_names(
metric_name=metric_name)
info['metadata'] = [d['dimension_name'] for d in dimension_names]
return info
# NOTE(lukapeschke) if anyone sees a better way to do this,
# please make a patch
@classmethod
def get_metadata(cls, resource_type, conf):
tmp = cls(period=conf['period'])
return tmp._get_metadata(resource_type, conf)
def _get_dimensions(self, metric_name, project_id, q_filter):
dimensions = {}
scope_key = CONF.collect.scope_key
if project_id:
dimensions[scope_key] = project_id
if q_filter:
dimensions.update(q_filter)
return dimensions
def _fetch_measures(self, metric_name, start, end,
project_id=None, q_filter=None):
"""Get measures for given metric during the timeframe.
:param metric_name: metric name to filter on.
:type metric_name: str
:param start: Start of the timeframe.
:param end: End of the timeframe if needed.
:param project_id: Filter on a specific tenant/project.
:type project_id: str
:param q_filter: Append a custom filter.
:type q_filter: list
"""
dimensions = self._get_dimensions(metric_name, project_id, q_filter)
group_by = self.conf[metric_name]['groupby']
# NOTE(lpeschke): One aggregated measure per collect period
period = int((end - start).total_seconds())
extra_args = self.conf[metric_name]['extra_args']
kwargs = {}
if extra_args['forced_project_id']:
if extra_args['forced_project_id'] == 'SCOPE_ID' and project_id:
kwargs['tenant_id'] = project_id
dimensions.pop(CONF.collect.scope_key, None)
else:
kwargs['tenant_id'] = extra_args['forced_project_id']
return self._conn.metrics.list_statistics(
name=metric_name,
merge_metrics=True,
dimensions=dimensions,
start_time=start,
end_time=end,
period=period,
statistics=extra_args['aggregation_method'],
group_by=group_by,
**kwargs)
def _fetch_metrics(self, metric_name, start, end,
project_id=None, q_filter=None):
"""List active metrics during the timeframe.
:param metric_name: metric name to filter on.
:type metric_name: str
:param start: Start of the timeframe.
:param end: End of the timeframe if needed.
:param project_id: Filter on a specific tenant/project.
:type project_id: str
:param q_filter: Append a custom filter.
:type q_filter: list
"""
dimensions = self._get_dimensions(metric_name, project_id, q_filter)
metrics = self._conn.metrics.list(
name=metric_name,
dimensions=dimensions,
start_time=start,
end_time=end,
)
resource_key = self.conf[metric_name]['extra_args']['resource_key']
return {metric['dimensions'][resource_key]:
metric['dimensions'] for metric in metrics}
def _format_data(self, metconf, data, resources_info=None):
"""Formats Monasca data to CK data.
Returns metadata, groupby and qty
"""
groupby = data['dimensions']
resource_key = metconf['extra_args']['resource_key']
metadata = dict()
if resources_info:
resource = resources_info[groupby[resource_key]]
for i in metconf['metadata']:
metadata[i] = resource.get(i, '')
qty = data['statistics'][0][1]
converted_qty = ck_utils.convert_unit(
qty, metconf['factor'], metconf['offset'])
mutate_map = metconf.get('mutate_map')
mutated_qty = ck_utils.mutate(converted_qty, metconf['mutate'],
mutate_map=mutate_map)
return metadata, groupby, mutated_qty
def fetch_all(self, metric_name, start, end,
project_id=None, q_filter=None):
met = self.conf[metric_name]
data = self._fetch_measures(
metric_name,
start,
end,
project_id=project_id,
q_filter=q_filter,
)
resources_info = None
if met['metadata']:
resources_info = self._fetch_metrics(
metric_name,
start,
end,
project_id=project_id,
q_filter=q_filter,
)
formated_resources = list()
for d in data:
if len(d['statistics']):
metadata, groupby, qty = self._format_data(
met, d, resources_info)
point = self._create_data_point(met, qty, 0, groupby,
metadata, start)
formated_resources.append(point)
return formated_resources

3
cloudkitty/common/config.py
View File

@ -18,7 +18,6 @@ import itertools
import cloudkitty.api.app
import cloudkitty.collector.gnocchi
import cloudkitty.collector.monasca
import cloudkitty.collector.prometheus
import cloudkitty.config
import cloudkitty.fetcher
@ -44,8 +43,6 @@ _opts = [
cloudkitty.collector.collect_opts))),
('collector_gnocchi', list(itertools.chain(
cloudkitty.collector.gnocchi.collector_gnocchi_opts))),
('collector_monasca', list(itertools.chain(
cloudkitty.collector.monasca.collector_monasca_opts))),
('collector_prometheus', list(itertools.chain(
cloudkitty.collector.prometheus.collector_prometheus_opts))),
('fetcher', list(itertools.chain(

61
cloudkitty/common/monasca_client.py
View File

@ -1,61 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from keystoneauth1 import loading as ks_loading
from keystoneclient.v3 import client as ks_client
from monascaclient import client as mclient
MONASCA_API_VERSION = '2_0'
class EndpointNotFound(Exception):
"""Exception raised if the Monasca endpoint is not found"""
# NOTE(lukapeschke) This function should be removed as soon as the endpoint
# it no longer required by monascaclient
def get_monasca_endpoint(cfg, keystone_client):
service_name = cfg.monasca_service_name
endpoint_interface_type = cfg.interface
service_list = keystone_client.services.list(name=service_name)
if not service_list:
return None
mon_service = service_list[0]
endpoints = keystone_client.endpoints.list(mon_service.id)
for endpoint in endpoints:
if endpoint.interface == endpoint_interface_type:
return endpoint.url
return None
def get_monasca_client(conf, conf_opts):
ks_auth = ks_loading.load_auth_from_conf_options(conf, conf_opts)
session = ks_loading.load_session_from_conf_options(
conf,
conf_opts,
auth=ks_auth)
keystone_client = ks_client.Client(
session=session,
interface=conf[conf_opts].interface)
mon_endpoint = get_monasca_endpoint(conf[conf_opts], keystone_client)
if not mon_endpoint:
raise EndpointNotFound()
return mclient.Client(
api_version=MONASCA_API_VERSION,
session=session,
endpoint=mon_endpoint)

71
cloudkitty/fetcher/monasca.py
View File

@ -1,71 +0,0 @@
# -*- coding: utf-8 -*-
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from keystoneauth1 import loading as ks_loading
from oslo_config import cfg
from cloudkitty.common import monasca_client as mon_client_utils
from cloudkitty import fetcher
MONASCA_API_VERSION = '2_0'
FETCHER_MONASCA_OPTS = 'fetcher_monasca'
fetcher_monasca_opts = [
cfg.StrOpt('dimension_name',
default='project_id',
help='Monasca dimension from which scope_ids should be'
' collected.'),
cfg.StrOpt('monasca_tenant_id',
default=None,
help='If specified, monasca client will use this ID instead of'
' the default one.'),
cfg.StrOpt(
'monasca_service_name',
default='monasca',
help='Name of the Monasca service (defaults to monasca)',
),
cfg.StrOpt(
'interface',
default='internal',
help='Endpoint URL type (defaults to internal).',
),
]
CONF = cfg.CONF
cfg.CONF.register_opts(fetcher_monasca_opts, FETCHER_MONASCA_OPTS)
ks_loading.register_auth_conf_options(CONF, FETCHER_MONASCA_OPTS)
ks_loading.register_session_conf_options(CONF, FETCHER_MONASCA_OPTS)
class MonascaFetcher(fetcher.BaseFetcher):
"""Monasca fetcher"""
name = 'monasca'
def __init__(self):
self._conn = mon_client_utils.get_monasca_client(CONF,
FETCHER_MONASCA_OPTS)
def get_tenants(self):
kwargs = {
"tenant_id": CONF.fetcher_monasca.monasca_tenant_id,
"dimension_name": CONF.fetcher_monasca.dimension_name,
}
if kwargs['tenant_id'] is None:
del kwargs['tenant_id']
values = self._conn.metrics.list_dimension_values(**kwargs)
return [v['dimension_value'] for v in values]

96
cloudkitty/tests/collectors/test_monasca.py
View File

@ -1,96 +0,0 @@
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
import datetime
from unittest import mock
from cloudkitty.collector import monasca as mon_collector
from cloudkitty import tests
class MonascaCollectorTest(tests.TestCase):
def setUp(self):
super(MonascaCollectorTest, self).setUp()
self.conf.set_override('collector', 'monasca', 'collect')
conf = {
'metrics': {
'metric_one': {
'unit': 'GiB',
'groupby': ['project_id'],
'extra_args': {
'aggregation_method': 'max',
},
},
'metric_two': {
'unit': 'MiB',
'groupby': ['project_id'],
'extra_args': {
'aggregation_method': 'max',
'forced_project_id': 'project_x'
},
},
}
}
with mock.patch(
'cloudkitty.common.monasca_client.'
'get_monasca_endpoint',
return_value='http://noop'):
self.collector = mon_collector.MonascaCollector(
period=3600,
conf=conf,
)
def test_fetch_measures_kwargs_no_forced_project(self):
with mock.patch.object(self.collector._conn.metrics,
'list_statistics') as m:
start = datetime.datetime(2019, 1, 1)
end = datetime.datetime(2019, 1, 1, 1)
self.collector._fetch_measures(
'metric_one',
start,
end,
)
m.assert_called_once_with(
name='metric_one',
merge_metrics=True,
dimensions={},
start_time=start,
end_time=end,
period=3600,
statistics='max',
group_by=['project_id', 'resource_id'],
)
def test_fetch_measures_kwargs_with_forced_project(self):
with mock.patch.object(self.collector._conn.metrics,
'list_statistics') as m:
start = datetime.datetime(2019, 1, 1)
end = datetime.datetime(2019, 1, 1, 1)
self.collector._fetch_measures(
'metric_two',
start,
end,
)
m.assert_called_once_with(
name='metric_two',
merge_metrics=True,
dimensions={},
start_time=start,
end_time=end,
period=3600,
statistics='max',
group_by=['project_id', 'resource_id'],
tenant_id='project_x',
)

32
cloudkitty/tests/collectors/test_validation.py
View File

@ -95,38 +95,6 @@ class MetricConfigValidationTest(tests.TestCase):
data,
)
def test_monasca_minimal_config_no_extra_args(self):
data = copy.deepcopy(self.base_data)
expected_output = copy.deepcopy(self.base_output)
expected_output['metric_one']['groupby'].extend(
['project_id', 'resource_id'])
expected_output['metric_one']['extra_args'] = {
'resource_key': 'resource_id',
'aggregation_method': 'max',
'forced_project_id': ''
}
self.assertEqual(
collector.monasca.MonascaCollector.check_configuration(data),
expected_output,
)
def test_monasca_minimal_config_minimal_extra_args(self):
data = copy.deepcopy(self.base_data)
data['metrics']['metric_one']['extra_args'] = {}
expected_output = copy.deepcopy(self.base_output)
expected_output['metric_one']['groupby'].extend(
['project_id', 'resource_id'])
expected_output['metric_one']['extra_args'] = {
'aggregation_method': 'max',
'resource_key': 'resource_id',
'forced_project_id': '',
}
self.assertEqual(
collector.monasca.MonascaCollector.check_configuration(data),
expected_output,
)
def test_prometheus_minimal_config_empty_extra_args(self):
data = copy.deepcopy(self.base_data)
data['metrics']['metric_one']['extra_args'] = {}

48
cloudkitty/tests/fetchers/test_monasca.py
View File

@ -1,48 +0,0 @@
# Copyright 2019 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
from unittest import mock
from cloudkitty.fetcher import monasca as mon_fetcher
from cloudkitty import tests
class MonascaFetcherTest(tests.TestCase):
def setUp(self):
super(MonascaFetcherTest, self).setUp()
self.conf.set_override('dimension_name', 'dimension_name_test',
'fetcher_monasca')
self.conf.set_override('monasca_tenant_id', 'this_is_definitely_a_uid',
'fetcher_monasca')
self.conf.set_override('monasca_service_name', 'monasca-api-test',
'fetcher_monasca')
self.conf.set_override('interface', 'interface-test',
'fetcher_monasca')
with mock.patch(
'cloudkitty.common.monasca_client.'
'get_monasca_endpoint',
return_value='http://noop'):
self.fetcher = mon_fetcher.MonascaFetcher()
def test_get_tenants(self):
with mock.patch.object(self.fetcher._conn.metrics,
'list_dimension_values') as m:
self.fetcher.get_tenants()
m.assert_called_once_with(
tenant_id='this_is_definitely_a_uid',
dimension_name='dimension_name_test',
)

7
doc/source/admin/architecture.rst
View File

@ -73,9 +73,6 @@ Five fetchers are available in CloudKitty:
discover new projects from Gnocchi when it is used with OpenStack. It can be
used in an OpenStack context or with a standalone Gnocchi deployment.
* The ``monasca`` fetcher retrieves from `Monasca`_ all values from a
configurable metric dimension (``project_id`` by default).
* The ``prometheus`` fetcher works in a similar way to the Gnocchi fetcher,
which allows to discover scopes from `Prometheus`_.
@ -95,9 +92,6 @@ There are three collectors available in CloudKitty:
* The ``gnocchi`` collector retrieves data from `Gnocchi`_. It can be used in
an OpenStack context or with a standalone Gnocchi deployment.
* The ``monasca`` collector retrieves data from `Monasca`_. Keystone
authentication is required for this collector.
* The ``prometheus`` collector retrieves data from `Prometheus`_.
Details about the configuration of each collector are available in the
@ -109,7 +103,6 @@ the `developer documentation`_.
.. _developer documentation: ../developer/collector.html
.. _collector configuration guide: configuration/collector.html
.. _Gnocchi: https://gnocchi.xyz/
.. _Monasca: https://docs.openstack.org/monasca-api/latest/
.. _Prometheus: https://prometheus.io/docs/introduction/overview/
Rating

39
doc/source/admin/configuration/collector.rst
View File

@ -10,7 +10,7 @@ Options common to all collectors are specified in the ``[collect]`` section of
the configuration file. The following options are available:
* ``collector``: Defaults to ``gnocchi``. The name of the collector to load.
Must be one of [``gnocchi``, ``monasca``, ``prometheus``].
Must be one of [``gnocchi``, ``prometheus``].
* ``period``: Default to 3600. Duration (in seconds) of the collect period.
@ -54,25 +54,6 @@ Section: ``collector_gnocchi``.
Region name.
Monasca
-------
Section: ``collector_monasca``.
* ``interface``: Defaults to ``internal``. The interface to use for keystone
URL discovery.
* ``monasca_service_name``: Defaults to ``monasca``. Name of the Monasca
service in Keystone.
.. note:: By default, cloudkitty retrieves all metrics from Monasca in the
project it is identified in. However, some metrics may need to be
fetched from another tenant (for example if ceilometer is publishing
metrics to monasca in the ``service`` tenant but monasca-agent is
publishing metrics to the ``admin`` tenant). See the monasca-specific
section in "Metric collection" below for details on how to configure
this.
Prometheus
----------
@ -413,24 +394,6 @@ CloudKitty.
RE_AGGREGATION_METHOD (metric METRIC_NAME AGGREGATION_METHOD)))) 2)``: this
custom query would return ``0`` when the value of the series swap.
Monasca
~~~~~~~
* ``resource_key``: Defaults to ``resource_id``. The attribute containing the
unique resource identifier. This is an advanced option, do not modify it
unless you know what you're doing.
* ``aggregation_method``: Defaults to ``max``. The aggregation method to use
when retrieving measures from monasca. Must be one of ``min``, ``max``,
``mean``.
* ``forced_project_id``: Defaults to None. Force the given metric to be
fetched from a specific tenant instead of the one cloudkitty is identified
in. For example, if cloudkitty is identified in the ``service`` project, but
needs to fetch a metric from the ``admin`` project, its ID should be
specified through this option. If this option is set to ``SCOPE_ID``,
the metric will be fetched from the current project (this assumes that
scopes are configured to be projects/tenants).
Prometheus
~~~~~~~~~~

14
doc/source/admin/configuration/fetcher.rst
View File

@ -67,20 +67,6 @@ fetcher using regular Keystone authentication options as found here:
projects that are disabled in Keystone. Defaults to false.
Monasca
-------
Section ``fetcher_monasca``.
* ``dimension_name``: Monasca dimension from which scope_ids is collected.
* ``monasca_tenant_id``: If specified, monasca client will use this ID
instead of the default one.
* ``monasca_service_name``: Name of the Monasca service (defaults to monasca).
* ``interface``: Endpoint URL type (defaults to internal).
Prometheus
----------

2
doc/source/common-index.rst
View File

@ -30,7 +30,7 @@ What can be done with CloudKitty ? What can't ?
**With Cloudkitty, it is possible to:**
- Collect metrics from OpenStack (through Gnocchi and Monasca) or from
- Collect metrics from OpenStack (through Gnocchi) or from
somewhere else (through Gnocchi in standalone mode and Prometheus). Metric
collection is **highly customizable**.

5
releasenotes/notes/remove-monasca-429122691d0e5d52.yaml Normal file
View File

@ -0,0 +1,5 @@
---
upgrade:
- |
The Monasca collector and fetcher are removed due to the unmaintained
state of Monasca.

1
requirements.txt
View File

@ -10,7 +10,6 @@ pbr>=5.5.1 # Apache-2.0
alembic>=1.4.3 # MIT
keystonemiddleware>=9.1.0 # Apache-2.0
gnocchiclient>=7.0.6 # Apache-2.0
python-monascaclient>=2.2.1 # Apache-2.0
python-keystoneclient>=4.1.1 # Apache-2.0
keystoneauth1>=4.2.1 # Apache-2.0
iso8601>=0.1.13 # MIT

2
setup.cfg
View File

@ -49,7 +49,6 @@ oslo.config.opts.defaults =
cloudkitty.collector.backends =
gnocchi = cloudkitty.collector.gnocchi:GnocchiCollector
monasca = cloudkitty.collector.monasca:MonascaCollector
prometheus = cloudkitty.collector.prometheus:PrometheusCollector
cloudkitty.fetchers =
@ -57,7 +56,6 @@ cloudkitty.fetchers =
source = cloudkitty.fetcher.source:SourceFetcher
gnocchi = cloudkitty.fetcher.gnocchi:GnocchiFetcher
prometheus = cloudkitty.fetcher.prometheus:PrometheusFetcher
monasca = cloudkitty.fetcher.monasca:MonascaFetcher
cloudkitty.rating.processors =
noop = cloudkitty.rating.noop:Noop