Split metrology configuration from CK config file

To avoid source code updates with metrology conf update,
the metrology configuration is separated from Cloudkitty
configuration file and placed in a yaml one.

Task: 5724
Story: 2001215

Change-Id: Icc098c40bc52c2589e89d705d9d711d0ce2fb557

cloudkitty/api/v1/controllers/info.py

@@ -26,8 +26,12 @@ from cloudkitty.api.v1.datamodels import info as info_models
 from cloudkitty.api.v1 import types as ck_types
 from cloudkitty import collector
 from cloudkitty.common import policy
+from cloudkitty import utils as ck_utils
 
 CONF = cfg.CONF
+
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 METADATA = collector.get_collector_metadata()
 
 
@@ -79,5 +83,5 @@ class InfoController(rest.RestController):
         """Return current configuration."""
         policy.enforce(pecan.request.context, 'info:get_config', {})
         info = {}
-        info["collect"] = {key: value for key, value in CONF.collect.items()}
+        info["collect"] = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
         return info

cloudkitty/api/v1/datamodels/info.py

@@ -18,10 +18,14 @@
 from oslo_config import cfg
 from wsme import types as wtypes
 
+from cloudkitty import utils as ck_utils
+
 CONF = cfg.CONF
-CONF.import_opt('services', 'cloudkitty.collector', 'collect')
+
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
-                                  *CONF.collect.services)
+                                  *METRICS_CONF['services'])
 
 
 class CloudkittyServiceInfo(wtypes.Base):

cloudkitty/api/v1/datamodels/rating.py

@@ -21,11 +21,14 @@ from oslo_config import cfg
 from wsme import types as wtypes
 
 from cloudkitty.api.v1 import types as cktypes
+from cloudkitty import utils as ck_utils
 
 CONF = cfg.CONF
-CONF.import_opt('services', 'cloudkitty.collector', 'collect')
+
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
-                                  *CONF.collect.services)
+                                  *METRICS_CONF['services'])
 
 
 class CloudkittyResource(wtypes.Base):

cloudkitty/cli/processor.py

@@ -15,12 +15,18 @@
 #
 # @author: Stéphane Albert
 #
-from cloudkitty import orchestrator
 from cloudkitty import service
 
 
 def main():
     service.prepare_service()
+
+    # NOTE(mc): This import is done here to ensure that the prepare_service()
+    # fonction is called before any cfg option. By importing the orchestrator
+    # file, the utils one is imported too, and then some cfg option are read
+    # before the prepare_service(), making cfg.CONF returning default values
+    # systematically.
+    from cloudkitty import orchestrator
     processor = orchestrator.Orchestrator()
     try:
         processor.process()

cloudkitty/collector/__init__.py

@@ -22,32 +22,12 @@ import six
 from stevedore import driver
 
 from cloudkitty import transformer
-import cloudkitty.utils as ck_utils
+from cloudkitty import utils as ck_utils
 
-collect_opts = [
-    cfg.StrOpt('collector',
-               default='ceilometer',
-               help='Data collector.'),
-    cfg.IntOpt('window',
-               default=1800,
-               help='Number of samples to collect per call.'),
-    cfg.IntOpt('period',
-               default=3600,
-               help='Rating period in seconds.'),
-    cfg.IntOpt('wait_periods',
-               default=2,
-               help='Wait for N periods before collecting new data.'),
-    cfg.ListOpt('services',
-                default=['compute',
-                         'image',
-                         'volume',
-                         'network.bw.in',
-                         'network.bw.out',
-                         'network.floating'],
-                help='Services to monitor.'), ]
 
 CONF = cfg.CONF
-CONF.register_opts(collect_opts, 'collect')
+
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
 
 COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
 
@@ -56,11 +36,11 @@ def get_collector(transformers=None):
     if not transformers:
         transformers = transformer.get_transformers()
     collector_args = {
-        'period': CONF.collect.period,
+        'period': METRICS_CONF['period'],
         'transformers': transformers}
     collector = driver.DriverManager(
         COLLECTORS_NAMESPACE,
-        CONF.collect.collector,
+        METRICS_CONF['collector'],
         invoke_on_load=True,
         invoke_kwds=collector_args).driver
     return collector
@@ -73,10 +53,10 @@ def get_collector_metadata():
     """
     transformers = transformer.get_transformers()
     collector = driver.DriverManager(
-        COLLECTORS_NAMESPACE, CONF.collect.collector,
+        COLLECTORS_NAMESPACE, METRICS_CONF['collector'],
         invoke_on_load=False).driver
     metadata = {}
-    for service in CONF.collect.services:
+    for service in METRICS_CONF['services']:
         metadata[service] = collector.get_metadata(service, transformers)
     return metadata
 

cloudkitty/collector/ceilometer.py

@@ -20,12 +20,15 @@ import decimal
 from ceilometerclient import client as cclient
 from keystoneauth1 import loading as ks_loading
 from oslo_config import cfg
+from oslo_log import log as logging
 from oslo_utils import units
 
 from cloudkitty import collector
 from cloudkitty import utils as ck_utils
 
 
+LOG = logging.getLogger(__name__)
+
 CEILOMETER_COLLECTOR_OPTS = 'ceilometer_collector'
 ceilometer_collector_opts = ks_loading.get_auth_common_conf_options()
 
@@ -38,6 +41,8 @@ ks_loading.register_auth_conf_options(
     CEILOMETER_COLLECTOR_OPTS)
 CONF = cfg.CONF
 
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 
 class ResourceNotFound(Exception):
     """Raised when the resource doesn't exist."""
@@ -113,7 +118,15 @@ class CeilometerCollector(collector.BaseCollector):
         try:
             info["metadata"].extend(transformers['CeilometerTransformer']
                                     .get_metadata(resource_name))
-            info["unit"] = cls.units_mappings[resource_name]
+
+            try:
+                info["unit"] = METRICS_CONF['services_units'][resource_name][1]
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                info["unit"] = cls.units_mappings[resource_name]
+
         except KeyError:
             pass
         return info
@@ -201,9 +214,22 @@ class CeilometerCollector(collector.BaseCollector):
                                                  instance)
             instance = self._cacher.get_resource_detail('compute',
                                                         instance_id)
-            compute_data.append(
+
-                self.t_cloudkitty.format_item(instance, self.units_mappings[
+            try:
-                    "compute"], 1))
+                compute_data.append(self.t_cloudkitty.format_item(
+                    instance,
+                    METRICS_CONF['services_units']['compute'],
+                    1,
+                ))
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                compute_data.append(self.t_cloudkitty.format_item(
+                    instance,
+                    self.units_mappings['compute'],
+                    1,
+                ))
         if not compute_data:
             raise collector.NoDataCollected(self.collector_name, 'compute')
         return self.t_cloudkitty.format_service('compute', compute_data)
@@ -228,9 +254,22 @@ class CeilometerCollector(collector.BaseCollector):
                                                      image_id)
 
             image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
-            image_data.append(
+
-                self.t_cloudkitty.format_item(image, self.units_mappings[
+            try:
-                    "image"], image_size_mb))
+                image_data.append(self.t_cloudkitty.format_item(
+                    image,
+                    METRICS_CONF['services_units']['image'],
+                    image_size_mb,
+                ))
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                image_data.append(self.t_cloudkitty.format_item(
+                    image,
+                    self.units_mappings['image'],
+                    image_size_mb,
+                ))
 
         if not image_data:
             raise collector.NoDataCollected(self.collector_name, 'image')
@@ -255,9 +294,23 @@ class CeilometerCollector(collector.BaseCollector):
                                                  volume)
             volume = self._cacher.get_resource_detail('volume',
                                                       volume_id)
-            volume_data.append(
+
-                self.t_cloudkitty.format_item(volume, self.units_mappings[
+            try:
-                    "volume"], volume_stats.max))
+                volume_data.append(self.t_cloudkitty.format_item(
+                    volume,
+                    METRICS_CONF['services_units']['volume'],
+                    volume_stats.max,
+                ))
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                volume_data.append(self.t_cloudkitty.format_item(
+                    volume,
+                    self.units_mappings['volume'],
+                    volume_stats.max,
+                ))
+
         if not volume_data:
             raise collector.NoDataCollected(self.collector_name, 'volume')
         return self.t_cloudkitty.format_service('volume', volume_data)
@@ -294,9 +347,22 @@ class CeilometerCollector(collector.BaseCollector):
                                                    tap_id)
 
             tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
-            bw_data.append(
+
-                self.t_cloudkitty.format_item(tap, self.units_mappings[
+            try:
-                    "network.bw." + direction], tap_bw_mb))
+                bw_data.append(self.t_cloudkitty.format_item(
+                    tap,
+                    METRICS_CONF['services_units']['network.bw.' + direction],
+                    tap_bw_mb,
+                ))
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                bw_data.append(self.t_cloudkitty.format_item(
+                    tap,
+                    self.units_mappings['network.bw.' + direction],
+                    tap_bw_mb,
+                ))
 
         ck_res_name = 'network.bw.{}'.format(direction)
         if not bw_data:
@@ -342,9 +408,23 @@ class CeilometerCollector(collector.BaseCollector):
                                                  floating)
             floating = self._cacher.get_resource_detail('network.floating',
                                                         floating_id)
-            floating_data.append(
+
-                self.t_cloudkitty.format_item(floating, self.units_mappings[
+            try:
-                    "network.floating"], 1))
+                floating_data.append(self.t_cloudkitty.format_item(
+                    floating,
+                    METRICS_CONF['services_units']['network.floating'],
+                    1,
+                ))
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                floating_data.append(self.t_cloudkitty.format_item(
+                    floating,
+                    self.units_mappings['network.floating'],
+                    1,
+                ))
+
         if not floating_data:
             raise collector.NoDataCollected(self.collector_name,
                                             'network.floating')

cloudkitty/collector/gnocchi.py

@@ -18,11 +18,15 @@ import decimal
 from gnocchiclient import client as gclient
 from keystoneauth1 import loading as ks_loading
 from oslo_config import cfg
+from oslo_log import log as logging
 from oslo_utils import units
 
 from cloudkitty import collector
 from cloudkitty import utils as ck_utils
 
+
+LOG = logging.getLogger(__name__)
+
 GNOCCHI_COLLECTOR_OPTS = 'gnocchi_collector'
 gnocchi_collector_opts = ks_loading.get_auth_common_conf_options()
 
@@ -35,6 +39,8 @@ ks_loading.register_auth_conf_options(
     GNOCCHI_COLLECTOR_OPTS)
 CONF = cfg.CONF
 
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 
 class GnocchiCollector(collector.BaseCollector):
     collector_name = 'gnocchi'
@@ -102,7 +108,15 @@ class GnocchiCollector(collector.BaseCollector):
         try:
             info["metadata"].extend(transformers['GnocchiTransformer']
                                     .get_metadata(resource_name))
-            info["unit"] = cls.units_mappings[resource_name][1]
+
+            try:
+                info["unit"] = METRICS_CONF['services_units'][resource_name][1]
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                info["unit"] = cls.units_mappings[resource_name][1]
+
         except KeyError:
             pass
         return info
@@ -155,26 +169,50 @@ class GnocchiCollector(collector.BaseCollector):
             self.gen_filter(cop="<=", started_at=end))
         return time_filter
 
+    def _expand(self, metrics, resource, name, aggregate, start, end):
+        try:
+            values = self._conn.metric.get_measures(
+                metric=metrics[name],
+                start=ck_utils.ts2dt(start),
+                stop=ck_utils.ts2dt(end),
+                aggregation=aggregate)
+            # NOTE(sheeprine): Get the list of values for the current
+            # metric and get the first result value.
+            # [point_date, granularity, value]
+            # ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
+            resource[name] = values[0][2]
+        except IndexError:
+            resource[name] = 0
+        except KeyError:
+            # Skip metrics not found
+            pass
+
     def _expand_metrics(self, resources, mappings, start, end):
         for resource in resources:
             metrics = resource.get('metrics', {})
-            for name, aggregate in mappings:
+            try:
-                try:
+                for mapping in mappings:
-                    values = self._conn.metric.get_measures(
+                    self._expand(
-                        metric=metrics[name],
+                        metrics,
-                        start=ck_utils.ts2dt(start),
+                        resource,
-                        stop=ck_utils.ts2dt(end),
+                        mapping.keys()[0],
-                        aggregation=aggregate)
+                        mapping.values()[0],
-                    # NOTE(sheeprine): Get the list of values for the current
+                        start,
-                    # metric and get the first result value.
+                        end,
-                    # [point_date, granularity, value]
+                    )
-                    # ["2015-11-24T00:00:00+00:00", 86400.0, 64.0]
+            # NOTE(mc): deprecated except part kept for backward compatibility.
-                    resource[name] = values[0][2]
+            except AttributeError:
-                except IndexError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
-                    resource[name] = 0
+                LOG.warning('Fallback on the deprecated oslo config method.')
-                except KeyError:
+                for name, aggregate in mappings:
-                    # Skip metrics not found
+                    self._expand(
-                    pass
+                        metrics,
+                        resource,
+                        name,
+                        aggregate,
+                        start,
+                        end,
+                    )
 
     def get_resources(self, resource_name, start, end,
                       project_id, q_filter=None):
@@ -196,7 +234,15 @@ class GnocchiCollector(collector.BaseCollector):
 
         # Translating the resource name if needed
         query_parameters = self._generate_time_filter(start, end)
-        resource_type = self.retrieve_mappings.get(resource_name)
+
+        try:
+            resource_type = METRICS_CONF['services_objects'].get(resource_name)
+        # NOTE(mc): deprecated except part kept for backward compatibility.
+        except KeyError:
+            LOG.warning('Error when trying to use yaml metrology conf.')
+            LOG.warning('Fallback on the deprecated oslo config method.')
+            resource_type = self.retrieve_mappings.get(resource_name)
+
         query_parameters.append(
             self.gen_filter(cop="=", type=resource_type))
         query_parameters.append(
@@ -210,7 +256,18 @@ class GnocchiCollector(collector.BaseCollector):
 
     def resource_info(self, resource_name, start, end, project_id,
                       q_filter=None):
-        qty, unit = self.units_mappings.get(resource_name, self.default_unit)
+        try:
+            qty = METRICS_CONF['services_units'][resource_name].keys()[0]
+            unit = METRICS_CONF['services_units'][resource_name].values()[0]
+        # NOTE(mc): deprecated except part kept for backward compatibility.
+        except KeyError:
+            LOG.warning('Error when trying to use yaml metrology conf.')
+            LOG.warning('Fallback on the deprecated oslo config method.')
+            qty, unit = self.units_mappings.get(
+                resource_name,
+                self.default_unit,
+            )
+
         resources = self.get_resources(resource_name, start, end,
                                        project_id=project_id,
                                        q_filter=q_filter)
@@ -218,7 +275,15 @@ class GnocchiCollector(collector.BaseCollector):
         for resource in resources:
             resource_data = self.t_gnocchi.strip_resource_data(
                 resource_name, resource)
-            mappings = self.metrics_mappings[resource_name]
+
+            try:
+                mappings = METRICS_CONF['services_metrics'][resource_name]
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                mappings = self.metrics_mappings[resource_name]
+
             self._expand_metrics([resource_data], mappings, start, end)
             resource_data.pop('metrics', None)
             # Convert network.bw.in, network.bw.out and image unit to MB

cloudkitty/collector/monasca.py

@@ -21,12 +21,16 @@ from keystoneauth1 import loading as ks_loading
 from keystoneclient.v3 import client as ks_client
 from monascaclient import client as mclient
 from oslo_config import cfg
+from oslo_log import log as logging
 from oslo_utils import units
 
 from cloudkitty import collector
 from cloudkitty import transformer
 from cloudkitty import utils as ck_utils
 
+
+LOG = logging.getLogger(__name__)
+
 MONASCA_API_VERSION = '2_0'
 COLLECTOR_MONASCA_OPTS = 'collector_monasca'
 collector_monasca_opts = ks_loading.get_auth_common_conf_options()
@@ -40,6 +44,8 @@ ks_loading.register_auth_conf_options(
     COLLECTOR_MONASCA_OPTS)
 CONF = cfg.CONF
 
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 
 class EndpointNotFound(Exception):
     """Exception raised if the Monasca endpoint is not found"""
@@ -57,7 +63,7 @@ class MonascaCollector(collector.BaseCollector):
         'network.bw.in': 'network.incoming.bytes',
         'network.bw.out': 'network.outgoing.bytes',
     }
-    metric_mappings = {
+    metrics_mappings = {
         'compute': [
             ('cpu', 'max'),
             ('vpcus', 'max'),
@@ -77,7 +83,7 @@ class MonascaCollector(collector.BaseCollector):
     }
     # (qty, unit). qty must be either a metric name, an integer
     # or a decimal.Decimal object
-    unit_mappings = {
+    units_mappings = {
         'compute': (1, 'instance'),
         'image': ('image.size', 'MB'),
         'volume': ('volume.size', 'GB'),
@@ -127,9 +133,16 @@ class MonascaCollector(collector.BaseCollector):
     def _get_metadata(self, resource_type, transformers):
         info = {}
         try:
-            info['unit'] = self.unit_mappings[resource_type][1]
+            info['unit'] = METRICS_CONF['services_units']
-        except (KeyError, IndexError):
+        # NOTE(mc): deprecated second try kept for backward compatibility.
-            info['unit'] = self.default_unit[1]
+        except KeyError:
+            LOG.warning('Error when trying to use yaml metrology conf.')
+            LOG.warning('Fallback on the deprecated oslo config method.')
+            try:
+                info['unit'] = self.units_mappings[resource_type][1]
+            except (KeyError, IndexError):
+                info['unit'] = self.default_unit[1]
+
         start = ck_utils.dt2ts(ck_utils.get_month_start())
         end = ck_utils.dt2ts(ck_utils.get_month_end())
         try:
@@ -140,11 +153,19 @@ class MonascaCollector(collector.BaseCollector):
         metadata = self._get_resource_metadata(resource_type, start,
                                                end, resource_id)
         info['metadata'] = metadata.keys()
+
         try:
-            for metric, statistics in self.metric_mappings[resource_type]:
+            for metric, statistics in METRICS_CONF['services_metrics']:
                 info['metadata'].append(metric)
-        except (KeyError, IndexError):
+        # NOTE(mc): deprecated second try kept for backward compatibility.
-            pass
+        except KeyError:
+            LOG.warning('Error when trying to use yaml metrology conf.')
+            LOG.warning('Fallback on the deprecated oslo config method.')
+            try:
+                for metric, statistics in self.metrics_mappings[resource_type]:
+                    info['metadata'].append(metric)
+            except (KeyError, IndexError):
+                pass
         return info
 
     # NOTE(lukapeschke) if anyone sees a better way to do this,
@@ -158,7 +179,14 @@ class MonascaCollector(collector.BaseCollector):
         return tmp._get_metadata(resource_type, transformers)
 
     def _get_resource_metadata(self, resource_type, start, end, resource_id):
-        meter = self.retrieve_mappings.get(resource_type)
+        try:
+            meter = METRICS_CONF['services_objects'].get(resource_type)
+        # NOTE(mc): deprecated except part kept for backward compatibility.
+        except KeyError:
+            LOG.warning('Error when trying to use yaml metrology conf.')
+            LOG.warning('Fallback on the deprecated oslo config method.')
+            meter = self.retrieve_mappings.get(resource_type)
+
         if not meter:
             return {}
         measurements = self._conn.metrics.list_measurements(
@@ -209,7 +237,14 @@ class MonascaCollector(collector.BaseCollector):
 
     def active_resources(self, resource_type, start,
                          end, project_id, **kwargs):
-        meter = self.retrieve_mappings.get(resource_type)
+        try:
+            meter = METRICS_CONF['services_objects'].get(resource_type)
+        # NOTE(mc): deprecated except part kept for backward compatibility.
+        except KeyError:
+            LOG.warning('Error when trying to use yaml metrology conf.')
+            LOG.warning('Fallback on the deprecated oslo config method.')
+            meter = self.retrieve_mappings.get(resource_type)
+
         if not meter:
             return {}
         dimensions = {}
@@ -245,7 +280,21 @@ class MonascaCollector(collector.BaseCollector):
 
     def resource_info(self, resource_type, start, end,
                       project_id, q_filter=None):
-        qty, unit = self.unit_mappings.get(resource_type, self.default_unit)
+
+        try:
+            qty, unit = METRICS_CONF['services_units'].get(
+                resource_type,
+                self.default_unit
+            )
+        # NOTE(mc): deprecated except part kept for backward compatibility.
+        except KeyError:
+            LOG.warning('Error when trying to use yaml metrology conf.')
+            LOG.warning('Fallback on the deprecated oslo config method.')
+            qty, unit = self.units_mappings.get(
+                resource_type,
+                self.default_unit
+            )
+
         active_resource_ids = self.active_resources(
             resource_type, start, end, project_id
         )
@@ -253,11 +302,27 @@ class MonascaCollector(collector.BaseCollector):
         for resource_id in active_resource_ids:
             data = self._get_resource_metadata(resource_type, start,
                                                end, resource_id)
-            mappings = self.metric_mappings[resource_type]
+
+            try:
+                mappings = METRICS_CONF['services_metrics'][resource_type]
+            # NOTE(mc): deprecated except part kept for backward compatibility.
+            except KeyError:
+                LOG.warning('Error when trying to use yaml metrology conf.')
+                LOG.warning('Fallback on the deprecated oslo config method.')
+                mappings = self.metrics_mappings[resource_type]
+
             self._expand_metrics(data, resource_id, mappings, start, end)
             resource_qty = qty
             if not (isinstance(qty, int) or isinstance(qty, decimal.Decimal)):
-                resource_qty = data[self.retrieve_mappings[resource_type]]
+                try:
+                    resource_qty = METRICS_CONF['services_objects']
+                # NOTE(mc): deprecated except part kept for backward compat.
+                except KeyError:
+                    LOG.warning('Error when trying to use yaml metrology conf')
+                    msg = 'Fallback on the deprecated oslo config method'
+                    LOG.warning(msg)
+                    resource_qty = data[self.retrieve_mappings[resource_type]]
+
             resource = self.t_cloudkitty.format_item(data, unit, resource_qty)
             resource['desc']['resource_id'] = resource_id
             resource['resource_id'] = resource_id

cloudkitty/common/config.py

@@ -17,7 +17,6 @@ import copy
 import itertools
 
 import cloudkitty.api.app
-import cloudkitty.collector
 import cloudkitty.collector.ceilometer
 import cloudkitty.collector.gnocchi
 import cloudkitty.collector.monasca
@@ -28,6 +27,7 @@ import cloudkitty.storage
 import cloudkitty.storage.gnocchi
 import cloudkitty.tenant_fetcher
 import cloudkitty.tenant_fetcher.keystone
+import cloudkitty.utils
 
 __all__ = ['list_opts']
 
@@ -35,7 +35,7 @@ _opts = [
     ('api', list(itertools.chain(
         cloudkitty.api.app.api_opts,))),
     ('collect', list(itertools.chain(
-        cloudkitty.collector.collect_opts))),
+        cloudkitty.utils.collect_opts))),
     ('ceilometer_collector', list(itertools.chain(
         cloudkitty.collector.ceilometer.ceilometer_collector_opts))),
     ('collector_monasca', list(itertools.chain(

cloudkitty/orchestrator.py

@@ -51,6 +51,8 @@ orchestrator_opts = [
 ]
 CONF.register_opts(orchestrator_opts, group='orchestrator')
 
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 FETCHERS_NAMESPACE = 'cloudkitty.tenant.fetchers'
 PROCESSORS_NAMESPACE = 'cloudkitty.rating.processors'
 
@@ -152,8 +154,8 @@ class Worker(BaseWorker):
     def __init__(self, collector, storage, tenant_id=None):
         self._collector = collector
         self._storage = storage
-        self._period = CONF.collect.period
+        self._period = METRICS_CONF['period']
-        self._wait_time = CONF.collect.wait_periods * self._period
+        self._wait_time = METRICS_CONF['wait_periods'] * self._period
 
         super(Worker, self).__init__(tenant_id)
 
@@ -180,7 +182,7 @@ class Worker(BaseWorker):
             if not timestamp:
                 break
 
-            for service in CONF.collect.services:
+            for service in METRICS_CONF['services']:
                 try:
                     try:
                         data = self._collect(service, timestamp)
@@ -232,8 +234,8 @@ class Orchestrator(object):
             uuidutils.generate_uuid().encode('ascii'))
         self.coord.start()
 
-        self._period = CONF.collect.period
+        self._period = METRICS_CONF['period']
-        self._wait_time = CONF.collect.wait_periods * self._period
+        self._wait_time = METRICS_CONF['wait_periods'] * self._period
 
     def _lock(self, tenant_id):
         lock_name = b"cloudkitty-" + str(tenant_id).encode('ascii')

cloudkitty/storage/__init__.py

@@ -33,12 +33,14 @@ storage_opts = [
 CONF = cfg.CONF
 CONF.register_opts(storage_opts, group='storage')
 
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
+
 STORAGES_NAMESPACE = 'cloudkitty.storage.backends'
 
 
 def get_storage(collector=None):
     storage_args = {
-        'period': CONF.collect.period,
+        'period': METRICS_CONF['period'],
         'collector': collector if collector else ck_collector.get_collector()}
     backend = driver.DriverManager(
         STORAGES_NAMESPACE,
@@ -63,7 +65,7 @@ class BaseStorage(object):
         Handle incoming data from the global orchestrator, and store them.
     """
     def __init__(self, **kwargs):
-        self._period = kwargs.get('period', CONF.collect.period)
+        self._period = kwargs.get('period', METRICS_CONF['period'])
         self._collector = kwargs.get('collector')
 
         # State vars

cloudkitty/storage/gnocchi/__init__.py

@@ -34,7 +34,7 @@ from cloudkitty import utils as ck_utils
 LOG = log.getLogger(__name__)
 CONF = cfg.CONF
 
-CONF.import_opt('period', 'cloudkitty.collector', 'collect')
+METRICS_CONF = ck_utils.get_metrics_conf(CONF.collect.metrics_conf)
 
 GNOCCHI_STORAGE_OPTS = 'storage_gnocchi'
 gnocchi_storage_opts = [
@@ -44,7 +44,7 @@ gnocchi_storage_opts = [
     # The archive policy definition MUST include the collect period granularity
     cfg.StrOpt('archive_policy_definition',
                default='[{"granularity": '
-                       + six.text_type(CONF.collect.period) +
+                       + six.text_type(METRICS_CONF['period']) +
                        ', "timespan": "90 days"}, '
                        '{"granularity": 86400, "timespan": "360 days"}, '
                        '{"granularity": 2592000, "timespan": "1800 days"}]',
@@ -84,7 +84,7 @@ class GnocchiStorage(storage.BaseStorage):
             CONF.storage_gnocchi.archive_policy_name)
         self._archive_policy_definition = json.loads(
             CONF.storage_gnocchi.archive_policy_definition)
-        self._period = CONF.collect.period
+        self._period = METRICS_CONF['period']
         if "period" in kwargs:
             self._period = kwargs["period"]
 

cloudkitty/tests/gabbi/gabbits/v1-info.yaml

@@ -8,12 +8,12 @@ tests:
     response_json_paths:
       $.collect.services.`len`: 6
       $.collect.services[0]: compute
-      $.collect.services[1]: image
+      $.collect.services[1]: volume
-      $.collect.services[2]: volume
+      $.collect.services[2]: network.bw.in
-      $.collect.services[3]: network.bw.in
+      $.collect.services[3]: network.bw.out
-      $.collect.services[4]: network.bw.out
+      $.collect.services[4]: network.floating
-      $.collect.services[5]: network.floating
+      $.collect.services[5]: image
-      $.collect.collector: ceilometer
+      $.collect.collector: gnocchi
       $.collect.window: 1800
       $.collect.wait_periods: 2
       $.collect.period: 3600
@@ -42,4 +42,4 @@ tests:
     response_json_paths:
       $.service_id: compute
       $.unit: instance
-      $.metadata.`len`: 10
+      $.metadata.`len`: 8

cloudkitty/utils.py

@@ -24,7 +24,10 @@ to ease maintenance in case of library modifications.
 import calendar
 import datetime
 import sys
+import yaml
 
+from oslo_config import cfg
+from oslo_log import log as logging
 from oslo_utils import timeutils
 from six import moves
 from stevedore import extension
@@ -33,6 +36,43 @@ from stevedore import extension
 _ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
 _ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
 
+LOG = logging.getLogger(__name__)
+
+collect_opts = [
+    cfg.StrOpt('collector',
+               default='gnocchi',
+               deprecated_for_removal=True,
+               help='Data collector.'),
+    cfg.IntOpt('window',
+               default=1800,
+               deprecated_for_removal=True,
+               help='Number of samples to collect per call.'),
+    cfg.IntOpt('period',
+               default=3600,
+               deprecated_for_removal=True,
+               help='Rating period in seconds.'),
+    cfg.IntOpt('wait_periods',
+               default=2,
+               deprecated_for_removal=True,
+               help='Wait for N periods before collecting new data.'),
+    cfg.ListOpt('services',
+                default=[
+                    'compute',
+                    'volume',
+                    'network.bw.in',
+                    'network.bw.out',
+                    'network.floating',
+                    'image',
+                ],
+                deprecated_for_removal=True,
+                help='Services to monitor.'),
+    cfg.StrOpt('metrics_conf',
+               default='/etc/cloudkitty/metrics.yml',
+               help='Metrology configuration file.'),
+]
+CONF = cfg.CONF
+CONF.register_opts(collect_opts, 'collect')
+
 
 def isotime(at=None, subsecond=False):
     """Stringify time in ISO 8601 format."""
@@ -190,3 +230,30 @@ def check_time_state(timestamp=None, period=0, wait_time=0):
     if next_timestamp + wait_time < now:
         return next_timestamp
     return 0
+
+
+def get_metrics_conf(conf_path):
+    """Return loaded yaml metrology configuration.
+
+    In case of empty /etc/cloudkitty folder,
+    a fallback is done on the former deprecated oslo config method.
+    """
+    res = None
+    try:
+        with open(conf_path) as conf:
+            res = yaml.load(conf)
+            res = res[0]
+    except Exception as exc:
+        LOG.warning('Error when trying to retrieve yaml metrology conf file.')
+        LOG.warning(exc)
+        LOG.warning('Fallback on the deprecated oslo config method.')
+
+        try:
+            res = {key: val for key, val in CONF.collect.items()}
+        except Exception as exc:
+            err_msg = 'Error when trying to retrieve ' \
+                      'deprecated oslo config method.'
+            LOG.error(err_msg)
+            LOG.error(exc)
+
+    return res

devstack/plugin.sh

@@ -134,6 +134,7 @@ function configure_cloudkitty {
 
     cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/policy.json $CLOUDKITTY_CONF_DIR
     cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/api_paste.ini $CLOUDKITTY_CONF_DIR
+    cp $CLOUDKITTY_DIR$CLOUDKITTY_CONF_DIR/metrics.yml $CLOUDKITTY_CONF_DIR
     iniset_rpc_backend cloudkitty $CLOUDKITTY_CONF DEFAULT
 
     iniset $CLOUDKITTY_CONF DEFAULT notification_topics 'notifications'
@@ -161,6 +162,7 @@ function configure_cloudkitty {
     iniset $CLOUDKITTY_CONF collect collector $CLOUDKITTY_COLLECTOR
     iniset $CLOUDKITTY_CONF ${CLOUDKITTY_COLLECTOR}_collector auth_section authinfos
     iniset $CLOUDKITTY_CONF collect services $CLOUDKITTY_SERVICES
+    iniset $CLOUDKITTY_CONF collect metrics_conf $CLOUDKITTY_CONF_DIR/$CLOUDKITTY_METRICS_CONF
 
     # output
     iniset $CLOUDKITTY_CONF output backend $CLOUDKITTY_OUTPUT_BACKEND

devstack/settings

@@ -43,6 +43,7 @@ CLOUDKITTY_PRICING_TENANT=${CLOUDKITTY_PRICING_TENANT:-"demo"}
 # Set CloudKitty collect info
 CLOUDKITTY_COLLECTOR=${CLOUDKITTY_COLLECTOR:-ceilometer}
 CLOUDKITTY_SERVICES=${CLOUDKITTY_SERVICES:-compute}
+CLOUDKITTY_METRICS_CONF=metrics.yml
 
 # Set CloudKitty output info
 CLOUDKITTY_OUTPUT_BACKEND=${CLOUDKITTY_OUTPUT_BACKEND:-"cloudkitty.backend.file.FileBackend"}

doc/source/configuration/index.rst

@@ -167,18 +167,83 @@ Three collectors are available: Ceilometer (deprecated, see the Telemetry
 documentation), Gnocchi and Monasca. The Monasca collector collects metrics
 published by the Ceilometer agent to Monasca using Ceilosca_.
 
+The collect information, is separated from the Cloudkitty configuration file, in a yaml one.
+
+This allows Cloudkitty users to change metrology configuration,
+without modifying source code or Cloudkitty configuration file.
+
 .. code-block:: ini
 
     [collect]
-    collector = gnocchi
+    metrics_conf = /etc/cloudkitty/metrics.yml
-    # Metrics are collected every 3600 seconds
-    period = 3600
-    # By default, only the compute service is enabled
-    services = compute, volume, network.bw.in, network.bw.out, network.floating, image
 
     [gnocchi_collector]
     auth_section = ks_auth
 
+The ``/etc/cloudkitty/metrics.yml`` file looks like this:
+
+.. code-block:: yaml
+
+    - name: OpenStack
+
+      collector: gnocchi
+      period: 3600
+      wait_period: 2
+      window: 1800
+
+      services:
+        - compute
+        - volume
+        - network.bw.in
+        - network.bw.out
+        - network.floating
+        - image
+
+      services_objects:
+        compute: instance
+        volume: volume
+        network.bw.in: instance_network_interface
+        network.bw.out: instance_network_interface
+        network.floating: network
+        image: image
+
+      services_metrics:
+        compute:
+          - vcpus: max
+          - memory: max
+          - cpu: max
+          - disk.root.size: max
+          - disk.ephemeral.size: max
+        volume:
+          - volume.size: max
+        network.bw.in:
+          - network.incoming.bytes: max
+        network.bw.out:
+          - network.outgoing.bytes: max
+        network.floating:
+          - ip.floating: max
+        image:
+          - image.size: max
+          - image.download: max
+          - image.serve: max
+
+      services_units:
+        compute:
+          1: instance
+        volume:
+          volume.size: GB
+        network.bw.in:
+          network.incoming.bytes: MB
+        network.bw.out:
+          network.outgoing.bytes: MB
+        network.floating:
+          1: ip
+        image:
+          image.size: MB
+        default_unit:
+          1: unknown
+
+
 Setup the database and storage backend
 --------------------------------------
 
@@ -274,4 +339,4 @@ Choose and start the API server
     $ cloudkitty-api -p 8889
 
 
-.. _Ceilosca: https://github.com/openstack/monasca-ceilometer
+.. _Ceilosca: https://github.com/openstack/monasca-ceilometer

etc/cloudkitty/metrics.yml

@@ -0,0 +1,58 @@
+- name: OpenStack
+
+  collector: gnocchi
+  period: 3600
+  wait_periods: 2
+  window: 1800
+
+  services:
+    - compute
+    - volume
+    - network.bw.in
+    - network.bw.out
+    - network.floating
+    - image
+
+  services_objects:
+    compute: instance
+    volume: volume
+    network.bw.out: instance_network_interface
+    network.bw.in: instance_network_interface
+    network.floating: network
+    image: image
+
+  services_metrics:
+    compute:
+      - vcpus: max
+      - memory: max
+      - cpu: max
+      - disk.root.size: max
+      - disk.ephemeral.size: max
+    volume:
+      - volume.size: max
+    network.bw.in:
+      - network.incoming: max
+    network.bw.out:
+      - network.outgoing.bytes: max
+    network.floating:
+      - ip.floating: max
+    image:
+      - image.size: max
+      - image.download: max
+      - image.serve: max
+
+  services_units:
+    compute:
+      1: instance
+    volume:
+      volume.size: GB
+    network.bw.in:
+      network.incoming.bytes: MB
+    network.bw.out:
+      network.outgoing.bytes: MB
+    network.floating:
+      1: ip
+    image:
+      image.size: MB
+    default_unit:
+      1: unknown