Added support for Multiple Collectors

Added a new collector backend: MetaCollector.
Added new API endpoints for the MetaCollector configuration.
Modified documentation to add new API endpoints.

Change-Id: I0216fdb6829fdb2274edf693971c6730727f2cde
This commit is contained in:
Stéphane Albert 2014-09-24 18:01:39 +02:00 committed by Stéphane Albert
parent e1d8bdccf6
commit cc10eb6e08
8 changed files with 369 additions and 0 deletions

View File

@ -24,6 +24,7 @@ import wsmeext.pecan as wsme_pecan
from cloudkitty.api.controllers import types as cktypes
from cloudkitty import config # noqa
from cloudkitty.db import api as db_api
from cloudkitty.openstack.common import log as logging
CONF = cfg.CONF
@ -67,6 +68,115 @@ class ResourceDescriptor(wtypes.Base):
return sample
class ServiceToCollectorMapping(wtypes.Base):
"""Type describing a service to collector mapping.
"""
service = wtypes.text
"""Name of the service."""
collector = wtypes.text
"""Name of the collector."""
def to_json(self):
res_dict = {}
res_dict[self.service] = self.collector
return res_dict
@classmethod
def sample(cls):
sample = cls(service='compute',
collector='ceilometer')
return sample
class MappingController(rest.RestController):
"""REST Controller managing service to collector mapping.
"""
def __init__(self):
self._db = db_api.get_instance().get_service_to_collector_mapping()
@wsme_pecan.wsexpose([wtypes.text])
def get_all(self):
"""Return the list of every services mapped.
:return: List of every services mapped.
"""
return [mapping.service for mapping in self._db.list_services()]
@wsme_pecan.wsexpose(ServiceToCollectorMapping, wtypes.text)
def get_one(self, service):
"""Return a service to collector mapping.
:param service: Name of the service to filter on.
"""
try:
return self._db.get_mapping(service)
except db_api.NoSuchMapping as e:
pecan.abort(400, str(e))
pecan.response.status = 200
@wsme_pecan.wsexpose(ServiceToCollectorMapping,
wtypes.text,
body=wtypes.text)
def post(self, service, collector):
"""Create or modify a mapping.
:param service: Name of the service to map a collector to.
:param collector: Name of the collector.
"""
return self._db.set_mapping(service, collector)
@wsme_pecan.wsexpose(None, body=wtypes.text)
def delete(self, service):
"""Delete a mapping.
:param service: Name of the service to suppress the mapping from.
"""
try:
self._db.delete_mapping(service)
except db_api.NoSuchMapping as e:
pecan.abort(400, str(e))
pecan.response.status = 204
class CollectorController(rest.RestController):
"""REST Controller managing collector modules.
"""
mapping = MappingController()
_custom_actions = {
'state': ['GET', 'POST']
}
def __init__(self):
self._db = db_api.get_instance().get_module_enable_state()
@wsme_pecan.wsexpose(bool, wtypes.text)
def state(self, collector):
"""Query the enable state of a collector.
:param collector: Name of the collector.
:return: State of the collector.
"""
return self._db.get_state('collector_{}'.format(collector))
@wsme_pecan.wsexpose(bool, wtypes.text, body=bool)
def post_state(self, collector, state):
"""Set the enable state of a collector.
:param collector: Name of the collector.
:param state: New state for the collector.
:return: State of the collector.
"""
return self._db.set_state('collector_{}'.format(collector), state)
class ModulesController(rest.RestController):
"""REST Controller managing billing modules.
@ -149,5 +259,6 @@ class V1Controller(rest.RestController):
"""
collector = CollectorController()
billing = BillingController()
report = ReportController()

View File

@ -0,0 +1,92 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from stevedore import extension
from cloudkitty import collector
from cloudkitty.db import api as db_api
COLLECTORS_NAMESPACE = 'cloudkitty.collector.backends'
class MetaCollector(collector.BaseCollector):
def __init__(self, transformers, **kwargs):
super(MetaCollector, self).__init__(transformers, **kwargs)
self._db = db_api.get_instance().get_service_to_collector_mapping()
self._collectors = {}
self._load_collectors()
self._mappings = {}
self._load_mappings()
def _connect(self):
pass
def _load_mappings(self):
mappings = self._db.list_services()
for mapping in mappings:
db_mapping = self._db.get_mapping(mapping.service)
self._mappings[db_mapping.service] = db_mapping.collector
def _check_enabled(self, name):
enable_state = db_api.get_instance().get_module_enable_state()
return enable_state.get_state('collector_{}'.format(name))
def _load_collectors(self):
self._collectors = {}
collectors = extension.ExtensionManager(
COLLECTORS_NAMESPACE,
)
collectors_list = collectors.names()
collectors_list.remove('meta')
for name in collectors_list:
if self._check_enabled(name):
self._collectors[name] = collectors[name].plugin(
self.transformers,
user=self.user,
password=self.password,
tenant=self.tenant,
region=self.region,
keystone_url=self.keystone_url,
period=self.period)
def map_retrieve(self, trans_resource, res_collector=None):
if res_collector:
if hasattr(res_collector, trans_resource):
return getattr(res_collector, trans_resource)
for cur_collector in self._collectors.values():
if hasattr(cur_collector, trans_resource):
return getattr(cur_collector, trans_resource)
def retrieve(self, resource, start, end=None, project_id=None,
q_filter=None):
# Resource to function translation
trans_resource = 'get_'
trans_resource += resource.replace('.', '_')
# Resource to collector mapping processing
res_collector = None
if resource in self._mappings and resource in self._collectors:
res_collector = self._collectors[resource]
func = self.map_retrieve(trans_resource, res_collector)
if func is not None:
return func(start, end, project_id, q_filter)

View File

@ -89,3 +89,45 @@ class ModuleEnableState(object):
:param name: Name of the module
:param value: State of the module
"""
class NoSuchMapping(Exception):
"""Raised when the mapping doesn't exist."""
def __init__(self, service):
super(NoSuchMapping, self).__init__(
"No such mapping for service: %s" % service)
self.service = service
@six.add_metaclass(abc.ABCMeta)
class ServiceToCollectorMapping(object):
"""Base class for service to collector mapping."""
@abc.abstractmethod
def get_mapping(self, service):
"""Get a mapping.
:return mapping: service to collector object.
"""
@abc.abstractmethod
def set_mapping(self, service, collector):
"""Set a mapping.
:param service: Service to work on.
:param collector: Collector to prioritize.
:return mapping: Service to Collector object.
"""
@abc.abstractmethod
def list_services(self):
"""Retrieve the list of every services mapped.
:return list(str): List of services' name.
"""
@abc.abstractmethod
def delete_mapping(self, service):
"""Remove a mapping.
"""

View File

@ -0,0 +1,30 @@
"""Added support for meta collector
Revision ID: 2ac2217dcbd9
Revises: 464e951dc3b8
Create Date: 2014-09-25 12:41:28.585333
"""
# revision identifiers, used by Alembic.
revision = '2ac2217dcbd9'
down_revision = '464e951dc3b8'
from alembic import op
import sqlalchemy as sa
def upgrade():
### commands auto generated by Alembic - please adjust! ###
op.create_table('service_to_collector_mappings',
sa.Column('service', sa.String(length=255), nullable=False),
sa.Column('collector', sa.String(length=255), nullable=False),
sa.PrimaryKeyConstraint('service')
)
### end Alembic commands ###
def downgrade():
### commands auto generated by Alembic - please adjust! ###
op.drop_table('service_to_collector_mappings')
### end Alembic commands ###

View File

@ -124,6 +124,63 @@ class ModuleEnableState(api.ModuleEnableState):
return bool(db_state.state)
class ServiceToCollectorMapping(object):
"""Base class for service to collector mapping."""
def get_mapping(self, service):
session = db.get_session()
try:
res = utils.model_query(
models.ServiceToCollectorMapping,
session
).filter_by(
service=service,
).one()
return res
except sqlalchemy.orm.exc.NoResultFound:
raise api.NoSuchMapping(service)
def set_mapping(self, service, collector):
session = db.get_session()
with session.begin():
try:
q = utils.model_query(
models.ServiceToCollectorMapping,
session
).filter_by(
service=service,
).with_lockmode('update')
db_mapping = q.one()
db_mapping.collector = collector
except sqlalchemy.orm.exc.NoResultFound:
model = models.ServiceToCollectorMapping
db_mapping = model(service=service, collector=collector)
session.add(db_mapping)
return db_mapping
def list_services(self):
session = db.get_session()
q = utils.model_query(
models.ServiceToCollectorMapping,
session
)
res = q.distinct().values(
models.ServiceToCollectorMapping.service
)
return res
def delete_mapping(self, service):
session = db.get_session()
r = utils.model_query(
models.ServiceToCollectorMapping,
session
).filter_by(
service=service,
).delete()
if not r:
raise api.NoSuchMapping(service)
class DBAPIManager(object):
@staticmethod
@ -134,6 +191,10 @@ class DBAPIManager(object):
def get_module_enable_state():
return ModuleEnableState()
@staticmethod
def get_service_to_collector_mapping():
return ServiceToCollectorMapping()
@staticmethod
def get_migration():
return migration

View File

@ -66,3 +66,22 @@ class ModuleStateInfo(Base, models.ModelBase):
'enabled={state}>').format(
name=self.name,
state=self.state)
class ServiceToCollectorMapping(Base, models.ModelBase):
"""Collector module state.
"""
__tablename__ = 'service_to_collector_mappings'
service = sqlalchemy.Column(sqlalchemy.String(255),
primary_key=True)
collector = sqlalchemy.Column(sqlalchemy.String(255),
nullable=False)
def __repr__(self):
return ('<ServiceToCollectorMapping[{service}]: '
'collector={collector}>').format(
service=self.service,
collector=self.collector)

View File

@ -2,6 +2,19 @@
CloudKitty REST API (v1)
========================
Collector
=========
.. rest-controller:: cloudkitty.api.controllers.v1:CollectorController
:webprefix: /v1/collector
.. rest-controller:: cloudkitty.api.controllers.v1:MappingController
:webprefix: /v1/collector/mapping
.. autotype:: cloudkitty.api.controllers.v1.MetricToCollectorMapping
:members:
Billing
=======

View File

@ -26,6 +26,7 @@ console_scripts =
cloudkitty.collector.backends =
ceilometer = cloudkitty.collector.ceilometer:CeilometerCollector
meta = cloudkitty.collector.meta:MetaCollector
cloudkitty.transformers =
CloudKittyFormatTransformer = cloudkitty.transformer.format:CloudKittyFormatTransformer