openstack
/
cloudkitty
Code Issues Proposed changes

Improve User Experience by adding an info REST entrypoint 

This patch allow the user to query the API in order to:

* Get current collector configuration (period, services, etc)
* Get collected metadata list for a given service (using currently
  configured collector)

For this, the following work have been made:

* Each transformer can now export available metadata for a given
  resource (using a FakeData object and the strip_resource_data method)
* Each collector can now export information for a given resource
  about metadata (using associated transformer) and unit
* A new REST controller 'info' provides API entrypoints to those
  information:
  * configuration with GET /v1/info/config
  * all active services information with GET /v1/info/services
  * given active service information with GET /v1/info/services/SERVICE

Change-Id: I02b1bc5709371785748661b63c5e6f0705ce891b
Implements: blueprint user-experience-improvement (PARTIAL)
This commit is contained in:
Maxime Cottret 2016-12-02 16:24:05 +01:00
parent 0f90e9a6ea
commit 671fbb9fe3
13 changed files with 363 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cloudkitty/api/v1/controllers/__init__.py
View File

@ -18,6 +18,7 @@
from pecan import rest
from cloudkitty.api.v1.controllers import collector as collector_api
from cloudkitty.api.v1.controllers import info as info_api
from cloudkitty.api.v1.controllers import rating as rating_api
from cloudkitty.api.v1.controllers import report as report_api
from cloudkitty.api.v1.controllers import storage as storage_api
@ -33,3 +34,4 @@ class V1Controller(rest.RestController):
rating = rating_api.RatingController()
report = report_api.ReportController()
storage = storage_api.StorageController()
info = info_api.InfoController()

83
cloudkitty/api/v1/controllers/info.py Normal file
View File

@ -0,0 +1,83 @@
# -*- coding: utf-8 -*-
# Copyright 2016 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Maxime Cottret
#
from oslo_config import cfg
import pecan
from pecan import rest
import six
from wsme import types as wtypes
import wsmeext.pecan as wsme_pecan
from cloudkitty.api.v1.datamodels import info as info_models
from cloudkitty.api.v1 import types as ck_types
from cloudkitty import collector
from cloudkitty.common import policy
CONF = cfg.CONF
METADATA = collector.get_collector_metadata()
class ServiceInfoController(rest.RestController):
"""REST Controller mananging collected services information."""
@wsme_pecan.wsexpose(info_models.CloudkittyServiceInfoCollection)
def get_all(self):
"""Get the service list.
:return: List of every services.
"""
policy.enforce(pecan.request.context, 'info:list_services_info', {})
services_info_list = []
for service, metadata in METADATA.items():
info = metadata.copy()
info['service_id'] = service
services_info_list.append(
info_models.CloudkittyServiceInfo(**info))
return info_models.CloudkittyServiceInfoCollection(
services=services_info_list)
@wsme_pecan.wsexpose(info_models.CloudkittyServiceInfo, wtypes.text)
def get_one(self, service_name):
"""Return a service.
:param service_name: name of the service.
"""
policy.enforce(pecan.request.context, 'info:get_service_info', {})
try:
info = METADATA[service_name].copy()
info['service_id'] = service_name
return info_models.CloudkittyServiceInfo(**info)
except KeyError:
pecan.abort(404, six.text_type(service_name))
class InfoController(rest.RestController):
"""REST Controller managing Cloudkitty general information."""
services = ServiceInfoController()
_custom_actions = {'config': ['GET']}
@wsme_pecan.wsexpose({
str: ck_types.MultiType(wtypes.text, int, float, dict, list)
})
def config(self):
"""Return current configuration."""
policy.enforce(pecan.request.context, 'info:get_config', {})
info = {}
info["collect"] = {key: value for key, value in CONF.collect.items()}
return info

65
cloudkitty/api/v1/datamodels/info.py Normal file
View File

@ -0,0 +1,65 @@
# -*- coding: utf-8 -*-
# Copyright 2014 Objectif Libre
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
# @author: Stéphane Albert
#
from oslo_config import cfg
from wsme import types as wtypes
CONF = cfg.CONF
CONF.import_opt('services', 'cloudkitty.collector', 'collect')
CLOUDKITTY_SERVICES = wtypes.Enum(wtypes.text,
*CONF.collect.services)
class CloudkittyServiceInfo(wtypes.Base):
"""Type describing a service info in CloudKitty.
"""
service_id = CLOUDKITTY_SERVICES
"""Name of the service."""
metadata = [wtypes.text]
"""List of service metadata"""
unit = wtypes.text
"""service unit"""
def to_json(self):
res_dict = {}
res_dict[self.service_id] = [{
'metadata': self.metadata,
'unit': self.unit
}]
return res_dict
@classmethod
def sample(cls):
sample = cls(service_id='compute',
metadata=['resource_id', 'flavor', 'availability_zone'],
unit='instance')
return sample
class CloudkittyServiceInfoCollection(wtypes.Base):
"""A list of CloudKittyServiceInfo."""
services = [CloudkittyServiceInfo]
@classmethod
def sample(cls):
sample = CloudkittyServiceInfo.sample()
return cls(services=[sample])

25
cloudkitty/collector/__init__.py
View File

@ -66,6 +66,21 @@ def get_collector(transformers=None):
return collector
def get_collector_metadata():
"""Return dict of metadata.
Results are based on enabled collector and services in CONF.
"""
transformers = transformer.get_transformers()
collector = driver.DriverManager(
COLLECTORS_NAMESPACE, CONF.collect.collector,
invoke_on_load=False).driver
metadata = {}
for service in CONF.collect.services:
metadata[service] = collector.get_metadata(service, transformers)
return metadata
class TransformerDependencyError(Exception):
"""Raised when a collector can't find a mandatory transformer."""
@ -132,6 +147,16 @@ class BaseCollector(object):
trans_resource += resource_name.replace('.', '_')
return trans_resource
@classmethod
def get_metadata(cls, resource_name, transformers):
"""Return metadata about collected resource as a dict.
Dict object should contain:
- "metadata": available metadata list,
- "unit": collected quantity unit
"""
return {"metadata": [], "unit": "undefined"}
def retrieve(self,
resource,
start,

55
cloudkitty/collector/ceilometer.py
View File

@ -74,6 +74,15 @@ class CeilometerCollector(collector.BaseCollector):
dependencies = ('CeilometerTransformer',
'CloudKittyFormatTransformer')
units_mappings = {
'compute': 'instance',
'image': 'MB',
'volume': 'GB',
'network.bw.out': 'MB',
'network.bw.in': 'MB',
'network.floating': 'ip',
}
def __init__(self, transformers, **kwargs):
super(CeilometerCollector, self).__init__(transformers, **kwargs)
@ -93,6 +102,18 @@ class CeilometerCollector(collector.BaseCollector):
'2',
session=self.session)
@classmethod
def get_metadata(cls, resource_name, transformers):
info = super(CeilometerCollector, cls).get_metadata(resource_name,
transformers)
try:
info["metadata"].extend(transformers['CeilometerTransformer']
.get_metadata(resource_name))
info["unit"] = cls.units_mappings[resource_name]
except KeyError:
pass
return info
def gen_filter(self, op='eq', **kwargs):
"""Generate ceilometer filter from kwargs."""
q_filter = []
@ -176,9 +197,9 @@ class CeilometerCollector(collector.BaseCollector):
instance)
instance = self._cacher.get_resource_detail('compute',
instance_id)
compute_data.append(self.t_cloudkitty.format_item(instance,
'instance',
1))
compute_data.append(
self.t_cloudkitty.format_item(instance, self.units_mappings[
"compute"], 1))
if not compute_data:
raise collector.NoDataCollected(self.collector_name, 'compute')
return self.t_cloudkitty.format_service('compute', compute_data)
@ -201,10 +222,12 @@ class CeilometerCollector(collector.BaseCollector):
image)
image = self._cacher.get_resource_detail('image',
image_id)
image_size_mb = decimal.Decimal(image_stats.max) / units.Mi
image_data.append(self.t_cloudkitty.format_item(image,
'MB',
image_size_mb))
image_data.append(
self.t_cloudkitty.format_item(image, self.units_mappings[
"image"], image_size_mb))
if not image_data:
raise collector.NoDataCollected(self.collector_name, 'image')
return self.t_cloudkitty.format_service('image', image_data)
@ -228,9 +251,9 @@ class CeilometerCollector(collector.BaseCollector):
volume)
volume = self._cacher.get_resource_detail('volume',
volume_id)
volume_data.append(self.t_cloudkitty.format_item(volume,
'GB',
volume_stats.max))
volume_data.append(
self.t_cloudkitty.format_item(volume, self.units_mappings[
"volume"], volume_stats.max))
if not volume_data:
raise collector.NoDataCollected(self.collector_name, 'volume')
return self.t_cloudkitty.format_service('volume', volume_data)
@ -265,10 +288,12 @@ class CeilometerCollector(collector.BaseCollector):
tap)
tap = self._cacher.get_resource_detail('network.tap',
tap_id)
tap_bw_mb = decimal.Decimal(tap_stat.max) / units.M
bw_data.append(self.t_cloudkitty.format_item(tap,
'MB',
tap_bw_mb))
bw_data.append(
self.t_cloudkitty.format_item(tap, self.units_mappings[
"network.bw." + direction], tap_bw_mb))
ck_res_name = 'network.bw.{}'.format(direction)
if not bw_data:
raise collector.NoDataCollected(self.collector_name,
@ -313,9 +338,9 @@ class CeilometerCollector(collector.BaseCollector):
floating)
floating = self._cacher.get_resource_detail('network.floating',
floating_id)
floating_data.append(self.t_cloudkitty.format_item(floating,
'ip',
1))
floating_data.append(
self.t_cloudkitty.format_item(floating, self.units_mappings[
"network.floating"], 1))
if not floating_data:
raise collector.NoDataCollected(self.collector_name,
'network.floating')

18
cloudkitty/collector/fake.py
View File

@ -48,6 +48,24 @@ class CSVCollector(collector.BaseCollector):
self._file = csvfile
self._csv = reader
@classmethod
def get_metadata(cls, resource_name, transformers):
res = super(CSVCollector, cls).get_metadata(resource_name,
transformers)
try:
filename = cfg.CONF.fake_collector.file
csvfile = open(filename, 'rb')
reader = csv.DictReader(csvfile)
entry = None
for row in reader:
if row['type'] == resource_name:
entry = row
break
res['metadata'] = json.loads(entry['desc']).keys() if entry else {}
except IOError:
pass
return res
def filter_rows(self,
start,
end=None,

12
cloudkitty/collector/gnocchi.py
View File

@ -91,6 +91,18 @@ class GnocchiCollector(collector.BaseCollector):
'1',
session=self.session)
@classmethod
def get_metadata(cls, resource_name, transformers):
info = super(GnocchiCollector, cls).get_metadata(resource_name,
transformers)
try:
info["metadata"].extend(transformers['GnocchiTransformer']
.get_metadata(resource_name))
info["unit"] = cls.units_mappings[resource_name][1]
except KeyError:
pass
return info
@classmethod
def gen_filter(cls, cop='=', lop='and', **kwargs):
"""Generate gnocchi filter from kwargs.

45
cloudkitty/tests/gabbi/gabbits/v1_info.yaml Normal file
View File

@ -0,0 +1,45 @@
fixtures:
- ConfigFixture
tests:
- name: get config
url: /v1/info/config
status: 200
response_json_paths:
$.collect.services.`len`: 6
$.collect.services[0]: compute
$.collect.services[1]: image
$.collect.services[2]: volume
$.collect.services[3]: network.bw.in
$.collect.services[4]: network.bw.out
$.collect.services[5]: network.floating
$.collect.collector: ceilometer
$.collect.window: 1800
$.collect.wait_periods: 2
$.collect.period: 3600
- name: get services info
url: /v1/info/services
status: 200
response_json_paths:
$.services.`len`: 6
$.services[/service_id][0].service_id: compute
$.services[/service_id][0].unit: instance
$.services[/service_id][1].service_id: image
$.services[/service_id][1].unit: MB
$.services[/service_id][2].service_id: network.bw.in
$.services[/service_id][2].unit: MB
$.services[/service_id][3].service_id: network.bw.out
$.services[/service_id][3].unit: MB
$.services[/service_id][4].service_id: network.floating
$.services[/service_id][4].unit: ip
$.services[/service_id][5].service_id: volume
$.services[/service_id][5].unit: GB
- name: get compute service info
url: /v1/info/services/compute
status: 200
response_json_paths:
$.service_id: compute
$.unit: instance
$.metadata.`len`: 10

5
cloudkitty/transformer/__init__.py
View File

@ -64,3 +64,8 @@ class BaseTransformer(object):
if strip_func:
return strip_func(res_data)
return self.generic_strip(res_type, res_data) or res_data
def get_metadata(self, res_type):
"""Return list of metadata available for given resource type."""
return []

23
cloudkitty/transformer/ceilometer.py
View File

@ -104,3 +104,26 @@ class CeilometerTransformer(transformer.BaseTransformer):
res_data['project_id'] = data.project_id
res_data['floatingip_id'] = data.resource_id
return res_data
def get_metadata(self, res_type):
"""Return list of metadata available after transformation for given
resource type.
"""
class FakeData(dict):
"""FakeData object."""
def __getattr__(self, name, default=None):
try:
return super(FakeData, self).__getattr__(self, name)
except AttributeError:
return default or name
# list of metadata is built by applying the generic strip_resource_data
# function to a fake data object
fkdt = FakeData()
setattr(fkdt, self.metadata_item, FakeData())
res_data = self.strip_resource_data(res_type, fkdt)
return res_data.keys()

25
cloudkitty/transformer/gnocchi.py
View File

@ -55,3 +55,28 @@ class GnocchiTransformer(transformer.BaseTransformer):
res_data)
result.update(stripped_data)
return result
def get_metadata(self, res_type):
"""Return list of metadata available after transformation for
given resource type.
"""
class FakeData(dict):
"""FakeData object."""
def __getitem__(self, item):
try:
return super(FakeData, self).__getitem__(item)
except KeyError:
return item
def get(self, item, default=None):
return super(FakeData, self).get(item, item)
# list of metadata is built by applying the generic strip_resource_data
# function to a fake data object
fkdt = FakeData()
res_data = self.strip_resource_data(res_type, fkdt)
return res_data.keys()

16
doc/source/webapi/v1.rst
View File

@ -24,6 +24,22 @@ Collector
:members:
Info
====
.. rest-controller:: cloudkitty.api.v1.controllers.info:InfoController
:webprefix: /v1/info
.. rest-controller:: cloudkitty.api.v1.controllers.info:ServiceInfoController
:webprefix: /v1/info/services
.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyServiceInfo
:members:
.. autotype:: cloudkitty.api.v1.datamodels.info.CloudkittyServiceInfoCollection
:members:
Rating
======

4
etc/cloudkitty/policy.json
View File

@ -2,6 +2,10 @@
"context_is_admin": "role:admin",
"default": "",
"info:list_services_info": "",
"info:get_service_info": "",
"info:get_config":"",
"rating:list_modules": "role:admin",
"rating:get_module": "role:admin",
"rating:update_module": "role:admin",