Enable OpenStack pollster to configure Ids(project, user, and resource)
This pull request enables OpenStack pollsters to configure Ids(project, user, and resource) as the non-openstack pollster. This is useful as some of the OpenStack APIs use other keys to represent values such as project_id and user_id. Change-Id: Id5ae1720ad714675722bc857a142d7bd667fe7ed
This commit is contained in:
Rafael Weingärtner
parent
2dcd15a466
commit
1a4ab34df5
|
|
@ -362,7 +362,11 @@ class PollsterDefinitions(object):
|
|||
PollsterDefinition(name='metadata_mapping', default={}),
|
||||
PollsterDefinition(name='preserve_mapped_metadata', default=True),
|
||||
PollsterDefinition(name='response_entries_key'),
|
||||
PollsterDefinition(name='next_sample_url_attribute')]
|
||||
PollsterDefinition(name='next_sample_url_attribute'),
|
||||
PollsterDefinition(name='user_id_attribute', default="user_id"),
|
||||
PollsterDefinition(name='resource_id_attribute', default="id"),
|
||||
PollsterDefinition(name='project_id_attribute', default="project_id"),
|
||||
]
|
||||
|
||||
extra_definitions = []
|
||||
|
||||
|
|
@ -482,14 +486,52 @@ class PollsterSampleGatherer(object):
|
|||
response_json, url, self.definitions.configurations['name'])
|
||||
|
||||
if entry_size > 0:
|
||||
response = self.retrieve_entries_from_response(response_json)
|
||||
samples = self.retrieve_entries_from_response(response_json)
|
||||
url_to_next_sample = self.get_url_to_next_sample(response_json)
|
||||
if url_to_next_sample:
|
||||
kwargs['next_sample_url'] = url_to_next_sample
|
||||
response += self.execute_request_get_samples(**kwargs)
|
||||
return response
|
||||
samples += self.execute_request_get_samples(**kwargs)
|
||||
|
||||
self.execute_id_overrides(samples)
|
||||
return samples
|
||||
return []
|
||||
|
||||
def execute_id_overrides(self, samples):
|
||||
if samples:
|
||||
user_id_attribute = self.definitions.configurations[
|
||||
'user_id_attribute']
|
||||
project_id_attribute = self.definitions.configurations[
|
||||
'project_id_attribute']
|
||||
resource_id_attribute = self.definitions.configurations[
|
||||
'resource_id_attribute']
|
||||
|
||||
for request_sample in samples:
|
||||
self.generate_new_attributes_in_sample(
|
||||
request_sample, user_id_attribute, 'user_id')
|
||||
self.generate_new_attributes_in_sample(
|
||||
request_sample, project_id_attribute, 'project_id')
|
||||
self.generate_new_attributes_in_sample(
|
||||
request_sample, resource_id_attribute, 'id')
|
||||
|
||||
def generate_new_attributes_in_sample(
|
||||
self, sample, attribute_key, new_attribute_key):
|
||||
|
||||
if attribute_key == new_attribute_key:
|
||||
LOG.debug("We do not need to generate new attribute as the "
|
||||
"attribute_key[%s] and the new_attribute_key[%s] "
|
||||
"configurations are the same.",
|
||||
attribute_key, new_attribute_key)
|
||||
return
|
||||
|
||||
if attribute_key:
|
||||
attribute_value = self.definitions.sample_extractor.\
|
||||
retrieve_attribute_nested_value(sample, attribute_key)
|
||||
|
||||
LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
|
||||
attribute_key, attribute_value, sample)
|
||||
|
||||
sample[new_attribute_key] = attribute_value
|
||||
|
||||
def get_url_to_next_sample(self, resp):
|
||||
linked_sample_extractor = self.definitions.configurations[
|
||||
'next_sample_url_attribute']
|
||||
|
|
@ -545,11 +587,8 @@ class NonOpenStackApisPollsterDefinition(PollsterDefinitions):
|
|||
PollsterDefinition(name='value_attribute', required=True),
|
||||
PollsterDefinition(name='module', required=True),
|
||||
PollsterDefinition(name='authentication_object', required=True),
|
||||
PollsterDefinition(name='user_id_attribute'),
|
||||
PollsterDefinition(name='resource_id_attribute'),
|
||||
PollsterDefinition(name='barbican_secret_id', default=""),
|
||||
PollsterDefinition(name='authentication_parameters', default=""),
|
||||
PollsterDefinition(name='project_id_attribute'),
|
||||
PollsterDefinition(name='endpoint_type')]
|
||||
|
||||
def __init__(self, configurations):
|
||||
|
|
@ -602,28 +641,6 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
|
|||
|
||||
return resp, url
|
||||
|
||||
def execute_request_get_samples(self, **kwargs):
|
||||
samples = super(NonOpenStackApisSamplesGatherer,
|
||||
self).execute_request_get_samples(**kwargs)
|
||||
|
||||
if samples:
|
||||
user_id_attribute = self.definitions.configurations[
|
||||
'user_id_attribute']
|
||||
project_id_attribute = self.definitions.configurations[
|
||||
'project_id_attribute']
|
||||
resource_id_attribute = self.definitions.configurations[
|
||||
'resource_id_attribute']
|
||||
|
||||
for request_sample in samples:
|
||||
self.generate_new_attributes_in_sample(
|
||||
request_sample, user_id_attribute, 'user_id')
|
||||
self.generate_new_attributes_in_sample(
|
||||
request_sample, project_id_attribute, 'project_id')
|
||||
self.generate_new_attributes_in_sample(
|
||||
request_sample, resource_id_attribute, 'id')
|
||||
|
||||
return samples
|
||||
|
||||
def generate_new_attributes_in_sample(
|
||||
self, sample, attribute_key, new_attribute_key):
|
||||
if attribute_key:
|
||||
|
|
|
|||
|
|
@ -714,3 +714,48 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
self.assertEqual(46, get_obj_sample.volume)
|
||||
self.assertEqual(8, list_bucket_sample.volume)
|
||||
self.assertEqual(46, put_obj_sample.volume)
|
||||
|
||||
def test_execute_request_get_samples_custom_ids(self):
|
||||
sample = {'user_id_attribute': "1",
|
||||
'project_id_attribute': "2",
|
||||
'resource_id_attribute': "3",
|
||||
'user_id': "234",
|
||||
'project_id': "2334",
|
||||
'id': "35"}
|
||||
|
||||
def internal_execute_request_get_samples_mock(self, arg):
|
||||
class Response:
|
||||
def json(self):
|
||||
return [sample]
|
||||
return Response(), "url"
|
||||
|
||||
original_method = dynamic_pollster.PollsterSampleGatherer.\
|
||||
internal_execute_request_get_samples
|
||||
try:
|
||||
dynamic_pollster.PollsterSampleGatherer. \
|
||||
internal_execute_request_get_samples = \
|
||||
internal_execute_request_get_samples_mock
|
||||
|
||||
self.pollster_definition_all_fields[
|
||||
'user_id_attribute'] = 'user_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'project_id_attribute'] = 'project_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'resource_id_attribute'] = 'resource_id_attribute'
|
||||
|
||||
pollster = dynamic_pollster.DynamicPollster(
|
||||
self.pollster_definition_all_fields)
|
||||
|
||||
params = {"d": "d"}
|
||||
response = pollster.definitions.sample_gatherer. \
|
||||
execute_request_get_samples(**params)
|
||||
|
||||
self.assertEqual(sample['user_id_attribute'],
|
||||
response[0]['user_id'])
|
||||
self.assertEqual(sample['project_id_attribute'],
|
||||
response[0]['project_id'])
|
||||
self.assertEqual(sample['resource_id_attribute'],
|
||||
response[0]['id'])
|
||||
finally:
|
||||
dynamic_pollster.PollsterSampleGatherer. \
|
||||
internal_execute_request_get_samples = original_method
|
||||
|
|
|
|||
|
|
@ -25,6 +25,7 @@ from ceilometer.polling.dynamic_pollster import DynamicPollster
|
|||
from ceilometer.polling.dynamic_pollster import MultiMetricPollsterDefinitions
|
||||
from ceilometer.polling.dynamic_pollster import \
|
||||
NonOpenStackApisPollsterDefinition
|
||||
from ceilometer.polling.dynamic_pollster import NonOpenStackApisSamplesGatherer
|
||||
from ceilometer.polling.dynamic_pollster import PollsterSampleGatherer
|
||||
from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions
|
||||
|
||||
|
|
@ -176,9 +177,10 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
|
|||
|
||||
pollster_definitions = pollster.pollster_definitions
|
||||
|
||||
self.assertEqual(None, pollster_definitions['user_id_attribute'])
|
||||
self.assertEqual(None, pollster_definitions['project_id_attribute'])
|
||||
self.assertEqual(None, pollster_definitions['resource_id_attribute'])
|
||||
self.assertEqual("user_id", pollster_definitions['user_id_attribute'])
|
||||
self.assertEqual("project_id",
|
||||
pollster_definitions['project_id_attribute'])
|
||||
self.assertEqual("id", pollster_definitions['resource_id_attribute'])
|
||||
self.assertEqual('', pollster_definitions['barbican_secret_id'])
|
||||
self.assertEqual('', pollster_definitions['authentication_parameters'])
|
||||
|
||||
|
|
@ -300,33 +302,42 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
|
|||
'project_id_attribute': "dfghyt432345t",
|
||||
'resource_id_attribute': "sdfghjt543"}
|
||||
|
||||
def execute_request_get_samples_mock(self, **kwargs):
|
||||
samples = [sample]
|
||||
return samples
|
||||
def internal_execute_request_get_samples_mock(self, arg):
|
||||
class Response:
|
||||
def json(self):
|
||||
return [sample]
|
||||
return Response(), "url"
|
||||
|
||||
PollsterSampleGatherer.execute_request_get_samples = \
|
||||
execute_request_get_samples_mock
|
||||
original_method = NonOpenStackApisSamplesGatherer. \
|
||||
internal_execute_request_get_samples
|
||||
try:
|
||||
NonOpenStackApisSamplesGatherer. \
|
||||
internal_execute_request_get_samples = \
|
||||
internal_execute_request_get_samples_mock
|
||||
|
||||
self.pollster_definition_all_fields[
|
||||
'user_id_attribute'] = 'user_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'project_id_attribute'] = 'project_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'resource_id_attribute'] = 'resource_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'user_id_attribute'] = 'user_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'project_id_attribute'] = 'project_id_attribute'
|
||||
self.pollster_definition_all_fields[
|
||||
'resource_id_attribute'] = 'resource_id_attribute'
|
||||
|
||||
pollster = DynamicPollster(
|
||||
self.pollster_definition_all_fields)
|
||||
pollster = DynamicPollster(
|
||||
self.pollster_definition_all_fields)
|
||||
|
||||
params = {"d": "d"}
|
||||
response = pollster.definitions.sample_gatherer. \
|
||||
execute_request_get_samples(**params)
|
||||
params = {"d": "d"}
|
||||
response = pollster.definitions.sample_gatherer. \
|
||||
execute_request_get_samples(**params)
|
||||
|
||||
self.assertEqual(sample['user_id_attribute'],
|
||||
response[0]['user_id'])
|
||||
self.assertEqual(sample['project_id_attribute'],
|
||||
response[0]['project_id'])
|
||||
self.assertEqual(sample['resource_id_attribute'],
|
||||
response[0]['id'])
|
||||
self.assertEqual(sample['user_id_attribute'],
|
||||
response[0]['user_id'])
|
||||
self.assertEqual(sample['project_id_attribute'],
|
||||
response[0]['project_id'])
|
||||
self.assertEqual(sample['resource_id_attribute'],
|
||||
response[0]['id'])
|
||||
finally:
|
||||
NonOpenStackApisSamplesGatherer. \
|
||||
internal_execute_request_get_samples = original_method
|
||||
|
||||
def test_execute_request_get_samples_empty_keys(self):
|
||||
sample = {'user_id_attribute': "123456789",
|
||||
|
|
|
|||
|
|
@ -151,6 +151,20 @@ attributes to define a dynamic pollster:
|
|||
directly. We also accept nested values dictionaries. To use a nested value
|
||||
one can simply use ``attribute1.attribute2.<asMuchAsNeeded>.lastattribute``
|
||||
|
||||
* ``user_id_attribute``: optional parameter. The default value is ``user_id``.
|
||||
The name of the attribute in the entries that are processed from
|
||||
``response_entries_key`` elements that will be mapped to ``user_id``
|
||||
attribute that is sent to Gnocchi.
|
||||
|
||||
* ``project_id_attribute``: optional parameter. The default value is
|
||||
``project_id``. The name of the attribute in the entries that are
|
||||
processed from ``response_entries_key`` elements that will be mapped to
|
||||
``project_id`` attribute that is sent to Gnocchi.
|
||||
|
||||
* ``resource_id_attribute``: optional parameter. The default value is ``id``.
|
||||
The name of the attribute in the entries that are processed from
|
||||
``response_entries_key`` elements that will be mapped to ``id`` attribute
|
||||
that is sent to Gnocchi.
|
||||
|
||||
The complete YAML configuration to gather data from Magnum (that has been used
|
||||
as an example) is the following:
|
||||
|
|
@ -244,8 +258,8 @@ the Dynamic pollster system. The attribute that is not supported is
|
|||
the ``endpoint_type``. The dynamic pollster system for non-OpenStack APIs
|
||||
is activated automatically when one uses the configurations ``module``.
|
||||
|
||||
The extra parameters that are available when using the Non-OpenStack
|
||||
dynamic pollster sub-subsystem are the following:
|
||||
The extra parameters (in addition to the original ones) that are available
|
||||
when using the Non-OpenStack dynamic pollster sub-subsystem are the following:
|
||||
|
||||
* ``module``: required parameter. It is the python module name that Ceilometer
|
||||
has to load to use the authentication object when executing requests against
|
||||
|
|
@ -268,18 +282,6 @@ dynamic pollster sub-subsystem are the following:
|
|||
from which, Ceilometer can retrieve the comma separated values of the
|
||||
``authentication_parameters``.
|
||||
|
||||
* ``user_id_attribute``: optional parameter. The name of the attribute in the
|
||||
entries that are processed from ``response_entries_key`` elements that
|
||||
will be mapped to ``user_id`` attribute that is sent to Gnocchi.
|
||||
|
||||
* ``project_id_attribute``: optional parameter. The name of the attribute in
|
||||
the entries that are processed from ``response_entries_key`` elements that
|
||||
will be mapped to ``project_id`` attribute that is sent to Gnocchi.
|
||||
|
||||
* ``resource_id_attribute``: optional parameter. The name of the attribute
|
||||
in the entries that are processed from ``response_entries_key`` elements that
|
||||
will be mapped to ``id`` attribute that is sent to Gnocchi.
|
||||
|
||||
As follows we present an example on how to convert the hard-coded pollster
|
||||
for `radosgw.api.request` metric to the dynamic pollster model:
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue