diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py index 18a39d122c..f230c8337e 100644 --- a/ceilometer/polling/dynamic_pollster.py +++ b/ceilometer/polling/dynamic_pollster.py @@ -362,7 +362,11 @@ class PollsterDefinitions(object): PollsterDefinition(name='metadata_mapping', default={}), PollsterDefinition(name='preserve_mapped_metadata', default=True), PollsterDefinition(name='response_entries_key'), - PollsterDefinition(name='next_sample_url_attribute')] + PollsterDefinition(name='next_sample_url_attribute'), + PollsterDefinition(name='user_id_attribute', default="user_id"), + PollsterDefinition(name='resource_id_attribute', default="id"), + PollsterDefinition(name='project_id_attribute', default="project_id"), + ] extra_definitions = [] @@ -482,14 +486,52 @@ class PollsterSampleGatherer(object): response_json, url, self.definitions.configurations['name']) if entry_size > 0: - response = self.retrieve_entries_from_response(response_json) + samples = self.retrieve_entries_from_response(response_json) url_to_next_sample = self.get_url_to_next_sample(response_json) if url_to_next_sample: kwargs['next_sample_url'] = url_to_next_sample - response += self.execute_request_get_samples(**kwargs) - return response + samples += self.execute_request_get_samples(**kwargs) + + self.execute_id_overrides(samples) + return samples return [] + def execute_id_overrides(self, samples): + if samples: + user_id_attribute = self.definitions.configurations[ + 'user_id_attribute'] + project_id_attribute = self.definitions.configurations[ + 'project_id_attribute'] + resource_id_attribute = self.definitions.configurations[ + 'resource_id_attribute'] + + for request_sample in samples: + self.generate_new_attributes_in_sample( + request_sample, user_id_attribute, 'user_id') + self.generate_new_attributes_in_sample( + request_sample, project_id_attribute, 'project_id') + self.generate_new_attributes_in_sample( + request_sample, resource_id_attribute, 'id') + + def generate_new_attributes_in_sample( + self, sample, attribute_key, new_attribute_key): + + if attribute_key == new_attribute_key: + LOG.debug("We do not need to generate new attribute as the " + "attribute_key[%s] and the new_attribute_key[%s] " + "configurations are the same.", + attribute_key, new_attribute_key) + return + + if attribute_key: + attribute_value = self.definitions.sample_extractor.\ + retrieve_attribute_nested_value(sample, attribute_key) + + LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].", + attribute_key, attribute_value, sample) + + sample[new_attribute_key] = attribute_value + def get_url_to_next_sample(self, resp): linked_sample_extractor = self.definitions.configurations[ 'next_sample_url_attribute'] @@ -545,11 +587,8 @@ class NonOpenStackApisPollsterDefinition(PollsterDefinitions): PollsterDefinition(name='value_attribute', required=True), PollsterDefinition(name='module', required=True), PollsterDefinition(name='authentication_object', required=True), - PollsterDefinition(name='user_id_attribute'), - PollsterDefinition(name='resource_id_attribute'), PollsterDefinition(name='barbican_secret_id', default=""), PollsterDefinition(name='authentication_parameters', default=""), - PollsterDefinition(name='project_id_attribute'), PollsterDefinition(name='endpoint_type')] def __init__(self, configurations): @@ -602,28 +641,6 @@ class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer): return resp, url - def execute_request_get_samples(self, **kwargs): - samples = super(NonOpenStackApisSamplesGatherer, - self).execute_request_get_samples(**kwargs) - - if samples: - user_id_attribute = self.definitions.configurations[ - 'user_id_attribute'] - project_id_attribute = self.definitions.configurations[ - 'project_id_attribute'] - resource_id_attribute = self.definitions.configurations[ - 'resource_id_attribute'] - - for request_sample in samples: - self.generate_new_attributes_in_sample( - request_sample, user_id_attribute, 'user_id') - self.generate_new_attributes_in_sample( - request_sample, project_id_attribute, 'project_id') - self.generate_new_attributes_in_sample( - request_sample, resource_id_attribute, 'id') - - return samples - def generate_new_attributes_in_sample( self, sample, attribute_key, new_attribute_key): if attribute_key: diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py index a9c7bc016a..7276cbe8e5 100644 --- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py @@ -714,3 +714,48 @@ class TestDynamicPollster(base.BaseTestCase): self.assertEqual(46, get_obj_sample.volume) self.assertEqual(8, list_bucket_sample.volume) self.assertEqual(46, put_obj_sample.volume) + + def test_execute_request_get_samples_custom_ids(self): + sample = {'user_id_attribute': "1", + 'project_id_attribute': "2", + 'resource_id_attribute': "3", + 'user_id': "234", + 'project_id': "2334", + 'id': "35"} + + def internal_execute_request_get_samples_mock(self, arg): + class Response: + def json(self): + return [sample] + return Response(), "url" + + original_method = dynamic_pollster.PollsterSampleGatherer.\ + internal_execute_request_get_samples + try: + dynamic_pollster.PollsterSampleGatherer. \ + internal_execute_request_get_samples = \ + internal_execute_request_get_samples_mock + + self.pollster_definition_all_fields[ + 'user_id_attribute'] = 'user_id_attribute' + self.pollster_definition_all_fields[ + 'project_id_attribute'] = 'project_id_attribute' + self.pollster_definition_all_fields[ + 'resource_id_attribute'] = 'resource_id_attribute' + + pollster = dynamic_pollster.DynamicPollster( + self.pollster_definition_all_fields) + + params = {"d": "d"} + response = pollster.definitions.sample_gatherer. \ + execute_request_get_samples(**params) + + self.assertEqual(sample['user_id_attribute'], + response[0]['user_id']) + self.assertEqual(sample['project_id_attribute'], + response[0]['project_id']) + self.assertEqual(sample['resource_id_attribute'], + response[0]['id']) + finally: + dynamic_pollster.PollsterSampleGatherer. \ + internal_execute_request_get_samples = original_method diff --git a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py index 278eebb6c3..1376796330 100644 --- a/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py +++ b/ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py @@ -25,6 +25,7 @@ from ceilometer.polling.dynamic_pollster import DynamicPollster from ceilometer.polling.dynamic_pollster import MultiMetricPollsterDefinitions from ceilometer.polling.dynamic_pollster import \ NonOpenStackApisPollsterDefinition +from ceilometer.polling.dynamic_pollster import NonOpenStackApisSamplesGatherer from ceilometer.polling.dynamic_pollster import PollsterSampleGatherer from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions @@ -176,9 +177,10 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): pollster_definitions = pollster.pollster_definitions - self.assertEqual(None, pollster_definitions['user_id_attribute']) - self.assertEqual(None, pollster_definitions['project_id_attribute']) - self.assertEqual(None, pollster_definitions['resource_id_attribute']) + self.assertEqual("user_id", pollster_definitions['user_id_attribute']) + self.assertEqual("project_id", + pollster_definitions['project_id_attribute']) + self.assertEqual("id", pollster_definitions['resource_id_attribute']) self.assertEqual('', pollster_definitions['barbican_secret_id']) self.assertEqual('', pollster_definitions['authentication_parameters']) @@ -300,33 +302,42 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase): 'project_id_attribute': "dfghyt432345t", 'resource_id_attribute': "sdfghjt543"} - def execute_request_get_samples_mock(self, **kwargs): - samples = [sample] - return samples + def internal_execute_request_get_samples_mock(self, arg): + class Response: + def json(self): + return [sample] + return Response(), "url" - PollsterSampleGatherer.execute_request_get_samples = \ - execute_request_get_samples_mock + original_method = NonOpenStackApisSamplesGatherer. \ + internal_execute_request_get_samples + try: + NonOpenStackApisSamplesGatherer. \ + internal_execute_request_get_samples = \ + internal_execute_request_get_samples_mock - self.pollster_definition_all_fields[ - 'user_id_attribute'] = 'user_id_attribute' - self.pollster_definition_all_fields[ - 'project_id_attribute'] = 'project_id_attribute' - self.pollster_definition_all_fields[ - 'resource_id_attribute'] = 'resource_id_attribute' + self.pollster_definition_all_fields[ + 'user_id_attribute'] = 'user_id_attribute' + self.pollster_definition_all_fields[ + 'project_id_attribute'] = 'project_id_attribute' + self.pollster_definition_all_fields[ + 'resource_id_attribute'] = 'resource_id_attribute' - pollster = DynamicPollster( - self.pollster_definition_all_fields) + pollster = DynamicPollster( + self.pollster_definition_all_fields) - params = {"d": "d"} - response = pollster.definitions.sample_gatherer. \ - execute_request_get_samples(**params) + params = {"d": "d"} + response = pollster.definitions.sample_gatherer. \ + execute_request_get_samples(**params) - self.assertEqual(sample['user_id_attribute'], - response[0]['user_id']) - self.assertEqual(sample['project_id_attribute'], - response[0]['project_id']) - self.assertEqual(sample['resource_id_attribute'], - response[0]['id']) + self.assertEqual(sample['user_id_attribute'], + response[0]['user_id']) + self.assertEqual(sample['project_id_attribute'], + response[0]['project_id']) + self.assertEqual(sample['resource_id_attribute'], + response[0]['id']) + finally: + NonOpenStackApisSamplesGatherer. \ + internal_execute_request_get_samples = original_method def test_execute_request_get_samples_empty_keys(self): sample = {'user_id_attribute': "123456789", diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst index f7569fa0c4..fb174201ba 100644 --- a/doc/source/admin/telemetry-dynamic-pollster.rst +++ b/doc/source/admin/telemetry-dynamic-pollster.rst @@ -151,6 +151,20 @@ attributes to define a dynamic pollster: directly. We also accept nested values dictionaries. To use a nested value one can simply use ``attribute1.attribute2..lastattribute`` +* ``user_id_attribute``: optional parameter. The default value is ``user_id``. + The name of the attribute in the entries that are processed from + ``response_entries_key`` elements that will be mapped to ``user_id`` + attribute that is sent to Gnocchi. + +* ``project_id_attribute``: optional parameter. The default value is + ``project_id``. The name of the attribute in the entries that are + processed from ``response_entries_key`` elements that will be mapped to + ``project_id`` attribute that is sent to Gnocchi. + +* ``resource_id_attribute``: optional parameter. The default value is ``id``. + The name of the attribute in the entries that are processed from + ``response_entries_key`` elements that will be mapped to ``id`` attribute + that is sent to Gnocchi. The complete YAML configuration to gather data from Magnum (that has been used as an example) is the following: @@ -244,8 +258,8 @@ the Dynamic pollster system. The attribute that is not supported is the ``endpoint_type``. The dynamic pollster system for non-OpenStack APIs is activated automatically when one uses the configurations ``module``. -The extra parameters that are available when using the Non-OpenStack -dynamic pollster sub-subsystem are the following: +The extra parameters (in addition to the original ones) that are available +when using the Non-OpenStack dynamic pollster sub-subsystem are the following: * ``module``: required parameter. It is the python module name that Ceilometer has to load to use the authentication object when executing requests against @@ -268,18 +282,6 @@ dynamic pollster sub-subsystem are the following: from which, Ceilometer can retrieve the comma separated values of the ``authentication_parameters``. -* ``user_id_attribute``: optional parameter. The name of the attribute in the - entries that are processed from ``response_entries_key`` elements that - will be mapped to ``user_id`` attribute that is sent to Gnocchi. - -* ``project_id_attribute``: optional parameter. The name of the attribute in - the entries that are processed from ``response_entries_key`` elements that - will be mapped to ``project_id`` attribute that is sent to Gnocchi. - -* ``resource_id_attribute``: optional parameter. The name of the attribute - in the entries that are processed from ``response_entries_key`` elements that - will be mapped to ``id`` attribute that is sent to Gnocchi. - As follows we present an example on how to convert the hard-coded pollster for `radosgw.api.request` metric to the dynamic pollster model: