# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. """Tests for OpenStack dynamic pollster """ import copy import json import logging from unittest import mock import requests from urllib import parse as urlparse from ceilometer import declarative from ceilometer.polling import dynamic_pollster from ceilometer import sample from oslotest import base LOG = logging.getLogger(__name__) REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', 'value_attribute', 'endpoint_type', 'url_path'] class SampleGenerator(object): def __init__(self, samples_dict, turn_to_list=False): self.turn_to_list = turn_to_list self.samples_dict = {} for k, v in samples_dict.items(): if isinstance(v, list): self.samples_dict[k] = [0, v] else: self.samples_dict[k] = [0, [v]] def get_next_sample_dict(self): _dict = {} for key in self.samples_dict.keys(): _dict[key] = self.get_next_sample(key) if self.turn_to_list: _dict = [_dict] return _dict def get_next_sample(self, key): samples = self.samples_dict[key][1] samples_next_iteration = self.samples_dict[key][0] % len(samples) self.samples_dict[key][0] += 1 _sample = samples[samples_next_iteration] if isinstance(_sample, SampleGenerator): return _sample.get_next_sample_dict() return _sample class PagedSamplesGenerator(SampleGenerator): def __init__(self, samples_dict, dict_name, page_link_name): super(PagedSamplesGenerator, self).__init__(samples_dict) self.dict_name = dict_name self.page_link_name = page_link_name self.response = {} def generate_samples(self, page_base_link, page_links, last_page_size): self.response.clear() current_page_link = page_base_link for page_link, page_size in page_links.items(): page_link = page_base_link + "/" + page_link self.response[current_page_link] = { self.page_link_name: [{'href': page_link, 'rel': 'next'}], self.dict_name: self.populate_page(page_size) } current_page_link = page_link self.response[current_page_link] = { self.dict_name: self.populate_page(last_page_size) } def populate_page(self, page_size): page = [] for item_number in range(0, page_size): page.append(self.get_next_sample_dict()) return page class PagedSamplesGeneratorHttpRequestMock(PagedSamplesGenerator): def mock_request(self, url, **kwargs): return_value = TestDynamicPollster.FakeResponse() return_value.status_code = requests.codes.ok return_value.json_object = self.response[url] return return_value class TestDynamicPollster(base.BaseTestCase): class FakeResponse(object): status_code = None json_object = None _text = None @property def text(self): return self._text or json.dumps(self.json_object) def json(self): return self.json_object def raise_for_status(self): raise requests.HTTPError("Mock HTTP error.", response=self) class FakeManager(object): def __init__(self, keystone=None): self._keystone = keystone def setUp(self): super(TestDynamicPollster, self).setUp() self.pollster_definition_only_required_fields = { 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", 'value_attribute': "volume", 'endpoint_type': "test", 'url_path': "v1/test/endpoint/fake"} self.pollster_definition_all_fields = { 'metadata_fields': "metadata-field-name", 'skip_sample_values': ["I-do-not-want-entries-with-this-value"], 'value_mapping': { 'value-to-map': 'new-value', 'value-to-map-to-numeric': 12 }, 'default_value_mapping': 0, 'metadata_mapping': { 'old-metadata-name': "new-metadata-name" }, 'preserve_mapped_metadata': False} self.pollster_definition_all_fields.update( self.pollster_definition_only_required_fields) self.multi_metric_pollster_definition = { 'name': "test-pollster.{category}", 'sample_type': "gauge", 'unit': "test", 'value_attribute': "[categories].ops", 'endpoint_type': "test", 'url_path': "v1/test/endpoint/fake"} def execute_basic_asserts(self, pollster, pollster_definition): self.assertEqual(pollster, pollster.obj) self.assertEqual(pollster_definition['name'], pollster.name) for key in REQUIRED_POLLSTER_FIELDS: self.assertEqual(pollster_definition[key], pollster.pollster_definitions[key]) self.assertEqual(pollster_definition, pollster.pollster_definitions) @mock.patch('keystoneclient.v2_0.client.Client') def test_skip_samples_with_linked_samples(self, keystone_mock): generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={ 'volume': SampleGenerator(samples_dict={ 'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], 'tmp': ['ra', 'rb', 'rc', 'rd', 're', 'rf', 'rg', 'rh']}, turn_to_list=True), 'id': [1, 2, 3, 4, 5, 6, 7, 8], 'name': ['a1', 'b2', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'] }, dict_name='servers', page_link_name='server_link') generator.generate_samples('http://test.com/v1/test-volumes', { 'marker=c3': 3, 'marker=f6': 3 }, 2) keystone_mock.session.get.side_effect = generator.mock_request fake_manager = self.FakeManager(keystone=keystone_mock) pollster_definition = dict(self.multi_metric_pollster_definition) pollster_definition['name'] = 'test-pollster.{name}' pollster_definition['value_attribute'] = '[volume].tmp' pollster_definition['skip_sample_values'] = ['rb'] pollster_definition['url_path'] = 'v1/test-volumes' pollster_definition['response_entries_key'] = 'servers' pollster_definition['next_sample_url_attribute'] = \ 'server_link | filter(lambda v: v.get("rel") == "next", value) |' \ 'list(value) | value[0] | value.get("href")' pollster = dynamic_pollster.DynamicPollster(pollster_definition) samples = pollster.get_samples(fake_manager, None, ['http://test.com']) self.assertEqual(['ra', 'rc', 'rd', 're', 'rf', 'rg', 'rh'], list(map(lambda s: s.volume, samples))) generator.generate_samples('http://test.com/v1/test-volumes', { 'marker=c3': 3, 'marker=f6': 3 }, 2) pollster_definition['name'] = 'test-pollster' pollster_definition['value_attribute'] = 'name' pollster_definition['skip_sample_values'] = ['b2'] pollster = dynamic_pollster.DynamicPollster(pollster_definition) samples = pollster.get_samples(fake_manager, None, ['http://test.com']) self.assertEqual(['a1', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8'], list(map(lambda s: s.volume, samples))) def test_all_required_fields_ok(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) self.execute_basic_asserts( pollster, self.pollster_definition_only_required_fields) self.assertEqual( 0, len(pollster.pollster_definitions['skip_sample_values'])) self.assertEqual( 0, len(pollster.pollster_definitions['value_mapping'])) self.assertEqual( -1, pollster.pollster_definitions['default_value']) self.assertEqual( 0, len(pollster.pollster_definitions['metadata_mapping'])) self.assertEqual( True, pollster.pollster_definitions['preserve_mapped_metadata']) def test_all_fields_ok(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_all_fields) self.execute_basic_asserts(pollster, self.pollster_definition_all_fields) self.assertEqual( 1, len(pollster.pollster_definitions['skip_sample_values'])) self.assertEqual( 2, len(pollster.pollster_definitions['value_mapping'])) self.assertEqual( 0, pollster.pollster_definitions['default_value_mapping']) self.assertEqual( 1, len(pollster.pollster_definitions['metadata_mapping'])) self.assertEqual( False, pollster.pollster_definitions['preserve_mapped_metadata']) def test_all_required_fields_exceptions(self): for key in REQUIRED_POLLSTER_FIELDS: pollster_definition = copy.deepcopy( self.pollster_definition_only_required_fields) pollster_definition.pop(key) exception = self.assertRaises( declarative.DynamicPollsterDefinitionException, dynamic_pollster.DynamicPollster, pollster_definition) self.assertEqual("Required fields ['%s'] not specified." % key, exception.brief_message) def test_invalid_sample_type(self): self.pollster_definition_only_required_fields[ 'sample_type'] = "invalid_sample_type" exception = self.assertRaises( declarative.DynamicPollsterDefinitionException, dynamic_pollster.DynamicPollster, self.pollster_definition_only_required_fields) self.assertEqual("Invalid sample type [invalid_sample_type]. " "Valid ones are [('gauge', 'delta', 'cumulative')].", exception.brief_message) def test_all_valid_sample_type(self): for sample_type in sample.TYPES: self.pollster_definition_only_required_fields[ 'sample_type'] = sample_type pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) self.execute_basic_asserts( pollster, self.pollster_definition_only_required_fields) def test_default_discovery_method(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) self.assertEqual("endpoint:test", pollster.definitions.sample_gatherer .default_discovery) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_get_samples_empty_response(self, client_mock): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value.json_object = {} client_mock.session.get.return_value = return_value samples = pollster.definitions.sample_gatherer. \ execute_request_get_samples( keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual(0, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_get_samples_response_non_empty( self, client_mock): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value.json_object = {"firstElement": [{}, {}, {}]} client_mock.session.get.return_value = return_value samples = pollster.definitions.sample_gatherer. \ execute_request_get_samples( keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual(3, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_json_response_handler( self, client_mock): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = '{"test": [1,2,3]}' client_mock.session.get.return_value = return_value samples = pollster.definitions.sample_gatherer. \ execute_request_get_samples( keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual(3, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_xml_response_handler( self, client_mock): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['response_handlers'] = ['xml'] pollster = dynamic_pollster.DynamicPollster(definitions) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = '123' client_mock.session.get.return_value = return_value samples = pollster.definitions.sample_gatherer. \ execute_request_get_samples( keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual(3, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_xml_json_response_handler( self, client_mock): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['response_handlers'] = ['xml', 'json'] pollster = dynamic_pollster.DynamicPollster(definitions) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = '123' client_mock.session.get.return_value = return_value samples = pollster.definitions.sample_gatherer. \ execute_request_get_samples( keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual(3, len(samples)) return_value._text = '{"test": [1,2,3,4]}' samples = pollster.definitions.sample_gatherer. \ execute_request_get_samples( keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual(4, len(samples)) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_extra_metadata_fields_cache_disabled( self, client_mock): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) extra_metadata_fields = { 'extra_metadata_fields_cache_seconds': 0, 'name': "project_name", 'endpoint_type': "identity", 'url_path': "'/v3/projects/' + str(sample['project_id'])", 'value': "name", } definitions['value_attribute'] = 'project_id' definitions['extra_metadata_fields'] = extra_metadata_fields pollster = dynamic_pollster.DynamicPollster(definitions) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = ''' {"projects": [ {"project_id": 9999, "name": "project1"}, {"project_id": 8888, "name": "project2"}, {"project_id": 7777, "name": "project3"}, {"project_id": 9999, "name": "project1"}, {"project_id": 8888, "name": "project2"}, {"project_id": 7777, "name": "project3"}, {"project_id": 9999, "name": "project1"}, {"project_id": 8888, "name": "project2"}, {"project_id": 7777, "name": "project3"}] } ''' return_value9999 = self.FakeResponse() return_value9999.status_code = requests.codes.ok return_value9999._text = ''' {"project": {"project_id": 9999, "name": "project1"} } ''' return_value8888 = self.FakeResponse() return_value8888.status_code = requests.codes.ok return_value8888._text = ''' {"project": {"project_id": 8888, "name": "project2"} } ''' return_value7777 = self.FakeResponse() return_value7777.status_code = requests.codes.ok return_value7777._text = ''' {"project": {"project_id": 7777, "name": "project3"} } ''' def get(url, *args, **kwargs): if '9999' in url: return return_value9999 if '8888' in url: return return_value8888 if '7777' in url: return return_value7777 return return_value client_mock.session.get.side_effect = get manager = mock.Mock manager._keystone = client_mock def discover(*args, **kwargs): return ["https://endpoint.server.name/"] manager.discover = discover samples = pollster.get_samples( manager=manager, cache=None, resources=["https://endpoint.server.name/"]) samples = list(samples) n_calls = client_mock.session.get.call_count self.assertEqual(9, len(samples)) self.assertEqual(10, n_calls) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_extra_metadata_fields_cache_enabled( self, client_mock): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) extra_metadata_fields = { 'extra_metadata_fields_cache_seconds': 3600, 'name': "project_name", 'endpoint_type': "identity", 'url_path': "'/v3/projects/' + str(sample['project_id'])", 'value': "name", } definitions['value_attribute'] = 'project_id' definitions['extra_metadata_fields'] = extra_metadata_fields pollster = dynamic_pollster.DynamicPollster(definitions) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = ''' {"projects": [ {"project_id": 9999, "name": "project1"}, {"project_id": 8888, "name": "project2"}, {"project_id": 7777, "name": "project3"}, {"project_id": 9999, "name": "project4"}, {"project_id": 8888, "name": "project5"}, {"project_id": 7777, "name": "project6"}, {"project_id": 9999, "name": "project7"}, {"project_id": 8888, "name": "project8"}, {"project_id": 7777, "name": "project9"}] } ''' return_value9999 = self.FakeResponse() return_value9999.status_code = requests.codes.ok return_value9999._text = ''' {"project": {"project_id": 9999, "name": "project1"} } ''' return_value8888 = self.FakeResponse() return_value8888.status_code = requests.codes.ok return_value8888._text = ''' {"project": {"project_id": 8888, "name": "project2"} } ''' return_value7777 = self.FakeResponse() return_value7777.status_code = requests.codes.ok return_value7777._text = ''' {"project": {"project_id": 7777, "name": "project3"} } ''' def get(url, *args, **kwargs): if '9999' in url: return return_value9999 if '8888' in url: return return_value8888 if '7777' in url: return return_value7777 return return_value client_mock.session.get.side_effect = get manager = mock.Mock manager._keystone = client_mock def discover(*args, **kwargs): return ["https://endpoint.server.name/"] manager.discover = discover samples = pollster.get_samples( manager=manager, cache=None, resources=["https://endpoint.server.name/"]) samples = list(samples) n_calls = client_mock.session.get.call_count self.assertEqual(9, len(samples)) self.assertEqual(4, n_calls) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_extra_metadata_fields( self, client_mock): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) extra_metadata_fields = [{ 'name': "project_name", 'endpoint_type': "identity", 'url_path': "'/v3/projects/' + str(sample['project_id'])", 'value': "name", 'metadata_fields': ['meta'] }, { 'name': "project_alias", 'endpoint_type': "identity", 'url_path': "'/v3/projects/' + " "str(extra_metadata_captured['project_name'])", 'value': "name", 'metadata_fields': ['meta'] }, { 'name': "project_meta", 'endpoint_type': "identity", 'url_path': "'/v3/projects/' + " "str(extra_metadata_by_name['project_name']" "['metadata']['meta'])", 'value': "project_id", 'metadata_fields': ['meta'] }] definitions['value_attribute'] = 'project_id' definitions['extra_metadata_fields'] = extra_metadata_fields pollster = dynamic_pollster.DynamicPollster(definitions) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = ''' {"projects": [ {"project_id": 9999, "name": "project1"}, {"project_id": 8888, "name": "project2"}, {"project_id": 7777, "name": "project3"}] } ''' return_value9999 = self.FakeResponse() return_value9999.status_code = requests.codes.ok return_value9999._text = ''' {"project": {"project_id": 9999, "name": "project1", "meta": "m1"} } ''' return_value8888 = self.FakeResponse() return_value8888.status_code = requests.codes.ok return_value8888._text = ''' {"project": {"project_id": 8888, "name": "project2", "meta": "m2"} } ''' return_value7777 = self.FakeResponse() return_value7777.status_code = requests.codes.ok return_value7777._text = ''' {"project": {"project_id": 7777, "name": "project3", "meta": "m3"} } ''' return_valueP1 = self.FakeResponse() return_valueP1.status_code = requests.codes.ok return_valueP1._text = ''' {"project": {"project_id": 7777, "name": "p1", "meta": null} } ''' return_valueP2 = self.FakeResponse() return_valueP2.status_code = requests.codes.ok return_valueP2._text = ''' {"project": {"project_id": 7777, "name": "p2", "meta": null} } ''' return_valueP3 = self.FakeResponse() return_valueP3.status_code = requests.codes.ok return_valueP3._text = ''' {"project": {"project_id": 7777, "name": "p3", "meta": null} } ''' return_valueM1 = self.FakeResponse() return_valueM1.status_code = requests.codes.ok return_valueM1._text = ''' {"project": {"project_id": "META1", "name": "p3", "meta": null} } ''' return_valueM2 = self.FakeResponse() return_valueM2.status_code = requests.codes.ok return_valueM2._text = ''' {"project": {"project_id": "META2", "name": "p3", "meta": null} } ''' return_valueM3 = self.FakeResponse() return_valueM3.status_code = requests.codes.ok return_valueM3._text = ''' {"project": {"project_id": "META3", "name": "p3", "meta": null} } ''' def get(url, *args, **kwargs): if '9999' in url: return return_value9999 if '8888' in url: return return_value8888 if '7777' in url: return return_value7777 if 'project1' in url: return return_valueP1 if 'project2' in url: return return_valueP2 if 'project3' in url: return return_valueP3 if 'm1' in url: return return_valueM1 if 'm2' in url: return return_valueM2 if 'm3' in url: return return_valueM3 return return_value client_mock.session.get = get manager = mock.Mock manager._keystone = client_mock def discover(*args, **kwargs): return ["https://endpoint.server.name/"] manager.discover = discover samples = pollster.get_samples( manager=manager, cache=None, resources=["https://endpoint.server.name/"]) samples = list(samples) self.assertEqual(3, len(samples)) self.assertEqual(samples[0].volume, 9999) self.assertEqual(samples[1].volume, 8888) self.assertEqual(samples[2].volume, 7777) self.assertEqual(samples[0].resource_metadata, {'project_name': 'project1', 'project_alias': 'p1', 'meta': 'm1', 'project_meta': 'META1'}) self.assertEqual(samples[1].resource_metadata, {'project_name': 'project2', 'project_alias': 'p2', 'meta': 'm2', 'project_meta': 'META2'}) self.assertEqual(samples[2].resource_metadata, {'project_name': 'project3', 'project_alias': 'p3', 'meta': 'm3', 'project_meta': 'META3'}) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_extra_metadata_fields_different_requests( self, client_mock): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) command = ''' \'\'\'echo '{"project": {"project_id": \'\'\'+ str(sample['project_id']) +\'\'\' , "name": "project1"}}' \'\'\' '''.replace('\n', '') command2 = ''' \'\'\'echo '{"project": {"project_id": \'\'\'+ str(sample['project_id']) +\'\'\' , "name": "project2"}}' \'\'\' '''.replace('\n', '') extra_metadata_fields_embedded = { 'name': "project_name2", 'host_command': command2, 'value': "name", } extra_metadata_fields = { 'name': "project_id2", 'host_command': command, 'value': "project_id", 'extra_metadata_fields': extra_metadata_fields_embedded } definitions['value_attribute'] = 'project_id' definitions['extra_metadata_fields'] = extra_metadata_fields pollster = dynamic_pollster.DynamicPollster(definitions) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = ''' {"projects": [ {"project_id": 9999, "name": "project1"}, {"project_id": 8888, "name": "project2"}, {"project_id": 7777, "name": "project3"}] } ''' def get(url, *args, **kwargs): return return_value client_mock.session.get = get manager = mock.Mock manager._keystone = client_mock def discover(*args, **kwargs): return ["https://endpoint.server.name/"] manager.discover = discover samples = pollster.get_samples( manager=manager, cache=None, resources=["https://endpoint.server.name/"]) samples = list(samples) self.assertEqual(3, len(samples)) self.assertEqual(samples[0].volume, 9999) self.assertEqual(samples[1].volume, 8888) self.assertEqual(samples[2].volume, 7777) self.assertEqual(samples[0].resource_metadata, {'project_id2': 9999, 'project_name2': 'project2'}) self.assertEqual(samples[1].resource_metadata, {'project_id2': 8888, 'project_name2': 'project2'}) self.assertEqual(samples[2].resource_metadata, {'project_id2': 7777, 'project_name2': 'project2'}) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_xml_json_response_handler_invalid_response( self, client_mock): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['response_handlers'] = ['xml', 'json'] pollster = dynamic_pollster.DynamicPollster(definitions) return_value = self.FakeResponse() return_value.status_code = requests.codes.ok return_value._text = 'Invalid response' client_mock.session.get.return_value = return_value with self.assertLogs('ceilometer.polling.dynamic_pollster', level='DEBUG') as logs: gatherer = pollster.definitions.sample_gatherer exception = self.assertRaises( declarative.InvalidResponseTypeException, gatherer.execute_request_get_samples, keystone_client=client_mock, resource="https://endpoint.server.name/") xml_handling_error = logs.output[3] json_handling_error = logs.output[4] self.assertIn( 'DEBUG:ceilometer.polling.dynamic_pollster:' 'Error handling response [Invalid response] ' 'with handler [XMLResponseHandler]', xml_handling_error) self.assertIn( 'DEBUG:ceilometer.polling.dynamic_pollster:' 'Error handling response [Invalid response] ' 'with handler [JsonResponseHandler]', json_handling_error) self.assertEqual( "InvalidResponseTypeException None: " "No remaining handlers to handle the response " "[Invalid response], used handlers " "[XMLResponseHandler, JsonResponseHandler]. " "[{'url_path': 'v1/test/endpoint/fake'}].", str(exception)) def test_configure_response_handler_definition_invalid_value(self): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['response_handlers'] = ['jason'] exception = self.assertRaises( declarative.DynamicPollsterDefinitionException, dynamic_pollster.DynamicPollster, pollster_definitions=definitions) self.assertEqual("DynamicPollsterDefinitionException None: " "Invalid response_handler value [jason]. " "Accepted values are [json, xml, text]", str(exception)) def test_configure_response_handler_definition_invalid_type(self): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['response_handlers'] = 'json' exception = self.assertRaises( declarative.DynamicPollsterDefinitionException, dynamic_pollster.DynamicPollster, pollster_definitions=definitions) self.assertEqual("DynamicPollsterDefinitionException None: " "Invalid response_handlers configuration. " "It must be a list. Provided value type: str", str(exception)) @mock.patch('keystoneclient.v2_0.client.Client') def test_execute_request_get_samples_exception_on_request( self, client_mock): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) return_value = self.FakeResponse() return_value.status_code = requests.codes.bad client_mock.session.get.return_value = return_value exception = self.assertRaises(requests.HTTPError, pollster.definitions.sample_gatherer. execute_request_get_samples, keystone_client=client_mock, resource="https://endpoint.server.name/") self.assertEqual("Mock HTTP error.", str(exception)) def test_execute_host_command_paged_responses(self): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['host_command'] = ''' echo '{"server": [{"status": "ACTIVE"}], "next": ""}' ''' str_json = "'{\\\"server\\\": [{\\\"status\\\": \\\"INACTIVE\\\"}]}'" definitions['next_sample_url_attribute'] = \ "next|\"echo \"+value+\"" + str_json + '"' pollster = dynamic_pollster.DynamicPollster(definitions) samples = pollster.definitions.sample_gatherer. \ execute_request_get_samples() resp_json = [{'status': 'ACTIVE'}, {'status': 'INACTIVE'}] self.assertEqual(resp_json, samples) def test_execute_host_command_response_handler(self): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['response_handlers'] = ['xml', 'json'] definitions['host_command'] = 'echo "xml\nxml"' entry = 'a' definitions['response_entries_key'] = entry definitions.pop('url_path') definitions.pop('endpoint_type') pollster = dynamic_pollster.DynamicPollster(definitions) samples_xml = pollster.definitions.sample_gatherer. \ execute_request_get_samples() definitions['host_command'] = 'echo \'{"a": {"y":"json",' \ '\n"s":"json"}}\'' samples_json = pollster.definitions.sample_gatherer. \ execute_request_get_samples() resp_xml = {'a': {'y': 'xml', 's': 'xml'}} resp_json = {'a': {'y': 'json', 's': 'json'}} self.assertEqual(resp_xml[entry], samples_xml) self.assertEqual(resp_json[entry], samples_json) def test_execute_host_command_invalid_command(self): definitions = copy.deepcopy( self.pollster_definition_only_required_fields) definitions['host_command'] = 'invalid-command' definitions.pop('url_path') definitions.pop('endpoint_type') pollster = dynamic_pollster.DynamicPollster(definitions) self.assertRaises( declarative.InvalidResponseTypeException, pollster.definitions.sample_gatherer.execute_request_get_samples) def test_generate_new_metadata_fields_no_metadata_mapping(self): metadata = {'name': 'someName', 'value': 1} metadata_before_call = copy.deepcopy(metadata) self.pollster_definition_only_required_fields['metadata_mapping'] = {} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) pollster.definitions.sample_extractor.generate_new_metadata_fields( metadata, self.pollster_definition_only_required_fields) self.assertEqual(metadata_before_call, metadata) def test_generate_new_metadata_fields_preserve_old_key(self): metadata = {'name': 'someName', 'value': 2} expected_metadata = copy.deepcopy(metadata) expected_metadata['balance'] = metadata['value'] self.pollster_definition_only_required_fields[ 'metadata_mapping'] = {'value': 'balance'} self.pollster_definition_only_required_fields[ 'preserve_mapped_metadata'] = True pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) pollster.definitions.sample_extractor.generate_new_metadata_fields( metadata, self.pollster_definition_only_required_fields) self.assertEqual(expected_metadata, metadata) def test_generate_new_metadata_fields_preserve_old_key_equals_false(self): metadata = {'name': 'someName', 'value': 1} expected_clean_metadata = copy.deepcopy(metadata) expected_clean_metadata['balance'] = metadata['value'] expected_clean_metadata.pop('value') self.pollster_definition_only_required_fields[ 'metadata_mapping'] = {'value': 'balance'} self.pollster_definition_only_required_fields[ 'preserve_mapped_metadata'] = False pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) pollster.definitions.sample_extractor.generate_new_metadata_fields( metadata, self.pollster_definition_only_required_fields) self.assertEqual(expected_clean_metadata, metadata) def test_execute_value_mapping_no_value_mapping(self): self.pollster_definition_only_required_fields['value_mapping'] = {} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) value_to_be_mapped = "test" expected_value = value_to_be_mapped value = pollster.definitions.value_mapper. \ execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) def test_execute_value_mapping_no_value_mapping_found_with_default(self): self.pollster_definition_only_required_fields[ 'value_mapping'] = {'some-possible-value': 15} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) value_to_be_mapped = "test" expected_value = -1 value = pollster.definitions.value_mapper. \ execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) def test_execute_value_mapping_no_value_mapping_found_with_custom_default( self): self.pollster_definition_only_required_fields[ 'value_mapping'] = {'some-possible-value': 5} self.pollster_definition_only_required_fields[ 'default_value'] = 0 pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) value_to_be_mapped = "test" expected_value = 0 value = pollster.definitions.value_mapper. \ execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) def test_execute_value_mapping(self): self.pollster_definition_only_required_fields[ 'value_mapping'] = {'test': 'new-value'} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) value_to_be_mapped = "test" expected_value = 'new-value' value = pollster.definitions.value_mapper. \ execute_value_mapping(value_to_be_mapped) self.assertEqual(expected_value, value) def test_get_samples_no_resources(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) samples = pollster.get_samples(None, None, None) self.assertEqual(None, next(samples)) @mock.patch('ceilometer.polling.dynamic_pollster.' 'PollsterSampleGatherer.execute_request_get_samples') def test_get_samples_empty_samples(self, execute_request_get_samples_mock): execute_request_get_samples_mock.side_effect = [] pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) fake_manager = self.FakeManager() samples = pollster.get_samples( fake_manager, None, ["https://endpoint.server.name.com/"]) samples_list = list() try: for s in samples: samples_list.append(s) except RuntimeError as e: LOG.debug("Generator threw a StopIteration " "and we need to catch it [%s]." % e) self.assertEqual(0, len(samples_list)) def fake_sample_list(self, **kwargs): samples_list = list() samples_list.append( {'name': "sample5", 'volume': 5, 'description': "desc-sample-5", 'user_id': "924d1f77-5d75-4b96-a755-1774d6be17af", 'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", 'id': "e335c317-dfdd-4f22-809a-625bd9a5992d" } ) samples_list.append( {'name': "sample1", 'volume': 2, 'description': "desc-sample-2", 'user_id': "20b5a704-b481-4603-a99e-2636c144b876", 'project_id': "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", 'id': "2e350554-6c05-4fda-8109-e47b595a714c" } ) return samples_list @mock.patch.object( dynamic_pollster.PollsterSampleGatherer, 'execute_request_get_samples', fake_sample_list) def test_get_samples(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) fake_manager = self.FakeManager() samples = pollster.get_samples( fake_manager, None, ["https://endpoint.server.name.com/"]) samples_list = list(samples) self.assertEqual(2, len(samples_list)) first_element = [ s for s in samples_list if s.resource_id == "e335c317-dfdd-4f22-809a-625bd9a5992d"][0] self.assertEqual(5, first_element.volume) self.assertEqual( "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", first_element.project_id) self.assertEqual( "924d1f77-5d75-4b96-a755-1774d6be17af", first_element.user_id) second_element = [ s for s in samples_list if s.resource_id == "2e350554-6c05-4fda-8109-e47b595a714c"][0] self.assertEqual(2, second_element.volume) self.assertEqual( "6c7a0e87-7f2e-45d3-89ca-5a2dbba71a0e", second_element.project_id) self.assertEqual( "20b5a704-b481-4603-a99e-2636c144b876", second_element.user_id) def test_retrieve_entries_from_response_response_is_a_list(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) response = [{"object1-attr1": 1}, {"object1-attr2": 2}] entries = pollster.definitions.sample_gatherer. \ retrieve_entries_from_response(response, pollster.definitions) self.assertEqual(response, entries) def test_retrieve_entries_using_first_entry_from_response(self): self.pollster_definition_only_required_fields[ 'response_entries_key'] = "first" pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) first_entries_from_response = [{"object1-attr1": 1}, {"object1-attr2": 2}] second_entries_from_response = [{"object1-attr3": 3}, {"object1-attr4": 33}] response = {"first": first_entries_from_response, "second": second_entries_from_response} entries = pollster.definitions.sample_gatherer.\ retrieve_entries_from_response( response, pollster.definitions.configurations) self.assertEqual(first_entries_from_response, entries) def test_retrieve_entries_using_second_entry_from_response(self): self.pollster_definition_only_required_fields[ 'response_entries_key'] = "second" pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) first_entries_from_response = [{"object1-attr1": 1}, {"object1-attr2": 2}] second_entries_from_response = [{"object1-attr3": 3}, {"object1-attr4": 33}] response = {"first": first_entries_from_response, "second": second_entries_from_response} entries = pollster.definitions.sample_gatherer. \ retrieve_entries_from_response(response, pollster.definitions.configurations) self.assertEqual(second_entries_from_response, entries) def test_retrieve_attribute_nested_value_non_nested_key(self): key = "key" value = [{"d": 2}, {"g": {"h": "val"}}] json_object = {"key": value} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) returned_value = pollster.definitions.sample_extractor.\ retrieve_attribute_nested_value(json_object, key) self.assertEqual(value, returned_value) def test_retrieve_attribute_nested_value_nested_key(self): key = "key.subKey" value1 = [{"d": 2}, {"g": {"h": "val"}}] sub_value = [{"r": 245}, {"h": {"yu": "yu"}}] json_object = {"key": {"subKey": sub_value, "subkey2": value1}} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) returned_value = pollster.definitions.sample_extractor. \ retrieve_attribute_nested_value(json_object, key) self.assertEqual(sub_value, returned_value) def test_retrieve_attribute_nested_value_with_operation_on_attribute(self): # spaces here are added on purpose at the end to make sure we # execute the strip in the code before the eval key = "key.subKey | value + 1|value / 2 | value * 3" value1 = [{"d": 2}, {"g": {"h": "val"}}] sub_value = 1 expected_value_after_operations = 3 json_object = {"key": {"subKey": sub_value, "subkey2": value1}} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) returned_value = pollster.definitions.sample_extractor.\ retrieve_attribute_nested_value(json_object, key) self.assertEqual(expected_value_after_operations, returned_value) def test_retrieve_attribute_nested_value_simulate_radosgw_processing(self): key = "user | value.split('$') | value[0] | value.strip()" json_object = {"categories": [ { "bytes_received": 0, "bytes_sent": 357088, "category": "complete_multipart", "ops": 472, "successful_ops": 472 }], "total": { "bytes_received": 206739531986, "bytes_sent": 273793180, "ops": 119690, "successful_ops": 119682 }, "user": " 00ab8d7e76fc4$00ab8d7e76fc45a37776732" } expected_value_after_operations = "00ab8d7e76fc4" pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) returned_value = pollster.definitions.sample_extractor.\ retrieve_attribute_nested_value(json_object, key) self.assertEqual(expected_value_after_operations, returned_value) def fake_sample_multi_metric(self, **kwargs): multi_metric_sample_list = [ {"categories": [ { "bytes_received": 0, "bytes_sent": 0, "category": "create_bucket", "ops": 2, "successful_ops": 2 }, { "bytes_received": 0, "bytes_sent": 2120428, "category": "get_obj", "ops": 46, "successful_ops": 46 }, { "bytes_received": 0, "bytes_sent": 21484, "category": "list_bucket", "ops": 8, "successful_ops": 8 }, { "bytes_received": 6889056, "bytes_sent": 0, "category": "put_obj", "ops": 46, "successful_ops": 6 }], "total": { "bytes_received": 6889056, "bytes_sent": 2141912, "ops": 102, "successful_ops": 106 }, "user": "test-user"}] return multi_metric_sample_list @mock.patch.object( dynamic_pollster.PollsterSampleGatherer, 'execute_request_get_samples', fake_sample_multi_metric) def test_get_samples_multi_metric_pollster(self): pollster = dynamic_pollster.DynamicPollster( self.multi_metric_pollster_definition) fake_manager = self.FakeManager() samples = pollster.get_samples( fake_manager, None, ["https://endpoint.server.name.com/"]) samples_list = list(samples) self.assertEqual(4, len(samples_list)) create_bucket_sample = [ s for s in samples_list if s.name == "test-pollster.create_bucket"][0] get_obj_sample = [ s for s in samples_list if s.name == "test-pollster.get_obj"][0] list_bucket_sample = [ s for s in samples_list if s.name == "test-pollster.list_bucket"][0] put_obj_sample = [ s for s in samples_list if s.name == "test-pollster.put_obj"][0] self.assertEqual(2, create_bucket_sample.volume) self.assertEqual(46, get_obj_sample.volume) self.assertEqual(8, list_bucket_sample.volume) self.assertEqual(46, put_obj_sample.volume) def test_execute_request_get_samples_custom_ids(self): sample = {'user_id_attribute': "1", 'project_id_attribute': "2", 'resource_id_attribute': "3", 'user_id': "234", 'project_id': "2334", 'id': "35"} def internal_execute_request_get_samples_mock(self, **kwargs): class Response: @property def text(self): return json.dumps([sample]) def json(self): return [sample] return Response(), "url" original_method = dynamic_pollster.PollsterSampleGatherer.\ _internal_execute_request_get_samples try: dynamic_pollster.PollsterSampleGatherer. \ _internal_execute_request_get_samples = \ internal_execute_request_get_samples_mock self.pollster_definition_all_fields[ 'user_id_attribute'] = 'user_id_attribute' self.pollster_definition_all_fields[ 'project_id_attribute'] = 'project_id_attribute' self.pollster_definition_all_fields[ 'resource_id_attribute'] = 'resource_id_attribute' pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_all_fields) params = {"d": "d"} response = pollster.definitions.sample_gatherer. \ execute_request_get_samples(**params) self.assertEqual(sample['user_id_attribute'], response[0]['user_id']) self.assertEqual(sample['project_id_attribute'], response[0]['project_id']) self.assertEqual(sample['resource_id_attribute'], response[0]['id']) finally: dynamic_pollster.PollsterSampleGatherer. \ _internal_execute_request_get_samples = original_method def test_retrieve_attribute_self_reference_sample(self): key = " . | value['key1']['subKey1'][0]['d'] if 'key1' in value else 0" sub_value1 = [{"d": 2}, {"g": {"h": "val"}}] sub_value2 = [{"r": 245}, {"h": {"yu": "yu"}}] json_object = {"key1": {"subKey1": sub_value1}, "key2": {"subkey2": sub_value2}} pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) returned_value = pollster.definitions.sample_extractor.\ retrieve_attribute_nested_value(json_object, key) self.assertEqual(2, returned_value) del json_object['key1'] returned_value = pollster.definitions.sample_extractor.\ retrieve_attribute_nested_value(json_object, key) self.assertEqual(0, returned_value) def test_create_request_arguments_NonOpenStackApisSamplesGatherer(self): pollster_definition = { 'name': "test-pollster", 'sample_type': "gauge", 'unit': "test", 'value_attribute': "volume", 'url_path': "https://test.com/v1/test/endpoint/fake", "module": "someModule", "authentication_object": "objectAuthentication", "authentication_parameters": "authParam", "headers": [{"header1": "val1"}, {"header2": "val2"}]} pollster = dynamic_pollster.DynamicPollster(pollster_definition) request_args = pollster.definitions.sample_gatherer\ .create_request_arguments(pollster.definitions.configurations) self.assertTrue("headers" in request_args) self.assertEqual(2, len(request_args["headers"])) self.assertEqual(['header1', 'header2'], list(map(lambda h: list(h.keys())[0], request_args["headers"]))) self.assertEqual(['val1', 'val2'], list(map(lambda h: list(h.values())[0], request_args["headers"]))) self.assertTrue("authenticated" not in request_args) def test_create_request_arguments_PollsterSampleGatherer(self): pollster_definition = copy.deepcopy( self.pollster_definition_only_required_fields) pollster_definition["headers"] = [ {"x-openstack-nova-api-version": "2.46"}, {"custom_header": "custom"}, {"some_other_header": "something"}] pollster = dynamic_pollster.DynamicPollster(pollster_definition) request_args = pollster.definitions.sample_gatherer\ .create_request_arguments(pollster.definitions.configurations) self.assertTrue("headers" in request_args) self.assertTrue("authenticated" in request_args) self.assertTrue(request_args["authenticated"]) self.assertEqual(3, len(request_args["headers"])) self.assertEqual(['x-openstack-nova-api-version', 'custom_header', "some_other_header"], list(map(lambda h: list(h.keys())[0], request_args["headers"]))) self.assertEqual(['2.46', 'custom', 'something'], list(map(lambda h: list(h.values())[0], request_args["headers"]))) def test_create_request_arguments_PollsterSampleGatherer_no_headers(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) request_args =\ pollster.definitions.sample_gatherer.create_request_arguments( pollster.definitions.configurations) self.assertTrue("headers" not in request_args) self.assertTrue("authenticated" in request_args) self.assertTrue(request_args["authenticated"]) @mock.patch('keystoneclient.v2_0.client.Client') def test_metadata_nested_objects(self, keystone_mock): generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={ 'flavor': [{"name": "a", "ram": 1}, {"name": "b", "ram": 2}, {"name": "c", "ram": 3}, {"name": "d", "ram": 4}, {"name": "e", "ram": 5}, {"name": "f", "ram": 6}, {"name": "g", "ram": 7}, {"name": "h", "ram": 8}], 'name': ['s1', 's2', 's3', 's4', 's5', 's6', 's7', 's8'], 'state': ['Active', 'Error', 'Down', 'Active', 'Active', 'Migrating', 'Active', 'Error'] }, dict_name='servers', page_link_name='server_link') generator.generate_samples('http://test.com/v1/test-servers', { 'marker=c3': 3, 'marker=f6': 3 }, 2) keystone_mock.session.get.side_effect = generator.mock_request fake_manager = self.FakeManager(keystone=keystone_mock) pollster_definition = dict(self.multi_metric_pollster_definition) pollster_definition['name'] = 'test-pollster' pollster_definition['value_attribute'] = 'state' pollster_definition['url_path'] = 'v1/test-servers' pollster_definition['response_entries_key'] = 'servers' pollster_definition['metadata_fields'] = ['flavor.name', 'flavor.ram'] pollster_definition['next_sample_url_attribute'] = \ 'server_link | filter(lambda v: v.get("rel") == "next", value) |' \ 'list(value)| value [0] | value.get("href")' pollster = dynamic_pollster.DynamicPollster(pollster_definition) samples = pollster.get_samples(fake_manager, None, ['http://test.com']) samples = list(samples) self.assertEqual(['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'], list(map(lambda s: s.resource_metadata["flavor.name"], samples))) self.assertEqual(list(range(1, 9)), list(map(lambda s: s.resource_metadata["flavor.ram"], samples))) def test_get_request_linked_samples_url_no_next_sample(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) base_url = "http://test.com/something_that_we_do_not_care" expected_url = urlparse.urljoin( base_url, self.pollster_definition_only_required_fields[ 'url_path']) kwargs = {'resource': base_url} url = pollster.definitions.sample_gatherer\ .get_request_linked_samples_url( kwargs, pollster.definitions.configurations) self.assertEqual(expected_url, url) def test_get_request_linked_samples_url_next_sample_url(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) base_url = "http://test.com/something_that_we_do_not_care" expected_url = "http://test.com/next_page" kwargs = {'resource': base_url, 'next_sample_url': expected_url} url = pollster.definitions.sample_gatherer\ .get_request_linked_samples_url(kwargs, pollster.definitions) self.assertEqual(expected_url, url) def test_get_request_linked_samples_url_next_sample_only_url_path(self): pollster = dynamic_pollster.DynamicPollster( self.pollster_definition_only_required_fields) base_url = "http://test.com/something_that_we_do_not_care" expected_url = "http://test.com/next_page" kwargs = {'resource': base_url, 'next_sample_url': "/next_page"} url = pollster.definitions.sample_gatherer\ .get_request_linked_samples_url( kwargs, pollster.definitions.configurations) self.assertEqual(expected_url, url) def test_generate_sample_and_extract_metadata(self): definition = self.pollster_definition_only_required_fields.copy() definition['metadata_fields'] = ["metadata1", 'metadata2'] pollster = dynamic_pollster.DynamicPollster(definition) pollster_sample = {'metadata1': 'metadata1', 'metadata2': 'metadata2', 'value': 1} sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(2, len(sample.resource_metadata)) self.assertEqual('metadata1', sample.resource_metadata['metadata1']) self.assertEqual('metadata2', sample.resource_metadata['metadata2']) def test_generate_sample_and_extract_metadata_false_value(self): definition = self.pollster_definition_only_required_fields.copy() definition['metadata_fields'] = ["metadata1", 'metadata2', 'metadata3_false'] pollster = dynamic_pollster.DynamicPollster(definition) pollster_sample = {'metadata1': 'metadata1', 'metadata2': 'metadata2', 'metadata3_false': False, 'value': 1} sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) self.assertEqual('metadata1', sample.resource_metadata['metadata1']) self.assertEqual('metadata2', sample.resource_metadata['metadata2']) self.assertIs(False, sample.resource_metadata['metadata3_false']) def test_generate_sample_and_extract_metadata_none_value(self): definition = self.pollster_definition_only_required_fields.copy() definition['metadata_fields'] = ["metadata1", 'metadata2', 'metadata3'] pollster = dynamic_pollster.DynamicPollster(definition) pollster_sample = {'metadata1': 'metadata1', 'metadata2': 'metadata2', 'metadata3': None, 'value': 1} sample = pollster.definitions.sample_extractor.generate_sample( pollster_sample, pollster.definitions.configurations, manager=mock.Mock(), conf={}) self.assertEqual(1, sample.volume) self.assertEqual(3, len(sample.resource_metadata)) self.assertEqual('metadata1', sample.resource_metadata['metadata1']) self.assertEqual('metadata2', sample.resource_metadata['metadata2']) self.assertIsNone(sample.resource_metadata['metadata3'])