Merge "Add extra metadata fields skip"
This commit is contained in:
Zuul
Gerrit Code Review
commit
5106f4acc4
|
|
@ -93,6 +93,15 @@ def validate_response_handler(val):
|
|||
"are [%s]" % (value, ', '.join(list(VALID_HANDLERS))))
|
||||
|
||||
|
||||
def validate_extra_metadata_skip_samples(val):
|
||||
if not isinstance(val, list) or next(
|
||||
filter(lambda v: not isinstance(v, dict), val), None):
|
||||
raise declarative.DynamicPollsterDefinitionException(
|
||||
"Invalid extra_metadata_fields_skip configuration."
|
||||
" It must be a list of maps. Provided value: %s,"
|
||||
" value type: %s." % (val, type(val).__name__))
|
||||
|
||||
|
||||
class ResponseHandlerChain(object):
|
||||
"""Tries to convert a string to a dict using the response handlers"""
|
||||
|
||||
|
|
@ -205,6 +214,7 @@ class PollsterSampleExtractor(object):
|
|||
self.generate_new_metadata_fields(
|
||||
metadata=metadata, pollster_definitions=pollster_definitions)
|
||||
|
||||
pollster_sample['metadata'] = metadata
|
||||
extra_metadata = self.definitions.retrieve_extra_metadata(
|
||||
kwargs['manager'], pollster_sample, kwargs['conf'])
|
||||
|
||||
|
|
@ -518,6 +528,8 @@ class PollsterDefinitions(object):
|
|||
PollsterDefinition(name='extra_metadata_fields_cache_seconds',
|
||||
default=3600),
|
||||
PollsterDefinition(name='extra_metadata_fields'),
|
||||
PollsterDefinition(name='extra_metadata_fields_skip', default=[{}],
|
||||
validator=validate_extra_metadata_skip_samples),
|
||||
PollsterDefinition(name='response_handlers', default=['json'],
|
||||
validator=validate_response_handler),
|
||||
PollsterDefinition(name='base_metadata', default={})
|
||||
|
|
@ -574,6 +586,39 @@ class PollsterDefinitions(object):
|
|||
"Required fields %s not specified."
|
||||
% missing, self.configurations)
|
||||
|
||||
def should_skip_extra_metadata(self, skip, sample):
|
||||
match_msg = "Sample [%s] %smatches with configured" \
|
||||
" extra_metadata_fields_skip [%s]."
|
||||
if skip == sample:
|
||||
LOG.debug(match_msg, sample, "", skip)
|
||||
return True
|
||||
if not isinstance(skip, dict) or not isinstance(sample, dict):
|
||||
LOG.debug(match_msg, sample, "not ", skip)
|
||||
return False
|
||||
|
||||
for key in skip:
|
||||
if key not in sample:
|
||||
LOG.debug(match_msg, sample, "not ", skip)
|
||||
return False
|
||||
if not self.should_skip_extra_metadata(skip[key], sample[key]):
|
||||
LOG.debug(match_msg, sample, "not ", skip)
|
||||
return False
|
||||
|
||||
LOG.debug(match_msg, sample, "", skip)
|
||||
return True
|
||||
|
||||
def skip_sample(self, request_sample, skips):
|
||||
for skip in skips:
|
||||
if not skip:
|
||||
continue
|
||||
if self.should_skip_extra_metadata(skip, request_sample):
|
||||
LOG.debug("Skipping extra_metadata_field gathering for "
|
||||
"sample [%s] as defined in the "
|
||||
"extra_metadata_fields_skip [%s]", request_sample,
|
||||
skip)
|
||||
return True
|
||||
return False
|
||||
|
||||
def retrieve_extra_metadata(self, manager, request_sample, pollster_conf):
|
||||
extra_metadata_fields = self.configurations['extra_metadata_fields']
|
||||
if extra_metadata_fields:
|
||||
|
|
@ -582,6 +627,9 @@ class PollsterDefinitions(object):
|
|||
if not isinstance(extra_metadata_fields, (list, tuple)):
|
||||
extra_metadata_fields = [extra_metadata_fields]
|
||||
for ext_metadata in extra_metadata_fields:
|
||||
ext_metadata.setdefault(
|
||||
'extra_metadata_fields_skip',
|
||||
self.configurations['extra_metadata_fields_skip'])
|
||||
ext_metadata.setdefault(
|
||||
'sample_type', self.configurations['sample_type'])
|
||||
ext_metadata.setdefault('unit', self.configurations['unit'])
|
||||
|
|
@ -603,6 +651,11 @@ class PollsterDefinitions(object):
|
|||
ext_metadata, conf=pollster_conf, cache_ttl=cache_ttl,
|
||||
extra_metadata_responses_cache=response_cache,
|
||||
)
|
||||
|
||||
skips = ext_metadata['extra_metadata_fields_skip']
|
||||
if self.skip_sample(request_sample, skips):
|
||||
continue
|
||||
|
||||
resources = [None]
|
||||
if ext_metadata.get('endpoint_type'):
|
||||
resources = manager.discover([
|
||||
|
|
|
|||
|
|
@ -733,6 +733,171 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
'meta': 'm3',
|
||||
'project_meta': 'META3'})
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_execute_request_extra_metadata_fields_skip(
|
||||
self, client_mock):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
extra_metadata_fields = [{
|
||||
'name': "project_name",
|
||||
'endpoint_type': "identity",
|
||||
'url_path': "'/v3/projects/' + str(sample['project_id'])",
|
||||
'value': "name",
|
||||
}, {
|
||||
'name': "project_alias",
|
||||
'endpoint_type': "identity",
|
||||
'extra_metadata_fields_skip': [{
|
||||
'value': 7777
|
||||
}],
|
||||
'url_path': "'/v3/projects/' + "
|
||||
"str(sample['p_name'])",
|
||||
'value': "name",
|
||||
}]
|
||||
definitions['value_attribute'] = 'project_id'
|
||||
definitions['metadata_fields'] = ['to_skip', 'p_name']
|
||||
definitions['extra_metadata_fields'] = extra_metadata_fields
|
||||
definitions['extra_metadata_fields_skip'] = [{
|
||||
'metadata': {
|
||||
'to_skip': 'skip1'
|
||||
}
|
||||
}, {
|
||||
'value': 8888
|
||||
}]
|
||||
pollster = dynamic_pollster.DynamicPollster(definitions)
|
||||
|
||||
return_value = self.FakeResponse()
|
||||
return_value.status_code = requests.codes.ok
|
||||
return_value._text = '''
|
||||
{"projects": [
|
||||
{"project_id": 9999, "p_name": "project1",
|
||||
"to_skip": "skip1"},
|
||||
{"project_id": 8888, "p_name": "project2",
|
||||
"to_skip": "skip2"},
|
||||
{"project_id": 7777, "p_name": "project3",
|
||||
"to_skip": "skip3"},
|
||||
{"project_id": 6666, "p_name": "project4",
|
||||
"to_skip": "skip4"}]
|
||||
}
|
||||
'''
|
||||
|
||||
return_value9999 = self.FakeResponse()
|
||||
return_value9999.status_code = requests.codes.ok
|
||||
return_value9999._text = '''
|
||||
{"project":
|
||||
{"project_id": 9999, "name": "project1"}
|
||||
}
|
||||
'''
|
||||
|
||||
return_value8888 = self.FakeResponse()
|
||||
return_value8888.status_code = requests.codes.ok
|
||||
return_value8888._text = '''
|
||||
{"project":
|
||||
{"project_id": 8888, "name": "project2"}
|
||||
}
|
||||
'''
|
||||
|
||||
return_value7777 = self.FakeResponse()
|
||||
return_value7777.status_code = requests.codes.ok
|
||||
return_value7777._text = '''
|
||||
{"project":
|
||||
{"project_id": 7777, "name": "project3"}
|
||||
}
|
||||
'''
|
||||
|
||||
return_value6666 = self.FakeResponse()
|
||||
return_value6666.status_code = requests.codes.ok
|
||||
return_value6666._text = '''
|
||||
{"project":
|
||||
{"project_id": 6666, "name": "project4"}
|
||||
}
|
||||
'''
|
||||
|
||||
return_valueP1 = self.FakeResponse()
|
||||
return_valueP1.status_code = requests.codes.ok
|
||||
return_valueP1._text = '''
|
||||
{"project":
|
||||
{"project_id": 7777, "name": "p1"}
|
||||
}
|
||||
'''
|
||||
|
||||
return_valueP2 = self.FakeResponse()
|
||||
return_valueP2.status_code = requests.codes.ok
|
||||
return_valueP2._text = '''
|
||||
{"project":
|
||||
{"project_id": 7777, "name": "p2"}
|
||||
}
|
||||
'''
|
||||
|
||||
return_valueP3 = self.FakeResponse()
|
||||
return_valueP3.status_code = requests.codes.ok
|
||||
return_valueP3._text = '''
|
||||
{"project":
|
||||
{"project_id": 7777, "name": "p3"}
|
||||
}
|
||||
'''
|
||||
|
||||
return_valueP4 = self.FakeResponse()
|
||||
return_valueP4.status_code = requests.codes.ok
|
||||
return_valueP4._text = '''
|
||||
{"project":
|
||||
{"project_id": 6666, "name": "p4"}
|
||||
}
|
||||
'''
|
||||
|
||||
def get(url, *args, **kwargs):
|
||||
if '9999' in url:
|
||||
return return_value9999
|
||||
if '8888' in url:
|
||||
return return_value8888
|
||||
if '7777' in url:
|
||||
return return_value7777
|
||||
if '6666' in url:
|
||||
return return_value6666
|
||||
if 'project1' in url:
|
||||
return return_valueP1
|
||||
if 'project2' in url:
|
||||
return return_valueP2
|
||||
if 'project3' in url:
|
||||
return return_valueP3
|
||||
if 'project4' in url:
|
||||
return return_valueP4
|
||||
|
||||
return return_value
|
||||
|
||||
client_mock.session.get = get
|
||||
manager = mock.Mock
|
||||
manager._keystone = client_mock
|
||||
|
||||
def discover(*args, **kwargs):
|
||||
return ["https://endpoint.server.name/"]
|
||||
|
||||
manager.discover = discover
|
||||
samples = pollster.get_samples(
|
||||
manager=manager, cache=None,
|
||||
resources=["https://endpoint.server.name/"])
|
||||
|
||||
samples = list(samples)
|
||||
self.assertEqual(4, len(samples))
|
||||
|
||||
self.assertEqual(samples[0].volume, 9999)
|
||||
self.assertEqual(samples[1].volume, 8888)
|
||||
self.assertEqual(samples[2].volume, 7777)
|
||||
|
||||
self.assertEqual(samples[0].resource_metadata,
|
||||
{'p_name': 'project1', 'project_alias': 'p1',
|
||||
'to_skip': 'skip1'})
|
||||
self.assertEqual(samples[1].resource_metadata,
|
||||
{'p_name': 'project2', 'project_alias': 'p2',
|
||||
'to_skip': 'skip2'})
|
||||
self.assertEqual(samples[2].resource_metadata,
|
||||
{'p_name': 'project3', 'project_name': 'project3',
|
||||
'to_skip': 'skip3'})
|
||||
self.assertEqual(samples[3].resource_metadata,
|
||||
{'p_name': 'project4',
|
||||
'project_alias': 'p4',
|
||||
'project_name': 'project4',
|
||||
'to_skip': 'skip4'})
|
||||
|
||||
@mock.patch('keystoneclient.v2_0.client.Client')
|
||||
def test_execute_request_extra_metadata_fields_different_requests(
|
||||
self, client_mock):
|
||||
|
|
@ -865,6 +1030,39 @@ class TestDynamicPollster(base.BaseTestCase):
|
|||
"Accepted values are [json, xml, text]",
|
||||
str(exception))
|
||||
|
||||
def test_configure_extra_metadata_field_skip_invalid_value(self):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
definitions['extra_metadata_fields_skip'] = 'teste'
|
||||
|
||||
exception = self.assertRaises(
|
||||
declarative.DynamicPollsterDefinitionException,
|
||||
dynamic_pollster.DynamicPollster,
|
||||
pollster_definitions=definitions)
|
||||
self.assertEqual("DynamicPollsterDefinitionException None: "
|
||||
"Invalid extra_metadata_fields_skip configuration."
|
||||
" It must be a list of maps. Provided value: teste,"
|
||||
" value type: str.",
|
||||
str(exception))
|
||||
|
||||
def test_configure_extra_metadata_field_skip_invalid_sub_value(self):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
definitions['extra_metadata_fields_skip'] = [{'test': '1'},
|
||||
{'test': '2'},
|
||||
'teste']
|
||||
|
||||
exception = self.assertRaises(
|
||||
declarative.DynamicPollsterDefinitionException,
|
||||
dynamic_pollster.DynamicPollster,
|
||||
pollster_definitions=definitions)
|
||||
self.assertEqual("DynamicPollsterDefinitionException None: "
|
||||
"Invalid extra_metadata_fields_skip configuration."
|
||||
" It must be a list of maps. Provided value: "
|
||||
"[{'test': '1'}, {'test': '2'}, 'teste'], "
|
||||
"value type: list.",
|
||||
str(exception))
|
||||
|
||||
def test_configure_response_handler_definition_invalid_type(self):
|
||||
definitions = copy.deepcopy(
|
||||
self.pollster_definition_only_required_fields)
|
||||
|
|
|
|||
|
|
@ -983,6 +983,13 @@ project name, domain ID, and domain name.
|
|||
"locked": "dynamic_locked"
|
||||
"tags | ','.join(value)": "dynamic_tags"
|
||||
extra_metadata_fields_cache_seconds: 3600
|
||||
extra_metadata_fields_skip:
|
||||
- value: '1'
|
||||
metadata:
|
||||
dynamic_flavor_vcpus: 4
|
||||
- value: '1'
|
||||
metadata:
|
||||
dynamic_flavor_vcpus: 2
|
||||
extra_metadata_fields:
|
||||
- name: "project_name"
|
||||
endpoint_type: "identity"
|
||||
|
|
@ -1065,4 +1072,10 @@ The metadata enrichment feature has the following options:
|
|||
dynamic pollster configuration is not set in the `extra_metadata_fields`,
|
||||
will be used the parent pollster configuration, except the `name`.
|
||||
|
||||
* ``extra_metadata_fields_skip``: optional parameter. This option is a list
|
||||
of objects or a single one, where each one of its elements is a set of
|
||||
key/value pairs. When defined, if any set of key/value pairs is a subset
|
||||
of the collected sample, then the extra_metadata_fields gathering of this
|
||||
sample will be skipped.
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue