diff --git a/ceilometer/polling/dynamic_pollster.py b/ceilometer/polling/dynamic_pollster.py
index 3a37c9eec0..c42b1848eb 100644
--- a/ceilometer/polling/dynamic_pollster.py
+++ b/ceilometer/polling/dynamic_pollster.py
@@ -20,6 +20,7 @@
import copy
import json
import re
+import subprocess
import time
import xmltodict
@@ -205,7 +206,7 @@ class PollsterSampleExtractor(object):
metadata=metadata, pollster_definitions=pollster_definitions)
extra_metadata = self.definitions.retrieve_extra_metadata(
- kwargs['manager'], pollster_sample)
+ kwargs['manager'], pollster_sample, kwargs['conf'])
for key in extra_metadata.keys():
if key in metadata.keys():
@@ -518,7 +519,8 @@ class PollsterDefinitions(object):
default=3600),
PollsterDefinition(name='extra_metadata_fields'),
PollsterDefinition(name='response_handlers', default=['json'],
- validator=validate_response_handler)
+ validator=validate_response_handler),
+ PollsterDefinition(name='base_metadata', default={})
]
extra_definitions = []
@@ -572,114 +574,75 @@ class PollsterDefinitions(object):
"Required fields %s not specified."
% missing, self.configurations)
- def retrieve_extra_metadata(self, manager, request_sample):
+ def retrieve_extra_metadata(self, manager, request_sample, pollster_conf):
extra_metadata_fields = self.configurations['extra_metadata_fields']
if extra_metadata_fields:
- if isinstance(self, NonOpenStackApisPollsterDefinition):
- raise declarative.NonOpenStackApisDynamicPollsterException(
- "Not supported the use of extra metadata gathering for "
- "non-openstack pollsters [%s] (yet)."
- % self.configurations['name'])
-
- return self._retrieve_extra_metadata(
- extra_metadata_fields, manager, request_sample)
+ extra_metadata_samples = {}
+ extra_metadata_by_name = {}
+ if not isinstance(extra_metadata_fields, (list, tuple)):
+ extra_metadata_fields = [extra_metadata_fields]
+ for ext_metadata in extra_metadata_fields:
+ ext_metadata.setdefault(
+ 'sample_type', self.configurations['sample_type'])
+ ext_metadata.setdefault('unit', self.configurations['unit'])
+ ext_metadata.setdefault(
+ 'value_attribute', ext_metadata.get(
+ 'value', self.configurations['value_attribute']))
+ ext_metadata['base_metadata'] = {
+ 'extra_metadata_captured': extra_metadata_samples,
+ 'extra_metadata_by_name': extra_metadata_by_name,
+ 'sample': request_sample
+ }
+ parent_cache_ttl = self.configurations[
+ 'extra_metadata_fields_cache_seconds']
+ cache_ttl = ext_metadata.get(
+ 'extra_metadata_fields_cache_seconds', parent_cache_ttl
+ )
+ response_cache = self.response_cache
+ extra_metadata_pollster = DynamicPollster(
+ ext_metadata, conf=pollster_conf, cache_ttl=cache_ttl,
+ extra_metadata_responses_cache=response_cache,
+ )
+ resources = [None]
+ if ext_metadata.get('endpoint_type'):
+ resources = manager.discover([
+ extra_metadata_pollster.default_discovery], {})
+ samples = extra_metadata_pollster.get_samples(
+ manager, None, resources)
+ for sample in samples:
+ self.fill_extra_metadata_samples(
+ extra_metadata_by_name,
+ extra_metadata_samples,
+ sample)
+ return extra_metadata_samples
LOG.debug("No extra metadata to be captured for pollsters [%s] and "
"request sample [%s].", self.definitions, request_sample)
return {}
- def _retrieve_extra_metadata(
- self, extra_metadata_fields, manager, request_sample):
- LOG.debug("Processing extra metadata fields [%s] for "
- "sample [%s].", extra_metadata_fields,
- request_sample)
-
- extra_metadata_captured = {}
- for extra_metadata in extra_metadata_fields:
- extra_metadata_name = extra_metadata['name']
-
- if extra_metadata_name in extra_metadata_captured.keys():
- LOG.warning("Duplicated extra metadata name [%s]. Therefore, "
- "we do not process this iteration [%s].",
- extra_metadata_name, extra_metadata)
+ def fill_extra_metadata_samples(self, extra_metadata_by_name,
+ extra_metadata_samples, sample):
+ extra_metadata_samples[sample.name] = sample.volume
+ LOG.debug("Merging the sample metadata [%s] of the "
+ "extra_metadata_field [%s], with the "
+ "extra_metadata_samples [%s].",
+ sample.resource_metadata,
+ sample.name,
+ extra_metadata_samples)
+ for key, value in sample.resource_metadata.items():
+ if value is None and key in extra_metadata_samples:
+ LOG.debug("Metadata [%s] for extra_metadata_field [%s] "
+ "is None, skipping metadata override by None "
+ "value", key, sample.name)
continue
+ extra_metadata_samples[key] = value
+ extra_metadata_by_name[sample.name] = {
+ 'value': sample.volume,
+ 'metadata': sample.resource_metadata
+ }
- LOG.debug("Processing extra metadata [%s] for sample [%s].",
- extra_metadata_name, request_sample)
-
- endpoint_type = 'endpoint:' + extra_metadata['endpoint_type']
- if not endpoint_type.endswith(
- PollsterDefinitions.EXTERNAL_ENDPOINT_TYPE):
- response = self.execute_openstack_extra_metadata_gathering(
- endpoint_type, extra_metadata, manager, request_sample,
- extra_metadata_captured)
- else:
- raise declarative.NonOpenStackApisDynamicPollsterException(
- "Not supported the use of extra metadata gathering for "
- "non-openstack endpoints [%s] (yet)." % extra_metadata)
-
- extra_metadata_extractor_kwargs = {
- 'value_attribute': extra_metadata['value'],
- 'sample': request_sample}
-
- extra_metadata_value = \
- self.sample_extractor.retrieve_attribute_nested_value(
- response, **extra_metadata_extractor_kwargs)
-
- LOG.debug("Generated extra metadata [%s] with value [%s].",
- extra_metadata_name, extra_metadata_value)
- extra_metadata_captured[extra_metadata_name] = extra_metadata_value
-
- return extra_metadata_captured
-
- def execute_openstack_extra_metadata_gathering(self, endpoint_type,
- extra_metadata, manager,
- request_sample,
- extra_metadata_captured):
- url_for_endpoint_type = manager.discover(
- [endpoint_type], self.response_cache)
-
- LOG.debug("URL [%s] found for endpoint type [%s].",
- url_for_endpoint_type, endpoint_type)
-
- if url_for_endpoint_type:
- url_for_endpoint_type = url_for_endpoint_type[0]
-
- self.sample_gatherer.generate_url_path(
- extra_metadata, request_sample, extra_metadata_captured)
-
- cached_response, max_ttl_for_cache = self.response_cache.get(
- extra_metadata['url_path'], (None, None))
-
- extra_metadata_fields_cache_seconds = extra_metadata.get(
- 'extra_metadata_fields_cache_seconds',
- self.configurations['extra_metadata_fields_cache_seconds'])
-
- current_time = time.time()
- if cached_response and max_ttl_for_cache >= current_time:
- LOG.debug("Returning response [%s] for request [%s] as the TTL "
- "[max=%s, current_time=%s] has not expired yet.",
- cached_response, extra_metadata['url_path'],
- max_ttl_for_cache, current_time)
- return cached_response
-
- if cached_response:
- LOG.debug("Cleaning cached response [%s] for request [%s] "
- "as the TTL [max=%s, current_time=%s] has expired.",
- cached_response, extra_metadata['url_path'],
- max_ttl_for_cache, current_time)
-
- response = self.sample_gatherer.execute_request_for_definitions(
- extra_metadata, **{'manager': manager,
- 'keystone_client': manager._keystone,
- 'resource': url_for_endpoint_type,
- 'execute_id_overrides': False})
-
- max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds
-
- cache_tuple = (response, max_ttl_for_cache)
- self.response_cache[extra_metadata['url_path']] = cache_tuple
- return response
+ LOG.debug("extra_metadata_samples after merging: [%s].",
+ extra_metadata_samples)
class MultiMetricPollsterDefinitions(PollsterDefinitions):
@@ -739,6 +702,42 @@ class PollsterSampleGatherer(object):
url_path=definitions.configurations['url_path']
)
+ def get_cache_key(self, definitions, **kwargs):
+ return self.get_request_linked_samples_url(kwargs, definitions)
+
+ def get_cached_response(self, definitions, **kwargs):
+ if self.definitions.cache_ttl == 0:
+ return
+ cache_key = self.get_cache_key(definitions, **kwargs)
+ response_cache = self.definitions.response_cache
+ cached_response, max_ttl_for_cache = response_cache.get(
+ cache_key, (None, None))
+
+ current_time = time.time()
+ if cached_response and max_ttl_for_cache >= current_time:
+ LOG.debug("Returning response [%s] for request [%s] as the TTL "
+ "[max=%s, current_time=%s] has not expired yet.",
+ cached_response, definitions,
+ max_ttl_for_cache, current_time)
+ return cached_response
+
+ if cached_response and max_ttl_for_cache < current_time:
+ LOG.debug("Cleaning cached response [%s] for request [%s] "
+ "as the TTL [max=%s, current_time=%s] has expired.",
+ cached_response, definitions,
+ max_ttl_for_cache, current_time)
+ response_cache.pop(cache_key, None)
+
+ def store_cached_response(self, definitions, resp, **kwargs):
+ if self.definitions.cache_ttl == 0:
+ return
+ cache_key = self.get_cache_key(definitions, **kwargs)
+ extra_metadata_fields_cache_seconds = self.definitions.cache_ttl
+ max_ttl_for_cache = time.time() + extra_metadata_fields_cache_seconds
+
+ cache_tuple = (resp, max_ttl_for_cache)
+ self.definitions.response_cache[cache_key] = cache_tuple
+
@property
def default_discovery(self):
return 'endpoint:' + self.definitions.configurations['endpoint_type']
@@ -748,10 +747,14 @@ class PollsterSampleGatherer(object):
self.definitions.configurations, **kwargs)
def execute_request_for_definitions(self, definitions, **kwargs):
- resp, url = self._internal_execute_request_get_samples(
- definitions=definitions, **kwargs)
+ if response_dict := self.get_cached_response(definitions, **kwargs):
+ url = 'cached'
+ else:
+ resp, url = self._internal_execute_request_get_samples(
+ definitions=definitions, **kwargs)
+ response_dict = self.response_handler_chain.handle(resp.text)
+ self.store_cached_response(definitions, response_dict, **kwargs)
- response_dict = self.response_handler_chain.handle(resp.text)
entry_size = len(response_dict)
LOG.debug("Entries [%s] in the DICT for request [%s] "
"for dynamic pollster [%s].",
@@ -790,21 +793,6 @@ class PollsterSampleGatherer(object):
self.generate_new_attributes_in_sample(
request_sample, resource_id_attribute, 'id')
- def generate_url_path(self, extra_metadata, sample,
- extra_metadata_captured):
- if not extra_metadata.get('url_path_original'):
- extra_metadata[
- 'url_path_original'] = extra_metadata['url_path']
-
- extra_metadata['url_path'] = eval(
- extra_metadata['url_path_original'])
-
- LOG.debug("URL [%s] generated for pattern [%s] for sample [%s] and "
- "extra metadata captured [%s].",
- extra_metadata['url_path'],
- extra_metadata['url_path_original'], sample,
- extra_metadata_captured)
-
def generate_new_attributes_in_sample(
self, sample, attribute_key, new_attribute_key):
@@ -881,6 +869,15 @@ class PollsterSampleGatherer(object):
def get_request_url(self, kwargs, url_path):
endpoint = kwargs['resource']
+ params = copy.deepcopy(
+ self.definitions.configurations.get(
+ 'base_metadata', {}))
+ try:
+ url_path = eval(url_path, params)
+ except Exception:
+ LOG.debug("Cannot eval path [%s] with params [%s],"
+ " using [%s] instead.",
+ url_path, params, url_path)
return urlparse.urljoin(endpoint, url_path)
def retrieve_entries_from_response(self, response_json, definitions):
@@ -919,6 +916,57 @@ class NonOpenStackApisPollsterDefinition(PollsterDefinitions):
return configurations.get('module')
+class HostCommandPollsterDefinition(PollsterDefinitions):
+
+ extra_definitions = [
+ PollsterDefinition(name='endpoint_type', required=False),
+ PollsterDefinition(name='url_path', required=False),
+ PollsterDefinition(name='host_command', required=True)]
+
+ def __init__(self, configurations):
+ super(HostCommandPollsterDefinition, self).__init__(
+ configurations)
+ self.sample_gatherer = HostCommandSamplesGatherer(self)
+
+ @staticmethod
+ def is_field_applicable_to_definition(configurations):
+ return configurations.get('host_command')
+
+
+class HostCommandSamplesGatherer(PollsterSampleGatherer):
+
+ class Response(object):
+ def __init__(self, text):
+ self.text = text
+
+ def get_cache_key(self, definitions, **kwargs):
+ return self.get_command(definitions)
+
+ def _internal_execute_request_get_samples(self, definitions, **kwargs):
+ command = self.get_command(definitions, **kwargs)
+ LOG.debug('Running Host command: [%s]', command)
+ result = subprocess.getoutput(command)
+ LOG.debug('Host command [%s] result: [%s]', command, result)
+ return self.Response(result), command
+
+ def get_command(self, definitions, next_sample_url=None, **kwargs):
+ command = next_sample_url or definitions['host_command']
+ params = copy.deepcopy(
+ self.definitions.configurations.get(
+ 'base_metadata', {}))
+ try:
+ command = eval(command, params)
+ except Exception:
+ LOG.debug("Cannot eval command [%s] with params [%s],"
+ " using [%s] instead.",
+ command, params, command)
+ return command
+
+ @property
+ def default_discovery(self):
+ return 'local_node'
+
+
class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
@property
@@ -996,8 +1044,10 @@ class DynamicPollster(plugin_base.PollsterBase):
# Mandatory name field
name = ""
- def __init__(self, pollster_definitions={}, conf=None,
- supported_definitions=[NonOpenStackApisPollsterDefinition,
+ def __init__(self, pollster_definitions={}, conf=None, cache_ttl=0,
+ extra_metadata_responses_cache=None,
+ supported_definitions=[HostCommandPollsterDefinition,
+ NonOpenStackApisPollsterDefinition,
MultiMetricPollsterDefinitions,
SingleMetricPollsterDefinitions]):
super(DynamicPollster, self).__init__(conf)
@@ -1007,6 +1057,10 @@ class DynamicPollster(plugin_base.PollsterBase):
self.definitions = PollsterDefinitionBuilder(
self.supported_definitions).build_definitions(pollster_definitions)
+ self.definitions.cache_ttl = cache_ttl
+ self.definitions.response_cache = extra_metadata_responses_cache
+ if extra_metadata_responses_cache is None:
+ self.definitions.response_cache = {}
self.pollster_definitions = self.definitions.configurations
if 'metadata_fields' in self.pollster_definitions:
LOG.debug("Metadata fields configured to [%s].",
@@ -1040,9 +1094,12 @@ class DynamicPollster(plugin_base.PollsterBase):
for r in resources:
LOG.debug("Executing get sample for resource [%s].", r)
samples = self.load_samples(r, manager)
+ if not isinstance(samples, (list, tuple)):
+ samples = [samples]
for pollster_sample in samples:
- kwargs = {'manager': manager, 'resource': r}
- sample = self.extract_sample(pollster_sample, **kwargs)
+ sample = self.extract_sample(
+ pollster_sample, manager=manager,
+ resource=r, conf=self.conf)
if isinstance(sample, SkippedSample):
continue
yield from sample
diff --git a/ceilometer/tests/unit/polling/test_dynamic_pollster.py b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
index dfc0c576f2..653f2df2e9 100644
--- a/ceilometer/tests/unit/polling/test_dynamic_pollster.py
+++ b/ceilometer/tests/unit/polling/test_dynamic_pollster.py
@@ -388,6 +388,424 @@ class TestDynamicPollster(base.BaseTestCase):
self.assertEqual(4, len(samples))
+ @mock.patch('keystoneclient.v2_0.client.Client')
+ def test_execute_request_extra_metadata_fields_cache_disabled(
+ self, client_mock):
+ definitions = copy.deepcopy(
+ self.pollster_definition_only_required_fields)
+ extra_metadata_fields = {
+ 'extra_metadata_fields_cache_seconds': 0,
+ 'name': "project_name",
+ 'endpoint_type': "identity",
+ 'url_path': "'/v3/projects/' + str(sample['project_id'])",
+ 'value': "name",
+ }
+ definitions['value_attribute'] = 'project_id'
+ definitions['extra_metadata_fields'] = extra_metadata_fields
+ pollster = dynamic_pollster.DynamicPollster(definitions)
+
+ return_value = self.FakeResponse()
+ return_value.status_code = requests.codes.ok
+ return_value._text = '''
+ {"projects": [
+ {"project_id": 9999, "name": "project1"},
+ {"project_id": 8888, "name": "project2"},
+ {"project_id": 7777, "name": "project3"},
+ {"project_id": 9999, "name": "project1"},
+ {"project_id": 8888, "name": "project2"},
+ {"project_id": 7777, "name": "project3"},
+ {"project_id": 9999, "name": "project1"},
+ {"project_id": 8888, "name": "project2"},
+ {"project_id": 7777, "name": "project3"}]
+ }
+ '''
+
+ return_value9999 = self.FakeResponse()
+ return_value9999.status_code = requests.codes.ok
+ return_value9999._text = '''
+ {"project":
+ {"project_id": 9999, "name": "project1"}
+ }
+ '''
+
+ return_value8888 = self.FakeResponse()
+ return_value8888.status_code = requests.codes.ok
+ return_value8888._text = '''
+ {"project":
+ {"project_id": 8888, "name": "project2"}
+ }
+ '''
+
+ return_value7777 = self.FakeResponse()
+ return_value7777.status_code = requests.codes.ok
+ return_value7777._text = '''
+ {"project":
+ {"project_id": 7777, "name": "project3"}
+ }
+ '''
+
+ def get(url, *args, **kwargs):
+ if '9999' in url:
+ return return_value9999
+ if '8888' in url:
+ return return_value8888
+ if '7777' in url:
+ return return_value7777
+ return return_value
+
+ client_mock.session.get.side_effect = get
+ manager = mock.Mock
+ manager._keystone = client_mock
+
+ def discover(*args, **kwargs):
+ return ["https://endpoint.server.name/"]
+
+ manager.discover = discover
+ samples = pollster.get_samples(
+ manager=manager, cache=None,
+ resources=["https://endpoint.server.name/"])
+
+ samples = list(samples)
+
+ n_calls = client_mock.session.get.call_count
+ self.assertEqual(9, len(samples))
+ self.assertEqual(10, n_calls)
+
+ @mock.patch('keystoneclient.v2_0.client.Client')
+ def test_execute_request_extra_metadata_fields_cache_enabled(
+ self, client_mock):
+ definitions = copy.deepcopy(
+ self.pollster_definition_only_required_fields)
+ extra_metadata_fields = {
+ 'extra_metadata_fields_cache_seconds': 3600,
+ 'name': "project_name",
+ 'endpoint_type': "identity",
+ 'url_path': "'/v3/projects/' + str(sample['project_id'])",
+ 'value': "name",
+ }
+ definitions['value_attribute'] = 'project_id'
+ definitions['extra_metadata_fields'] = extra_metadata_fields
+ pollster = dynamic_pollster.DynamicPollster(definitions)
+
+ return_value = self.FakeResponse()
+ return_value.status_code = requests.codes.ok
+ return_value._text = '''
+ {"projects": [
+ {"project_id": 9999, "name": "project1"},
+ {"project_id": 8888, "name": "project2"},
+ {"project_id": 7777, "name": "project3"},
+ {"project_id": 9999, "name": "project4"},
+ {"project_id": 8888, "name": "project5"},
+ {"project_id": 7777, "name": "project6"},
+ {"project_id": 9999, "name": "project7"},
+ {"project_id": 8888, "name": "project8"},
+ {"project_id": 7777, "name": "project9"}]
+ }
+ '''
+
+ return_value9999 = self.FakeResponse()
+ return_value9999.status_code = requests.codes.ok
+ return_value9999._text = '''
+ {"project":
+ {"project_id": 9999, "name": "project1"}
+ }
+ '''
+
+ return_value8888 = self.FakeResponse()
+ return_value8888.status_code = requests.codes.ok
+ return_value8888._text = '''
+ {"project":
+ {"project_id": 8888, "name": "project2"}
+ }
+ '''
+
+ return_value7777 = self.FakeResponse()
+ return_value7777.status_code = requests.codes.ok
+ return_value7777._text = '''
+ {"project":
+ {"project_id": 7777, "name": "project3"}
+ }
+ '''
+
+ def get(url, *args, **kwargs):
+ if '9999' in url:
+ return return_value9999
+ if '8888' in url:
+ return return_value8888
+ if '7777' in url:
+ return return_value7777
+ return return_value
+
+ client_mock.session.get.side_effect = get
+ manager = mock.Mock
+ manager._keystone = client_mock
+
+ def discover(*args, **kwargs):
+ return ["https://endpoint.server.name/"]
+
+ manager.discover = discover
+ samples = pollster.get_samples(
+ manager=manager, cache=None,
+ resources=["https://endpoint.server.name/"])
+
+ samples = list(samples)
+
+ n_calls = client_mock.session.get.call_count
+ self.assertEqual(9, len(samples))
+ self.assertEqual(4, n_calls)
+
+ @mock.patch('keystoneclient.v2_0.client.Client')
+ def test_execute_request_extra_metadata_fields(
+ self, client_mock):
+ definitions = copy.deepcopy(
+ self.pollster_definition_only_required_fields)
+ extra_metadata_fields = [{
+ 'name': "project_name",
+ 'endpoint_type': "identity",
+ 'url_path': "'/v3/projects/' + str(sample['project_id'])",
+ 'value': "name",
+ 'metadata_fields': ['meta']
+ }, {
+ 'name': "project_alias",
+ 'endpoint_type': "identity",
+ 'url_path': "'/v3/projects/' + "
+ "str(extra_metadata_captured['project_name'])",
+ 'value': "name",
+ 'metadata_fields': ['meta']
+ }, {
+ 'name': "project_meta",
+ 'endpoint_type': "identity",
+ 'url_path': "'/v3/projects/' + "
+ "str(extra_metadata_by_name['project_name']"
+ "['metadata']['meta'])",
+ 'value': "project_id",
+ 'metadata_fields': ['meta']
+ }]
+ definitions['value_attribute'] = 'project_id'
+ definitions['extra_metadata_fields'] = extra_metadata_fields
+ pollster = dynamic_pollster.DynamicPollster(definitions)
+
+ return_value = self.FakeResponse()
+ return_value.status_code = requests.codes.ok
+ return_value._text = '''
+ {"projects": [
+ {"project_id": 9999, "name": "project1"},
+ {"project_id": 8888, "name": "project2"},
+ {"project_id": 7777, "name": "project3"}]
+ }
+ '''
+
+ return_value9999 = self.FakeResponse()
+ return_value9999.status_code = requests.codes.ok
+ return_value9999._text = '''
+ {"project":
+ {"project_id": 9999, "name": "project1",
+ "meta": "m1"}
+ }
+ '''
+
+ return_value8888 = self.FakeResponse()
+ return_value8888.status_code = requests.codes.ok
+ return_value8888._text = '''
+ {"project":
+ {"project_id": 8888, "name": "project2",
+ "meta": "m2"}
+ }
+ '''
+
+ return_value7777 = self.FakeResponse()
+ return_value7777.status_code = requests.codes.ok
+ return_value7777._text = '''
+ {"project":
+ {"project_id": 7777, "name": "project3",
+ "meta": "m3"}
+ }
+ '''
+
+ return_valueP1 = self.FakeResponse()
+ return_valueP1.status_code = requests.codes.ok
+ return_valueP1._text = '''
+ {"project":
+ {"project_id": 7777, "name": "p1",
+ "meta": null}
+ }
+ '''
+
+ return_valueP2 = self.FakeResponse()
+ return_valueP2.status_code = requests.codes.ok
+ return_valueP2._text = '''
+ {"project":
+ {"project_id": 7777, "name": "p2",
+ "meta": null}
+ }
+ '''
+
+ return_valueP3 = self.FakeResponse()
+ return_valueP3.status_code = requests.codes.ok
+ return_valueP3._text = '''
+ {"project":
+ {"project_id": 7777, "name": "p3",
+ "meta": null}
+ }
+ '''
+
+ return_valueM1 = self.FakeResponse()
+ return_valueM1.status_code = requests.codes.ok
+ return_valueM1._text = '''
+ {"project":
+ {"project_id": "META1", "name": "p3",
+ "meta": null}
+ }
+ '''
+
+ return_valueM2 = self.FakeResponse()
+ return_valueM2.status_code = requests.codes.ok
+ return_valueM2._text = '''
+ {"project":
+ {"project_id": "META2", "name": "p3",
+ "meta": null}
+ }
+ '''
+
+ return_valueM3 = self.FakeResponse()
+ return_valueM3.status_code = requests.codes.ok
+ return_valueM3._text = '''
+ {"project":
+ {"project_id": "META3", "name": "p3",
+ "meta": null}
+ }
+ '''
+
+ def get(url, *args, **kwargs):
+ if '9999' in url:
+ return return_value9999
+ if '8888' in url:
+ return return_value8888
+ if '7777' in url:
+ return return_value7777
+ if 'project1' in url:
+ return return_valueP1
+ if 'project2' in url:
+ return return_valueP2
+ if 'project3' in url:
+ return return_valueP3
+ if 'm1' in url:
+ return return_valueM1
+ if 'm2' in url:
+ return return_valueM2
+ if 'm3' in url:
+ return return_valueM3
+
+ return return_value
+
+ client_mock.session.get = get
+ manager = mock.Mock
+ manager._keystone = client_mock
+
+ def discover(*args, **kwargs):
+ return ["https://endpoint.server.name/"]
+
+ manager.discover = discover
+ samples = pollster.get_samples(
+ manager=manager, cache=None,
+ resources=["https://endpoint.server.name/"])
+
+ samples = list(samples)
+ self.assertEqual(3, len(samples))
+
+ self.assertEqual(samples[0].volume, 9999)
+ self.assertEqual(samples[1].volume, 8888)
+ self.assertEqual(samples[2].volume, 7777)
+
+ self.assertEqual(samples[0].resource_metadata,
+ {'project_name': 'project1',
+ 'project_alias': 'p1',
+ 'meta': 'm1',
+ 'project_meta': 'META1'})
+ self.assertEqual(samples[1].resource_metadata,
+ {'project_name': 'project2',
+ 'project_alias': 'p2',
+ 'meta': 'm2',
+ 'project_meta': 'META2'})
+ self.assertEqual(samples[2].resource_metadata,
+ {'project_name': 'project3',
+ 'project_alias': 'p3',
+ 'meta': 'm3',
+ 'project_meta': 'META3'})
+
+ @mock.patch('keystoneclient.v2_0.client.Client')
+ def test_execute_request_extra_metadata_fields_different_requests(
+ self, client_mock):
+ definitions = copy.deepcopy(
+ self.pollster_definition_only_required_fields)
+
+ command = ''' \'\'\'echo '{"project":
+ {"project_id": \'\'\'+ str(sample['project_id'])
+ +\'\'\' , "name": "project1"}}' \'\'\' '''.replace('\n', '')
+
+ command2 = ''' \'\'\'echo '{"project":
+ {"project_id": \'\'\'+ str(sample['project_id'])
+ +\'\'\' , "name": "project2"}}' \'\'\' '''.replace('\n', '')
+
+ extra_metadata_fields_embedded = {
+ 'name': "project_name2",
+ 'host_command': command2,
+ 'value': "name",
+ }
+
+ extra_metadata_fields = {
+ 'name': "project_id2",
+ 'host_command': command,
+ 'value': "project_id",
+ 'extra_metadata_fields': extra_metadata_fields_embedded
+ }
+
+ definitions['value_attribute'] = 'project_id'
+ definitions['extra_metadata_fields'] = extra_metadata_fields
+ pollster = dynamic_pollster.DynamicPollster(definitions)
+
+ return_value = self.FakeResponse()
+ return_value.status_code = requests.codes.ok
+ return_value._text = '''
+ {"projects": [
+ {"project_id": 9999, "name": "project1"},
+ {"project_id": 8888, "name": "project2"},
+ {"project_id": 7777, "name": "project3"}]
+ }
+ '''
+
+ def get(url, *args, **kwargs):
+ return return_value
+
+ client_mock.session.get = get
+ manager = mock.Mock
+ manager._keystone = client_mock
+
+ def discover(*args, **kwargs):
+ return ["https://endpoint.server.name/"]
+
+ manager.discover = discover
+ samples = pollster.get_samples(
+ manager=manager, cache=None,
+ resources=["https://endpoint.server.name/"])
+
+ samples = list(samples)
+ self.assertEqual(3, len(samples))
+
+ self.assertEqual(samples[0].volume, 9999)
+ self.assertEqual(samples[1].volume, 8888)
+ self.assertEqual(samples[2].volume, 7777)
+
+ self.assertEqual(samples[0].resource_metadata,
+ {'project_id2': 9999,
+ 'project_name2': 'project2'})
+ self.assertEqual(samples[1].resource_metadata,
+ {'project_id2': 8888,
+ 'project_name2': 'project2'})
+ self.assertEqual(samples[2].resource_metadata,
+ {'project_id2': 7777,
+ 'project_name2': 'project2'})
+
@mock.patch('keystoneclient.v2_0.client.Client')
def test_execute_request_xml_json_response_handler_invalid_response(
self, client_mock):
@@ -410,8 +828,8 @@ class TestDynamicPollster(base.BaseTestCase):
keystone_client=client_mock,
resource="https://endpoint.server.name/")
- xml_handling_error = logs.output[2]
- json_handling_error = logs.output[3]
+ xml_handling_error = logs.output[3]
+ json_handling_error = logs.output[4]
self.assertIn(
'DEBUG:ceilometer.polling.dynamic_pollster:'
@@ -479,6 +897,57 @@ class TestDynamicPollster(base.BaseTestCase):
resource="https://endpoint.server.name/")
self.assertEqual("Mock HTTP error.", str(exception))
+ def test_execute_host_command_paged_responses(self):
+ definitions = copy.deepcopy(
+ self.pollster_definition_only_required_fields)
+ definitions['host_command'] = '''
+ echo '{"server": [{"status": "ACTIVE"}], "next": ""}'
+ '''
+ str_json = "'{\\\"server\\\": [{\\\"status\\\": \\\"INACTIVE\\\"}]}'"
+ definitions['next_sample_url_attribute'] = \
+ "next|\"echo \"+value+\"" + str_json + '"'
+ pollster = dynamic_pollster.DynamicPollster(definitions)
+ samples = pollster.definitions.sample_gatherer. \
+ execute_request_get_samples()
+ resp_json = [{'status': 'ACTIVE'}, {'status': 'INACTIVE'}]
+ self.assertEqual(resp_json, samples)
+
+ def test_execute_host_command_response_handler(self):
+ definitions = copy.deepcopy(
+ self.pollster_definition_only_required_fields)
+ definitions['response_handlers'] = ['xml', 'json']
+ definitions['host_command'] = 'echo "xml\nxml"'
+ entry = 'a'
+ definitions['response_entries_key'] = entry
+ definitions.pop('url_path')
+ definitions.pop('endpoint_type')
+ pollster = dynamic_pollster.DynamicPollster(definitions)
+
+ samples_xml = pollster.definitions.sample_gatherer. \
+ execute_request_get_samples()
+
+ definitions['host_command'] = 'echo \'{"a": {"y":"json",' \
+ '\n"s":"json"}}\''
+ samples_json = pollster.definitions.sample_gatherer. \
+ execute_request_get_samples()
+
+ resp_xml = {'a': {'y': 'xml', 's': 'xml'}}
+ resp_json = {'a': {'y': 'json', 's': 'json'}}
+ self.assertEqual(resp_xml[entry], samples_xml)
+ self.assertEqual(resp_json[entry], samples_json)
+
+ def test_execute_host_command_invalid_command(self):
+ definitions = copy.deepcopy(
+ self.pollster_definition_only_required_fields)
+ definitions['host_command'] = 'invalid-command'
+ definitions.pop('url_path')
+ definitions.pop('endpoint_type')
+ pollster = dynamic_pollster.DynamicPollster(definitions)
+
+ self.assertRaises(
+ declarative.InvalidResponseTypeException,
+ pollster.definitions.sample_gatherer.execute_request_get_samples)
+
def test_generate_new_metadata_fields_no_metadata_mapping(self):
metadata = {'name': 'someName',
'value': 1}
@@ -1105,7 +1574,7 @@ class TestDynamicPollster(base.BaseTestCase):
sample = pollster.definitions.sample_extractor.generate_sample(
pollster_sample, pollster.definitions.configurations,
- manager=mock.Mock())
+ manager=mock.Mock(), conf={})
self.assertEqual(1, sample.volume)
self.assertEqual(2, len(sample.resource_metadata))
@@ -1127,7 +1596,7 @@ class TestDynamicPollster(base.BaseTestCase):
sample = pollster.definitions.sample_extractor.generate_sample(
pollster_sample, pollster.definitions.configurations,
- manager=mock.Mock())
+ manager=mock.Mock(), conf={})
self.assertEqual(1, sample.volume)
self.assertEqual(3, len(sample.resource_metadata))
@@ -1150,7 +1619,7 @@ class TestDynamicPollster(base.BaseTestCase):
sample = pollster.definitions.sample_extractor.generate_sample(
pollster_sample, pollster.definitions.configurations,
- manager=mock.Mock())
+ manager=mock.Mock(), conf={})
self.assertEqual(1, sample.volume)
self.assertEqual(3, len(sample.resource_metadata))
diff --git a/doc/source/admin/telemetry-dynamic-pollster.rst b/doc/source/admin/telemetry-dynamic-pollster.rst
index f91187926a..9ac51f5f55 100644
--- a/doc/source/admin/telemetry-dynamic-pollster.rst
+++ b/doc/source/admin/telemetry-dynamic-pollster.rst
@@ -466,6 +466,62 @@ ones), we can use the `successful_ops`.
resource_id_attribute: "user"
response_entries_key: "summary"
+The dynamic pollsters system configuration (for local host commands)
+--------------------------------------------------------------------
+
+The dynamic pollster system can also be used for local host commands,
+these commands must be installed in the system that is running the
+Ceilometer compute agent.
+To configure local hosts commands, one can use all but two attributes of
+the Dynamic pollster system. The attributes that are not supported are
+the ``endpoint_type`` and ``url_path``. The dynamic pollster system for
+local host commands is activated automatically when one uses the
+configuration ``host_command``.
+
+The extra parameter (in addition to the original ones) that is available
+when using the local host commands dynamic pollster sub-subsystem is the
+following:
+
+* ``host_command``: required parameter. It is the host command that will
+ be executed in the same host the Ceilometer dynamic pollster agent is
+ running. The output of the command will be processed by the pollster and
+ stored in the configured backend.
+
+As follows we present an example on how to use the local host command:
+
+.. code-block:: yaml
+
+ ---
+
+ - name: "dynamic.host.command"
+ sample_type: "gauge"
+ unit: "request"
+ value_attribute: "value"
+ response_entries_key: "test"
+ host_command: "echo 'id1_uid1_pid1meta-data-to-store1'"
+ metadata_fields:
+ - "meta"
+ response_handlers:
+ - xml
+
+To execute multi page host commands, the `next_sample_url_attribute`
+must generate the next sample command, like the following example:
+
+.. code-block:: yaml
+
+ ---
+
+ - name: "dynamic.s3.objects.size"
+ sample_type: "gauge"
+ unit: "request"
+ value_attribute: "Size"
+ project_id_attribute: "Owner.ID"
+ user_id_attribute: "Owner.ID"
+ resource_id_attribute: "Key"
+ response_entries_key: "Contents"
+ host_command: "aws s3api list-objects"
+ next_sample_url_attribute: NextToken | 'aws s3api list-objects --starting-token "' + value + '"'
+
Operations on extracted attributes
----------------------------------
@@ -871,12 +927,10 @@ we only have the `tenant_id`, which must be used as the `project_id`. However,
for billing and later invoicing one might need/want the project name, domain
id, and other metadata that are available in Keystone (and maybe some others
that are scattered over other components). To achieve that, one can use the
-OpenStack metadata enrichment option. This feature is only available
-to *OpenStack pollsters*, and can only gather extra metadata from OpenStack
-APIs. As follows we present an example that shows a dynamic pollster
-configuration to gather virtual machine (VM) status, and to enrich the data
-pushed to the storage backend (e.g. Gnocchi) with project name, domain ID,
-and domain name.
+OpenStack metadata enrichment option. As follows we present an example that
+shows a dynamic pollster configuration to gather virtual machine (VM) status,
+and to enrich the data pushed to the storage backend (e.g. Gnocchi) with
+project name, domain ID, and domain name.
.. code-block:: yaml
@@ -932,26 +986,59 @@ and domain name.
"Openstack-API-Version": "identity latest"
value: "name"
extra_metadata_fields_cache_seconds: 1800 # overriding the default cache policy
+ metadata_fields:
+ - id
- name: "domain_id"
endpoint_type: "identity"
url_path: "'/v3/projects/' + str(sample['project_id'])"
headers:
"Openstack-API-Version": "identity latest"
value: "domain_id"
+ metadata_fields:
+ - id
- name: "domain_name"
endpoint_type: "identity"
url_path: "'/v3/domains/' + str(extra_metadata_captured['domain_id'])"
headers:
"Openstack-API-Version": "identity latest"
value: "name"
+ metadata_fields:
+ - id
+ - name: "operating-system"
+ host_command: "'get-vm --vm-name ' + str(extra_metadata_by_name['project_name']['metadata']['id'])"
+ value: "os"
+
The above example can be used to gather and persist in the backend the
status of VMs. It will persist `1` in the backend as a measure for every
collecting period if the VM's status is `ACTIVE`, and `0` otherwise. This is
quite useful to create hashmap rating rules for running VMs in CloudKitty.
Then, to enrich the resource in the storage backend, we are adding extra
-metadata that are collected in Keystone via the `extra_metadata_fields`
-options.
+metadata that are collected in Keystone and in the local host via the
+`extra_metadata_fields` options. If you have multiples `extra_metadata_fields`
+defining the same `metadata_field`, the last not `None` metadata value will
+be used.
+
+To operate values in the `extra_metadata_fields`, you can access 3 local
+variables:
+
+* ``sample``: it is a dictionary which holds the current data of the root
+ sample. The root sample is the final sample that will be persisted in the
+ configured storage backend.
+
+* ``extra_metadata_captured``: it is a dictionary which holds the current
+ data of all `extra_metadata_fields` processed before this one.
+ If you have multiples `extra_metadata_fields` defining the same
+ `metadata_field`, the last not `None` metadata value will be used.
+
+* ``extra_metadata_by_name``: it is a dictionary which holds the data of
+ all `extra_metadata_fields` processed before this one. No data is
+ overwritten in this variable. To access an specific `extra_metadata_field`
+ using this variable, you can do
+ `extra_metadata_by_name['']['value']` to get
+ its value, or
+ `extra_metadata_by_name['']['metadata']['']`
+ to get its metadata.
The metadata enrichment feature has the following options:
@@ -964,41 +1051,13 @@ The metadata enrichment feature has the following options:
value can be increased of decreased.
* ``extra_metadata_fields``: optional parameter. This option is a list of
- objects, where each one of its elements is an extra metadata definition.
- Each one of the extra metadata definition can have the options defined in
- the dynamic pollsters such as to handle paged responses, operations on the
- extracted values, headers and so on. The basic options that must be
- defined for an extra metadata definitions are the following:
+ objects or a single one, where each one of its elements is an
+ dynamic pollster configuration set. Each one of the extra metadata
+ definition can have the same options defined in the dynamic pollsters,
+ including the `extra_metadata_fields` option, so this option is a
+ multi-level option. When defined, the result of the collected data will
+ be merged in the final sample resource metadata. If some of the required
+ dynamic pollster configuration is not set in the `extra_metadata_fields`,
+ will be used the parent pollster configuration, except the `name`.
- * ``name``: This option is mandatory. The name of the extra metadata.
- This is the name that is going to be used by the metadata. If there is
- already any other metadata gathered via `metadata_fields` option or
- transformed via `metadata_mapping` configuration, this metadata is
- going to be discarded.
- * ``endpoint_type``: The endpoint type that we want to execute the
- call against. This option is mandatory. It works similarly to the
- `endpoint_type` option in the dynamic pollster definition.
-
- * ``url_path``: This option is mandatory. It works similarly to the
- `url_path` option in the dynamic pollster definition. However, this
- `one enables operators to execute/evaluate expressions in runtime, which
- `allows one to retrieve the information from previously gathered
- metadata via ``extra_metadata_captured` dictionary, or via the
- `sample` itself.
-
- * ``value``: This configuration is mandatory. It works similarly to the
- `value_attribute` option in the dynamic pollster definition. It is
- the value we want to extract from the response, and assign in the
- metadata being generated.
-
- * ``headers``: This option is optional. It works similarly to the
- `headers` option in the dynamic pollster definition.
-
- * ``next_sample_url_attribute``: This option is optional. It works
- similarly to the `next_sample_url_attribute` option in the dynamic
- pollster definition.
-
- * ``response_entries_key``: This option is optional. It works
- similarly to the `response_entries_key` option in the dynamic
- pollster definition.
diff --git a/setup.cfg b/setup.cfg
index a4dbdd49f4..2e8999ce97 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -45,6 +45,7 @@ ceilometer.sample.endpoint =
ceilometer.discover.compute =
local_instances = ceilometer.compute.discovery:InstanceDiscovery
+ local_node = ceilometer.polling.discovery.localnode:LocalNodeDiscovery
ceilometer.discover.central =
barbican = ceilometer.polling.discovery.non_openstack_credentials_discovery:NonOpenStackCredentialsDiscovery