openstack
/
ceilometer
Code Issues Proposed changes

Multi metric dynamic pollsters (handling attribute values with list of objects) 

The initial idea for this feature comes from the `categories` fields that we
can find in the `summary` object of the RadosGW API. Each user has a
`categories` attribute in the response; in the `categories` list, we can find
the object that presents ain a granular fashion the consumption of different
RadosGW API operations such as GET, PUT, POST, and may others.

In that context, and having in mind that we have APIs with similar data
structures, we developed an extension for the dynamic pollster that enables
multi-metric processing for a single pollster. It works as follows.

The pollster name will contain a placeholder for the variable that
identifies the "submetric". E.g. `dynamic.radosgw.api.request.{category}`.
The placeholder `{category}` indicates the object attribute that is in the list
of objects that we use to load the sub metric name. Then, we must use a special
notation in the `value_attribute` configuration to indicate that we are dealing
with a list of objects. This is achieved via `[]` (brackets); for instance, in
the `dynamic.radosgw.api.request.{category}`, we can use `[categories].ops`
as the `value_attribute`. This indicates that the value we retrieve is a list
of objects, and when the dynamic pollster processes it, we want it (the
pollster) to load the `ops` value for the sub metrics being generated

Depends-On: https://review.opendev.org/#/c/694519/
Change-Id: I6ed4632f209ac51a07687476ca316212659d72bb
Signed-off-by: Rafael Weingärtner <rafael@apache.org>
This commit is contained in:
rwe 2019-11-19 15:41:26 +01:00 committed by Rafael Weingärtner
parent d0e8f95fe4
commit 4e3c12968d
8 changed files with 1214 additions and 415 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
ceilometer/declarative.py
View File

@ -42,11 +42,16 @@ class ResourceDefinitionException(DefinitionException):
pass
class DynamicPollsterDefinitionException(DefinitionException):
class DynamicPollsterException(DefinitionException):
pass
class NonOpenStackApisDynamicPollsterException(DefinitionException):
class DynamicPollsterDefinitionException(DynamicPollsterException):
pass
class NonOpenStackApisDynamicPollsterException\
(DynamicPollsterDefinitionException):
pass

746
ceilometer/polling/dynamic_pollster.py
View File

@ -17,6 +17,9 @@
'/etc/ceilometer/pollsters.d/'. The pollster are defined in YAML files
similar to the idea used for handling notifications.
"""
import copy
import re
from oslo_log import log
from oslo_utils import timeutils
@ -24,7 +27,7 @@ from requests import RequestException
from ceilometer import declarative
from ceilometer.polling import plugin_base
from ceilometer import sample
from ceilometer import sample as ceilometer_sample
from functools import reduce
import operator
@ -35,171 +38,57 @@ from six.moves.urllib import parse as url_parse
LOG = log.getLogger(__name__)
class DynamicPollster(plugin_base.PollsterBase):
def validate_sample_type(sample_type):
if sample_type not in ceilometer_sample.TYPES:
raise declarative.DynamicPollsterDefinitionException(
"Invalid sample type [%s]. Valid ones are [%s]."
% (sample_type, ceilometer_sample.TYPES))
OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
'value_mapping', 'default_value',
'metadata_mapping',
'preserve_mapped_metadata',
'response_entries_key']
REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
'value_attribute', 'endpoint_type',
'url_path']
class PollsterDefinitionBuilder(object):
# Mandatory name field
name = ""
def __init__(self, definitions):
self.definitions = definitions
def __init__(self, pollster_definitions, conf=None):
super(DynamicPollster, self).__init__(conf)
def build_definitions(self, configurations):
supported_definitions = []
for definition in self.definitions:
if definition.is_field_applicable_to_definition(configurations):
supported_definitions.append(definition)
self.ALL_POLLSTER_FIELDS =\
self.OPTIONAL_POLLSTER_FIELDS + self.REQUIRED_POLLSTER_FIELDS
LOG.debug("%s instantiated with [%s]", __name__,
pollster_definitions)
self.pollster_definitions = pollster_definitions
self.validate_pollster_definition()
if 'metadata_fields' in self.pollster_definitions:
LOG.debug("Metadata fields configured to [%s].",
self.pollster_definitions['metadata_fields'])
self.set_default_values()
self.name = self.pollster_definitions['name']
self.obj = self
def set_default_values(self):
if 'skip_sample_values' not in self.pollster_definitions:
self.pollster_definitions['skip_sample_values'] = []
if 'value_mapping' not in self.pollster_definitions:
self.pollster_definitions['value_mapping'] = {}
if 'default_value' not in self.pollster_definitions:
self.pollster_definitions['default_value'] = -1
if 'preserve_mapped_metadata' not in self.pollster_definitions:
self.pollster_definitions['preserve_mapped_metadata'] = True
if 'metadata_mapping' not in self.pollster_definitions:
self.pollster_definitions['metadata_mapping'] = {}
if 'response_entries_key' not in self.pollster_definitions:
self.pollster_definitions['response_entries_key'] = None
def validate_pollster_definition(self):
missing_required_fields = \
[field for field in self.REQUIRED_POLLSTER_FIELDS
if field not in self.pollster_definitions]
if missing_required_fields:
if not supported_definitions:
raise declarative.DynamicPollsterDefinitionException(
"Required fields %s not specified."
% missing_required_fields, self.pollster_definitions)
"Your configurations do not fit any type of DynamicPollsters, "
"please recheck them. Used configurations = [%s]." %
configurations)
sample_type = self.pollster_definitions['sample_type']
if sample_type not in sample.TYPES:
raise declarative.DynamicPollsterDefinitionException(
"Invalid sample type [%s]. Valid ones are [%s]."
% (sample_type, sample.TYPES), self.pollster_definitions)
definition_name = self.join_supported_definitions_names(
supported_definitions)
for definition_key in self.pollster_definitions:
if definition_key not in self.ALL_POLLSTER_FIELDS:
LOG.warning(
"Field [%s] defined in [%s] is unknown "
"and will be ignored. Valid fields are [%s].",
definition_key, self.pollster_definitions,
self.ALL_POLLSTER_FIELDS)
definition_parents = tuple(supported_definitions)
definition_attribs = {'extra_definitions': reduce(
lambda d1, d2: d1 + d2, map(lambda df: df.extra_definitions,
supported_definitions))}
definition_type = type(definition_name, definition_parents,
definition_attribs)
return definition_type(configurations)
def get_samples(self, manager, cache, resources):
if not resources:
LOG.debug("No resources received for processing.")
yield None
@staticmethod
def join_supported_definitions_names(supported_definitions):
return ''.join(map(lambda df: df.__name__,
supported_definitions))
for r in resources:
LOG.debug("Executing get sample for resource [%s].", r)
samples = list([])
try:
samples = self.execute_request_get_samples(
keystone_client=manager._keystone, resource=r)
except RequestException as e:
LOG.warning("Error [%s] while loading samples for [%s] "
"for dynamic pollster [%s].",
e, r, self.name)
class PollsterSampleExtractor(object):
for pollster_sample in samples:
response_value_attribute_name = self.pollster_definitions[
'value_attribute']
value = self.retrieve_attribute_nested_value(
pollster_sample, response_value_attribute_name)
def __init__(self, definitions):
self.definitions = definitions
skip_sample_values = \
self.pollster_definitions['skip_sample_values']
if skip_sample_values and value in skip_sample_values:
LOG.debug("Skipping sample [%s] because value [%s] "
"is configured to be skipped in skip list [%s].",
pollster_sample, value, skip_sample_values)
continue
value = self.execute_value_mapping(value)
user_id = None
if 'user_id' in pollster_sample:
user_id = pollster_sample["user_id"]
project_id = None
if 'project_id' in pollster_sample:
project_id = pollster_sample["project_id"]
resource_id = None
if 'id' in pollster_sample:
resource_id = pollster_sample["id"]
metadata = []
if 'metadata_fields' in self.pollster_definitions:
metadata = dict((k, pollster_sample.get(k))
for k in self.pollster_definitions[
'metadata_fields'])
self.generate_new_metadata_fields(metadata=metadata)
yield sample.Sample(
timestamp=timeutils.isotime(),
name=self.pollster_definitions['name'],
type=self.pollster_definitions['sample_type'],
unit=self.pollster_definitions['unit'],
volume=value,
user_id=user_id,
project_id=project_id,
resource_id=resource_id,
resource_metadata=metadata
)
def execute_value_mapping(self, value):
value_mapping = self.pollster_definitions['value_mapping']
if value_mapping:
if value in value_mapping:
old_value = value
value = value_mapping[value]
LOG.debug("Value mapped from [%s] to [%s]",
old_value, value)
else:
default_value = \
self.pollster_definitions['default_value']
LOG.warning(
"Value [%s] was not found in value_mapping [%s]; "
"therefore, we will use the default [%s].",
value, value_mapping, default_value)
value = default_value
return value
def generate_new_metadata_fields(self, metadata=None):
metadata_mapping = self.pollster_definitions['metadata_mapping']
def generate_new_metadata_fields(self, metadata=None,
pollster_definitions=None):
pollster_definitions =\
pollster_definitions or self.definitions.configurations
metadata_mapping = pollster_definitions['metadata_mapping']
if not metadata_mapping or not metadata:
return
@ -212,26 +101,384 @@ class DynamicPollster(plugin_base.PollsterBase):
metadata[new_key] = metadata[k]
LOG.debug("Generating new key [%s] with content [%s] of key [%s]",
new_key, metadata[k], k)
if self.pollster_definitions['preserve_mapped_metadata']:
if pollster_definitions['preserve_mapped_metadata']:
continue
k_value = metadata.pop(k)
LOG.debug("Removed key [%s] with value [%s] from "
"metadata set that is sent to Gnocchi.", k, k_value)
def generate_sample(self, pollster_sample, pollster_definitons=None):
pollster_definitions =\
pollster_definitons or self.definitions.configurations
metadata = []
if 'metadata_fields' in pollster_definitions:
metadata = dict((k, pollster_sample.get(k))
for k in pollster_definitions['metadata_fields'])
self.generate_new_metadata_fields(
metadata=metadata, pollster_definitions=pollster_definitions)
return ceilometer_sample.Sample(
timestamp=timeutils.isotime(),
name=pollster_definitions['name'],
type=pollster_definitions['sample_type'],
unit=pollster_definitions['unit'],
volume=pollster_sample['value'],
user_id=pollster_sample.get("user_id"),
project_id=pollster_sample.get("project_id"),
resource_id=pollster_sample.get("id"),
resource_metadata=metadata)
def retrieve_attribute_nested_value(self, json_object,
value_attribute=None):
attribute_key = value_attribute or self.definitions.\
extract_attribute_key()
LOG.debug("Retrieving the nested keys [%s] from [%s].",
attribute_key, json_object)
keys_and_operations = attribute_key.split("|")
attribute_key = keys_and_operations[0].strip()
nested_keys = attribute_key.split(".")
value = reduce(operator.getitem, nested_keys, json_object)
return self.operate_value(keys_and_operations, value)
def operate_value(self, keys_and_operations, value):
# We do not have operations to be executed against the value extracted
if len(keys_and_operations) < 2:
return value
for operation in keys_and_operations[1::]:
# The operation must be performed onto the 'value' variable
if 'value' not in operation:
raise declarative.DynamicPollsterDefinitionException(
"The attribute field operation [%s] must use the ["
"value] variable." % operation,
self.definitions.configurations)
LOG.debug("Executing operation [%s] against value [%s].",
operation, value)
value = eval(operation.strip())
LOG.debug("Result [%s] of operation [%s].",
value, operation)
return value
class SimplePollsterSampleExtractor(PollsterSampleExtractor):
def generate_single_sample(self, pollster_sample):
value = self.retrieve_attribute_nested_value(pollster_sample)
value = self.definitions.value_mapper.map_or_skip_value(
value, pollster_sample)
if isinstance(value, SkippedSample):
return value
pollster_sample['value'] = value
return self.generate_sample(pollster_sample)
def extract_sample(self, pollster_sample):
sample = self.generate_single_sample(pollster_sample)
if isinstance(sample, SkippedSample):
return sample
yield sample
class MultiMetricPollsterSampleExtractor(PollsterSampleExtractor):
def extract_sample(self, pollster_sample):
pollster_definitions = self.definitions.configurations
value = self.retrieve_attribute_nested_value(pollster_sample)
LOG.debug("We are dealing with a multi metric pollster. The "
"value we are processing is the following: [%s].",
value)
self.validate_sample_is_list(value)
sub_metric_placeholder, pollster_name, sub_metric_attribute_name = \
self.extract_names_attrs()
value_attribute = \
self.extract_field_name_from_value_attribute_configuration()
LOG.debug("Using attribute [%s] to look for values in the "
"multi metric pollster [%s] with sample [%s]",
value_attribute, pollster_definitions, value)
pollster_definitions = copy.deepcopy(pollster_definitions)
yield from self.extract_sub_samples(value, sub_metric_attribute_name,
pollster_name, value_attribute,
sub_metric_placeholder,
pollster_definitions,
pollster_sample)
def extract_sub_samples(self, value, sub_metric_attribute_name,
pollster_name, value_attribute,
sub_metric_placeholder, pollster_definitions,
pollster_sample):
for sub_sample in value:
sub_metric_name = sub_sample[sub_metric_attribute_name]
new_metric_name = pollster_name.replace(
sub_metric_placeholder, sub_metric_name)
pollster_definitions['name'] = new_metric_name
actual_value = self.retrieve_attribute_nested_value(
sub_sample, value_attribute)
pollster_sample['value'] = actual_value
if self.should_skip_generate_sample(actual_value, sub_sample,
sub_metric_name):
continue
yield self.generate_sample(pollster_sample, pollster_definitions)
def extract_field_name_from_value_attribute_configuration(self):
value_attribute = self.definitions.configurations['value_attribute']
return self.definitions.pattern_pollster_value_attribute.match(
value_attribute).group(3)[1::]
def extract_names_attrs(self):
pollster_name = self.definitions.configurations['name']
sub_metric_placeholder = pollster_name.split(".").pop()
return (sub_metric_placeholder,
pollster_name,
self.definitions.pattern_pollster_name.match(
"." + sub_metric_placeholder).group(2))
def validate_sample_is_list(self, value):
pollster_definitions = self.definitions.configurations
if not isinstance(value, list):
raise declarative.DynamicPollsterException(
"Multi metric pollster defined, but the value [%s]"
" obtained with [%s] attribute is not a list"
" of objects."
% (value,
pollster_definitions['value_attribute']),
pollster_definitions)
def should_skip_generate_sample(self, actual_value, sub_sample,
sub_metric_name):
skip_sample_values = \
self.definitions.configurations['skip_sample_values']
if actual_value in skip_sample_values:
LOG.debug(
"Skipping multi metric sample [%s] because "
"value [%s] is configured to be skipped in "
"skip list [%s].", sub_sample, actual_value,
skip_sample_values)
return True
if sub_metric_name in skip_sample_values:
LOG.debug(
"Skipping sample [%s] because its sub-metric "
"name [%s] is configured to be skipped in "
"skip list [%s].", sub_sample, sub_metric_name,
skip_sample_values)
return True
return False
class PollsterValueMapper(object):
def __init__(self, definitions):
self.definitions = definitions
def map_or_skip_value(self, value, pollster_sample):
skip_sample_values = \
self.definitions.configurations['skip_sample_values']
if value in skip_sample_values:
LOG.debug("Skipping sample [%s] because value [%s] "
"is configured to be skipped in skip list [%s].",
pollster_sample, value, skip_sample_values)
return SkippedSample()
return self.execute_value_mapping(value)
def execute_value_mapping(self, value):
value_mapping = self.definitions.configurations['value_mapping']
if not value_mapping:
return value
if value in value_mapping:
old_value = value
value = value_mapping[value]
LOG.debug("Value mapped from [%s] to [%s]",
old_value, value)
else:
default_value = \
self.definitions.configurations['default_value']
LOG.warning(
"Value [%s] was not found in value_mapping [%s]; "
"therefore, we will use the default [%s].",
value, value_mapping, default_value)
value = default_value
return value
class PollsterDefinition(object):
def __init__(self, name, required=False, on_missing=lambda df: df.default,
default=None, validation_regex=None, creatable=True,
validator=None):
self.name = name
self.required = required
self.on_missing = on_missing
self.validation_regex = validation_regex
self.creatable = creatable
self.default = default
if self.validation_regex:
self.validation_pattern = re.compile(self.validation_regex)
self.validator = validator
def validate(self, val):
if val is None:
return self.on_missing(self)
if self.validation_regex and not self.validation_pattern.match(val):
raise declarative.DynamicPollsterDefinitionException(
"Pollster %s [%s] does not match [%s]."
% (self.name, val, self.validation_regex))
if self.validator:
self.validator(val)
return val
class PollsterDefinitions(object):
POLLSTER_VALID_NAMES_REGEXP = "^([\w-]+)(\.[\w-]+)*(\.{[\w-]+})?$"
standard_definitions = [
PollsterDefinition(name='name', required=True,
validation_regex=POLLSTER_VALID_NAMES_REGEXP),
PollsterDefinition(name='sample_type', required=True,
validator=validate_sample_type),
PollsterDefinition(name='unit', required=True),
PollsterDefinition(name='endpoint_type', required=True),
PollsterDefinition(name='url_path', required=True),
PollsterDefinition(name='metadata_fields', creatable=False),
PollsterDefinition(name='skip_sample_values', default=[]),
PollsterDefinition(name='value_mapping', default={}),
PollsterDefinition(name='default_value', default=-1),
PollsterDefinition(name='metadata_mapping', default={}),
PollsterDefinition(name='preserve_mapped_metadata', default=True),
PollsterDefinition(name='response_entries_key')]
extra_definitions = []
def __init__(self, configurations):
self.configurations = configurations
self.value_mapper = PollsterValueMapper(self)
self.definitions = self.map_definitions()
self.validate_configurations(configurations)
self.validate_missing()
self.sample_gatherer = PollsterSampleGatherer(self)
self.sample_extractor = SimplePollsterSampleExtractor(self)
def validate_configurations(self, configurations):
for k, v in self.definitions.items():
if configurations.get(k) is not None:
self.configurations[k] = self.definitions[k].validate(
self.configurations[k])
elif self.definitions[k].creatable:
self.configurations[k] = self.definitions[k].default
@staticmethod
def is_field_applicable_to_definition(configurations):
return True
def map_definitions(self):
definitions = dict(
map(lambda df: (df.name, df), self.standard_definitions))
extra_definitions = dict(
map(lambda df: (df.name, df), self.extra_definitions))
definitions.update(extra_definitions)
return definitions
def extract_attribute_key(self):
pass
def validate_missing(self):
required_configurations = map(lambda fdf: fdf.name,
filter(lambda df: df.required,
self.definitions.values()))
missing = list(filter(
lambda rf: rf not in map(lambda f: f[0],
filter(lambda f: f[1],
self.configurations.items())),
required_configurations))
if missing:
raise declarative.DynamicPollsterDefinitionException(
"Required fields %s not specified."
% missing, self.configurations)
class MultiMetricPollsterDefinitions(PollsterDefinitions):
MULTI_METRIC_POLLSTER_NAME_REGEXP = ".*(\.{(\w+)})$"
pattern_pollster_name = re.compile(
MULTI_METRIC_POLLSTER_NAME_REGEXP)
MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP = "^(\[(\w+)\])((\.\w+)+)$"
pattern_pollster_value_attribute = re.compile(
MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP)
extra_definitions = [
PollsterDefinition(
name='value_attribute', required=True,
validation_regex=MULTI_METRIC_POLLSTER_VALUE_ATTRIBUTE_REGEXP),
]
def __init__(self, configurations):
super(MultiMetricPollsterDefinitions, self).__init__(configurations)
self.sample_extractor = MultiMetricPollsterSampleExtractor(self)
@staticmethod
def is_field_applicable_to_definition(configurations):
return configurations.get(
'name') and MultiMetricPollsterDefinitions.\
pattern_pollster_name.match(configurations['name'])
def extract_attribute_key(self):
return self.pattern_pollster_value_attribute.match(
self.configurations['value_attribute']).group(2)
class SingleMetricPollsterDefinitions(PollsterDefinitions):
extra_definitions = [
PollsterDefinition(name='value_attribute', required=True)]
def __init__(self, configurations):
super(SingleMetricPollsterDefinitions, self).__init__(configurations)
def extract_attribute_key(self):
return self.configurations['value_attribute']
@staticmethod
def is_field_applicable_to_definition(configurations):
return not MultiMetricPollsterDefinitions. \
is_field_applicable_to_definition(configurations)
class PollsterSampleGatherer(object):
def __init__(self, definitions):
self.definitions = definitions
@property
def default_discovery(self):
return 'endpoint:' + self.pollster_definitions['endpoint_type']
return 'endpoint:' + self.definitions.configurations['endpoint_type']
def execute_request_get_samples(self, **kwargs):
resp, url = self.internal_execute_request_get_samples(kwargs)
resp, url = self.definitions.sample_gatherer. \
internal_execute_request_get_samples(kwargs)
response_json = resp.json()
entry_size = len(response_json)
LOG.debug("Entries [%s] in the JSON for request [%s] "
"for dynamic pollster [%s].",
response_json, url, self.name)
response_json, url, self.definitions.configurations['name'])
if entry_size > 0:
return self.retrieve_entries_from_response(response_json)
@ -241,7 +488,7 @@ class DynamicPollster(plugin_base.PollsterBase):
keystone_client = kwargs['keystone_client']
endpoint = kwargs['resource']
url = url_parse.urljoin(
endpoint, self.pollster_definitions['url_path'])
endpoint, self.definitions.configurations['url_path'])
resp = keystone_client.session.get(url, authenticated=True)
if resp.status_code != requests.codes.ok:
resp.raise_for_status()
@ -251,41 +498,172 @@ class DynamicPollster(plugin_base.PollsterBase):
if isinstance(response_json, list):
return response_json
first_entry_name = self.pollster_definitions['response_entries_key']
first_entry_name = \
self.definitions.configurations['response_entries_key']
if not first_entry_name:
try:
first_entry_name = next(iter(response_json))
except RuntimeError as e:
LOG.debug("Generator threw a StopIteration "
"and we need to catch it [%s].", e)
return self.retrieve_attribute_nested_value(response_json,
first_entry_name)
return self.definitions.sample_extractor. \
retrieve_attribute_nested_value(response_json, first_entry_name)
def retrieve_attribute_nested_value(self, json_object, attribute_key):
LOG.debug("Retrieving the nested keys [%s] from [%s].",
attribute_key, json_object)
keys_and_operations = attribute_key.split("|")
attribute_key = keys_and_operations[0].strip()
class NonOpenStackApisPollsterDefinition(PollsterDefinitions):
nested_keys = attribute_key.split(".")
value = reduce(operator.getitem, nested_keys, json_object)
extra_definitions = [
PollsterDefinition(name='value_attribute', required=True),
PollsterDefinition(name='module', required=True),
PollsterDefinition(name='authentication_object', required=True),
PollsterDefinition(name='user_id_attribute'),
PollsterDefinition(name='resource_id_attribute'),
PollsterDefinition(name='barbican_secret_id', default=""),
PollsterDefinition(name='authentication_parameters', default=""),
PollsterDefinition(name='project_id_attribute'),
PollsterDefinition(name='endpoint_type')]
# We have operations to be executed against the value extracted
if len(keys_and_operations) > 1:
for operation in keys_and_operations[1::]:
# The operation must be performed onto the 'value' variable
if 'value' not in operation:
raise declarative.DynamicPollsterDefinitionException(
"The attribute field operation [%s] must use the ["
"value] variable." % operation,
self.pollster_definitions)
def __init__(self, configurations):
super(NonOpenStackApisPollsterDefinition, self).__init__(
configurations)
self.sample_gatherer = NonOpenStackApisSamplesGatherer(self)
LOG.debug("Executing operation [%s] against value[%s].",
operation, value)
@staticmethod
def is_field_applicable_to_definition(configurations):
return configurations.get('module')
value = eval(operation.strip())
LOG.debug("Result [%s] of operation [%s].",
value, operation)
return value
class NonOpenStackApisSamplesGatherer(PollsterSampleGatherer):
@property
def default_discovery(self):
return 'barbican:' + \
self.definitions.configurations['barbican_secret_id']
def internal_execute_request_get_samples(self, kwargs):
credentials = kwargs['resource']
override_credentials = self.definitions.configurations[
'authentication_parameters']
if override_credentials:
credentials = override_credentials
url = self.definitions.configurations['url_path']
authenticator_module_name = self.definitions.configurations['module']
authenticator_class_name = \
self.definitions.configurations['authentication_object']
imported_module = __import__(authenticator_module_name)
authenticator_class = getattr(imported_module,
authenticator_class_name)
authenticator_arguments = list(map(str.strip, credentials.split(",")))
authenticator_instance = authenticator_class(*authenticator_arguments)
resp = requests.get(
url,
auth=authenticator_instance)
if resp.status_code != requests.codes.ok:
raise declarative.NonOpenStackApisDynamicPollsterException(
"Error while executing request[%s]."
" Status[%s] and reason [%s]."
% (url, resp.status_code, resp.reason))
return resp, url
def execute_request_get_samples(self, **kwargs):
samples = super(NonOpenStackApisSamplesGatherer,
self).execute_request_get_samples(**kwargs)
if samples:
user_id_attribute = self.definitions.configurations[
'user_id_attribute']
project_id_attribute = self.definitions.configurations[
'project_id_attribute']
resource_id_attribute = self.definitions.configurations[
'resource_id_attribute']
for request_sample in samples:
self.generate_new_attributes_in_sample(
request_sample, user_id_attribute, 'user_id')
self.generate_new_attributes_in_sample(
request_sample, project_id_attribute, 'project_id')
self.generate_new_attributes_in_sample(
request_sample, resource_id_attribute, 'id')
return samples
def generate_new_attributes_in_sample(
self, sample, attribute_key, new_attribute_key):
if attribute_key:
attribute_value = self.definitions.sample_extractor. \
retrieve_attribute_nested_value(sample, attribute_key)
LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
attribute_key, attribute_value, sample)
sample[new_attribute_key] = attribute_value
class SkippedSample(object):
pass
class DynamicPollster(plugin_base.PollsterBase):
# Mandatory name field
name = ""
def __init__(self, pollster_definitions={}, conf=None,
supported_definitions=[NonOpenStackApisPollsterDefinition,
MultiMetricPollsterDefinitions,
SingleMetricPollsterDefinitions]):
super(DynamicPollster, self).__init__(conf)
self.supported_definitions = supported_definitions
LOG.debug("%s instantiated with [%s]", __name__,
pollster_definitions)
self.definitions = PollsterDefinitionBuilder(
self.supported_definitions).build_definitions(pollster_definitions)
self.pollster_definitions = self.definitions.configurations
if 'metadata_fields' in self.pollster_definitions:
LOG.debug("Metadata fields configured to [%s].",
self.pollster_definitions['metadata_fields'])
self.name = self.pollster_definitions['name']
self.obj = self
@property
def default_discovery(self):
return self.definitions.sample_gatherer.default_discovery()
def load_samples(self, resource, manager):
try:
return self.definitions.sample_gatherer.\
execute_request_get_samples(keystone_client=manager._keystone,
resource=resource)
except RequestException as e:
LOG.warning("Error [%s] while loading samples for [%s] "
"for dynamic pollster [%s].",
e, resource, self.name)
return list([])
def get_samples(self, manager, cache, resources):
if not resources:
LOG.debug("No resources received for processing.")
yield None
for r in resources:
LOG.debug("Executing get sample for resource [%s].", r)
samples = self.load_samples(r, manager)
for pollster_sample in samples:
sample = self.extract_sample(pollster_sample)
if isinstance(sample, SkippedSample):
continue
yield from sample
def extract_sample(self, pollster_sample):
return self.definitions.sample_extractor.extract_sample(
pollster_sample)

11
ceilometer/polling/manager.py
View File

@ -40,7 +40,6 @@ from ceilometer import declarative
from ceilometer import keystone_client
from ceilometer import messaging
from ceilometer.polling import dynamic_pollster
from ceilometer.polling import non_openstack_dynamic_pollster
from ceilometer.polling import plugin_base
from ceilometer.publisher import utils as publisher_utils
from ceilometer import utils
@ -340,7 +339,8 @@ class AgentManager(cotyledon.Service):
pollster_name, pollsters_definitions_file)
try:
pollsters_definitions[pollster_name] =\
self.instantiate_dynamic_pollster(pollster_cfg)
dynamic_pollster.DynamicPollster(
pollster_cfg, self.conf)
except Exception as e:
LOG.error(
"Error [%s] while loading dynamic pollster [%s].",
@ -355,13 +355,6 @@ class AgentManager(cotyledon.Service):
len(pollsters_definitions))
return pollsters_definitions.values()
def instantiate_dynamic_pollster(self, pollster_cfg):
if 'module' in pollster_cfg:
return non_openstack_dynamic_pollster\
.NonOpenStackApisDynamicPollster(pollster_cfg, self.conf)
else:
return dynamic_pollster.DynamicPollster(pollster_cfg, self.conf)
@staticmethod
def _get_ext_mgr(namespace, *args, **kwargs):
def _catch_extension_load_error(mgr, ep, exc):

144
ceilometer/polling/non_openstack_dynamic_pollster.py
View File

@ -1,144 +0,0 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Non-OpenStack Dynamic pollster component
This component enables operators to create pollsters on the fly
via configuration for non-OpenStack APIs. This appraoch is quite
useful when adding metrics from APIs such as RadosGW into the Cloud
rating and billing modules.
"""
import copy
import requests
from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
from ceilometer.polling.dynamic_pollster import DynamicPollster
from oslo_log import log
LOG = log.getLogger(__name__)
class NonOpenStackApisDynamicPollster(DynamicPollster):
POLLSTER_REQUIRED_POLLSTER_FIELDS = ['module', 'authentication_object']
POLLSTER_OPTIONAL_POLLSTER_FIELDS = ['user_id_attribute',
'project_id_attribute',
'resource_id_attribute',
'barbican_secret_id',
'authentication_parameters'
]
def __init__(self, pollster_definitions, conf=None):
# Making sure that we do not change anything in parent classes
self.REQUIRED_POLLSTER_FIELDS = copy.deepcopy(
DynamicPollster.REQUIRED_POLLSTER_FIELDS)
self.OPTIONAL_POLLSTER_FIELDS = copy.deepcopy(
DynamicPollster.OPTIONAL_POLLSTER_FIELDS)
# Non-OpenStack dynamic pollster do not need the 'endpoint_type'.
self.REQUIRED_POLLSTER_FIELDS.remove('endpoint_type')
self.REQUIRED_POLLSTER_FIELDS += self.POLLSTER_REQUIRED_POLLSTER_FIELDS
self.OPTIONAL_POLLSTER_FIELDS += self.POLLSTER_OPTIONAL_POLLSTER_FIELDS
super(NonOpenStackApisDynamicPollster, self).__init__(
pollster_definitions, conf)
def set_default_values(self):
super(NonOpenStackApisDynamicPollster, self).set_default_values()
if 'user_id_attribute' not in self.pollster_definitions:
self.pollster_definitions['user_id_attribute'] = None
if 'project_id_attribute' not in self.pollster_definitions:
self.pollster_definitions['project_id_attribute'] = None
if 'resource_id_attribute' not in self.pollster_definitions:
self.pollster_definitions['resource_id_attribute'] = None
if 'barbican_secret_id' not in self.pollster_definitions:
self.pollster_definitions['barbican_secret_id'] = ""
if 'authentication_parameters' not in self.pollster_definitions:
self.pollster_definitions['authentication_parameters'] = ""
@property
def default_discovery(self):
return 'barbican:' + self.pollster_definitions['barbican_secret_id']
def internal_execute_request_get_samples(self, kwargs):
credentials = kwargs['resource']
override_credentials = self.pollster_definitions[
'authentication_parameters']
if override_credentials:
credentials = override_credentials
url = self.pollster_definitions['url_path']
authenticator_module_name = self.pollster_definitions['module']
authenticator_class_name = \
self.pollster_definitions['authentication_object']
imported_module = __import__(authenticator_module_name)
authenticator_class = getattr(imported_module,
authenticator_class_name)
authenticator_arguments = list(map(str.strip, credentials.split(",")))
authenticator_instance = authenticator_class(*authenticator_arguments)
resp = requests.get(
url,
auth=authenticator_instance)
if resp.status_code != requests.codes.ok:
raise NonOpenStackApisDynamicPollsterException(
"Error while executing request[%s]."
" Status[%s] and reason [%s]."
% (url, resp.status_code, resp.reason))
return resp, url
def execute_request_get_samples(self, **kwargs):
samples = super(NonOpenStackApisDynamicPollster,
self).execute_request_get_samples(**kwargs)
if samples:
user_id_attribute = self.pollster_definitions[
'user_id_attribute']
project_id_attribute = self.pollster_definitions[
'project_id_attribute']
resource_id_attribute = self.pollster_definitions[
'resource_id_attribute']
for sample in samples:
self.generate_new_attributes_in_sample(
sample, user_id_attribute, 'user_id')
self.generate_new_attributes_in_sample(
sample, project_id_attribute, 'project_id')
self.generate_new_attributes_in_sample(
sample, resource_id_attribute, 'id')
return samples
def generate_new_attributes_in_sample(
self, sample, attribute_key, new_attribute_key):
if attribute_key:
attribute_value = self.retrieve_attribute_nested_value(
sample, attribute_key)
LOG.debug("Mapped attribute [%s] to value [%s] in sample [%s].",
attribute_key, attribute_value, sample)
sample[new_attribute_key] = attribute_value

275
ceilometer/tests/unit/polling/test_dynamic_pollster.py
View File

@ -14,6 +14,7 @@
"""Tests for ceilometer/polling/dynamic_pollster.py
"""
from oslotest import base
from ceilometer.declarative import DynamicPollsterDefinitionException
from ceilometer.polling import dynamic_pollster
@ -23,13 +24,87 @@ import copy
import logging
import mock
from oslotest import base
import requests
LOG = logging.getLogger(__name__)
REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit',
'value_attribute', 'endpoint_type',
'url_path']
class SampleGenerator(object):
def __init__(self, samples_dict, turn_to_list=False):
self.turn_to_list = turn_to_list
self.samples_dict = {}
for k, v in samples_dict.items():
if isinstance(v, list):
self.samples_dict[k] = [0, v]
else:
self.samples_dict[k] = [0, [v]]
def get_next_sample_dict(self):
_dict = {}
for key in self.samples_dict.keys():
_dict[key] = self.get_next_sample(key)
if self.turn_to_list:
_dict = [_dict]
return _dict
def get_next_sample(self, key):
samples = self.samples_dict[key][1]
samples_next_iteration = self.samples_dict[key][0] % len(samples)
self.samples_dict[key][0] += 1
_sample = samples[samples_next_iteration]
if isinstance(_sample, SampleGenerator):
return _sample.get_next_sample_dict()
return _sample
class PagedSamplesGenerator(SampleGenerator):
def __init__(self, samples_dict, dict_name, page_link_name):
super(PagedSamplesGenerator, self).__init__(samples_dict)
self.dict_name = dict_name
self.page_link_name = page_link_name
self.response = {}
def generate_samples(self, page_base_link, page_links, last_page_size):
self.response.clear()
current_page_link = page_base_link
for page_link, page_size in page_links.items():
page_link = page_base_link + "/" + page_link
self.response[current_page_link] = {
self.page_link_name: page_link,
self.dict_name: self.populate_page(page_size)
}
current_page_link = page_link
self.response[current_page_link] = {
self.dict_name: self.populate_page(last_page_size)
}
def populate_page(self, page_size):
page = []
for item_number in range(0, page_size):
page.append(self.get_next_sample_dict())
return page
class PagedSamplesGeneratorHttpRequestMock(PagedSamplesGenerator):
def mock_request(self, url, **kwargs):
return_value = TestDynamicPollster.FakeResponse()
return_value.status_code = requests.codes.ok
return_value.json_object = self.response[url]
return return_value
class TestDynamicPollster(base.BaseTestCase):
class FakeResponse(object):
status_code = None
@ -42,7 +117,8 @@ class TestDynamicPollster(base.BaseTestCase):
raise requests.HTTPError("Mock HTTP error.", response=self)
class FakeManager(object):
_keystone = None
def __init__(self, keystone=None):
self._keystone = keystone
def setUp(self):
super(TestDynamicPollster, self).setUp()
@ -62,19 +138,61 @@ class TestDynamicPollster(base.BaseTestCase):
'old-metadata-name': "new-metadata-name"
},
'preserve_mapped_metadata': False}
self.pollster_definition_all_fields.update(
self.pollster_definition_only_required_fields)
self.multi_metric_pollster_definition = {
'name': "test-pollster.{category}", 'sample_type': "gauge",
'unit': "test", 'value_attribute': "[categories].ops",
'endpoint_type': "test", 'url_path': "v1/test/endpoint/fake"}
def execute_basic_asserts(self, pollster, pollster_definition):
self.assertEqual(pollster, pollster.obj)
self.assertEqual(pollster_definition['name'], pollster.name)
for key in pollster.REQUIRED_POLLSTER_FIELDS:
for key in REQUIRED_POLLSTER_FIELDS:
self.assertEqual(pollster_definition[key],
pollster.pollster_definitions[key])
self.assertEqual(pollster_definition, pollster.pollster_definitions)
@mock.patch('keystoneclient.v2_0.client.Client')
def test_skip_samples(self, keystone_mock):
generator = PagedSamplesGeneratorHttpRequestMock(samples_dict={
'volume': SampleGenerator(samples_dict={
'name': ['a', 'b', 'c', 'd', 'e', 'f', 'g', 'h'],
'tmp': ['ra', 'rb', 'rc', 'rd', 're', 'rf', 'rg', 'rh']},
turn_to_list=True),
'id': [1, 2, 3, 4, 5, 6, 7, 8],
'name': ['a1', 'b2', 'c3', 'd4', 'e5', 'f6', 'g7', 'h8']
}, dict_name='servers', page_link_name='server_link')
generator.generate_samples('http://test.com/v1/test-volumes', {
'marker=c3': 3,
'marker=f6': 3
}, 2)
keystone_mock.session.get.side_effect = generator.mock_request
fake_manager = self.FakeManager(keystone=keystone_mock)
pollster_definition = dict(self.multi_metric_pollster_definition)
pollster_definition['name'] = 'test-pollster.{name}'
pollster_definition['value_attribute'] = '[volume].tmp'
pollster_definition['skip_sample_values'] = ['rb']
pollster_definition['url_path'] = 'v1/test-volumes'
pollster_definition['response_entries_key'] = 'servers'
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['ra', 'rc'], list(map(lambda s: s.volume, samples)))
pollster_definition['name'] = 'test-pollster'
pollster_definition['value_attribute'] = 'name'
pollster_definition['skip_sample_values'] = ['b2']
pollster = dynamic_pollster.DynamicPollster(pollster_definition)
samples = pollster.get_samples(fake_manager, None, ['http://test.com'])
self.assertEqual(['a1', 'c3'], list(map(lambda s: s.volume, samples)))
def test_all_required_fields_ok(self):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
@ -112,8 +230,7 @@ class TestDynamicPollster(base.BaseTestCase):
False, pollster.pollster_definitions['preserve_mapped_metadata'])
def test_all_required_fields_exceptions(self):
for key in dynamic_pollster.\
DynamicPollster.REQUIRED_POLLSTER_FIELDS:
for key in REQUIRED_POLLSTER_FIELDS:
pollster_definition = copy.deepcopy(
self.pollster_definition_only_required_fields)
pollster_definition.pop(key)
@ -148,7 +265,8 @@ class TestDynamicPollster(base.BaseTestCase):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
self.assertEqual("endpoint:test", pollster.default_discovery)
self.assertEqual("endpoint:test", pollster.definitions.sample_gatherer
.default_discovery)
@mock.patch('keystoneclient.v2_0.client.Client')
def test_execute_request_get_samples_empty_response(self, client_mock):
@ -161,9 +279,10 @@ class TestDynamicPollster(base.BaseTestCase):
client_mock.session.get.return_value = return_value
samples = pollster.execute_request_get_samples(
keystone_client=client_mock,
resource="https://endpoint.server.name/")
samples = pollster.definitions.sample_gatherer. \
execute_request_get_samples(
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual(0, len(samples))
@ -179,9 +298,10 @@ class TestDynamicPollster(base.BaseTestCase):
client_mock.session.get.return_value = return_value
samples = pollster.execute_request_get_samples(
keystone_client=client_mock,
resource="https://endpoint.server.name/")
samples = pollster.definitions.sample_gatherer. \
execute_request_get_samples(
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual(3, len(samples))
@ -197,7 +317,8 @@ class TestDynamicPollster(base.BaseTestCase):
client_mock.session.get.return_value = return_value
exception = self.assertRaises(requests.HTTPError,
pollster.execute_request_get_samples,
pollster.definitions.sample_gatherer.
execute_request_get_samples,
keystone_client=client_mock,
resource="https://endpoint.server.name/")
self.assertEqual("Mock HTTP error.", str(exception))
@ -211,7 +332,8 @@ class TestDynamicPollster(base.BaseTestCase):
self.pollster_definition_only_required_fields['metadata_mapping'] = {}
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.generate_new_metadata_fields(metadata)
pollster.definitions.sample_extractor.generate_new_metadata_fields(
metadata, self.pollster_definition_only_required_fields)
self.assertEqual(metadata_before_call, metadata)
@ -227,7 +349,8 @@ class TestDynamicPollster(base.BaseTestCase):
'preserve_mapped_metadata'] = True
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.generate_new_metadata_fields(metadata)
pollster.definitions.sample_extractor.generate_new_metadata_fields(
metadata, self.pollster_definition_only_required_fields)
self.assertEqual(expected_metadata, metadata)
@ -244,7 +367,8 @@ class TestDynamicPollster(base.BaseTestCase):
'preserve_mapped_metadata'] = False
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
pollster.generate_new_metadata_fields(metadata)
pollster.definitions.sample_extractor.generate_new_metadata_fields(
metadata, self.pollster_definition_only_required_fields)
self.assertEqual(expected_clean_metadata, metadata)
@ -255,7 +379,8 @@ class TestDynamicPollster(base.BaseTestCase):
value_to_be_mapped = "test"
expected_value = value_to_be_mapped
value = pollster.execute_value_mapping(value_to_be_mapped)
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
@ -267,7 +392,8 @@ class TestDynamicPollster(base.BaseTestCase):
value_to_be_mapped = "test"
expected_value = -1
value = pollster.execute_value_mapping(value_to_be_mapped)
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
@ -282,7 +408,8 @@ class TestDynamicPollster(base.BaseTestCase):
value_to_be_mapped = "test"
expected_value = 0
value = pollster.execute_value_mapping(value_to_be_mapped)
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
@ -294,7 +421,8 @@ class TestDynamicPollster(base.BaseTestCase):
value_to_be_mapped = "test"
expected_value = 'new-value'
value = pollster.execute_value_mapping(value_to_be_mapped)
value = pollster.definitions.value_mapper. \
execute_value_mapping(value_to_be_mapped)
self.assertEqual(expected_value, value)
@ -306,7 +434,7 @@ class TestDynamicPollster(base.BaseTestCase):
self.assertEqual(None, next(samples))
@mock.patch('ceilometer.polling.dynamic_pollster.'
'DynamicPollster.execute_request_get_samples')
'PollsterSampleGatherer.execute_request_get_samples')
def test_get_samples_empty_samples(self, execute_request_get_samples_mock):
execute_request_get_samples_mock.side_effect = []
@ -346,7 +474,7 @@ class TestDynamicPollster(base.BaseTestCase):
return samples_list
@mock.patch.object(
dynamic_pollster.DynamicPollster,
dynamic_pollster.PollsterSampleGatherer,
'execute_request_get_samples',
fake_sample_list)
def test_get_samples(self):
@ -383,7 +511,8 @@ class TestDynamicPollster(base.BaseTestCase):
self.pollster_definition_only_required_fields)
response = [{"object1-attr1": 1}, {"object1-attr2": 2}]
entries = pollster.retrieve_entries_from_response(response)
entries = pollster.definitions.sample_gatherer. \
retrieve_entries_from_response(response)
self.assertEqual(response, entries)
@ -401,7 +530,8 @@ class TestDynamicPollster(base.BaseTestCase):
response = {"first": first_entries_from_response,
"second": second_entries_from_response}
entries = pollster.retrieve_entries_from_response(response)
entries = pollster.definitions.sample_gatherer. \
retrieve_entries_from_response(response)
self.assertEqual(first_entries_from_response, entries)
@ -418,7 +548,8 @@ class TestDynamicPollster(base.BaseTestCase):
response = {"first": first_entries_from_response,
"second": second_entries_from_response}
entries = pollster.retrieve_entries_from_response(response)
entries = pollster.definitions.sample_gatherer. \
retrieve_entries_from_response(response)
self.assertEqual(second_entries_from_response, entries)
@ -431,8 +562,8 @@ class TestDynamicPollster(base.BaseTestCase):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.retrieve_attribute_nested_value(
json_object, key)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(value, returned_value)
@ -447,8 +578,8 @@ class TestDynamicPollster(base.BaseTestCase):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.retrieve_attribute_nested_value(
json_object, key)
returned_value = pollster.definitions.sample_extractor. \
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(sub_value, returned_value)
@ -465,8 +596,8 @@ class TestDynamicPollster(base.BaseTestCase):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.retrieve_attribute_nested_value(
json_object, key)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(expected_value_after_operations, returned_value)
@ -496,7 +627,83 @@ class TestDynamicPollster(base.BaseTestCase):
pollster = dynamic_pollster.DynamicPollster(
self.pollster_definition_only_required_fields)
returned_value = pollster.retrieve_attribute_nested_value(json_object,
key)
returned_value = pollster.definitions.sample_extractor.\
retrieve_attribute_nested_value(json_object, key)
self.assertEqual(expected_value_after_operations, returned_value)
def fake_sample_multi_metric(self, keystone_client=None, resource=None):
multi_metric_sample_list = [
{"categories": [
{
"bytes_received": 0,
"bytes_sent": 0,
"category": "create_bucket",
"ops": 2,
"successful_ops": 2
},
{
"bytes_received": 0,
"bytes_sent": 2120428,
"category": "get_obj",
"ops": 46,
"successful_ops": 46
},
{
"bytes_received": 0,
"bytes_sent": 21484,
"category": "list_bucket",
"ops": 8,
"successful_ops": 8
},
{
"bytes_received": 6889056,
"bytes_sent": 0,
"category": "put_obj",
"ops": 46,
"successful_ops": 6
}],
"total": {
"bytes_received": 6889056,
"bytes_sent": 2141912,
"ops": 102,
"successful_ops": 106
},
"user": "test-user"}]
return multi_metric_sample_list
@mock.patch.object(
dynamic_pollster.PollsterSampleGatherer,
'execute_request_get_samples',
fake_sample_multi_metric)
def test_get_samples_multi_metric_pollster(self):
pollster = dynamic_pollster.DynamicPollster(
self.multi_metric_pollster_definition)
fake_manager = self.FakeManager()
samples = pollster.get_samples(
fake_manager, None, ["https://endpoint.server.name.com/"])
samples_list = list(samples)
self.assertEqual(4, len(samples_list))
create_bucket_sample = [
s for s in samples_list
if s.name == "test-pollster.create_bucket"][0]
get_obj_sample = [
s for s in samples_list
if s.name == "test-pollster.get_obj"][0]
list_bucket_sample = [
s for s in samples_list
if s.name == "test-pollster.list_bucket"][0]
put_obj_sample = [
s for s in samples_list
if s.name == "test-pollster.put_obj"][0]
self.assertEqual(2, create_bucket_sample.volume)
self.assertEqual(46, get_obj_sample.volume)
self.assertEqual(8, list_bucket_sample.volume)
self.assertEqual(46, put_obj_sample.volume)

17
ceilometer/tests/unit/polling/test_manager.py
View File

@ -27,9 +27,10 @@ from stevedore import extension
from ceilometer.compute import discovery as nova_discover
from ceilometer.hardware import discovery
from ceilometer.polling.dynamic_pollster import DynamicPollster
from ceilometer.polling.dynamic_pollster import \
NonOpenStackApisPollsterDefinition
from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions
from ceilometer.polling import manager
from ceilometer.polling.non_openstack_dynamic_pollster import \
NonOpenStackApisDynamicPollster
from ceilometer.polling import plugin_base
from ceilometer import sample
from ceilometer import service
@ -900,10 +901,10 @@ class TestPollingAgentPartitioned(BaseAgent):
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume", 'endpoint_type': "test",
'url_path': "v1/test/endpoint/fake"}
pollster = self.mgr.instantiate_dynamic_pollster(
pollster_definition_only_required_fields)
pollster = DynamicPollster(pollster_definition_only_required_fields)
self.assertIsInstance(pollster, DynamicPollster)
self.assertIsInstance(pollster.definitions,
SingleMetricPollsterDefinitions)
def test_instantiate_dynamic_pollster_non_openstack_api(self):
pollster_definition_only_required_fields = {
@ -911,7 +912,7 @@ class TestPollingAgentPartitioned(BaseAgent):
'value_attribute': "volume",
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
'authentication_object': "authentication_object"}
pollster = self.mgr.instantiate_dynamic_pollster(
pollster_definition_only_required_fields)
pollster = DynamicPollster(pollster_definition_only_required_fields)
self.assertIsInstance(pollster, NonOpenStackApisDynamicPollster)
self.assertIsInstance(pollster.definitions,
NonOpenStackApisPollsterDefinition)

220
ceilometer/tests/unit/polling/test_non_openstack_dynamic_pollster.py
View File

@ -22,13 +22,76 @@ import requests
from ceilometer.declarative import DynamicPollsterDefinitionException
from ceilometer.declarative import NonOpenStackApisDynamicPollsterException
from ceilometer.polling.dynamic_pollster import DynamicPollster
from ceilometer.polling.non_openstack_dynamic_pollster\
import NonOpenStackApisDynamicPollster
from ceilometer.polling.dynamic_pollster import MultiMetricPollsterDefinitions
from ceilometer.polling.dynamic_pollster import \
NonOpenStackApisPollsterDefinition
from ceilometer.polling.dynamic_pollster import PollsterSampleGatherer
from ceilometer.polling.dynamic_pollster import SingleMetricPollsterDefinitions
from oslotest import base
REQUIRED_POLLSTER_FIELDS = ['name', 'sample_type', 'unit', 'value_attribute',
'url_path', 'module', 'authentication_object']
OPTIONAL_POLLSTER_FIELDS = ['metadata_fields', 'skip_sample_values',
'value_mapping', 'default_value',
'metadata_mapping', 'preserve_mapped_metadata',
'response_entries_key', 'user_id_attribute',
'resource_id_attribute', 'barbican_secret_id',
'authentication_parameters',
'project_id_attribute']
ALL_POLLSTER_FIELDS = REQUIRED_POLLSTER_FIELDS + OPTIONAL_POLLSTER_FIELDS
def fake_sample_multi_metric(self, keystone_client=None, resource=None):
multi_metric_sample_list = [
{"user_id": "UID-U007",
"project_id": "UID-P007",
"id": "UID-007",
"categories": [
{
"bytes_received": 0,
"bytes_sent": 0,
"category": "create_bucket",
"ops": 2,
"successful_ops": 2
},
{
"bytes_received": 0,
"bytes_sent": 2120428,
"category": "get_obj",
"ops": 46,
"successful_ops": 46
},
{
"bytes_received": 0,
"bytes_sent": 21484,
"category": "list_bucket",
"ops": 8,
"successful_ops": 8
},
{
"bytes_received": 6889056,
"bytes_sent": 0,
"category": "put_obj",
"ops": 46,
"successful_ops": 6
}],
"total": {
"bytes_received": 6889056,
"bytes_sent": 2141912,
"ops": 102,
"successful_ops": 106
},
"user": "test-user"}]
return multi_metric_sample_list
class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
class FakeManager(object):
_keystone = None
class FakeResponse(object):
status_code = None
@ -42,6 +105,16 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
def setUp(self):
super(TestNonOpenStackApisDynamicPollster, self).setUp()
self.pollster_definition_only_openstack_required_single_metric = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume", "endpoint_type": "type",
'url_path': "v1/test/endpoint/fake"}
self.pollster_definition_only_openstack_required_multi_metric = {
'name': "test-pollster.{category}", 'sample_type': "gauge",
'unit': "test", 'value_attribute': "[categories].ops",
'url_path': "v1/test/endpoint/fake", "endpoint_type": "type"}
self.pollster_definition_only_required_fields = {
'name': "test-pollster", 'sample_type': "gauge", 'unit': "test",
'value_attribute': "volume",
@ -58,10 +131,17 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id',
'authentication_parameters': 'parameters'}
def test_all_fields(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)
self.pollster_definition_all_fields_multi_metrics = {
'name': "test-pollster.{category}", 'sample_type': "gauge",
'unit': "test", 'value_attribute': "[categories].ops",
'url_path': "v1/test/endpoint/fake", 'module': "module-name",
'authentication_object': "authentication_object",
'user_id_attribute': 'user_id',
'project_id_attribute': 'project_id',
'resource_id_attribute': 'id', 'barbican_secret_id': 'barbican_id',
'authentication_parameters': 'parameters'}
def test_all_fields(self):
all_required = ['module', 'authentication_object', 'name',
'sample_type', 'unit', 'value_attribute',
'url_path']
@ -74,27 +154,27 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
'response_entries_key'] + all_required
for field in all_required:
self.assertIn(field, pollster.REQUIRED_POLLSTER_FIELDS)
self.assertIn(field, REQUIRED_POLLSTER_FIELDS)
for field in all_optional:
self.assertIn(field, pollster.ALL_POLLSTER_FIELDS)
self.assertIn(field, ALL_POLLSTER_FIELDS)
def test_all_required_fields_exceptions(self):
pollster = NonOpenStackApisDynamicPollster(
self.pollster_definition_only_required_fields)
for key in pollster.REQUIRED_POLLSTER_FIELDS:
for key in REQUIRED_POLLSTER_FIELDS:
if key == 'module':
continue
pollster_definition = copy.deepcopy(
self.pollster_definition_only_required_fields)
pollster_definition.pop(key)
exception = self.assertRaises(DynamicPollsterDefinitionException,
NonOpenStackApisDynamicPollster,
pollster_definition)
exception = self.assertRaises(
DynamicPollsterDefinitionException, DynamicPollster,
pollster_definition, None,
[NonOpenStackApisPollsterDefinition])
self.assertEqual("Required fields ['%s'] not specified."
% key, exception.brief_message)
def test_set_default_values(self):
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
pollster_definitions = pollster.pollster_definitions
@ -106,7 +186,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
self.assertEqual('', pollster_definitions['authentication_parameters'])
def test_user_set_optional_parameters(self):
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_all_fields)
pollster_definitions = pollster.pollster_definitions
@ -122,23 +202,25 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
pollster_definitions['authentication_parameters'])
def test_default_discovery_empty_secret_id(self):
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
self.assertEqual("barbican:", pollster.default_discovery)
self.assertEqual("barbican:", pollster.definitions.sample_gatherer.
default_discovery)
def test_default_discovery_not_empty_secret_id(self):
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_all_fields)
self.assertEqual("barbican:barbican_id", pollster.default_discovery)
self.assertEqual("barbican:barbican_id", pollster.definitions.
sample_gatherer.default_discovery)
@mock.patch('requests.get')
def test_internal_execute_request_get_samples_status_code_ok(
self, get_mock):
sys.modules['module-name'] = mock.MagicMock()
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
return_value = self.FakeResponse()
@ -150,7 +232,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
kwargs = {'resource': "credentials"}
resp, url = pollster.internal_execute_request_get_samples(kwargs)
resp, url = pollster.definitions.sample_gatherer. \
internal_execute_request_get_samples(kwargs)
self.assertEqual(
self.pollster_definition_only_required_fields['url_path'], url)
@ -161,7 +244,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
self, get_mock):
sys.modules['module-name'] = mock.MagicMock()
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
for http_status_code in requests.status_codes._codes.keys():
@ -177,7 +260,8 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
kwargs = {'resource': "credentials"}
exception = self.assertRaises(
NonOpenStackApisDynamicPollsterException,
pollster.internal_execute_request_get_samples, kwargs)
pollster.definitions.sample_gatherer.
internal_execute_request_get_samples, kwargs)
self.assertEqual(
"NonOpenStackApisDynamicPollsterException"
@ -188,25 +272,28 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
http_status_code, return_value.reason), str(exception))
def test_generate_new_attributes_in_sample_attribute_key_none(self):
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
sample = {"test": "2"}
new_key = "new-key"
pollster.generate_new_attributes_in_sample(sample, None, new_key)
pollster.generate_new_attributes_in_sample(sample, "", new_key)
pollster.definitions.sample_gatherer. \
generate_new_attributes_in_sample(sample, None, new_key)
pollster.definitions.sample_gatherer. \
generate_new_attributes_in_sample(sample, "", new_key)
self.assertNotIn(new_key, sample)
def test_generate_new_attributes_in_sample(self):
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_only_required_fields)
sample = {"test": "2"}
new_key = "new-key"
pollster.generate_new_attributes_in_sample(sample, "test", new_key)
pollster.definitions.sample_gatherer. \
generate_new_attributes_in_sample(sample, "test", new_key)
self.assertIn(new_key, sample)
self.assertEqual(sample["test"], sample[new_key])
@ -220,7 +307,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
samples = [sample]
return samples
DynamicPollster.execute_request_get_samples =\
PollsterSampleGatherer.execute_request_get_samples = \
execute_request_get_samples_mock
self.pollster_definition_all_fields[
@ -230,11 +317,12 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
self.pollster_definition_all_fields[
'resource_id_attribute'] = 'resource_id_attribute'
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_all_fields)
params = {"d": "d"}
response = pollster.execute_request_get_samples(**params)
response = pollster.definitions.sample_gatherer. \
execute_request_get_samples(**params)
self.assertEqual(sample['user_id_attribute'],
response[0]['user_id'])
@ -252,7 +340,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
samples = [sample]
return samples
DynamicPollster.execute_request_get_samples =\
DynamicPollster.execute_request_get_samples = \
execute_request_get_samples_mock
self.pollster_definition_all_fields[
@ -262,7 +350,7 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
self.pollster_definition_all_fields[
'resource_id_attribute'] = None
pollster = NonOpenStackApisDynamicPollster(
pollster = DynamicPollster(
self.pollster_definition_all_fields)
params = {"d": "d"}
@ -271,3 +359,67 @@ class TestNonOpenStackApisDynamicPollster(base.BaseTestCase):
self.assertNotIn('user_id', response[0])
self.assertNotIn('project_id', response[0])
self.assertNotIn('id', response[0])
def test_pollster_defintions_instantiation(self):
def validate_definitions_instance(instance, isNonOpenstack,
isMultiMetric, isSingleMetric):
self.assertIs(
isinstance(instance, NonOpenStackApisPollsterDefinition),
isNonOpenstack)
self.assertIs(isinstance(instance, MultiMetricPollsterDefinitions),
isMultiMetric)
self.assertIs(
isinstance(instance, SingleMetricPollsterDefinitions),
isSingleMetric)
pollster = DynamicPollster(
self.pollster_definition_all_fields_multi_metrics)
validate_definitions_instance(pollster.definitions, True, True, False)
pollster = DynamicPollster(
self.pollster_definition_all_fields)
validate_definitions_instance(pollster.definitions, True, False, True)
pollster = DynamicPollster(
self.pollster_definition_only_openstack_required_multi_metric)
validate_definitions_instance(pollster.definitions, False, True, False)
pollster = DynamicPollster(
self.pollster_definition_only_openstack_required_single_metric)
validate_definitions_instance(pollster.definitions, False, False, True)
@mock.patch.object(
PollsterSampleGatherer,
'execute_request_get_samples',
fake_sample_multi_metric)
def test_get_samples_multi_metric_pollster(self):
pollster = DynamicPollster(
self.pollster_definition_all_fields_multi_metrics)
fake_manager = self.FakeManager()
samples = pollster.get_samples(
fake_manager, None, ["https://endpoint.server.name.com/"])
samples_list = list(samples)
self.assertEqual(4, len(samples_list))
create_bucket_sample = [
s for s in samples_list
if s.name == "test-pollster.create_bucket"][0]
get_obj_sample = [
s for s in samples_list
if s.name == "test-pollster.get_obj"][0]
list_bucket_sample = [
s for s in samples_list
if s.name == "test-pollster.list_bucket"][0]
put_obj_sample = [
s for s in samples_list
if s.name == "test-pollster.put_obj"][0]
self.assertEqual(2, create_bucket_sample.volume)
self.assertEqual(46, get_obj_sample.volume)
self.assertEqual(8, list_bucket_sample.volume)
self.assertEqual(46, put_obj_sample.volume)

207
doc/source/admin/telemetry-dynamic-pollster.rst
View File

@ -376,3 +376,210 @@ following:
* user_id_attribute
* project_id_attribute
* resource_id_attribute
Multi metric dynamic pollsters (handling attribute values with list of objects)
-------------------------------------------------------------------------------
The initial idea for this feature comes from the `categories` fields that we
can find in the `summary` object of the RadosGW API. Each user has a
`categories` attribute in the response; in the `categories` list, we can find
the object that presents in a granular fashion the consumption of different
RadosGW API operations such as GET, PUT, POST, and may others.
As follows we present an example of such a JSON response.
.. code-block:: json
{
"entries": [
{
"buckets": [
{
"bucket": "",
"categories": [
{
"bytes_received": 0,
"bytes_sent": 40,
"category": "list_buckets",
"ops": 2,
"successful_ops": 2
}
],
"epoch": 1572969600,
"owner": "user",
"time": "2019-11-21 00:00:00.000000Z"
},
{
"bucket": "-",
"categories": [
{
"bytes_received": 0,
"bytes_sent": 0,
"category": "get_obj",
"ops": 1,
"successful_ops": 0
}
],
"epoch": 1572969600,
"owner": "someOtherUser",
"time": "2019-11-21 00:00:00.000000Z"
}
]
}
]
"summary": [
{
"categories": [
{
"bytes_received": 0,
"bytes_sent": 0,
"category": "create_bucket",
"ops": 2,
"successful_ops": 2
},
{
"bytes_received": 0,
"bytes_sent": 2120428,
"category": "get_obj",
"ops": 46,
"successful_ops": 46
},
{
"bytes_received": 0,
"bytes_sent": 21484,
"category": "list_bucket",
"ops": 8,
"successful_ops": 8
},
{
"bytes_received": 6889056,
"bytes_sent": 0,
"category": "put_obj",
"ops": 46,
"successful_ops": 46
}
],
"total": {
"bytes_received": 6889056,
"bytes_sent": 2141912,
"ops": 102,
"successful_ops": 102
},
"user": "user"
},
{
"categories": [
{
"bytes_received": 0,
"bytes_sent": 0,
"category": "create_bucket",
"ops": 1,
"successful_ops": 1
},
{
"bytes_received": 0,
"bytes_sent": 0,
"category": "delete_obj",
"ops": 23,
"successful_ops": 23
},
{
"bytes_received": 0,
"bytes_sent": 5371,
"category": "list_bucket",
"ops": 2,
"successful_ops": 2
},
{
"bytes_received": 3444350,
"bytes_sent": 0,
"category": "put_obj",
"ops": 23,
"successful_ops": 23
}
],
"total": {
"bytes_received": 3444350,
"bytes_sent": 5371,
"ops": 49,
"successful_ops": 49
},
"user": "someOtherUser"
}
]
}
In that context, and having in mind that we have APIs with similar data
structures, we developed an extension for the dynamic pollster that enables
multi-metric processing for a single pollster. It works as follows.
The pollster name will contain a placeholder for the variable that
identifies the "submetric". E.g. `dynamic.radosgw.api.request.{category}`.
The placeholder `{category}` indicates the object's attribute that is in the
list of objects that we use to load the sub metric name. Then, we must use a
special notation in the `value_attribute` configuration to indicate that we are
dealing with a list of objects. This is achieved via `[]` (brackets); for
instance, in the `dynamic.radosgw.api.request.{category}`, we can use
`[categories].ops` as the `value_attribute`. This indicates that the value we
retrieve is a list of objects, and when the dynamic pollster processes it, we
want it (the pollster) to load the `ops` value for the sub metrics being
generated.
Examples on how to create multi-metric pollster to handle data from RadosGW API
are presented as follows:
.. code-block:: yaml
---
- name: "dynamic.radosgw.api.request.{category}"
sample_type: "gauge"
unit: "request"
value_attribute: "[categories].ops"
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
module: "awsauth"
authentication_object: "S3Auth"
authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>"
user_id_attribute: "user | value.split('$')[0]"
project_id_attribute: "user | value.split('$') | value[0]"
resource_id_attribute: "user | value.split('$') | value[0]"
response_entries_key: "summary"
- name: "dynamic.radosgw.api.request.successful_ops.{category}"
sample_type: "gauge"
unit: "request"
value_attribute: "[categories].successful_ops"
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
module: "awsauth"
authentication_object: "S3Auth"
authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>"
user_id_attribute: "user | value.split('$')[0]"
project_id_attribute: "user | value.split('$') | value[0]"
resource_id_attribute: "user | value.split('$') | value[0]"
response_entries_key: "summary"
- name: "dynamic.radosgw.api.bytes_sent.{category}"
sample_type: "gauge"
unit: "request"
value_attribute: "[categories].bytes_sent"
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
module: "awsauth"
authentication_object: "S3Auth"
authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>"
user_id_attribute: "user | value.split('$')[0]"
project_id_attribute: "user | value.split('$') | value[0]"
resource_id_attribute: "user | value.split('$') | value[0]"
response_entries_key: "summary"
- name: "dynamic.radosgw.api.bytes_received.{category}"
sample_type: "gauge"
unit: "request"
value_attribute: "[categories].bytes_received"
url_path: "http://rgw.service.stage.i.ewcs.ch/admin/usage"
module: "awsauth"
authentication_object: "S3Auth"
authentication_parameters: "<access_key>, <secret_key>,<rados_gateway_server>"
user_id_attribute: "user | value.split('$')[0]"
project_id_attribute: "user | value.split('$') | value[0]"
resource_id_attribute: "user | value.split('$') | value[0]"
response_entries_key: "summary"