openstack
/
ceilometer
Code Issues Proposed changes

Add configuration-driven conversion to Events 

Add configuration to control what traits get added to Events
from notifications, how those traits are defined, and what
notifications get converted (and eventually stored).

Change-Id: I2befc97b3d3b33851a5bfe709bf086895fd72cce
Implements: blueprint configurable-event-definitions
This commit is contained in:
Monsyne Dragon 2013-08-15 23:36:56 +00:00
parent a27d295bf0
commit 03ce87b300
19 changed files with 1930 additions and 91 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
bin/ceilometer-test-event.py Executable file
View File

@ -0,0 +1,87 @@
#!/usr/bin/env python
# -*- encoding: utf-8 -*-
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""Command line tool help you debug your event definitions.
Feed it a list of test notifications in json format, and it will show
you what events will be generated.
"""
import json
import sys
from oslo.config import cfg
from stevedore import extension
from ceilometer.event import converter
from ceilometer import service
cfg.CONF.register_cli_opts([
cfg.StrOpt('input-file',
short='i',
help='File to read test notifications from.'
' (Containing a json list of notifications.)'
' defaults to stdin.'),
cfg.StrOpt('output-file',
short='o',
help='File to write results to. Defaults to stdout'),
])
TYPES = {1: 'text',
2: 'int',
3: 'float',
4: 'datetime'}
service.prepare_service()
config_file = converter.get_config_file()
output_file = cfg.CONF.output_file
input_file = cfg.CONF.input_file
if output_file is None:
out = sys.stdout
else:
out = open(output_file, 'w')
if input_file is None:
notifications = json.load(sys.stdin)
else:
with open(input_file, 'r') as f:
notifications = json.load(f)
out.write("Definitions file: %s\n" % config_file)
out.write("Notifications tested: %s\n" % len(notifications))
event_converter = converter.setup_events(
extension.ExtensionManager(
namespace='ceilometer.event.trait_plugin'))
for notification in notifications:
event = event_converter.to_event(notification)
if event is None:
out.write("Dropped notification: %s\n" %
notification['message_id'])
continue
out.write("Event: %s at %s\n" % (event.event_name, event.generated))
for trait in event.traits:
dtype = TYPES[trait.dtype]
out.write(" Trait: name: %s, type: %s, value: %s\n" % (
trait.name, dtype, trait.value))

0
ceilometer/event/__init__.py Normal file
View File

397
ceilometer/event/converter.py Normal file
View File

@ -0,0 +1,397 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import fnmatch
import os
import jsonpath_rw
from oslo.config import cfg
import six
import yaml
from ceilometer.openstack.common import log
from ceilometer.openstack.common import timeutils
from ceilometer.storage import models
OPTS = [
cfg.StrOpt('definitions_cfg_file',
default="event_definitions.yaml",
help="Configuration file for event definitions"
),
cfg.BoolOpt('drop_unmatched_notifications',
default=False,
help='Drop notifications if no event definition matches. '
'(Otherwise, we convert them with just the default traits)'),
]
cfg.CONF.register_opts(OPTS, group='event')
LOG = log.getLogger(__name__)
class EventDefinitionException(Exception):
def __init__(self, message, definition_cfg):
super(EventDefinitionException, self).__init__(message)
self.definition_cfg = definition_cfg
def __str__(self):
return '%s %s: %s' % (self.__class__.__name__,
self.definition_cfg, self.message)
class TraitDefinition(object):
def __init__(self, name, trait_cfg, plugin_manager):
self.cfg = trait_cfg
self.name = name
type_name = trait_cfg.get('type', 'text')
if 'plugin' in trait_cfg:
plugin_cfg = trait_cfg['plugin']
if isinstance(plugin_cfg, six.string_types):
plugin_name = plugin_cfg
plugin_params = {}
else:
try:
plugin_name = plugin_cfg['name']
except KeyError:
raise EventDefinitionException(
_('Plugin specified, but no plugin name supplied for '
'trait %s') % name, self.cfg)
plugin_params = plugin_cfg.get('parameters')
if plugin_params is None:
plugin_params = {}
try:
plugin_ext = plugin_manager[plugin_name]
except KeyError:
raise EventDefinitionException(
_('No plugin named %(plugin)s available for '
'trait %(trait)s') % dict(plugin=plugin_name,
trait=name), self.cfg)
plugin_class = plugin_ext.plugin
self.plugin = plugin_class(**plugin_params)
else:
self.plugin = None
if 'fields' not in trait_cfg:
raise EventDefinitionException(
_("Required field in trait definition not specified: "
"'%s'") % 'fields',
self.cfg)
fields = trait_cfg['fields']
if not isinstance(fields, six.string_types):
# NOTE(mdragon): if not a string, we assume a list.
if len(fields) == 1:
fields = fields[0]
else:
fields = '|'.join('(%s)' % path for path in fields)
try:
self.fields = jsonpath_rw.parse(fields)
except Exception as e:
raise EventDefinitionException(
_("Parse error in JSONPath specification "
"'%(jsonpath)s' for %(trait)s: %(err)s")
% dict(jsonpath=fields, trait=name, err=e), self.cfg)
self.trait_type = models.Trait.get_type_by_name(type_name)
if self.trait_type is None:
raise EventDefinitionException(
_("Invalid trait type '%(type)s' for trait %(trait)s")
% dict(type=type_name, trait=name), self.cfg)
def _get_path(self, match):
if match.context is not None:
for path_element in self._get_path(match.context):
yield path_element
yield str(match.path)
def to_trait(self, notification_body):
values = [match for match in self.fields.find(notification_body)
if match.value is not None]
if self.plugin is not None:
value_map = [('.'.join(self._get_path(match)), match.value) for
match in values]
value = self.plugin.trait_value(value_map)
else:
value = values[0].value if values else None
if value is None:
return None
# NOTE(mdragon): some openstack projects (mostly Nova) emit ''
# for null fields for things like dates.
if self.trait_type != models.Trait.TEXT_TYPE and value == '':
return None
value = models.Trait.convert_value(self.trait_type, value)
return models.Trait(self.name, self.trait_type, value)
class EventDefinition(object):
DEFAULT_TRAITS = dict(
service=dict(type='text', fields='publisher_id'),
request_id=dict(type='text', fields='_context_request_id'),
tenant_id=dict(type='text', fields=['payload.tenant_id',
'_context_tenant']),
)
def __init__(self, definition_cfg, trait_plugin_mgr):
self._included_types = []
self._excluded_types = []
self.traits = dict()
self.cfg = definition_cfg
try:
event_type = definition_cfg['event_type']
traits = definition_cfg['traits']
except KeyError as err:
raise EventDefinitionException(
_("Required field %s not specified") % err.args[0], self.cfg)
if isinstance(event_type, six.string_types):
event_type = [event_type]
for t in event_type:
if t.startswith('!'):
self._excluded_types.append(t[1:])
else:
self._included_types.append(t)
if self._excluded_types and not self._included_types:
self._included_types.append('*')
for trait_name in self.DEFAULT_TRAITS:
self.traits[trait_name] = TraitDefinition(
trait_name,
self.DEFAULT_TRAITS[trait_name],
trait_plugin_mgr)
for trait_name in traits:
self.traits[trait_name] = TraitDefinition(
trait_name,
traits[trait_name],
trait_plugin_mgr)
def included_type(self, event_type):
for t in self._included_types:
if fnmatch.fnmatch(event_type, t):
return True
return False
def excluded_type(self, event_type):
for t in self._excluded_types:
if fnmatch.fnmatch(event_type, t):
return True
return False
def match_type(self, event_type):
return (self.included_type(event_type)
and not self.excluded_type(event_type))
@property
def is_catchall(self):
return '*' in self._included_types and not self._excluded_types
@staticmethod
def _extract_when(body):
"""Extract the generated datetime from the notification.
"""
# NOTE: I am keeping the logic the same as it was in the collector,
# However, *ALL* notifications should have a 'timestamp' field, it's
# part of the notification envelope spec. If this was put here because
# some openstack project is generating notifications without a
# timestamp, then that needs to be filed as a bug with the offending
# project (mdragon)
when = body.get('timestamp', body.get('_context_timestamp'))
if when:
return timeutils.normalize_time(timeutils.parse_isotime(when))
return timeutils.utcnow()
def to_event(self, notification_body):
event_type = notification_body['event_type']
message_id = notification_body['message_id']
when = self._extract_when(notification_body)
traits = (self.traits[t].to_trait(notification_body)
for t in self.traits)
# Only accept non-None value traits ...
traits = [trait for trait in traits if trait is not None]
event = models.Event(message_id, event_type, when, traits)
return event
class NotificationEventsConverter(object):
"""Notification Event Converter
The NotificationEventsConverter handles the conversion of Notifications
from openstack systems into Ceilometer Events.
The conversion is handled according to event definitions in a config file.
The config is a list of event definitions. Order is significant, a
notification will be processed according to the LAST definition that
matches it's event_type. (We use the last matching definition because that
allows you to use YAML merge syntax in the definitions file.)
Each definition is a dictionary with the following keys (all are required):
event_type: this is a list of notification event_types this definition
will handle. These can be wildcarded with unix shell glob
(not regex!) wildcards.
An exclusion listing (starting with a '!') will exclude any
types listed from matching. If ONLY exclusions are listed,
the definition will match anything not matching the
exclusions.
This item can also be a string, which will be taken as
equivalent to 1 item list.
Examples:
* ['compute.instance.exists'] will only match
compute.intance.exists notifications
* "compute.instance.exists" Same as above.
* ["image.create", "image.delete"] will match
image.create and image.delete, but not anything else.
* 'compute.instance.*" will match
compute.instance.create.start but not image.upload
* ['*.start','*.end', '!scheduler.*'] will match
compute.instance.create.start, and image.delete.end,
but NOT compute.instance.exists or
scheduler.run_instance.start
* '!image.*' matches any notification except image
notifications.
* ['*', '!image.*'] same as above.
traits: dictionary, The keys are trait names, the values are the trait
definitions
Each trait definiton is a dictionary with the following keys:
type (optional): The data type for this trait. (as a string)
Valid options are: 'text', 'int', 'float' and 'datetime'
defaults to 'text' if not specified.
fields: a path specification for the field(s) in the
notification you wish to extract. The paths can be
specified with a dot syntax (e.g. 'payload.host').
dictionary syntax (e.g. 'payload[host]') is also supported.
in either case, if the key for the field you are looking
for contains special charecters, like '.', it will need to
be quoted (with double or single quotes) like so:
"payload.image_meta.'org.openstack__1__architecture'"
The syntax used for the field specification is a variant
of JSONPath, and is fairly flexible.
(see: https://github.com/kennknowles/python-jsonpath-rw
for more info) Specifications can be written to match
multiple possible fields, the value for the trait will
be derived from the matching fields that exist and have
a non-null (i.e. is not None) values in the notification.
By default the value will be the first such field.
(plugins can alter that, if they wish)
This configuration value is normally a string, for
convenience, it can be specified as a list of
specifications, which will be OR'ed together (a union
query in jsonpath terms)
plugin (optional): (dictionary) with the following keys:
name: (string) name of a plugin to load
parameters: (optional) Dictionary of keyword args to pass
to the plugin on initialization.
See documentation on each plugin to see what
arguments it accepts.
For convenience, this value can also be specified as a
string, which is interpreted as a plugin name, which will
be loaded with no parameters.
"""
def __init__(self, events_config, trait_plugin_mgr, add_catchall=True):
self.definitions = [
EventDefinition(event_def, trait_plugin_mgr)
for event_def in reversed(events_config)]
if add_catchall and not any(d.is_catchall for d in self.definitions):
event_def = dict(event_type='*', traits={})
self.definitions.append(EventDefinition(event_def,
trait_plugin_mgr))
def to_event(self, notification_body):
event_type = notification_body['event_type']
message_id = notification_body['message_id']
edef = None
for d in self.definitions:
if d.match_type(event_type):
edef = d
break
if edef is None:
msg = (_('Dropping Notification %(type)s (uuid:%(msgid)s)')
% dict(type=event_type, msgid=message_id))
if cfg.CONF.event.drop_unmatched_notifications:
LOG.debug(msg)
else:
# If drop_unmatched_notifications is False, this should
# never happen. (mdragon)
LOG.error(msg)
return None
return edef.to_event(notification_body)
def get_config_file():
config_file = cfg.CONF.event.definitions_cfg_file
if not os.path.exists(config_file):
config_file = cfg.CONF.find_file(config_file)
return config_file
def setup_events(trait_plugin_mgr):
"""Setup the event definitions from yaml config file."""
config_file = get_config_file()
if config_file is not None:
LOG.debug(_("Event Definitions configuration file: %s"), config_file)
with open(config_file) as cf:
config = cf.read()
try:
events_config = yaml.safe_load(config)
except yaml.YAMLError as err:
if hasattr(err, 'problem_mark'):
mark = err.problem_mark
errmsg = (_("Invalid YAML syntax in Event Definitions file "
"%(file)s at line: %(line)s, column: %(column)s.")
% dict(file=config_file,
line=mark.line+1,
column=mark.column+1))
else:
errmsg = (_("YAML error reading Event Definitions file "
"%(file)s")
% dict(file=config_file))
LOG.error(errmsg)
raise
else:
LOG.debug(_("No Event Definitions configuration file found!"
" Using default config."))
events_config = []
LOG.info(_("Event Definitions: %s"), events_config)
allow_drop = cfg.CONF.event.drop_unmatched_notifications
return NotificationEventsConverter(events_config,
trait_plugin_mgr,
add_catchall=not allow_drop)

158
ceilometer/event/trait_plugins.py Normal file
View File

@ -0,0 +1,158 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import abc
import six
@six.add_metaclass(abc.ABCMeta)
class TraitPluginBase(object):
"""Base class for plugins that convert notification fields to
Trait values.
"""
def __init__(self, **kw):
"""Setup the trait plugin.
For each Trait definition a plugin is used on in a conversion
definition, a new instance of the plugin will be created, and
initialized with the parameters (if any) specified in the
config file.
:param kw: the parameters specified in the event definitions file.
"""
super(TraitPluginBase, self).__init__()
@abc.abstractmethod
def trait_value(self, match_list):
"""Convert a set of fields to a Trait value.
This method is called each time a trait is attempted to be extracted
from a notification. It will be called *even if* no matching fields
are found in the notification (in that case, the match_list will be
empty). If this method returns None, the trait *will not* be added to
the event. Any other value returned by this method will be used as
the value for the trait. Values returned will be coerced to the
appropriate type for the trait.
:param match_list: A list (may be empty if no matches) of *tuples*.
Each tuple is (field_path, value) where field_path
is the jsonpath for that specific field,
Example:
trait's fields definition: ['payload.foobar',
'payload.baz',
'payload.thing.*']
notification body:
{
'message_id': '12345',
'publisher': 'someservice.host',
'payload': {
'foobar': 'test',
'thing': {
'bar': 12,
'boing': 13,
}
}
}
match_list will be: [('payload.foobar','test'),
('payload.thing.bar',12),
('payload.thing.boing',13)]
Here is a plugin that emulates the default (no plugin) behavior:
class DefaultPlugin(TraitPluginBase):
"Plugin that returns the first field value"
def __init__(self, **kw):
super(DefaultPlugin, self).__init__()
def trait_value(self, match_list):
if not match_list:
return None
return match_list[0][1]
"""
class SplitterTraitPlugin(TraitPluginBase):
"""Plugin that splits a piece off of a string value."""
def __init__(self, separator=".", segment=0, max_split=None, **kw):
"""Setup how do split the field.
:param separator: String to split on. default "."
:param segment: Which segment to return. (int) default 0
:param max_split: Limit number of splits. Default: None (no limit)
"""
self.separator = separator
self.segment = segment
self.max_split = max_split
super(SplitterTraitPlugin, self).__init__(**kw)
def trait_value(self, match_list):
if not match_list:
return None
value = str(match_list[0][1])
if self.max_split is not None:
values = value.split(self.separator, self.max_split)
else:
values = value.split(self.separator)
try:
return values[self.segment]
except IndexError:
return None
class BitfieldTraitPlugin(TraitPluginBase):
"""Plugin to set flags on a bitfield."""
def __init__(self, initial_bitfield=0, flags=None, **kw):
"""Setup bitfield trait.
:param initial_bitfield: (int) initial value for the bitfield
Flags that are set will be OR'ed with this.
:param flags: List of dictionaries defining bitflags to set depending
on data in the notification. Each one has the following
keys:
path: jsonpath of field to match.
bit: (int) number of bit to set (lsb is bit 0)
value: set bit if corrosponding field's value
matches this. If value is not provided,
bit will be set if the field exists (and
is non-null), regardless of it's value.
"""
self.initial_bitfield = initial_bitfield
if flags is None:
flags = []
self.flags = flags
super(BitfieldTraitPlugin, self).__init__(**kw)
def trait_value(self, match_list):
matches = dict(match_list)
bitfield = self.initial_bitfield
for flagdef in self.flags:
path = flagdef['path']
bit = 2 ** int(flagdef['bit'])
if path in matches:
if 'value' in flagdef:
if matches[path] == flagdef['value']:
bitfield |= bit
else:
bitfield |= bit
return bitfield

55
ceilometer/notification.py
View File

@ -19,12 +19,12 @@
from oslo.config import cfg
from stevedore import extension
from ceilometer.event import converter as event_converter
from ceilometer.openstack.common import context
from ceilometer.openstack.common.gettextutils import _ # noqa
from ceilometer.openstack.common import log
from ceilometer.openstack.common.rpc import service as rpc_service
from ceilometer.openstack.common import service as os_service
from ceilometer.openstack.common import timeutils
from ceilometer import pipeline
from ceilometer import service
from ceilometer.storage import models
@ -73,6 +73,11 @@ class NotificationService(service.DispatchedService, rpc_service.Service):
),
)
LOG.debug('loading event definitions')
self.event_converter = event_converter.setup_events(
extension.ExtensionManager(
namespace='ceilometer.event.trait_plugin'))
self.notification_manager = \
extension.ExtensionManager(
namespace=self.NOTIFICATION_NAMESPACE,
@ -133,52 +138,24 @@ class NotificationService(service.DispatchedService, rpc_service.Service):
if cfg.CONF.notification.store_events:
self._message_to_event(notification)
@staticmethod
def _extract_when(body):
"""Extract the generated datetime from the notification.
"""
when = body.get('timestamp', body.get('_context_timestamp'))
if when:
return timeutils.normalize_time(timeutils.parse_isotime(when))
return timeutils.utcnow()
def _message_to_event(self, body):
"""Convert message to Ceilometer Event.
NOTE: this is currently based on the Nova notification format.
We will need to make this driver-based to support other formats.
NOTE: the rpc layer currently rips out the notification
delivery_info, which is critical to determining the
source of the notification. This will have to get added back later.
"""
message_id = body.get('message_id')
event_type = body['event_type']
when = self._extract_when(body)
LOG.debug(_('Saving event "%s"'), event_type)
event = self.event_converter.to_event(body)
publisher = body.get('publisher_id')
request_id = body.get('_context_request_id')
tenant_id = body.get('_context_tenant')
text = models.Trait.TEXT_TYPE
all_traits = [models.Trait('service', text, publisher),
models.Trait('request_id', text, request_id),
models.Trait('tenant_id', text, tenant_id),
]
# Only store non-None value traits ...
traits = [trait for trait in all_traits if trait.value is not None]
event = models.Event(message_id, event_type, when, traits)
problem_events = []
for dispatcher in self.dispatcher_manager:
problem_events.extend(dispatcher.obj.record_events(event))
if models.Event.UNKNOWN_PROBLEM in [x[0] for x in problem_events]:
# Don't ack the message, raise to requeue it
# if ack_on_error = False
raise UnableToSaveEventException()
if event is not None:
LOG.debug('Saving event "%s"', event.event_type)
problem_events = []
for dispatcher in self.dispatcher_manager:
problem_events.extend(dispatcher.obj.record_events(event))
if models.Event.UNKNOWN_PROBLEM in [x[0] for x in problem_events]:
# Don't ack the message, raise to requeue it
# if ack_on_error = False
raise UnableToSaveEventException()
def _process_notification_for_ext(self, ext, notification):
"""Wrapper for calling pipelines when a notification arrives

16
ceilometer/storage/models.py
View File

@ -18,6 +18,8 @@
"""Model classes for use in the storage API.
"""
from ceilometer.openstack.common import timeutils
class Model(object):
"""Base class for storage API models.
@ -97,6 +99,20 @@ class Trait(Model):
def __repr__(self):
return "<Trait: %s %d %s>" % (self.name, self.dtype, self.value)
@classmethod
def get_type_by_name(cls, type_name):
return getattr(cls, '%s_TYPE' % type_name.upper(), None)
@classmethod
def convert_value(cls, trait_type, value):
if trait_type is cls.INT_TYPE:
return int(value)
if trait_type is cls.FLOAT_TYPE:
return float(value)
if trait_type is cls.DATETIME_TYPE:
return timeutils.normalize_time(timeutils.parse_isotime(value))
return str(value)
class Resource(Model):
"""Something for which sample data has been collected.

0
ceilometer/tests/event/__init__.py Normal file
View File

734
ceilometer/tests/event/test_converter.py Normal file
View File

@ -0,0 +1,734 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
import jsonpath_rw
import mock
from oslo.config import cfg as oslo_cfg
import six
from ceilometer.event import converter
from ceilometer.openstack.common import timeutils
from ceilometer.storage import models
from ceilometer.tests import base
class ConverterBase(base.BaseTestCase):
def _create_test_notification(self, event_type, message_id, **kw):
return dict(event_type=event_type,
message_id=message_id,
priority="INFO",
publisher_id="compute.host-1-2-3",
timestamp="2013-08-08 21:06:37.803826",
payload=kw,
)
def assertIsValidEvent(self, event, notification):
self.assertIsNot(
None, event,
"Notification dropped unexpectedly:"
" %s" % str(notification))
self.assertIsInstance(event, models.Event)
def assertIsNotValidEvent(self, event, notification):
self.assertIs(
None, event,
"Notification NOT dropped when expected to be dropped:"
" %s" % str(notification))
def assertHasTrait(self, event, name, value=None, dtype=None):
traits = [trait for trait in event.traits if trait.name == name]
self.assertTrue(
len(traits) > 0,
"Trait %s not found in event %s" % (name, event))
trait = traits[0]
if value is not None:
self.assertEqual(trait.value, value)
if dtype is not None:
self.assertEqual(trait.dtype, dtype)
if dtype == models.Trait.INT_TYPE:
self.assertIsInstance(trait.value, int)
elif dtype == models.Trait.FLOAT_TYPE:
self.assertIsInstance(trait.value, float)
elif dtype == models.Trait.DATETIME_TYPE:
self.assertIsInstance(trait.value, datetime.datetime)
elif dtype == models.Trait.TEXT_TYPE:
self.assertIsInstance(trait.value, six.string_types)
def assertDoesNotHaveTrait(self, event, name):
traits = [trait for trait in event.traits if trait.name == name]
self.assertEqual(
len(traits), 0,
"Extra Trait %s found in event %s" % (name, event))
def assertHasDefaultTraits(self, event):
text = models.Trait.TEXT_TYPE
self.assertHasTrait(event, 'service', dtype=text)
def _cmp_tree(self, this, other):
if hasattr(this, 'right') and hasattr(other, 'right'):
return (self._cmp_tree(this.right, other.right) and
self._cmp_tree(this.left, other.left))
if not hasattr(this, 'right') and not hasattr(other, 'right'):
return this == other
return False
def assertPathsEqual(self, path1, path2):
self.assertTrue(self._cmp_tree(path1, path2),
'JSONPaths not equivalent %s %s' % (path1, path2))
class TestTraitDefinition(ConverterBase):
def setUp(self):
super(TestTraitDefinition, self).setUp()
self.n1 = self._create_test_notification(
"test.thing",
"uuid-for-notif-0001",
instance_uuid="uuid-for-instance-0001",
instance_id="id-for-instance-0001",
instance_uuid2=None,
instance_id2=None,
host='host-1-2-3',
bogus_date='',
image_meta=dict(
disk_gb='20',
thing='whatzit'),
foobar=50)
self.ext1 = mock.MagicMock(name='mock_test_plugin')
self.test_plugin_class = self.ext1.plugin
self.test_plugin = self.test_plugin_class()
self.test_plugin.trait_value.return_value = 'foobar'
self.ext1.reset_mock()
self.ext2 = mock.MagicMock(name='mock_nothing_plugin')
self.nothing_plugin_class = self.ext2.plugin
self.nothing_plugin = self.nothing_plugin_class()
self.nothing_plugin.trait_value.return_value = None
self.ext2.reset_mock()
self.fake_plugin_mgr = dict(test=self.ext1, nothing=self.ext2)
def test_to_trait_with_plugin(self):
cfg = dict(type='text',
fields=['payload.instance_id', 'payload.instance_uuid'],
plugin=dict(name='test'))
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.name, 'test_trait')
self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
self.assertEqual(t.value, 'foobar')
self.test_plugin_class.assert_called_once_with()
self.test_plugin.trait_value.assert_called_once_with([
('payload.instance_id', 'id-for-instance-0001'),
('payload.instance_uuid', 'uuid-for-instance-0001')])
def test_to_trait_null_match_with_plugin(self):
cfg = dict(type='text',
fields=['payload.nothere', 'payload.bogus'],
plugin=dict(name='test'))
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.name, 'test_trait')
self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
self.assertEqual(t.value, 'foobar')
self.test_plugin_class.assert_called_once_with()
self.test_plugin.trait_value.assert_called_once_with([])
def test_to_trait_with_plugin_null(self):
cfg = dict(type='text',
fields=['payload.instance_id', 'payload.instance_uuid'],
plugin=dict(name='nothing'))
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIs(None, t)
self.nothing_plugin_class.assert_called_once_with()
self.nothing_plugin.trait_value.assert_called_once_with([
('payload.instance_id', 'id-for-instance-0001'),
('payload.instance_uuid', 'uuid-for-instance-0001')])
def test_to_trait_with_plugin_with_parameters(self):
cfg = dict(type='text',
fields=['payload.instance_id', 'payload.instance_uuid'],
plugin=dict(name='test', parameters=dict(a=1, b='foo')))
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.name, 'test_trait')
self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
self.assertEqual(t.value, 'foobar')
self.test_plugin_class.assert_called_once_with(a=1, b='foo')
self.test_plugin.trait_value.assert_called_once_with([
('payload.instance_id', 'id-for-instance-0001'),
('payload.instance_uuid', 'uuid-for-instance-0001')])
def test_to_trait(self):
cfg = dict(type='text', fields='payload.instance_id')
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.name, 'test_trait')
self.assertEqual(t.dtype, models.Trait.TEXT_TYPE)
self.assertEqual(t.value, 'id-for-instance-0001')
cfg = dict(type='int', fields='payload.image_meta.disk_gb')
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.name, 'test_trait')
self.assertEqual(t.dtype, models.Trait.INT_TYPE)
self.assertEqual(t.value, 20)
def test_to_trait_multiple(self):
cfg = dict(type='text', fields=['payload.instance_id',
'payload.instance_uuid'])
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.value, 'id-for-instance-0001')
cfg = dict(type='text', fields=['payload.instance_uuid',
'payload.instance_id'])
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.value, 'uuid-for-instance-0001')
def test_to_trait_multiple_different_nesting(self):
cfg = dict(type='int', fields=['payload.foobar',
'payload.image_meta.disk_gb'])
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.value, 50)
cfg = dict(type='int', fields=['payload.image_meta.disk_gb',
'payload.foobar'])
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.value, 20)
def test_to_trait_some_null_multiple(self):
cfg = dict(type='text', fields=['payload.instance_id2',
'payload.instance_uuid'])
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.value, 'uuid-for-instance-0001')
def test_to_trait_some_missing_multiple(self):
cfg = dict(type='text', fields=['payload.not_here_boss',
'payload.instance_uuid'])
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIsInstance(t, models.Trait)
self.assertEqual(t.value, 'uuid-for-instance-0001')
def test_to_trait_missing(self):
cfg = dict(type='text', fields='payload.not_here_boss')
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIs(None, t)
def test_to_trait_null(self):
cfg = dict(type='text', fields='payload.instance_id2')
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIs(None, t)
def test_to_trait_empty_nontext(self):
cfg = dict(type='datetime', fields='payload.bogus_date')
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIs(None, t)
def test_to_trait_multiple_null_missing(self):
cfg = dict(type='text', fields=['payload.not_here_boss',
'payload.instance_id2'])
tdef = converter.TraitDefinition('test_trait', cfg,
self.fake_plugin_mgr)
t = tdef.to_trait(self.n1)
self.assertIs(None, t)
def test_missing_fields_config(self):
self.assertRaises(converter.EventDefinitionException,
converter.TraitDefinition,
'bogus_trait',
dict(),
self.fake_plugin_mgr)
def test_string_fields_config(self):
cfg = dict(fields='payload.test')
t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
self.assertPathsEqual(t.fields, jsonpath_rw.parse('payload.test'))
def test_list_fields_config(self):
cfg = dict(fields=['payload.test', 'payload.other'])
t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
self.assertPathsEqual(
t.fields,
jsonpath_rw.parse('(payload.test)|(payload.other)'))
def test_invalid_path_config(self):
#test invalid jsonpath...
cfg = dict(fields='payload.bogus(')
self.assertRaises(converter.EventDefinitionException,
converter.TraitDefinition,
'bogus_trait',
cfg,
self.fake_plugin_mgr)
def test_invalid_plugin_config(self):
#test invalid jsonpath...
cfg = dict(fields='payload.test', plugin=dict(bogus="true"))
self.assertRaises(converter.EventDefinitionException,
converter.TraitDefinition,
'test_trait',
cfg,
self.fake_plugin_mgr)
def test_unknown_plugin(self):
#test invalid jsonpath...
cfg = dict(fields='payload.test', plugin=dict(name='bogus'))
self.assertRaises(converter.EventDefinitionException,
converter.TraitDefinition,
'test_trait',
cfg,
self.fake_plugin_mgr)
def test_type_config(self):
cfg = dict(type='text', fields='payload.test')
t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
self.assertEqual(t.trait_type, models.Trait.TEXT_TYPE)
cfg = dict(type='int', fields='payload.test')
t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
self.assertEqual(t.trait_type, models.Trait.INT_TYPE)
cfg = dict(type='float', fields='payload.test')
t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
self.assertEqual(t.trait_type, models.Trait.FLOAT_TYPE)
cfg = dict(type='datetime', fields='payload.test')
t = converter.TraitDefinition('test_trait', cfg, self.fake_plugin_mgr)
self.assertEqual(t.trait_type, models.Trait.DATETIME_TYPE)
def test_invalid_type_config(self):
#test invalid jsonpath...
cfg = dict(type='bogus', fields='payload.test')
self.assertRaises(converter.EventDefinitionException,
converter.TraitDefinition,
'bogus_trait',
cfg,
self.fake_plugin_mgr)
class TestEventDefinition(ConverterBase):
def setUp(self):
super(TestEventDefinition, self).setUp()
self.traits_cfg = {
'instance_id': {
'type': 'text',
'fields': ['payload.instance_uuid',
'payload.instance_id'],
},
'host': {
'type': 'text',
'fields': 'payload.host',
},
}
self.test_notification1 = self._create_test_notification(
"test.thing",
"uuid-for-notif-0001",
instance_id="uuid-for-instance-0001",
host='host-1-2-3')
self.test_notification2 = self._create_test_notification(
"test.thing",
"uuid-for-notif-0002",
instance_id="uuid-for-instance-0002")
self.test_notification3 = self._create_test_notification(
"test.thing",
"uuid-for-notif-0003",
instance_id="uuid-for-instance-0003",
host=None)
self.fake_plugin_mgr = {}
def test_to_event(self):
dtype = models.Trait.TEXT_TYPE
cfg = dict(event_type='test.thing', traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
e = edef.to_event(self.test_notification1)
self.assertEqual(e.event_type, 'test.thing')
self.assertEqual(e.generated,
datetime.datetime(2013, 8, 8, 21, 6, 37, 803826))
self.assertHasDefaultTraits(e)
self.assertHasTrait(e, 'host', value='host-1-2-3', dtype=dtype)
self.assertHasTrait(e, 'instance_id',
value='uuid-for-instance-0001',
dtype=dtype)
def test_to_event_missing_trait(self):
dtype = models.Trait.TEXT_TYPE
cfg = dict(event_type='test.thing', traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
e = edef.to_event(self.test_notification2)
self.assertHasDefaultTraits(e)
self.assertHasTrait(e, 'instance_id',
value='uuid-for-instance-0002',
dtype=dtype)
self.assertDoesNotHaveTrait(e, 'host')
def test_to_event_null_trait(self):
dtype = models.Trait.TEXT_TYPE
cfg = dict(event_type='test.thing', traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
e = edef.to_event(self.test_notification3)
self.assertHasDefaultTraits(e)
self.assertHasTrait(e, 'instance_id',
value='uuid-for-instance-0003',
dtype=dtype)
self.assertDoesNotHaveTrait(e, 'host')
def test_bogus_cfg_no_traits(self):
bogus = dict(event_type='test.foo')
self.assertRaises(converter.EventDefinitionException,
converter.EventDefinition,
bogus,
self.fake_plugin_mgr)
def test_bogus_cfg_no_type(self):
bogus = dict(traits=self.traits_cfg)
self.assertRaises(converter.EventDefinitionException,
converter.EventDefinition,
bogus,
self.fake_plugin_mgr)
def test_included_type_string(self):
cfg = dict(event_type='test.thing', traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertEqual(len(edef._included_types), 1)
self.assertEqual(edef._included_types[0], 'test.thing')
self.assertEqual(len(edef._excluded_types), 0)
self.assertTrue(edef.included_type('test.thing'))
self.assertFalse(edef.excluded_type('test.thing'))
self.assertTrue(edef.match_type('test.thing'))
self.assertFalse(edef.match_type('random.thing'))
def test_included_type_list(self):
cfg = dict(event_type=['test.thing', 'other.thing'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertEqual(len(edef._included_types), 2)
self.assertEqual(len(edef._excluded_types), 0)
self.assertTrue(edef.included_type('test.thing'))
self.assertTrue(edef.included_type('other.thing'))
self.assertFalse(edef.excluded_type('test.thing'))
self.assertTrue(edef.match_type('test.thing'))
self.assertTrue(edef.match_type('other.thing'))
self.assertFalse(edef.match_type('random.thing'))
def test_excluded_type_string(self):
cfg = dict(event_type='!test.thing', traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertEqual(len(edef._included_types), 1)
self.assertEqual(edef._included_types[0], '*')
self.assertEqual(edef._excluded_types[0], 'test.thing')
self.assertEqual(len(edef._excluded_types), 1)
self.assertEqual(edef._excluded_types[0], 'test.thing')
self.assertTrue(edef.excluded_type('test.thing'))
self.assertTrue(edef.included_type('random.thing'))
self.assertFalse(edef.match_type('test.thing'))
self.assertTrue(edef.match_type('random.thing'))
def test_excluded_type_list(self):
cfg = dict(event_type=['!test.thing', '!other.thing'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertEqual(len(edef._included_types), 1)
self.assertEqual(len(edef._excluded_types), 2)
self.assertTrue(edef.excluded_type('test.thing'))
self.assertTrue(edef.excluded_type('other.thing'))
self.assertFalse(edef.excluded_type('random.thing'))
self.assertFalse(edef.match_type('test.thing'))
self.assertFalse(edef.match_type('other.thing'))
self.assertTrue(edef.match_type('random.thing'))
def test_mixed_type_list(self):
cfg = dict(event_type=['*.thing', '!test.thing', '!other.thing'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertEqual(len(edef._included_types), 1)
self.assertEqual(len(edef._excluded_types), 2)
self.assertTrue(edef.excluded_type('test.thing'))
self.assertTrue(edef.excluded_type('other.thing'))
self.assertFalse(edef.excluded_type('random.thing'))
self.assertFalse(edef.match_type('test.thing'))
self.assertFalse(edef.match_type('other.thing'))
self.assertFalse(edef.match_type('random.whatzit'))
self.assertTrue(edef.match_type('random.thing'))
def test_catchall(self):
cfg = dict(event_type=['*.thing', '!test.thing', '!other.thing'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertFalse(edef.is_catchall)
cfg = dict(event_type=['!other.thing'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertFalse(edef.is_catchall)
cfg = dict(event_type=['other.thing'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertFalse(edef.is_catchall)
cfg = dict(event_type=['*', '!other.thing'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertFalse(edef.is_catchall)
cfg = dict(event_type=['*'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertTrue(edef.is_catchall)
cfg = dict(event_type=['*', 'foo'],
traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
self.assertTrue(edef.is_catchall)
def test_extract_when(self):
now = timeutils.utcnow()
modified = now + datetime.timedelta(minutes=1)
timeutils.set_time_override(now)
body = {"timestamp": str(modified)}
when = converter.EventDefinition._extract_when(body)
self.assertTimestampEqual(modified, when)
body = {"_context_timestamp": str(modified)}
when = converter.EventDefinition._extract_when(body)
self.assertTimestampEqual(modified, when)
then = now + datetime.timedelta(hours=1)
body = {"timestamp": str(modified), "_context_timestamp": str(then)}
when = converter.EventDefinition._extract_when(body)
self.assertTimestampEqual(modified, when)
when = converter.EventDefinition._extract_when({})
self.assertTimestampEqual(now, when)
def test_default_traits(self):
cfg = dict(event_type='test.thing', traits={})
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
default_traits = converter.EventDefinition.DEFAULT_TRAITS.keys()
traits = set(edef.traits.keys())
for dt in default_traits:
self.assertIn(dt, traits)
self.assertEqual(len(edef.traits),
len(converter.EventDefinition.DEFAULT_TRAITS))
def test_traits(self):
cfg = dict(event_type='test.thing', traits=self.traits_cfg)
edef = converter.EventDefinition(cfg, self.fake_plugin_mgr)
default_traits = converter.EventDefinition.DEFAULT_TRAITS.keys()
traits = set(edef.traits.keys())
for dt in default_traits:
self.assertIn(dt, traits)
self.assertIn('host', traits)
self.assertIn('instance_id', traits)
self.assertEqual(len(edef.traits),
len(converter.EventDefinition.DEFAULT_TRAITS) + 2)
class TestNotificationConverter(ConverterBase):
def setUp(self):
super(TestNotificationConverter, self).setUp()
self.valid_event_def1 = [{
'event_type': 'compute.instance.create.*',
'traits': {
'instance_id': {
'type': 'text',
'fields': ['payload.instance_uuid',
'payload.instance_id'],
},
'host': {
'type': 'text',
'fields': 'payload.host',
},
},
}]
self.test_notification1 = self._create_test_notification(
"compute.instance.create.start",
"uuid-for-notif-0001",
instance_id="uuid-for-instance-0001",
host='host-1-2-3')
self.test_notification2 = self._create_test_notification(
"bogus.notification.from.mars",
"uuid-for-notif-0002",
weird='true',
host='cydonia')
self.fake_plugin_mgr = {}
def test_converter_missing_keys(self):
# test a malformed notification
now = timeutils.utcnow()
timeutils.set_time_override(now)
c = converter.NotificationEventsConverter(
[],
self.fake_plugin_mgr,
add_catchall=True)
message = {'event_type': "foo",
'message_id': "abc",
'publisher_id': "1"}
e = c.to_event(message)
self.assertIsValidEvent(e, message)
self.assertEqual(len(e.traits), 1)
self.assertEqual("foo", e.event_type)
self.assertEqual(now, e.generated)
def test_converter_with_catchall(self):
c = converter.NotificationEventsConverter(
self.valid_event_def1,
self.fake_plugin_mgr,
add_catchall=True)
self.assertEqual(len(c.definitions), 2)
e = c.to_event(self.test_notification1)
self.assertIsValidEvent(e, self.test_notification1)
self.assertEqual(len(e.traits), 3)
self.assertHasDefaultTraits(e)
self.assertHasTrait(e, 'instance_id')
self.assertHasTrait(e, 'host')
e = c.to_event(self.test_notification2)
self.assertIsValidEvent(e, self.test_notification2)
self.assertEqual(len(e.traits), 1)
self.assertHasDefaultTraits(e)
self.assertDoesNotHaveTrait(e, 'instance_id')
self.assertDoesNotHaveTrait(e, 'host')
def test_converter_without_catchall(self):
c = converter.NotificationEventsConverter(
self.valid_event_def1,
self.fake_plugin_mgr,
add_catchall=False)
self.assertEqual(len(c.definitions), 1)
e = c.to_event(self.test_notification1)
self.assertIsValidEvent(e, self.test_notification1)
self.assertEqual(len(e.traits), 3)
self.assertHasDefaultTraits(e)
self.assertHasTrait(e, 'instance_id')
self.assertHasTrait(e, 'host')
e = c.to_event(self.test_notification2)
self.assertIsNotValidEvent(e, self.test_notification2)
def test_converter_empty_cfg_with_catchall(self):
c = converter.NotificationEventsConverter(
[],
self.fake_plugin_mgr,
add_catchall=True)
self.assertEqual(len(c.definitions), 1)
e = c.to_event(self.test_notification1)
self.assertIsValidEvent(e, self.test_notification1)
self.assertEqual(len(e.traits), 1)
self.assertHasDefaultTraits(e)
e = c.to_event(self.test_notification2)
self.assertIsValidEvent(e, self.test_notification2)
self.assertEqual(len(e.traits), 1)
self.assertHasDefaultTraits(e)
def test_converter_empty_cfg_without_catchall(self):
c = converter.NotificationEventsConverter(
[],
self.fake_plugin_mgr,
add_catchall=False)
self.assertEqual(len(c.definitions), 0)
e = c.to_event(self.test_notification1)
self.assertIsNotValidEvent(e, self.test_notification1)
e = c.to_event(self.test_notification2)
self.assertIsNotValidEvent(e, self.test_notification2)
def test_setup_events_default_config(self):
def mock_exists(path):
return False
def mock_get_config_file():
return None
with mock.patch('ceilometer.event.converter.get_config_file',
mock_get_config_file):
oslo_cfg.CONF.set_override('drop_unmatched_notifications',
False, group='event')
with mock.patch('os.path.exists', mock_exists):
c = converter.setup_events(self.fake_plugin_mgr)
self.assertIsInstance(c, converter.NotificationEventsConverter)
self.assertEqual(len(c.definitions), 1)
self.assertTrue(c.definitions[0].is_catchall)
oslo_cfg.CONF.set_override('drop_unmatched_notifications',
True, group='event')
with mock.patch('os.path.exists', mock_exists):
c = converter.setup_events(self.fake_plugin_mgr)
self.assertIsInstance(c, converter.NotificationEventsConverter)
self.assertEqual(len(c.definitions), 0)

118
ceilometer/tests/event/test_trait_plugins.py Normal file
View File

@ -0,0 +1,118 @@
# -*- encoding: utf-8 -*-
#
# Copyright © 2013 Rackspace Hosting.
#
# Author: Monsyne Dragon <mdragon@rackspace.com>
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from ceilometer.event import trait_plugins
from ceilometer.openstack.common import test
class TestSplitterPlugin(test.BaseTestCase):
def setUp(self):
super(TestSplitterPlugin, self).setUp()
self.pclass = trait_plugins.SplitterTraitPlugin
def test_split(self):
param = dict(separator='-', segment=0)
plugin = self.pclass(**param)
match_list = [('test.thing', 'test-foobar-baz')]
value = plugin.trait_value(match_list)
self.assertEqual(value, 'test')
param = dict(separator='-', segment=1)
plugin = self.pclass(**param)
match_list = [('test.thing', 'test-foobar-baz')]
value = plugin.trait_value(match_list)
self.assertEqual(value, 'foobar')
param = dict(separator='-', segment=1, max_split=1)
plugin = self.pclass(**param)
match_list = [('test.thing', 'test-foobar-baz')]
value = plugin.trait_value(match_list)
self.assertEqual(value, 'foobar-baz')
def test_no_sep(self):
param = dict(separator='-', segment=0)
plugin = self.pclass(**param)
match_list = [('test.thing', 'test.foobar.baz')]
value = plugin.trait_value(match_list)
self.assertEqual(value, 'test.foobar.baz')
def test_no_segment(self):
param = dict(separator='-', segment=5)
plugin = self.pclass(**param)
match_list = [('test.thing', 'test-foobar-baz')]
value = plugin.trait_value(match_list)
self.assertIs(None, value)
def test_no_match(self):
param = dict(separator='-', segment=0)
plugin = self.pclass(**param)
match_list = []
value = plugin.trait_value(match_list)
self.assertIs(None, value)
class TestBitfieldPlugin(test.BaseTestCase):
def setUp(self):
super(TestBitfieldPlugin, self).setUp()
self.pclass = trait_plugins.BitfieldTraitPlugin
self.init = 0
self.params = dict(initial_bitfield=self.init,
flags=[dict(path='payload.foo', bit=0, value=42),
dict(path='payload.foo', bit=1, value=12),
dict(path='payload.thud', bit=1, value=23),
dict(path='thingy.boink', bit=4),
dict(path='thingy.quux', bit=6,
value="wokka"),
dict(path='payload.bar', bit=10,
value='test')])
def test_bitfield(self):
match_list = [('payload.foo', 12),
('payload.bar', 'test'),
('thingy.boink', 'testagain')]
plugin = self.pclass(**self.params)
value = plugin.trait_value(match_list)
self.assertEqual(value, 0x412)
def test_initial(self):
match_list = [('payload.foo', 12),
('payload.bar', 'test'),
('thingy.boink', 'testagain')]
self.params['initial_bitfield'] = 0x2000
plugin = self.pclass(**self.params)
value = plugin.trait_value(match_list)
self.assertEqual(value, 0x2412)
def test_no_match(self):
match_list = []
plugin = self.pclass(**self.params)
value = plugin.trait_value(match_list)
self.assertEqual(value, self.init)
def test_multi(self):
match_list = [('payload.foo', 12),
('payload.thud', 23),
('payload.bar', 'test'),
('thingy.boink', 'testagain')]
plugin = self.pclass(**self.params)
value = plugin.trait_value(match_list)
self.assertEqual(value, 0x412)

25
ceilometer/tests/storage/test_models.py
View File

@ -16,6 +16,8 @@
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from ceilometer.openstack.common import test
from ceilometer.storage import models
@ -54,3 +56,26 @@ class ModelTest(test.BaseTestCase):
def test_event_repr_no_traits(self):
x = models.Event("1", "name", "now", None)
self.assertEqual("<Event: 1, name, now, >", repr(x))
class TestTraitModel(test.BaseTestCase):
def test_convert_value(self):
v = models.Trait.convert_value(
models.Trait.INT_TYPE, '10')
self.assertEqual(v, 10)
self.assertIsInstance(v, int)
v = models.Trait.convert_value(
models.Trait.FLOAT_TYPE, '10')
self.assertEqual(v, 10.0)
self.assertIsInstance(v, float)
v = models.Trait.convert_value(
models.Trait.DATETIME_TYPE, '2013-08-08 21:05:37.123456')
self.assertEqual(v, datetime.datetime(2013, 8, 8, 21, 5, 37, 123456))
self.assertIsInstance(v, datetime.datetime)
v = models.Trait.convert_value(
models.Trait.TEXT_TYPE, 10)
self.assertEqual(v, "10")
self.assertIsInstance(v, str)

1
ceilometer/tests/test_collector.py
View File

@ -147,6 +147,7 @@ class TestCollector(tests_base.BaseTestCase):
self._verify_udp_socket(udp_socket)
@patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
@patch('ceilometer.event.converter.setup_events', mock.MagicMock())
def test_init_host(self):
# If we try to create a real RPC connection, init_host() never
# returns. Mock it out so we can establish the service

54
ceilometer/tests/test_notification.py
View File

@ -17,7 +17,6 @@
# under the License.
"""Tests for Ceilometer notify daemon."""
import datetime
import mock
from stevedore import extension
@ -26,7 +25,6 @@ from stevedore.tests import manager as test_manager
from ceilometer.compute import notifications
from ceilometer import notification
from ceilometer.openstack.common.fixture import config
from ceilometer.openstack.common import timeutils
from ceilometer.storage import models
from ceilometer.tests import base as tests_base
@ -89,6 +87,7 @@ class TestNotification(tests_base.BaseTestCase):
self.CONF.set_override("connection", "log://", group='database')
@mock.patch('ceilometer.pipeline.setup_pipeline', mock.MagicMock())
@mock.patch('ceilometer.event.converter.setup_events', mock.MagicMock())
def test_process_notification(self):
# If we try to create a real RPC connection, init_host() never
# returns. Mock it out so we can establish the service
@ -124,33 +123,12 @@ class TestNotification(tests_base.BaseTestCase):
self.srv.process_notification({})
self.assertTrue(fake_msg_to_event.called)
def test_message_to_event_missing_keys(self):
now = timeutils.utcnow()
timeutils.set_time_override(now)
message = {'event_type': "foo",
'message_id': "abc",
'publisher_id': "1"}
mock_dispatcher = mock.MagicMock()
self.srv.dispatcher_manager = test_manager.TestExtensionManager(
[extension.Extension('test',
None,
None,
mock_dispatcher
),
])
self.srv._message_to_event(message)
events = mock_dispatcher.record_events.call_args[0]
self.assertEqual(1, len(events))
event = events[0]
self.assertEqual("foo", event.event_type)
self.assertEqual(now, event.generated)
self.assertEqual(1, len(event.traits))
def test_message_to_event_duplicate(self):
self.CONF.set_override("store_events", True, group="notification")
mock_dispatcher = mock.MagicMock()
self.srv.event_converter = mock.MagicMock()
self.srv.event_converter.to_event.return_value = mock.MagicMock(
event_type='test.test')
self.srv.dispatcher_manager = test_manager.TestExtensionManager(
[extension.Extension('test',
None,
@ -166,6 +144,9 @@ class TestNotification(tests_base.BaseTestCase):
def test_message_to_event_bad_event(self):
self.CONF.set_override("store_events", True, group="notification")
mock_dispatcher = mock.MagicMock()
self.srv.event_converter = mock.MagicMock()
self.srv.event_converter.to_event.return_value = mock.MagicMock(
event_type='test.test')
self.srv.dispatcher_manager = test_manager.TestExtensionManager(
[extension.Extension('test',
None,
@ -178,24 +159,3 @@ class TestNotification(tests_base.BaseTestCase):
message = {'event_type': "foo", 'message_id': "abc"}
self.assertRaises(notification.UnableToSaveEventException,
self.srv._message_to_event, message)
def test_extract_when(self):
now = timeutils.utcnow()
modified = now + datetime.timedelta(minutes=1)
timeutils.set_time_override(now)
body = {"timestamp": str(modified)}
when = notification.NotificationService._extract_when(body)
self.assertTimestampEqual(modified, when)
body = {"_context_timestamp": str(modified)}
when = notification.NotificationService._extract_when(body)
self.assertTimestampEqual(modified, when)
then = now + datetime.timedelta(hours=1)
body = {"timestamp": str(modified), "_context_timestamp": str(then)}
when = notification.NotificationService._extract_when(body)
self.assertTimestampEqual(modified, when)
when = notification.NotificationService._extract_when({})
self.assertTimestampEqual(now, when)

16
doc/source/configuration.rst
View File

@ -169,6 +169,22 @@ database_connection hbase://$hbase-thrift-server:9090 Database conn
the Ceilometer services that use the database to allow the changes to take
affect, i.e. the collector and API services.
Event Conversion
================
The following options in the [event] configuration section affect the extraction of Event data from notifications.
================================== ====================================== ==============================================================
Parameter Default Note
================================== ====================================== ==============================================================
drop_unmatched_notifications False If set to True, then notifications with no matching event
definition will be dropped.
(Notifications will *only* be dropped if this is True)
definitions_cfg_file event_definitions.yaml Name of event definitions config file (yaml format)
================================== ====================================== ==============================================================
General options
===============

266
doc/source/events.rst Normal file
View File

@ -0,0 +1,266 @@
..
Copyright 2013 Rackspace Hosting.
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
=============================
Events and Event Processing
=============================
Events vs. Samples
==================
In addition to Meters, and related Sample data, Ceilometer can also process
Events. While a Sample represents a single numeric datapoint, driving a Meter
that represents the changes in that value over time, an Event represents the
state of an object in an OpenStack service (such as an Instance in Nova, or
an Image in Glance) at a point in time when something of interest has occurred.
This can include non-numeric data, such as an instance's flavor, or network
address.
In general, Events let you know when something has changed about an
object in an OpenStack system, such as the resize of an instance, or creation
of an image.
While Samples can be relatively cheap (small),
disposable (losing an individual sample datapoint won't matter much),
and fast, Events are larger, more informative, and should be handled more
consistently (you do not want to lose one).
Event Structure
===============
Events contain the following information:
event_type
A dotted string defining what event occured, such as "compute.instance.resize.start"
message_id
A UUID for this event.
generated
A timestamp of when the event occurred on the source system.
traits
A flat mapping of key-value pairs.
The event's Traits contain most of the details of the event. Traits are
typed, and can be strings, ints, floats, or datetimes.
Events from Notifications
=========================
Events are primarily created via the notifications system in OpenStack.
OpenStack systems, such as Nova, Glance, Neutron, etc. will emit
notifications in a JSON format to message queue when some notable action is
taken by that system. Ceilometer will consume such notifications from the
message queue, and process them.
The general philosophy of notifications in OpenStack is to emit any and all
data someone might need, and let the consumer filter out what they are not
interested in. In order to make processing simpler and more efficient,
the notifications are stored and processed within Ceilometer as Events.
The notification payload, which can be an arbitrarily complex JSON data
structure, is converted to a flat set of key-value pairs known as Traits.
This conversion is specified by a config file, so that only the specific
fields within the notification that are actually needed for processing the
event will have to be stored as Traits.
Note that the Event format is meant for efficient processing and querying,
there are other means available for archiving notifications (i.e. for audit
purposes, etc), possibly to different datastores.
Converting Notifications to Events
----------------------------------
In order to make it easier to allow users to extract what they need,
the conversion from Notifications to Events is driven by a
configuration file (specified by the flag definitions_cfg_file_ in
ceilometer.conf).
This includes descriptions of how to map fields in the notification body
to Traits, and optional plugins for doing any programmatic translations
(splitting a string, forcing case, etc.)
The mapping of notifications to events is defined per event_type, which
can be wildcarded. Traits are added to events if the corresponding fields
in the notification exist and are non-null. (As a special case, an empty
string is considered null for non-text traits. This is due to some openstack
projects (mostly Nova) using empty string for null dates.)
If the definitions file is not present, a warning will be logged, but an empty
set of definitions will be assumed. By default, any notifications that
do not have an event definition in the definitions file for them will be
converted to events with a set of minimal, default traits. This can be
changed by setting the flag drop_unmatched_notifications_ in the
ceilometer.conf file. If this is set to True, then any notifications
that don't have events defined for them in the file will be dropped.
This can be what you want, the notification system is quite chatty by design
(notifications philosophy is "tell us everything, we'll ignore what we don't
need"), so you may want to ignore the noisier ones if you don't use them.
.. _definitions_cfg_file: configuration.html#event-conversion
.. _drop_unmatched_notifications: configuration.html#event-conversion
There is a set of default traits (all are TEXT type) that will be added to
all events if the notification has the relevant data:
* service: (All notifications should have this) notification's publisher
* tenant_id
* request_id
These do not have to be specified in the event definition, they are
automatically added, but their definitions can be overridden for a given
event_type.
Definitions file format
-----------------------
The event definitions file is in YAML format. It consists of a list of event
definitions, which are mappings. Order is significant, the list of definitions
is scanned in *reverse* order (last definition in the file to the first),
to find a definition which matches the notification's event_type. That
definition will be used to generate the Event. The reverse ordering is done
because it is common to want to have a more general wildcarded definition
(such as "compute.instance.*" ) with a set of traits common to all of those
events, with a few more specific event definitions (like
"compute.instance.exists") afterward that have all of the above traits, plus
a few more. This lets you put the general definition first, followed by the
specific ones, and use YAML mapping include syntax to avoid copying all of the
trait definitions.
Event Definitions
-----------------
Each event definition is a mapping with two keys (both required):
event_type
This is a list (or a string, which will be taken as a 1 element
list) of event_types this definition will handle. These can be
wildcarded with unix shell glob syntax. An exclusion listing
(starting with a '!') will exclude any types listed from matching.
If ONLY exclusions are listed, the definition will match anything
not matching the exclusions.
traits
This is a mapping, the keys are the trait names, and the values are
trait definitions.
Trait Definitions
-----------------
Each trait definition is a mapping with the following keys:
type
(optional) The data type for this trait. (as a string). Valid
options are: *text*, *int*, *float*, and *datetime*.
defaults to *text* if not specified.
fields
A path specification for the field(s) in the notification you wish
to extract for this trait. Specifications can be written to match
multiple possible fields, the value for the trait will be derived
from the matching fields that exist and have a non-null values in
the notification. By default the value will be the first such field.
(plugins can alter that, if they wish). This is normally a string,
but, for convenience, it can be specified as a list of
specifications, which will match the fields for all of them. (See
`Field Path Specifications`_ for more info on this syntax.)
plugin
(optional) This is a mapping (For convenience, this value can also
be specified as a string, which is interpreted as the name of a
plugin to be loaded with no parameters) with the following keys
name
(string) name of a plugin to load
parameters
(optional) Mapping of keyword arguments to pass to the plugin on
initialization. (See documentation on each plugin to see what
arguments it accepts.)
Field Path Specifications
-------------------------
The path specifications define which fields in the JSON notification
body are extracted to provide the value for a given trait. The paths
can be specified with a dot syntax (e.g. "payload.host"). Square
bracket syntax (e.g. "payload[host]") is also supported. In either
case, if the key for the field you are looking for contains special
characters, like '.', it will need to be quoted (with double or single
quotes) like so:
payload.image_meta.'org.openstack__1__architecture'
The syntax used for the field specification is a variant of JSONPath,
and is fairly flexible. (see: https://github.com/kennknowles/python-jsonpath-rw for more info)
Example Definitions file
------------------------
::
---
- event_type: compute.instance.*
traits: &instance_traits
user_id:
fields: payload.user_id
instance_id:
fields: payload.instance_id
host:
fields: publisher_id
plugin:
name: split
parameters:
segment: 1
max_split: 1
service_name:
fields: publisher_id
plugin: split
instance_type_id:
type: int
fields: payload.instance_type_id
os_architecture:
fields: payload.image_meta.'org.openstack__1__architecture'
launched_at:
type: datetime
fields: payload.launched_at
deleted_at:
type: datetime
fields: payload.deleted_at
- event_type:
- compute.instance.exists
- compute.instance.update
traits:
<<: *instance_traits
audit_period_beginning:
type: datetime
fields: payload.audit_period_beginning
audit_period_ending:
type: datetime
fields: payload.audit_period_ending
Trait plugins
-------------
Trait plugins can be used to do simple programmatic conversions on the value in
a notification field, like splitting a string, lowercasing a value, converting
a screwball date into ISO format, or the like. They are initialized with the
parameters from the trait definition, if any, which can customize their
behavior for a given trait. They are called with a list of all matching fields
from the notification, so they can derive a value from multiple fields. The
plugin will be called even if there is no fields found matching the field
path(s), this lets a plugin set a default value, if needed. A plugin can also
reject a value by returning *None*, which will cause the trait not to be
added. If the plugin returns anything other than *None*, the trait's value
will be set from whatever the plugin returned (coerced to the appropriate type
for the trait).

1
doc/source/index.rst
View File

@ -48,6 +48,7 @@ Table of contents
architecture
measurements
events
install/index
configuration
webapi/index

25
etc/ceilometer/ceilometer.conf.sample
View File

@ -709,6 +709,17 @@
#record_history=true
[matchmaker_ring]
#
# Options defined in ceilometer.openstack.common.rpc.matchmaker_ring
#
# Matchmaker ring file (JSON) (string value)
# Deprecated group/name - [DEFAULT]/matchmaker_ringfile
#ringfile=/etc/oslo/matchmaker_ring.json
[rpc_notifier2]
#
@ -803,15 +814,19 @@
#udp_port=4952
[matchmaker_ring]
[event]
#
# Options defined in ceilometer.openstack.common.rpc.matchmaker_ring
# Options defined in ceilometer.event.converter
#
# Matchmaker ring file (JSON) (string value)
# Deprecated group/name - [DEFAULT]/matchmaker_ringfile
#ringfile=/etc/oslo/matchmaker_ring.json
# Configuration file for event definitions (string value)
#definitions_cfg_file=event_definitions.yaml
# Drop notifications if no event definition matches.
# (Otherwise, we convert them with just the default traits)
# (boolean value)
#drop_unmatched_notifications=false
[matchmaker_redis]

63
etc/ceilometer/event_definitions.yaml Normal file
View File

@ -0,0 +1,63 @@
---
- event_type: compute.instance.*
traits: &instance_traits
tenant_id:
fields: payload.tenant_id
user_id:
fields: payload.user_id
instance_id:
fields: payload.instance_id
host:
fields: publisher_id
plugin:
name: split
parameters:
segment: 1
max_split: 1
service:
fields: publisher_id
plugin: split
memory_mb:
type: int
fields: payload.memory_mb
disk_gb:
type: int
fields: payload.disk_gb
root_gb:
type: int
fields: payload.root_gb
ephemeral_gb:
type: int
fields: payload.ephemeral_gb
vcpus:
type: int
fields: payload.vcpus
instance_type_id:
type: int
fields: payload.instance_type_id
instance_type:
fields: payload.instance_type
state:
fields: payload.state
os_architecture:
fields: payload.image_meta.'org.openstack__1__architecture'
os_version:
fields: payload.image_meta.'org.openstack__1__os_version'
os_distro:
fields: payload.image_meta.'org.openstack__1__os_distro'
launched_at:
type: datetime
fields: payload.launched_at
deleted_at:
type: datetime
fields: payload.deleted_at
- event_type: compute.instance.exists
traits:
<<: *instance_traits
audit_period_beginning:
type: datetime
fields: payload.audit_period_beginning
audit_period_ending:
type: datetime
fields: payload.audit_period_ending

1
requirements.txt
View File

@ -5,6 +5,7 @@ eventlet>=0.13.0
Flask>=0.10,<1.0
happybase>=0.4
iso8601>=0.1.8
jsonpath-rw>=1.2.0,<2.0
kombu>=2.4.8
lockfile>=0.8
lxml>=2.3

4
setup.cfg
View File

@ -112,6 +112,10 @@ ceilometer.alarm.notifier =
http = ceilometer.alarm.notifier.rest:RestAlarmNotifier
https = ceilometer.alarm.notifier.rest:RestAlarmNotifier
ceilometer.event.trait_plugin =
split = ceilometer.event.trait_plugins:SplitterTraitPlugin
bitfield = ceilometer.event.trait_plugins:BitfieldTraitPlugin
paste.filter_factory =
swift = ceilometer.objectstore.swift_middleware:filter_factory