openstack
/
panko
Code Issues Proposed changes

remove event redundancy 

- we don't need to prefix 'event/storage' because that's all panko
does. cleanup inline with aodh.
- test_hbase_table_utils is not used anywhere and last reference
was from icehouse.
- drop a few places that use 'event_*' prefix since it wastes line
space

Change-Id: I348f5a3dfb59682589b4912eaed4e4b8abb15ebb
This commit is contained in:
gord chung 2017-01-12 19:19:25 +00:00
parent 4a498085c2
commit 41f345f552
25 changed files with 270 additions and 336 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
panko/api/controllers/v2/events.py
View File

@ -31,8 +31,8 @@ import wsmeext.pecan as wsme_pecan
from panko.api.controllers.v2 import base
from panko.api.controllers.v2 import utils as v2_utils
from panko.api import rbac
from panko.event import storage
from panko.event.storage import models as event_models
from panko import storage
from panko.storage import models as event_models
from panko.i18n import _, _LE
LOG = log.getLogger(__name__)

2
panko/dispatcher/database.py
View File

@ -16,10 +16,10 @@
from oslo_log import log
from oslo_utils import timeutils
from panko.event.storage import models
from panko.i18n import _LE
from panko import service
from panko import storage
from panko.storage import models
LOG = log.getLogger(__name__)

0
panko/event/__init__.py
View File

57
panko/event/storage/__init__.py
View File

@ -1,57 +0,0 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import six
from panko import utils
class EventFilter(object):
"""Properties for building an Event query.
:param start_timestamp: UTC start datetime (mandatory)
:param end_timestamp: UTC end datetime (mandatory)
:param event_type: the name of the event. None for all.
:param message_id: the message_id of the event. None for all.
:param admin_proj: the project_id of admin role. None if non-admin user.
:param traits_filter: the trait filter dicts, all of which are optional.
This parameter is a list of dictionaries that specify trait values:
.. code-block:: python
{'key': <key>,
'string': <value>,
'integer': <value>,
'datetime': <value>,
'float': <value>,
'op': <eq, lt, le, ne, gt or ge> }
"""
def __init__(self, start_timestamp=None, end_timestamp=None,
event_type=None, message_id=None, traits_filter=None,
admin_proj=None):
self.start_timestamp = utils.sanitize_timestamp(start_timestamp)
self.end_timestamp = utils.sanitize_timestamp(end_timestamp)
self.message_id = message_id
self.event_type = event_type
self.traits_filter = traits_filter or []
self.admin_proj = admin_proj
def __repr__(self):
return ("<EventFilter(start_timestamp: %s,"
" end_timestamp: %s,"
" event_type: %s,"
" traits: %s)>" %
(self.start_timestamp,
self.end_timestamp,
self.event_type,
six.text_type(self.traits_filter)))

100
panko/event/storage/base.py
View File

@ -1,100 +0,0 @@
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import panko
class Connection(object):
"""Base class for event storage system connections."""
# A dictionary representing the capabilities of this driver.
CAPABILITIES = {
'events': {'query': {'simple': False}},
}
STORAGE_CAPABILITIES = {
'storage': {'production_ready': False},
}
@staticmethod
def __init__(url, conf):
pass
@staticmethod
def upgrade():
"""Migrate the database to `version` or the most recent version."""
@staticmethod
def clear():
"""Clear database."""
@staticmethod
def record_events(events):
"""Write the events to the backend storage system.
:param events: a list of model.Event objects.
"""
raise panko.NotImplementedError('Events not implemented.')
@staticmethod
def get_events(event_filter, pagination=None):
"""Return an iterable of model.Event objects."""
raise panko.NotImplementedError('Events not implemented.')
@staticmethod
def get_event_types():
"""Return all event types as an iterable of strings."""
raise panko.NotImplementedError('Events not implemented.')
@staticmethod
def get_trait_types(event_type):
"""Return a dictionary containing the name and data type of the trait.
Only trait types for the provided event_type are
returned.
:param event_type: the type of the Event
"""
raise panko.NotImplementedError('Events not implemented.')
@staticmethod
def get_traits(event_type, trait_type=None):
"""Return all trait instances associated with an event_type.
If trait_type is specified, only return instances of that trait type.
:param event_type: the type of the Event to filter by
:param trait_type: the name of the Trait to filter by
"""
raise panko.NotImplementedError('Events not implemented.')
@classmethod
def get_capabilities(cls):
"""Return an dictionary with the capabilities of each driver."""
return cls.CAPABILITIES
@classmethod
def get_storage_capabilities(cls):
"""Return a dictionary representing the performance capabilities.
This is needed to evaluate the performance of each driver.
"""
return cls.STORAGE_CAPABILITIES
@staticmethod
def clear_expired_event_data(ttl):
"""Clear expired data from the backend storage system.
Clearing occurs according to the time-to-live.
:param ttl: Number of seconds to keep records for.
"""
raise panko.NotImplementedError('Clearing events not implemented')

48
panko/storage/__init__.py
View File

@ -17,10 +17,12 @@
from oslo_config import cfg
from oslo_log import log
import six
import six.moves.urllib.parse as urlparse
from stevedore import driver
import tenacity
from panko import utils
LOG = log.getLogger(__name__)
@ -77,7 +79,49 @@ def get_connection(url, conf):
# 'dialect+driver'. Handle the case where driver is specified.
engine_name = connection_scheme.split('+')[0]
# NOTE: translation not applied bug #1446983
LOG.debug('looking for %(name)r driver in panko.event.storage',
LOG.debug('looking for %(name)r driver in panko.storage',
{'name': engine_name})
mgr = driver.DriverManager('panko.event.storage', engine_name)
mgr = driver.DriverManager('panko.storage', engine_name)
return mgr.driver(url, conf)
class EventFilter(object):
"""Properties for building an Event query.
:param start_timestamp: UTC start datetime (mandatory)
:param end_timestamp: UTC end datetime (mandatory)
:param event_type: the name of the event. None for all.
:param message_id: the message_id of the event. None for all.
:param admin_proj: the project_id of admin role. None if non-admin user.
:param traits_filter: the trait filter dicts, all of which are optional.
This parameter is a list of dictionaries that specify trait values:
.. code-block:: python
{'key': <key>,
'string': <value>,
'integer': <value>,
'datetime': <value>,
'float': <value>,
'op': <eq, lt, le, ne, gt or ge> }
"""
def __init__(self, start_timestamp=None, end_timestamp=None,
event_type=None, message_id=None, traits_filter=None,
admin_proj=None):
self.start_timestamp = utils.sanitize_timestamp(start_timestamp)
self.end_timestamp = utils.sanitize_timestamp(end_timestamp)
self.message_id = message_id
self.event_type = event_type
self.traits_filter = traits_filter or []
self.admin_proj = admin_proj
def __repr__(self):
return ("<EventFilter(start_timestamp: %s,"
" end_timestamp: %s,"
" event_type: %s,"
" traits: %s)>" %
(self.start_timestamp,
self.end_timestamp,
self.event_type,
six.text_type(self.traits_filter)))

87
panko/storage/base.py
View File

@ -17,6 +17,8 @@
import six
import panko
class Model(object):
"""Base class for storage API models."""
@ -42,3 +44,88 @@ class Model(object):
def __ne__(self, other):
return not self.__eq__(other)
class Connection(object):
"""Base class for event storage system connections."""
# A dictionary representing the capabilities of this driver.
CAPABILITIES = {
'events': {'query': {'simple': False}},
}
STORAGE_CAPABILITIES = {
'storage': {'production_ready': False},
}
@staticmethod
def __init__(url, conf):
pass
@staticmethod
def upgrade():
"""Migrate the database to `version` or the most recent version."""
@staticmethod
def clear():
"""Clear database."""
@staticmethod
def record_events(events):
"""Write the events to the backend storage system.
:param events: a list of model.Event objects.
"""
raise panko.NotImplementedError('Events not implemented.')
@staticmethod
def get_events(event_filter, pagination=None):
"""Return an iterable of model.Event objects."""
@staticmethod
def get_event_types():
"""Return all event types as an iterable of strings."""
raise panko.NotImplementedError('Events not implemented.')
@staticmethod
def get_trait_types(event_type):
"""Return a dictionary containing the name and data type of the trait.
Only trait types for the provided event_type are
returned.
:param event_type: the type of the Event
"""
raise panko.NotImplementedError('Events not implemented.')
@staticmethod
def get_traits(event_type, trait_type=None):
"""Return all trait instances associated with an event_type.
If trait_type is specified, only return instances of that trait type.
:param event_type: the type of the Event to filter by
:param trait_type: the name of the Trait to filter by
"""
raise panko.NotImplementedError('Events not implemented.')
@classmethod
def get_capabilities(cls):
"""Return an dictionary with the capabilities of each driver."""
return cls.CAPABILITIES
@classmethod
def get_storage_capabilities(cls):
"""Return a dictionary representing the performance capabilities.
This is needed to evaluate the performance of each driver.
"""
return cls.STORAGE_CAPABILITIES
@staticmethod
def clear_expired_event_data(ttl):
"""Clear expired data from the backend storage system.
Clearing occurs according to the time-to-live.
:param ttl: Number of seconds to keep records for.
"""
raise panko.NotImplementedError('Clearing events not implemented')

4
panko/event/storage/impl_elasticsearch.py → panko/storage/impl_elasticsearch.py
View File

@ -21,8 +21,8 @@ from oslo_utils import netutils
from oslo_utils import timeutils
import six
from panko.event.storage import base
from panko.event.storage import models
from panko.storage import base
from panko.storage import models
from panko.i18n import _LE, _LI, _LW
from panko import storage
from panko import utils

4
panko/event/storage/impl_hbase.py → panko/storage/impl_hbase.py
View File

@ -15,8 +15,8 @@ import operator
from oslo_log import log
from panko.event.storage import base
from panko.event.storage import models
from panko.storage import base
from panko.storage import models
from panko.i18n import _LE, _LW
from panko.storage.hbase import base as hbase_base
from panko.storage.hbase import utils as hbase_utils

2
panko/event/storage/impl_log.py → panko/storage/impl_log.py
View File

@ -13,8 +13,8 @@
from oslo_log import log
from panko.event.storage import base
from panko.i18n import _LI
from panko.storage import base
LOG = log.getLogger(__name__)

2
panko/event/storage/impl_mongodb.py → panko/storage/impl_mongodb.py
View File

@ -15,9 +15,9 @@
from oslo_log import log
import pymongo
from panko.event.storage import pymongo_base
from panko import storage
from panko.storage.mongo import utils as pymongo_utils
from panko.storage import pymongo_base
LOG = log.getLogger(__name__)

7
panko/event/storage/impl_sqlalchemy.py → panko/storage/impl_sqlalchemy.py
View File

@ -24,9 +24,8 @@ from oslo_log import log
from oslo_utils import timeutils
import sqlalchemy as sa
from panko.event import storage as event_storage
from panko.event.storage import base
from panko.event.storage import models as api_models
from panko.storage import base
from panko.storage import models as api_models
from panko.i18n import _LE, _LI
from panko import storage
from panko.storage.sqlalchemy import models
@ -213,7 +212,7 @@ class Connection(base.Connection):
marker = None
if pagination.get('marker'):
marker_filter = event_storage.EventFilter(
marker_filter = storage.EventFilter(
message_id=pagination.get('marker'))
markers = list(self.get_events(marker_filter))
if markers:

0
panko/event/storage/models.py → panko/storage/models.py
View File

4
panko/event/storage/pymongo_base.py → panko/storage/pymongo_base.py
View File

@ -15,8 +15,8 @@
from oslo_log import log
import pymongo
from panko.event.storage import base
from panko.event.storage import models
from panko.storage import base
from panko.storage import models
from panko.i18n import _LE, _LI, _LW
from panko.storage.mongo import utils as pymongo_utils
from panko import utils

10
panko/tests/functional/api/v2/test_acl_scenarios.py
View File

@ -24,7 +24,7 @@ import six
import webtest
from panko.api import app
from panko.event.storage import models as ev_model
from panko.storage import models
from panko.tests.functional.api import v2
VALID_TOKEN = uuidutils.generate_uuid(dashed=False)
@ -87,11 +87,11 @@ class TestBaseApiEventRBAC(v2.FunctionalTest):
def setUp(self):
super(TestBaseApiEventRBAC, self).setUp()
traits = [ev_model.Trait('project_id', 1, 'project-good'),
ev_model.Trait('user_id', 1, 'user-good')]
traits = [models.Trait('project_id', 1, 'project-good'),
models.Trait('user_id', 1, 'user-good')]
self.message_id = uuidutils.generate_uuid()
ev = ev_model.Event(self.message_id, 'event_type',
datetime.datetime.now(), traits, {})
ev = models.Event(self.message_id, 'event_type',
datetime.datetime.now(), traits, {})
self.event_conn.record_events([ev])
def test_get_events_without_project(self):

2
panko/tests/functional/api/v2/test_event_scenarios.py
View File

@ -20,7 +20,7 @@ from oslo_serialization import jsonutils
from oslo_utils import uuidutils
import webtest.app
from panko.event.storage import models
from panko.storage import models
from panko.tests import db as tests_db
from panko.tests.functional.api import v2

2
panko/tests/functional/gabbi/fixtures.py
View File

@ -29,9 +29,9 @@ from six.moves.urllib import parse as urlparse
import sqlalchemy_utils
from panko.api import app
from panko.event.storage import models
from panko import service
from panko import storage
from panko.storage import models
# NOTE(chdent): Hack to restore semblance of global configuration to
# pass to the WSGI app used per test suite. LOAD_APP_KWARGS are the olso

2
panko/tests/functional/storage/test_impl_mongodb.py
View File

@ -21,7 +21,7 @@
"""
from panko.event.storage import impl_mongodb
from panko.storage import impl_mongodb
from panko.tests import base as test_base
from panko.tests import db as tests_db

4
panko/tests/functional/storage/test_impl_sqlalchemy.py
View File

@ -22,8 +22,8 @@ import datetime
from six.moves import reprlib
from panko.event.storage import impl_sqlalchemy as impl_sqla_event
from panko.event.storage import models
from panko.storage import impl_sqlalchemy as impl_sqla_event
from panko.storage import models
from panko.storage.sqlalchemy import models as sql_models
from panko.tests import base as test_base
from panko.tests import db as tests_db

196
panko/tests/functional/storage/test_storage_scenarios.py
View File

@ -20,8 +20,8 @@ import operator
import mock
from oslo_utils import timeutils
from panko.event import storage as event_storage
from panko.event.storage import models as event_models
from panko import storage
from panko.storage import models
from panko.tests import db as tests_db
@ -36,30 +36,30 @@ class EventTestBase(tests_db.TestBase):
self.prepare_data()
def prepare_data(self):
self.event_models = []
self.models = []
base = 0
self.start = datetime.datetime(2013, 12, 31, 5, 0)
now = self.start
for event_type in ['Foo', 'Bar', 'Zoo', 'Foo', 'Bar', 'Zoo']:
trait_models = [event_models.Trait(name, dtype, value)
trait_models = [models.Trait(name, dtype, value)
for name, dtype, value in [
('trait_A', event_models.Trait.TEXT_TYPE,
('trait_A', models.Trait.TEXT_TYPE,
"my_%s_text" % event_type),
('trait_B', event_models.Trait.INT_TYPE,
('trait_B', models.Trait.INT_TYPE,
base + 1),
('trait_C', event_models.Trait.FLOAT_TYPE,
('trait_C', models.Trait.FLOAT_TYPE,
float(base) + 0.123456),
('trait_D', event_models.Trait.DATETIME_TYPE,
('trait_D', models.Trait.DATETIME_TYPE,
now)]]
self.event_models.append(
event_models.Event("id_%s_%d" % (event_type, base),
event_type, now, trait_models,
{'status': {'nested': 'started'}}))
self.models.append(
models.Event("id_%s_%d" % (event_type, base),
event_type, now, trait_models,
{'status': {'nested': 'started'}}))
base += 100
now = now + datetime.timedelta(hours=1)
self.end = now
self.event_conn.record_events(self.event_models)
self.event_conn.record_events(self.models)
@tests_db.run_with('sqlite', 'mysql', 'pgsql')
@ -70,7 +70,7 @@ class EventTTLTest(EventTestBase):
mock_utcnow.return_value = datetime.datetime(2013, 12, 31, 10, 0)
self.event_conn.clear_expired_event_data(3600)
events = list(self.event_conn.get_events(event_storage.EventFilter()))
events = list(self.event_conn.get_events(storage.EventFilter()))
self.assertEqual(2, len(events))
event_types = list(self.event_conn.get_event_types())
self.assertEqual(['Bar', 'Zoo'], event_types)
@ -85,8 +85,8 @@ class EventTTLTest(EventTestBase):
class EventTest(EventTestBase):
def test_duplicate_message_id(self):
now = datetime.datetime.utcnow()
m = [event_models.Event("1", "Foo", now, None, {}),
event_models.Event("1", "Zoo", now, [], {})]
m = [models.Event("1", "Foo", now, None, {}),
models.Event("1", "Zoo", now, [], {})]
with mock.patch('%s.LOG' %
self.event_conn.record_events.__module__) as log:
self.event_conn.record_events(m)
@ -94,7 +94,7 @@ class EventTest(EventTestBase):
def test_bad_event(self):
now = datetime.datetime.utcnow()
broken_event = event_models.Event("1", "Foo", now, None, {})
broken_event = models.Event("1", "Foo", now, None, {})
del(broken_event.__dict__['raw'])
m = [broken_event, broken_event]
with mock.patch('%s.LOG' %
@ -108,21 +108,21 @@ class EventTest(EventTestBase):
class GetEventTest(EventTestBase):
def test_generated_is_datetime(self):
event_filter = event_storage.EventFilter(self.start, self.end)
event_filter = storage.EventFilter(self.start, self.end)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(6, len(events))
for i, event in enumerate(events):
self.assertIsInstance(event.generated, datetime.datetime)
self.assertEqual(event.generated,
self.event_models[i].generated)
model_traits = self.event_models[i].traits
self.models[i].generated)
model_traits = self.models[i].traits
for j, trait in enumerate(event.traits):
if trait.dtype == event_models.Trait.DATETIME_TYPE:
if trait.dtype == models.Trait.DATETIME_TYPE:
self.assertIsInstance(trait.value, datetime.datetime)
self.assertEqual(trait.value, model_traits[j].value)
def test_simple_get(self):
event_filter = event_storage.EventFilter(self.start, self.end)
event_filter = storage.EventFilter(self.start, self.end)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(6, len(events))
start_time = None
@ -151,7 +151,7 @@ class GetEventTest(EventTestBase):
}
}
event_filter = event_storage.EventFilter(self.start, self.end, "Bar")
event_filter = storage.EventFilter(self.start, self.end, "Bar")
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(2, len(events))
self.assertEqual("Bar", events[0].event_type)
@ -172,8 +172,8 @@ class GetEventTest(EventTestBase):
def test_get_event_trait_filter(self):
trait_filters = [{'key': 'trait_B', 'integer': 101}]
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
self.assertEqual("Bar", events[0].event_type)
@ -182,117 +182,117 @@ class GetEventTest(EventTestBase):
def test_get_event_trait_filter_op_string(self):
trait_filters = [{'key': 'trait_A', 'string': 'my_Foo_text',
'op': 'eq'}]
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(2, len(events))
self.assertEqual("Foo", events[0].event_type)
self.assertEqual(4, len(events[0].traits))
trait_filters[0].update({'key': 'trait_A', 'op': 'lt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(2, len(events))
self.assertEqual("Bar", events[0].event_type)
trait_filters[0].update({'key': 'trait_A', 'op': 'le'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(4, len(events))
self.assertEqual("Bar", events[1].event_type)
trait_filters[0].update({'key': 'trait_A', 'op': 'ne'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(4, len(events))
self.assertEqual("Zoo", events[3].event_type)
trait_filters[0].update({'key': 'trait_A', 'op': 'gt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(2, len(events))
self.assertEqual("Zoo", events[0].event_type)
trait_filters[0].update({'key': 'trait_A', 'op': 'ge'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(4, len(events))
self.assertEqual("Foo", events[2].event_type)
def test_get_event_trait_filter_op_integer(self):
trait_filters = [{'key': 'trait_B', 'integer': 101, 'op': 'eq'}]
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
self.assertEqual("Bar", events[0].event_type)
self.assertEqual(4, len(events[0].traits))
trait_filters[0].update({'key': 'trait_B', 'op': 'lt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
self.assertEqual("Foo", events[0].event_type)
trait_filters[0].update({'key': 'trait_B', 'op': 'le'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(2, len(events))
self.assertEqual("Bar", events[1].event_type)
trait_filters[0].update({'key': 'trait_B', 'op': 'ne'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(5, len(events))
self.assertEqual("Zoo", events[4].event_type)
trait_filters[0].update({'key': 'trait_B', 'op': 'gt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(4, len(events))
self.assertEqual("Zoo", events[0].event_type)
trait_filters[0].update({'key': 'trait_B', 'op': 'ge'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(5, len(events))
self.assertEqual("Foo", events[2].event_type)
def test_get_event_trait_filter_op_float(self):
trait_filters = [{'key': 'trait_C', 'float': 300.123456, 'op': 'eq'}]
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
self.assertEqual("Foo", events[0].event_type)
self.assertEqual(4, len(events[0].traits))
trait_filters[0].update({'key': 'trait_C', 'op': 'lt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(3, len(events))
self.assertEqual("Zoo", events[2].event_type)
trait_filters[0].update({'key': 'trait_C', 'op': 'le'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(4, len(events))
self.assertEqual("Bar", events[1].event_type)
trait_filters[0].update({'key': 'trait_C', 'op': 'ne'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(5, len(events))
self.assertEqual("Zoo", events[2].event_type)
trait_filters[0].update({'key': 'trait_C', 'op': 'gt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(2, len(events))
self.assertEqual("Bar", events[0].event_type)
trait_filters[0].update({'key': 'trait_C', 'op': 'ge'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(3, len(events))
self.assertEqual("Zoo", events[2].event_type)
@ -301,39 +301,39 @@ class GetEventTest(EventTestBase):
trait_filters = [{'key': 'trait_D',
'datetime': self.start + datetime.timedelta(hours=2),
'op': 'eq'}]
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
self.assertEqual("Zoo", events[0].event_type)
self.assertEqual(4, len(events[0].traits))
trait_filters[0].update({'key': 'trait_D', 'op': 'lt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(2, len(events))
trait_filters[0].update({'key': 'trait_D', 'op': 'le'})
self.assertEqual("Bar", events[1].event_type)
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(3, len(events))
self.assertEqual("Bar", events[1].event_type)
trait_filters[0].update({'key': 'trait_D', 'op': 'ne'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(5, len(events))
self.assertEqual("Foo", events[2].event_type)
trait_filters[0].update({'key': 'trait_D', 'op': 'gt'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(3, len(events))
self.assertEqual("Zoo", events[2].event_type)
trait_filters[0].update({'key': 'trait_D', 'op': 'ge'})
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(4, len(events))
self.assertEqual("Bar", events[2].event_type)
@ -342,8 +342,8 @@ class GetEventTest(EventTestBase):
trait_filters = [{'key': 'trait_B', 'integer': 1},
{'key': 'trait_A', 'string': 'my_Foo_text'},
{'key': 'trait_C', 'float': 0.123456}]
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
self.assertEqual("Foo", events[0].event_type)
@ -352,8 +352,8 @@ class GetEventTest(EventTestBase):
def test_get_event_multiple_trait_filter_expect_none(self):
trait_filters = [{'key': 'trait_B', 'integer': 1},
{'key': 'trait_A', 'string': 'my_Zoo_text'}]
event_filter = event_storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
event_filter = storage.EventFilter(self.start, self.end,
traits_filter=trait_filters)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(0, len(events))
@ -389,13 +389,13 @@ class GetEventTest(EventTestBase):
trait_dict[trait.name] = trait.dtype
self.assertIn("trait_A", trait_dict)
self.assertEqual(event_models.Trait.TEXT_TYPE, trait_dict["trait_A"])
self.assertEqual(models.Trait.TEXT_TYPE, trait_dict["trait_A"])
self.assertIn("trait_B", trait_dict)
self.assertEqual(event_models.Trait.INT_TYPE, trait_dict["trait_B"])
self.assertEqual(models.Trait.INT_TYPE, trait_dict["trait_B"])
self.assertIn("trait_C", trait_dict)
self.assertEqual(event_models.Trait.FLOAT_TYPE, trait_dict["trait_C"])
self.assertEqual(models.Trait.FLOAT_TYPE, trait_dict["trait_C"])
self.assertIn("trait_D", trait_dict)
self.assertEqual(event_models.Trait.DATETIME_TYPE,
self.assertEqual(models.Trait.DATETIME_TYPE,
trait_dict["trait_D"])
def test_get_all_traits(self):
@ -404,13 +404,13 @@ class GetEventTest(EventTestBase):
self.assertEqual(8, len(traits))
trait = traits[0]
self.assertEqual("trait_A", trait.name)
self.assertEqual(event_models.Trait.TEXT_TYPE, trait.dtype)
self.assertEqual(models.Trait.TEXT_TYPE, trait.dtype)
def test_simple_get_event_no_traits(self):
new_events = [event_models.Event("id_notraits", "NoTraits",
new_events = [models.Event("id_notraits", "NoTraits",
self.start, [], {})]
self.event_conn.record_events(new_events)
event_filter = event_storage.EventFilter(
event_filter = storage.EventFilter(
self.start, self.end, "NoTraits")
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
@ -419,41 +419,39 @@ class GetEventTest(EventTestBase):
self.assertEqual(0, len(events[0].traits))
def test_simple_get_no_filters(self):
event_filter = event_storage.EventFilter(None, None, None)
event_filter = storage.EventFilter(None, None, None)
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(6, len(events))
def test_get_by_message_id(self):
new_events = [event_models.Event("id_testid",
"MessageIDTest",
self.start,
[], {})]
new_events = [models.Event("id_testid", "MessageIDTest",
self.start, [], {})]
self.event_conn.record_events(new_events)
event_filter = event_storage.EventFilter(message_id="id_testid")
event_filter = storage.EventFilter(message_id="id_testid")
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertEqual(1, len(events))
event = events[0]
self.assertEqual("id_testid", event.message_id)
def test_simple_get_raw(self):
event_filter = event_storage.EventFilter()
event_filter = storage.EventFilter()
events = [event for event in self.event_conn.get_events(event_filter)]
self.assertTrue(events)
self.assertEqual({'status': {'nested': 'started'}}, events[0].raw)
def test_trait_type_enforced_on_none(self):
new_events = [event_models.Event(
new_events = [models.Event(
"id_testid", "MessageIDTest", self.start,
[event_models.Trait('text', event_models.Trait.TEXT_TYPE, ''),
event_models.Trait('int', event_models.Trait.INT_TYPE, 0),
event_models.Trait('float', event_models.Trait.FLOAT_TYPE, 0.0)],
[models.Trait('text', models.Trait.TEXT_TYPE, ''),
models.Trait('int', models.Trait.INT_TYPE, 0),
models.Trait('float', models.Trait.FLOAT_TYPE, 0.0)],
{})]
self.event_conn.record_events(new_events)
event_filter = event_storage.EventFilter(message_id="id_testid")
event_filter = storage.EventFilter(message_id="id_testid")
events = [event for event in self.event_conn.get_events(event_filter)]
options = [(event_models.Trait.TEXT_TYPE, ''),
(event_models.Trait.INT_TYPE, 0.0),
(event_models.Trait.FLOAT_TYPE, 0.0)]
options = [(models.Trait.TEXT_TYPE, ''),
(models.Trait.INT_TYPE, 0.0),
(models.Trait.FLOAT_TYPE, 0.0)]
for trait in events[0].traits:
options.remove((trait.dtype, trait.value))

8
panko/tests/unit/dispatcher/test_db.py
View File

@ -19,8 +19,8 @@ from oslo_utils import uuidutils
from oslotest import base
from panko.dispatcher import database
from panko.event.storage import models as event_models
from panko import service
from panko.storage import models
class TestDispatcherDB(base.BaseTestCase):
@ -34,9 +34,9 @@ class TestDispatcherDB(base.BaseTestCase):
self.dispatcher = database.DatabaseDispatcher(None)
def test_event_conn(self):
event = event_models.Event(uuidutils.generate_uuid(), 'test',
datetime.datetime(2012, 7, 2, 13, 53, 40),
[], {}).serialize()
event = models.Event(uuidutils.generate_uuid(), 'test',
datetime.datetime(2012, 7, 2, 13, 53, 40),
[], {}).serialize()
with mock.patch.object(self.dispatcher.event_conn,
'record_events') as record_events:
self.dispatcher.record_events(event)

7
panko/tests/unit/storage/test_get_connection.py
View File

@ -17,13 +17,12 @@
import mock
from oslotest import base
import six
from panko.event.storage import impl_log
from panko.event.storage import impl_sqlalchemy
from panko import service
from panko import storage
import six
from panko.storage import impl_log
from panko.storage import impl_sqlalchemy
class EngineTest(base.BaseTestCase):

16
setup.cfg
View File

@ -29,14 +29,14 @@ packages =
panko
[entry_points]
panko.event.storage =
es = panko.event.storage.impl_elasticsearch:Connection
log = panko.event.storage.impl_log:Connection
mongodb = panko.event.storage.impl_mongodb:Connection
mysql = panko.event.storage.impl_sqlalchemy:Connection
postgresql = panko.event.storage.impl_sqlalchemy:Connection
sqlite = panko.event.storage.impl_sqlalchemy:Connection
hbase = panko.event.storage.impl_hbase:Connection
panko.storage =
es = panko.storage.impl_elasticsearch:Connection
log = panko.storage.impl_log:Connection
mongodb = panko.storage.impl_mongodb:Connection
mysql = panko.storage.impl_sqlalchemy:Connection
postgresql = panko.storage.impl_sqlalchemy:Connection
sqlite = panko.storage.impl_sqlalchemy:Connection
hbase = panko.storage.impl_hbase:Connection
console_scripts =
panko-dbsync = panko.cmd.storage:dbsync

2
tools/make_test_event_data.py
View File

@ -28,9 +28,9 @@ import random
from oslo_utils import timeutils
from oslo_utils import uuidutils
from panko.event.storage import models
from panko import service
from panko import storage
from panko.storage import models
def make_test_data(conn, start, end, interval, event_types):

36
tools/test_hbase_table_utils.py
View File

@ -1,36 +0,0 @@
#!/usr/bin/env python
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
import sys
from panko import service
from panko import storage
def main(argv):
if os.getenv("PANKO_TEST_STORAGE_URL", "").startswith("hbase://"):
url = ("%s?table_prefix=%s" %
(os.getenv("PANKO_TEST_STORAGE_URL"),
os.getenv("PANKO_TEST_HBASE_TABLE_PREFIX", "test")))
event_conn = storage.get_connection(url, service.prepare_service())
for arg in argv:
if arg == "--upgrade":
event_conn.upgrade()
if arg == "--clear":
event_conn.clear()
if __name__ == '__main__':
main(sys.argv[1:])