diff --git a/ceilometer/storage/impl_mongodb.py b/ceilometer/storage/impl_mongodb.py index 77ca0b5d12..6a76f34cb4 100644 --- a/ceilometer/storage/impl_mongodb.py +++ b/ceilometer/storage/impl_mongodb.py @@ -1,8 +1,10 @@ # -*- encoding: utf-8 -*- # # Copyright © 2012 New Dream Network, LLC (DreamHost) +# Copyright © 2013 eNovance # # Author: Doug Hellmann +# Julien Danjou # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -20,7 +22,9 @@ import copy import datetime +import nose import operator +import os import re import urlparse @@ -126,6 +130,8 @@ class Connection(base.Connection): """MongoDB connection. """ + _mim_instance = None + # JavaScript function for doing map-reduce to get a counter volume # total. MAP_COUNTER_VOLUME = bson.code.Code(""" @@ -240,7 +246,36 @@ class Connection(base.Connection): def __init__(self, conf): opts = self._parse_connection_url(conf.database_connection) LOG.info('connecting to MongoDB on %s:%s', opts['host'], opts['port']) - self.conn = self._get_connection(opts) + + if opts['host'] == '__test__': + live_tests = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0))) + if live_tests: + url = os.environ.get('CEILOMETER_TEST_MONGO_URL') + if not url: + raise RuntimeError("CEILOMETER_TEST_LIVE is on, but " + "CEILOMETER_TEST_MONGO_URL " + "is not defined") + opts = self._parse_connection_url(url) + self.conn = pymongo.Connection(opts['host'], + opts['port'], + safe=True) + else: + # MIM will die if we have too many connections, so use a + # Singleton + if Connection._mim_instance is None: + try: + from ming import mim + except ImportError: + raise nose.SkipTest("Ming not found") + LOG.debug('Creating a new MIM Connection object') + Connection._mim_instance = mim.Connection() + self.conn = Connection._mim_instance + LOG.debug('Using MIM for test connection') + else: + self.conn = pymongo.Connection(opts['host'], + opts['port'], + safe=True) + self.db = getattr(self.conn, opts['dbname']) if 'username' in opts: self.db.authenticate(opts['username'], opts['password']) @@ -269,17 +304,13 @@ class Connection(base.Connection): pass def clear(self): - self.conn.drop_database(self.db) - - def _get_connection(self, opts): - """Return a connection to the database. - - .. note:: - - The tests use a subclass to override this and return an - in-memory connection. - """ - return pymongo.Connection(opts['host'], opts['port'], safe=True) + if self._mim_instance is not None: + # Don't want to use drop_database() because + # may end up running out of spidermonkey instances. + # http://davisp.lighthouseapp.com/projects/26898/tickets/22 + self.db.clear() + else: + self.conn.drop_database(self.db) def _parse_connection_url(self, url): opts = {} @@ -592,3 +623,16 @@ class Connection(base.Connection): answer = results['results'][0]['value'] return self._fix_interval_min_max(answer['min'], answer['max']) return (None, None) + + +def require_map_reduce(conn): + """Raises SkipTest if the connection is using mim. + """ + # NOTE(dhellmann): mim requires spidermonkey to implement the + # map-reduce functions, so if we can't import it then just + # skip these tests unless we aren't using mim. + try: + import spidermonkey + except BaseException: + if isinstance(conn.conn, mim.Connection): + raise nose.SkipTest('requires spidermonkey') diff --git a/ceilometer/storage/impl_test.py b/ceilometer/storage/impl_test.py deleted file mode 100644 index c6e2d7e89e..0000000000 --- a/ceilometer/storage/impl_test.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- encoding: utf-8 -*- -# -# Copyright © 2012 New Dream Network, LLC (DreamHost) -# -# Author: Doug Hellmann -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -"""In-memory storage driver for use with tests. - -This driver is based on MIM, an in-memory version of MongoDB. -""" - -import os -import nose - -try: - from ming import mim -except ImportError: - mim = None - -from ceilometer.openstack.common import log as logging - -from ceilometer.storage import base -from ceilometer.storage import impl_mongodb - - -LOG = logging.getLogger(__name__) - - -class TestDBStorage(base.StorageEngine): - """Put the data into an in-memory database for testing - - This driver is based on MIM, an in-memory version of MongoDB. - - Collections:: - - - user - - { _id: user id - source: [ array of source ids reporting for the user ] - } - - project - - { _id: project id - source: [ array of source ids reporting for the project ] - } - - meter - - the raw incoming data - - resource - - the metadata for resources - - { _id: uuid of resource, - metadata: metadata dictionaries - timestamp: datetime of last update - user_id: uuid - project_id: uuid - meter: [ array of {counter_name: string, counter_type: string, - counter_unit: string} ] - } - """ - - OPTIONS = [] - - def register_opts(self, conf): - """Register any configuration options used by this engine. - """ - conf.register_opts(self.OPTIONS) - - def get_connection(self, conf): - """Return a Connection instance based on the configuration settings. - """ - return TestConnection(conf) - - -class TestConnection(impl_mongodb.Connection): - - _mim_instance = None - FORCE_MONGO = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0))) - - def clear(self): - if TestConnection._mim_instance is not None: - # Don't want to use drop_database() because - # may end up running out of spidermonkey instances. - # http://davisp.lighthouseapp.com/projects/26898/tickets/22 - self.db.clear() - else: - super(TestConnection, self).clear() - - def _get_connection(self, conf): - # Use a real MongoDB server if we can connect, but fall back - # to a Mongo-in-memory connection if we cannot. - if self.FORCE_MONGO: - try: - return super(TestConnection, self)._get_connection(conf) - except: - LOG.debug('Unable to connect to mongodb') - raise - else: - LOG.debug('Using MIM for test connection') - - # MIM will die if we have too many connections, so use a - # Singleton - if TestConnection._mim_instance is None: - if mim: - LOG.debug('Creating a new MIM Connection object') - TestConnection._mim_instance = mim.Connection() - else: - raise nose.SkipTest("Ming not found") - return TestConnection._mim_instance - - -def require_map_reduce(conn): - """Raises SkipTest if the connection is using mim. - """ - # NOTE(dhellmann): mim requires spidermonkey to implement the - # map-reduce functions, so if we can't import it then just - # skip these tests unless we aren't using mim. - try: - import spidermonkey - except BaseException: - if isinstance(conn.conn, mim.Connection): - raise skip.SkipTest('requires spidermonkey') diff --git a/ceilometer/tests/api.py b/ceilometer/tests/api.py index 51a877c48a..0fd7118714 100644 --- a/ceilometer/tests/api.py +++ b/ceilometer/tests/api.py @@ -30,8 +30,6 @@ import pecan.testing from ceilometer.api import acl from ceilometer.api.v1 import app as v1_app from ceilometer.api.v1 import blueprint as v1_blueprint -from ceilometer import storage -from ceilometer.tests import base from ceilometer.tests import db as db_test_base diff --git a/ceilometer/tests/db.py b/ceilometer/tests/db.py index df1f13968b..4151c800c4 100644 --- a/ceilometer/tests/db.py +++ b/ceilometer/tests/db.py @@ -33,8 +33,10 @@ class BaseException(Exception): class TestBase(test_base.TestCase): - # Default tests use test:// (MIM) - database_connection = 'test://' + # Default tests use mongodb://__test__ (MIM) + # TODO(jd) remove it, so we're sure we run test on the backend we want, + # not this default one by mistake + database_connection = 'mongodb://__test__' def setUp(self): super(TestBase, self).setUp() diff --git a/setup.py b/setup.py index 382c0d9274..f03009e864 100755 --- a/setup.py +++ b/setup.py @@ -130,7 +130,6 @@ setuptools.setup( mysql = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage postgresql = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage sqlite = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage - test = ceilometer.storage.impl_test:TestDBStorage hbase = ceilometer.storage.impl_hbase:HBaseStorage [ceilometer.compute.virt] diff --git a/tests/api/v1/max_project_volume.py b/tests/api/v1/max_project_volume.py index 121513ff05..cc2c7a1a4a 100644 --- a/tests/api/v1/max_project_volume.py +++ b/tests/api/v1/max_project_volume.py @@ -27,7 +27,7 @@ from ceilometer.collector import meter from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class TestMaxProjectVolume(tests_api.TestBase): diff --git a/tests/api/v1/max_resource_volume.py b/tests/api/v1/max_resource_volume.py index 85f0d007ef..ddba198264 100644 --- a/tests/api/v1/max_resource_volume.py +++ b/tests/api/v1/max_resource_volume.py @@ -26,7 +26,7 @@ from ceilometer.collector import meter from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class TestMaxResourceVolume(tests_api.TestBase): diff --git a/tests/api/v1/test_impl_mongodb.py b/tests/api/v1/test_impl_mongodb.py index 565a1edb49..298105e8f1 100644 --- a/tests/api/v1/test_impl_mongodb.py +++ b/tests/api/v1/test_impl_mongodb.py @@ -29,64 +29,64 @@ from . import max_resource_volume class TestListEvents(list_events.TestListEvents): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListEventsMetaQuery(list_events.TestListEventsMetaquery): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListEmptyMeters(list_meters.TestListEmptyMeters): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListMeters(list_meters.TestListMeters): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListMetersMetaquery(list_meters.TestListMetersMetaquery): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListEmptyUsers(list_users.TestListEmptyUsers): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListUsers(list_users.TestListUsers): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListEmptyProjects(list_projects.TestListEmptyProjects): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListProjects(list_projects.TestListProjects): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestComputeDurationByResource(cdbr.TestComputeDurationByResource): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListEmptyResources(list_resources.TestListEmptyResources): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListResources(list_resources.TestListResources): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListResourcesMetaquery(list_resources.TestListResourcesMetaquery): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestListSource(list_sources.TestListSource): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestMaxProjectVolume(max_project_volume.TestMaxProjectVolume): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class TestMaxProjectVolume(max_resource_volume.TestMaxResourceVolume): - database_connection = 'test://' + database_connection = 'mongodb://__test__' diff --git a/tests/api/v1/test_sum_project_volume.py b/tests/api/v1/test_sum_project_volume.py index 7ee6e10a8e..fd043475fa 100644 --- a/tests/api/v1/test_sum_project_volume.py +++ b/tests/api/v1/test_sum_project_volume.py @@ -27,7 +27,7 @@ from ceilometer.collector import meter from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class TestSumProjectVolume(tests_api.TestBase): diff --git a/tests/api/v1/test_sum_resource_volume.py b/tests/api/v1/test_sum_resource_volume.py index 3af09e2956..aee4242c65 100644 --- a/tests/api/v1/test_sum_resource_volume.py +++ b/tests/api/v1/test_sum_resource_volume.py @@ -27,7 +27,7 @@ from ceilometer.collector import meter from ceilometer import counter from ceilometer.tests import api as tests_api -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class TestSumResourceVolume(tests_api.TestBase): diff --git a/tests/api/v2/test_compute_duration_by_resource.py b/tests/api/v2/test_compute_duration_by_resource.py index d3d787f3a6..2cee14d245 100644 --- a/tests/api/v2/test_compute_duration_by_resource.py +++ b/tests/api/v2/test_compute_duration_by_resource.py @@ -22,7 +22,7 @@ import datetime import logging from ceilometer.openstack.common import timeutils -from ceilometer.storage import impl_test +from ceilometer.storage import impl_mongodb from .base import FunctionalTest LOG = logging.getLogger(__name__) @@ -50,7 +50,7 @@ class TestComputeDurationByResource(FunctionalTest): self.late2 = datetime.datetime(2012, 8, 29, 19, 0) def _stub_interval_func(self, func): - self.stubs.Set(impl_test.TestConnection, + self.stubs.Set(impl_mongodb.Connection, 'get_meter_statistics', func) diff --git a/tests/api/v2/test_max_project_volume.py b/tests/api/v2/test_max_project_volume.py index ee0c83204d..f7f9d7911f 100644 --- a/tests/api/v2/test_max_project_volume.py +++ b/tests/api/v2/test_max_project_volume.py @@ -25,7 +25,7 @@ from oslo.config import cfg from ceilometer.collector import meter from ceilometer import counter -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce from .base import FunctionalTest diff --git a/tests/api/v2/test_max_resource_volume.py b/tests/api/v2/test_max_resource_volume.py index a8fd65da93..34d327d460 100644 --- a/tests/api/v2/test_max_resource_volume.py +++ b/tests/api/v2/test_max_resource_volume.py @@ -26,7 +26,7 @@ from ceilometer.collector import meter from ceilometer import counter from .base import FunctionalTest -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class TestMaxResourceVolume(FunctionalTest): diff --git a/tests/api/v2/test_sum_project_volume.py b/tests/api/v2/test_sum_project_volume.py index 92876c6ea4..9d3fabf5a7 100644 --- a/tests/api/v2/test_sum_project_volume.py +++ b/tests/api/v2/test_sum_project_volume.py @@ -26,7 +26,7 @@ from ceilometer.collector import meter from ceilometer import counter from .base import FunctionalTest -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class TestSumProjectVolume(FunctionalTest): diff --git a/tests/api/v2/test_sum_resource_volume.py b/tests/api/v2/test_sum_resource_volume.py index 7d69254438..60171b1947 100644 --- a/tests/api/v2/test_sum_resource_volume.py +++ b/tests/api/v2/test_sum_resource_volume.py @@ -26,7 +26,7 @@ from ceilometer.collector import meter from ceilometer import counter from .base import FunctionalTest -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class TestSumResourceVolume(FunctionalTest): diff --git a/tests/storage/test_impl_mongodb.py b/tests/storage/test_impl_mongodb.py index 9d3e107c53..babf5cedb8 100644 --- a/tests/storage/test_impl_mongodb.py +++ b/tests/storage/test_impl_mongodb.py @@ -53,11 +53,11 @@ from tests.storage import base from ceilometer.collector import meter from ceilometer import counter -from ceilometer.storage.impl_test import require_map_reduce +from ceilometer.storage.impl_mongodb import require_map_reduce class MongoDBEngineTestBase(base.DBTestBase): - database_connection = 'test://' + database_connection = 'mongodb://__test__' class IndexTest(MongoDBEngineTestBase):