Remove impl_test

Change-Id: If939bbaebf35a326076b6263f70248ff13975b45
Signed-off-by: Julien Danjou <julien@danjou.info>

ceilometer/storage/impl_mongodb.py

@@ -1,8 +1,10 @@
 # -*- encoding: utf-8 -*-
 #
 # Copyright © 2012 New Dream Network, LLC (DreamHost)
+# Copyright © 2013 eNovance
 #
 # Author: Doug Hellmann <doug.hellmann@dreamhost.com>
+#         Julien Danjou <julien@danjou.info>
 #
 # Licensed under the Apache License, Version 2.0 (the "License"); you may
 # not use this file except in compliance with the License. You may obtain
@@ -20,7 +22,9 @@
 
 import copy
 import datetime
+import nose
 import operator
+import os
 import re
 import urlparse
 
@@ -126,6 +130,8 @@ class Connection(base.Connection):
     """MongoDB connection.
     """
 
+    _mim_instance = None
+
     # JavaScript function for doing map-reduce to get a counter volume
     # total.
     MAP_COUNTER_VOLUME = bson.code.Code("""
@@ -240,7 +246,36 @@ class Connection(base.Connection):
     def __init__(self, conf):
         opts = self._parse_connection_url(conf.database_connection)
         LOG.info('connecting to MongoDB on %s:%s', opts['host'], opts['port'])
-        self.conn = self._get_connection(opts)
+
+        if opts['host'] == '__test__':
+            live_tests = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0)))
+            if live_tests:
+                url = os.environ.get('CEILOMETER_TEST_MONGO_URL')
+                if not url:
+                    raise RuntimeError("CEILOMETER_TEST_LIVE is on, but "
+                                       "CEILOMETER_TEST_MONGO_URL "
+                                       "is not defined")
+                opts = self._parse_connection_url(url)
+                self.conn = pymongo.Connection(opts['host'],
+                                               opts['port'],
+                                               safe=True)
+            else:
+                # MIM will die if we have too many connections, so use a
+                # Singleton
+                if Connection._mim_instance is None:
+                    try:
+                        from ming import mim
+                    except ImportError:
+                        raise nose.SkipTest("Ming not found")
+                    LOG.debug('Creating a new MIM Connection object')
+                    Connection._mim_instance = mim.Connection()
+                self.conn = Connection._mim_instance
+                LOG.debug('Using MIM for test connection')
+        else:
+            self.conn = pymongo.Connection(opts['host'],
+                                           opts['port'],
+                                           safe=True)
+
         self.db = getattr(self.conn, opts['dbname'])
         if 'username' in opts:
             self.db.authenticate(opts['username'], opts['password'])
@@ -269,17 +304,13 @@ class Connection(base.Connection):
         pass
 
     def clear(self):
-        self.conn.drop_database(self.db)
-
-    def _get_connection(self, opts):
-        """Return a connection to the database.
-
-        .. note::
-
-          The tests use a subclass to override this and return an
-          in-memory connection.
-        """
-        return pymongo.Connection(opts['host'], opts['port'], safe=True)
+        if self._mim_instance is not None:
+            # Don't want to use drop_database() because
+            # may end up running out of spidermonkey instances.
+            # http://davisp.lighthouseapp.com/projects/26898/tickets/22
+            self.db.clear()
+        else:
+            self.conn.drop_database(self.db)
 
     def _parse_connection_url(self, url):
         opts = {}
@@ -592,3 +623,16 @@ class Connection(base.Connection):
             answer = results['results'][0]['value']
             return self._fix_interval_min_max(answer['min'], answer['max'])
         return (None, None)
+
+
+def require_map_reduce(conn):
+    """Raises SkipTest if the connection is using mim.
+    """
+    # NOTE(dhellmann): mim requires spidermonkey to implement the
+    # map-reduce functions, so if we can't import it then just
+    # skip these tests unless we aren't using mim.
+    try:
+        import spidermonkey
+    except BaseException:
+        if isinstance(conn.conn, mim.Connection):
+            raise nose.SkipTest('requires spidermonkey')

ceilometer/storage/impl_test.py

@@ -1,129 +0,0 @@
-# -*- encoding: utf-8 -*-
-#
-# Copyright © 2012 New Dream Network, LLC (DreamHost)
-#
-# Author: Doug Hellmann <doug.hellmann@dreamhost.com>
-#
-# Licensed under the Apache License, Version 2.0 (the "License"); you may
-# not use this file except in compliance with the License. You may obtain
-# a copy of the License at
-#
-#      http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-# License for the specific language governing permissions and limitations
-# under the License.
-"""In-memory storage driver for use with tests.
-
-This driver is based on MIM, an in-memory version of MongoDB.
-"""
-
-import os
-import nose
-
-try:
-    from ming import mim
-except ImportError:
-    mim = None
-
-from ceilometer.openstack.common import log as logging
-
-from ceilometer.storage import base
-from ceilometer.storage import impl_mongodb
-
-
-LOG = logging.getLogger(__name__)
-
-
-class TestDBStorage(base.StorageEngine):
-    """Put the data into an in-memory database for testing
-
-    This driver is based on MIM, an in-memory version of MongoDB.
-
-    Collections::
-
-        - user
-          - { _id: user id
-              source: [ array of source ids reporting for the user ]
-              }
-        - project
-          - { _id: project id
-              source: [ array of source ids reporting for the project ]
-              }
-        - meter
-          - the raw incoming data
-        - resource
-          - the metadata for resources
-          - { _id: uuid of resource,
-              metadata: metadata dictionaries
-              timestamp: datetime of last update
-              user_id: uuid
-              project_id: uuid
-              meter: [ array of {counter_name: string, counter_type: string,
-                                 counter_unit: string} ]
-            }
-    """
-
-    OPTIONS = []
-
-    def register_opts(self, conf):
-        """Register any configuration options used by this engine.
-        """
-        conf.register_opts(self.OPTIONS)
-
-    def get_connection(self, conf):
-        """Return a Connection instance based on the configuration settings.
-        """
-        return TestConnection(conf)
-
-
-class TestConnection(impl_mongodb.Connection):
-
-    _mim_instance = None
-    FORCE_MONGO = bool(int(os.environ.get('CEILOMETER_TEST_LIVE', 0)))
-
-    def clear(self):
-        if TestConnection._mim_instance is not None:
-            # Don't want to use drop_database() because
-            # may end up running out of spidermonkey instances.
-            # http://davisp.lighthouseapp.com/projects/26898/tickets/22
-            self.db.clear()
-        else:
-            super(TestConnection, self).clear()
-
-    def _get_connection(self, conf):
-        # Use a real MongoDB server if we can connect, but fall back
-        # to a Mongo-in-memory connection if we cannot.
-        if self.FORCE_MONGO:
-            try:
-                return super(TestConnection, self)._get_connection(conf)
-            except:
-                LOG.debug('Unable to connect to mongodb')
-                raise
-        else:
-            LOG.debug('Using MIM for test connection')
-
-            # MIM will die if we have too many connections, so use a
-            # Singleton
-            if TestConnection._mim_instance is None:
-                if mim:
-                    LOG.debug('Creating a new MIM Connection object')
-                    TestConnection._mim_instance = mim.Connection()
-                else:
-                    raise nose.SkipTest("Ming not found")
-            return TestConnection._mim_instance
-
-
-def require_map_reduce(conn):
-    """Raises SkipTest if the connection is using mim.
-    """
-    # NOTE(dhellmann): mim requires spidermonkey to implement the
-    # map-reduce functions, so if we can't import it then just
-    # skip these tests unless we aren't using mim.
-    try:
-        import spidermonkey
-    except BaseException:
-        if isinstance(conn.conn, mim.Connection):
-            raise skip.SkipTest('requires spidermonkey')

ceilometer/tests/api.py

@@ -30,8 +30,6 @@ import pecan.testing
 from ceilometer.api import acl
 from ceilometer.api.v1 import app as v1_app
 from ceilometer.api.v1 import blueprint as v1_blueprint
-from ceilometer import storage
-from ceilometer.tests import base
 from ceilometer.tests import db as db_test_base
 
 

ceilometer/tests/db.py

@@ -33,8 +33,10 @@ class BaseException(Exception):
 
 class TestBase(test_base.TestCase):
 
-    # Default tests use test:// (MIM)
-    database_connection = 'test://'
+    # Default tests use mongodb://__test__ (MIM)
+    # TODO(jd) remove it, so we're sure we run test on the backend we want,
+    # not this default one by mistake
+    database_connection = 'mongodb://__test__'
 
     def setUp(self):
         super(TestBase, self).setUp()

setup.py

@@ -130,7 +130,6 @@ setuptools.setup(
     mysql = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage
     postgresql = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage
     sqlite = ceilometer.storage.impl_sqlalchemy:SQLAlchemyStorage
-    test = ceilometer.storage.impl_test:TestDBStorage
     hbase = ceilometer.storage.impl_hbase:HBaseStorage
 
     [ceilometer.compute.virt]

tests/api/v1/max_project_volume.py

@@ -27,7 +27,7 @@ from ceilometer.collector import meter
 from ceilometer import counter
 
 from ceilometer.tests import api as tests_api
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class TestMaxProjectVolume(tests_api.TestBase):

tests/api/v1/max_resource_volume.py

@@ -26,7 +26,7 @@ from ceilometer.collector import meter
 from ceilometer import counter
 
 from ceilometer.tests import api as tests_api
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class TestMaxResourceVolume(tests_api.TestBase):

tests/api/v1/test_impl_mongodb.py

@@ -29,64 +29,64 @@ from . import max_resource_volume
 
 
 class TestListEvents(list_events.TestListEvents):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListEventsMetaQuery(list_events.TestListEventsMetaquery):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListEmptyMeters(list_meters.TestListEmptyMeters):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListMeters(list_meters.TestListMeters):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListMetersMetaquery(list_meters.TestListMetersMetaquery):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListEmptyUsers(list_users.TestListEmptyUsers):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListUsers(list_users.TestListUsers):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListEmptyProjects(list_projects.TestListEmptyProjects):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListProjects(list_projects.TestListProjects):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestComputeDurationByResource(cdbr.TestComputeDurationByResource):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListEmptyResources(list_resources.TestListEmptyResources):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListResources(list_resources.TestListResources):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListResourcesMetaquery(list_resources.TestListResourcesMetaquery):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestListSource(list_sources.TestListSource):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestMaxProjectVolume(max_project_volume.TestMaxProjectVolume):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class TestMaxProjectVolume(max_resource_volume.TestMaxResourceVolume):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'

tests/api/v1/test_sum_project_volume.py

@@ -27,7 +27,7 @@ from ceilometer.collector import meter
 from ceilometer import counter
 
 from ceilometer.tests import api as tests_api
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class TestSumProjectVolume(tests_api.TestBase):

tests/api/v1/test_sum_resource_volume.py

@@ -27,7 +27,7 @@ from ceilometer.collector import meter
 from ceilometer import counter
 
 from ceilometer.tests import api as tests_api
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class TestSumResourceVolume(tests_api.TestBase):

tests/api/v2/test_compute_duration_by_resource.py

@@ -22,7 +22,7 @@ import datetime
 import logging
 
 from ceilometer.openstack.common import timeutils
-from ceilometer.storage import impl_test
+from ceilometer.storage import impl_mongodb
 from .base import FunctionalTest
 
 LOG = logging.getLogger(__name__)
@@ -50,7 +50,7 @@ class TestComputeDurationByResource(FunctionalTest):
         self.late2 = datetime.datetime(2012, 8, 29, 19, 0)
 
     def _stub_interval_func(self, func):
-        self.stubs.Set(impl_test.TestConnection,
+        self.stubs.Set(impl_mongodb.Connection,
                        'get_meter_statistics',
                        func)
 

tests/api/v2/test_max_project_volume.py

@@ -25,7 +25,7 @@ from oslo.config import cfg
 from ceilometer.collector import meter
 from ceilometer import counter
 
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 from .base import FunctionalTest
 

tests/api/v2/test_max_resource_volume.py

@@ -26,7 +26,7 @@ from ceilometer.collector import meter
 from ceilometer import counter
 
 from .base import FunctionalTest
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class TestMaxResourceVolume(FunctionalTest):

tests/api/v2/test_sum_project_volume.py

@@ -26,7 +26,7 @@ from ceilometer.collector import meter
 from ceilometer import counter
 
 from .base import FunctionalTest
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class TestSumProjectVolume(FunctionalTest):

tests/api/v2/test_sum_resource_volume.py

@@ -26,7 +26,7 @@ from ceilometer.collector import meter
 from ceilometer import counter
 
 from .base import FunctionalTest
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class TestSumResourceVolume(FunctionalTest):

tests/storage/test_impl_mongodb.py

@@ -53,11 +53,11 @@ from tests.storage import base
 
 from ceilometer.collector import meter
 from ceilometer import counter
-from ceilometer.storage.impl_test import require_map_reduce
+from ceilometer.storage.impl_mongodb import require_map_reduce
 
 
 class MongoDBEngineTestBase(base.DBTestBase):
-    database_connection = 'test://'
+    database_connection = 'mongodb://__test__'
 
 
 class IndexTest(MongoDBEngineTestBase):