Added separate MongoDB database for each test

This allows to run all the tests concurrently

Change-Id: If29c09e6f8e5e0e7c3ae98bd9a71842dcfb43011

ceilometer/storage/impl_mongodb.py

@@ -24,7 +24,6 @@ import calendar
 import copy
 import datetime
 import operator
-import os
 import uuid
 import weakref
 
@@ -247,13 +246,6 @@ class Connection(base.Connection):
     def __init__(self, conf):
         url = conf.database.connection
 
-        if url == 'mongodb://__test__':
-            url = os.environ.get('CEILOMETER_TEST_MONGODB_URL')
-            if not url:
-                raise RuntimeError(
-                    "No MongoDB test URL set,"
-                    "export CEILOMETER_TEST_MONGODB_URL environment variable")
-
         # NOTE(jd) Use our own connection pooling on top of the Pymongo one.
         # We need that otherwise we overflow the MongoDB instance with new
         # connection since we instanciate a Pymongo client each time someone
@@ -326,6 +318,8 @@ class Connection(base.Connection):
 
     def clear(self):
         self.conn.drop_database(self.db)
+        # Connection will be reopened automatically if needed
+        self.conn.close()
 
     def record_metering_data(self, data):
         """Write the data to the backend storage system.

ceilometer/tests/db.py

@@ -19,6 +19,8 @@
 # under the License.
 
 """Base classes for API tests."""
+import os
+import uuid
 
 from oslo.config import cfg
 
@@ -29,19 +31,37 @@ from ceilometer.tests import base as test_base
 class TestBase(test_base.TestCase):
     def setUp(self):
         super(TestBase, self).setUp()
-        cfg.CONF.set_override('connection', self.database_connection,
+        cfg.CONF.set_override('connection', str(self.database_connection),
                               group='database')
         self.conn = storage.get_connection(cfg.CONF)
         self.conn.upgrade()
         self.conn.clear()
         self.conn.upgrade()
 
+    def tearDown(self):
+        self.conn.clear()
+        self.conn = None
+        super(TestBase, self).tearDown()
+
+
+class MongoDBFakeConnectionUrl(object):
+
+    def __init__(self):
+        self.url = os.environ.get('CEILOMETER_TEST_MONGODB_URL')
+        if not self.url:
+            raise RuntimeError(
+                "No MongoDB test URL set,"
+                "export CEILOMETER_TEST_MONGODB_URL environment variable")
+
+    def __str__(self):
+        return '%(url)s_%(db)s' % dict(url=self.url, db=uuid.uuid4().hex)
+
 
 class MixinTestsWithBackendScenarios(object):
     __metaclass__ = test_base.SkipNotImplementedMeta
 
     scenarios = [
         ('sqlalchemy', dict(database_connection='sqlite://')),
-        ('mongodb', dict(database_connection='mongodb://__test__')),
+        ('mongodb', dict(database_connection=MongoDBFakeConnectionUrl())),
         ('hbase', dict(database_connection='hbase://__test__')),
     ]

run-tests.sh

@@ -25,7 +25,7 @@ fi
 MONGO_DATA=`mktemp -d`
 trap "clean_exit" EXIT
 mkfifo ${MONGO_DATA}/out
-mongod --maxConns 32 --noprealloc --smallfiles --quiet --noauth --port 29000 --dbpath "${MONGO_DATA}" --bind_ip localhost &>${MONGO_DATA}/out &
+mongod --maxConns 32 --nojournal --noprealloc --smallfiles --quiet --noauth --port 29000 --dbpath "${MONGO_DATA}" --bind_ip localhost &>${MONGO_DATA}/out &
 MONGO_PID=$!
 # Wait for Mongo to start listening to connections
 while read line
@@ -34,6 +34,6 @@ do
 done < ${MONGO_DATA}/out
 # Read the fifo for ever otherwise mongod would block
 # + that gives us the log on screen
-cat ${MONGO_DATA}/out &
+cat ${MONGO_DATA}/out > /dev/null &
 export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:29000/ceilometer"
-python setup.py testr --slowest --testr-args="--concurrency=1 $*" $COVERAGE_ARG
+python setup.py testr --slowest --testr-args="$*" $COVERAGE_ARG

tests/api/v2/test_app.py

@@ -25,6 +25,7 @@ from ceilometer.api import app
 from ceilometer.api import acl
 from ceilometer import service
 from ceilometer.tests import base
+from ceilometer.tests import db as tests_db
 from .base import FunctionalTest
 
 
@@ -62,7 +63,7 @@ class TestApp(base.TestCase):
 class TestApiMiddleware(FunctionalTest):
 
     # This doesn't really matter
-    database_connection = 'mongodb://__test__'
+    database_connection = tests_db.MongoDBFakeConnectionUrl()
 
     def test_json_parsable_error_middleware_404(self):
         response = self.get_json('/invalid_path',

tests/storage/test_impl_mongodb.py

@@ -36,10 +36,11 @@ from ceilometer.publisher import rpc
 from ceilometer import counter
 from ceilometer.storage import impl_mongodb
 from ceilometer.storage import models
+from ceilometer.tests import db as tests_db
 
 
 class MongoDBEngineTestBase(base.DBTestBase):
-    database_connection = 'mongodb://__test__'
+    database_connection = tests_db.MongoDBFakeConnectionUrl()
 
 
 class MongoDBConnection(MongoDBEngineTestBase):
@@ -48,10 +49,9 @@ class MongoDBConnection(MongoDBEngineTestBase):
                          impl_mongodb.Connection(cfg.CONF).conn)
 
     def test_replica_set(self):
-        # FIXME(Alexei_987) should not hardcode URL here
         cfg.CONF.set_override(
             'connection',
-            'mongodb://localhost:29000/ceilometer?replicaSet=foobar',
+            str(tests_db.MongoDBFakeConnectionUrl()) + '?replicaSet=foobar',
             group='database')
         conn = impl_mongodb.Connection(cfg.CONF)
         self.assertTrue(conn.conn)