Improved MongoClient pooling to avoid out of connections error

Use 1 MongoClient instance per database server determined by host:port +
connection options

Fixes bug #1218488

Change-Id: If06844f6bf09674216b029310c1a5f445c4476fe

ceilometer/storage/impl_db2.py

@@ -22,7 +22,6 @@
 """
 
 import copy
-import urlparse
 import weakref
 
 import bson.code
@@ -138,21 +137,22 @@ class ConnectionPool(object):
         self._pool = {}
 
     def connect(self, url):
-        if url in self._pool:
-            client = self._pool.get(url)()
+        connection_options = pymongo.uri_parser.parse_uri(url)
+        del connection_options['database']
+        del connection_options['username']
+        del connection_options['password']
+        del connection_options['collection']
+        pool_key = tuple(connection_options)
+
+        if pool_key in self._pool:
+            client = self._pool.get(pool_key)()
             if client:
                 return client
-        LOG.info('connecting to DB2 on %s', url)
-        url_parsed = urlparse.urlparse(url)
-        if url_parsed.path.startswith('/ceilometer_for_tox_testing_'):
-            #note(sileht): this is a workaround for running tests without reach
-            #the maximum allowed connection of mongod in gate
-            #this only work with pymongo >= 2.6, this is not in the
-            #requirements file because is not needed for normal use of mongo
-            client = pymongo.MongoClient(url, safe=True, max_pool_size=None)
-        else:
-            client = pymongo.MongoClient(url, safe=True)
-        self._pool[url] = weakref.ref(client)
+        LOG.info('connecting to MongoDB on %s', url)
+        client = pymongo.MongoClient(
+            url,
+            safe=True)
+        self._pool[pool_key] = weakref.ref(client)
         return client
 
 
@@ -205,6 +205,9 @@ class Connection(base.Connection):
 
         connection_options = pymongo.uri_parser.parse_uri(url)
         self.db = getattr(self.conn, connection_options['database'])
+        if connection_options.get('username'):
+            self.db.authenticate(connection_options['username'],
+                                 connection_options['password'])
 
         self.upgrade()
 

ceilometer/storage/impl_mongodb.py

@@ -23,7 +23,6 @@
 import calendar
 import copy
 import operator
-import urlparse
 import weakref
 
 import bson.code
@@ -145,21 +144,22 @@ class ConnectionPool(object):
         self._pool = {}
 
     def connect(self, url):
-        if url in self._pool:
-            client = self._pool.get(url)()
+        connection_options = pymongo.uri_parser.parse_uri(url)
+        del connection_options['database']
+        del connection_options['username']
+        del connection_options['password']
+        del connection_options['collection']
+        pool_key = tuple(connection_options)
+
+        if pool_key in self._pool:
+            client = self._pool.get(pool_key)()
             if client:
                 return client
         LOG.info('connecting to MongoDB on %s', url)
-        url_parsed = urlparse.urlparse(url)
-        if url_parsed.path.startswith('/ceilometer_for_tox_testing_'):
-            #note(sileht): this is a workaround for running tests without reach
-            #the maximum allowed connection of mongod in gate
-            #this only work with pymongo >= 2.6, this is not in the
-            #requirements file because is not needed for normal use of mongo
-            client = pymongo.MongoClient(url, safe=True, max_pool_size=None)
-        else:
-            client = pymongo.MongoClient(url, safe=True)
-        self._pool[url] = weakref.ref(client)
+        client = pymongo.MongoClient(
+            url,
+            safe=True)
+        self._pool[pool_key] = weakref.ref(client)
         return client
 
 
@@ -313,6 +313,9 @@ class Connection(base.Connection):
 
         connection_options = pymongo.uri_parser.parse_uri(url)
         self.db = getattr(self.conn, connection_options['database'])
+        if connection_options.get('username'):
+            self.db.authenticate(connection_options['username'],
+                                 connection_options['password'])
 
         # NOTE(jd) Upgrading is just about creating index, so let's do this
         # on connection to be sure at least the TTL is correcly updated if

run-tests.sh

@@ -36,5 +36,5 @@ done < ${MONGO_DATA}/out
 # Read the fifo for ever otherwise mongod would block
 # + that gives us the log on screen
 cat ${MONGO_DATA}/out > /dev/null &
-export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:29000/ceilometer_for_tox_testing"
+export CEILOMETER_TEST_MONGODB_URL="mongodb://localhost:29000/ceilometer"
 python setup.py testr --slowest --testr-args="$*" $COVERAGE_ARG