Dispose db connections pool on disconnect

When a disconnect happens (e.g. when VIP moves to another node), all
existing connections in the pool become invalid. There is no sense to
check every single connection in the pool, as this may take a long
time (e.g. when python-mysqldb driver is used with eventlet green
threads, there is no context switch on db IO, so the whole process is
blocked until socket read times out). Dispose all connections in the
pool when disconnect is detected, they will be recreated on demand
then.

Closes-Bug: #1288438

Change-Id: Ia357da2b3092d306a86b6d1787bab374a335f28c

openstack/common/db/sqlalchemy/session.py

@@ -505,6 +505,14 @@ def _ping_listener(engine, dbapi_conn, connection_rec, connection_proxy):
         if engine.dialect.is_disconnect(ex, dbapi_conn, cursor):
             msg = _LW('Database server has gone away: %s') % ex
             LOG.warning(msg)
+
+            # if the database server has gone away, all connections in the pool
+            # have become invalid and we can safely close all of them here,
+            # rather than waste time on checking of every single connection
+            engine.dispose()
+
+            # this will be handled by SQLAlchemy and will force it to create
+            # a new connection and retry the original action
             raise sqla_exc.DisconnectionError(msg)
         else:
             raise

tests/unit/db/sqlalchemy/test_sqlalchemy.py

@@ -209,6 +209,9 @@ class FakeDB2Engine(object):
     dialect = Dialect()
     name = 'ibm_db_sa'
 
+    def dispose(self):
+        pass
+
 
 class TestDBDisconnected(test.BaseTestCase):
 
@@ -219,11 +222,12 @@ class TestDBDisconnected(test.BaseTestCase):
             'convert_unicode': True}
 
         engine = sqlalchemy.create_engine(connection, **engine_args)
-
-        self.assertRaises(sqlalchemy.exc.DisconnectionError,
-                          session._ping_listener, engine,
-                          FakeDBAPIConnection(), FakeConnectionRec(),
-                          FakeConnectionProxy())
+        with mock.patch.object(engine, 'dispose') as dispose_mock:
+            self.assertRaises(sqlalchemy.exc.DisconnectionError,
+                              session._ping_listener, engine,
+                              FakeDBAPIConnection(), FakeConnectionRec(),
+                              FakeConnectionProxy())
+            dispose_mock.assert_called_once_with()
 
     def test_mysql_ping_listener_disconnected(self):
         def fake_execute(sql):