Ensure a functional database connection

Allow retrying database connection in get_engine() at an interval.  Resolves
the issue of nova components erroring at startup if a database connection is
unavailable, particularly at boot. Borrowed from a similar commit to glance,
(https://review.openstack.org/#change,5552).

This also fixes code duplication due to a half-backport of
commit 155ef7daab

Fixes Bug #959426 for nova.

Change-Id: Ifea94da8347714887c8cae02cc48288f3fa4fa7f

etc/nova/nova.conf.sample

@@ -215,6 +215,8 @@
 # sql_connection="sqlite:///$state_path/$sqlite_db"
 ###### (IntOpt) timeout before idle sql connections are reaped
 # sql_idle_timeout=3600
+###### (IntOpt) maximum db connection retries during startup. (setting -1 implies an infinite retry count)
+# sql_max_retries=10
 ###### (IntOpt) interval between retries of opening a sql connection
 # sql_retry_interval=10
 ###### (StrOpt) the filename to use with sqlite
@@ -1105,4 +1107,4 @@
 ###### (StrOpt) The ZFS path under which to create zvols for volumes.
 # san_zfs_volume_base="rpool/"
 
-# Total option count: 466
+# Total option count: 467

nova/db/sqlalchemy/session.py

@@ -22,7 +22,7 @@ import time
 
 import sqlalchemy.interfaces
 import sqlalchemy.orm
-from sqlalchemy.exc import DisconnectionError
+from sqlalchemy.exc import DisconnectionError, OperationalError
 from sqlalchemy.pool import NullPool, StaticPool
 
 import nova.exception
@@ -39,11 +39,11 @@ _MAKER = None
 
 def get_session(autocommit=True, expire_on_commit=False):
     """Return a SQLAlchemy session."""
-    global _ENGINE, _MAKER
+    global _MAKER
 
-    if _MAKER is None or _ENGINE is None:
+    if _MAKER is None:
-        _ENGINE = get_engine()
+        engine = get_engine()
-        _MAKER = get_maker(_ENGINE, autocommit, expire_on_commit)
+        _MAKER = get_maker(engine, autocommit, expire_on_commit)
 
     session = _MAKER()
     session.query = nova.exception.wrap_db_error(session.query)
@@ -80,6 +80,17 @@ class MySQLPingListener(object):
                 raise
 
 
+def is_db_connection_error(args):
+    """Return True if error in connecting to db."""
+    # NOTE(adam_g): This is currently MySQL specific and needs to be extended
+    #               to support Postgres and others.
+    conn_err_codes = ('2002', '2003', '2006')
+    for err_code in conn_err_codes:
+        if args.find(err_code) != -1:
+            return True
+    return False
+
+
 def get_engine():
     """Return a SQLAlchemy engine."""
     global _ENGINE
@@ -105,21 +116,37 @@ def get_engine():
                 engine_args["poolclass"] = StaticPool
                 engine_args["connect_args"] = {'check_same_thread': False}
 
-    engine_args = {
+            if not FLAGS.sqlite_synchronous:
-        "pool_recycle": FLAGS.sql_idle_timeout,
+                engine_args["listeners"] = [SynchronousSwitchListener()]
-        "echo": False,
-        'convert_unicode': True,
-    }
 
-    if "sqlite" in connection_dict.drivername:
+        if 'mysql' in connection_dict.drivername:
-        engine_args["poolclass"] = sqlalchemy.pool.NullPool
+            engine_args['listeners'] = [MySQLPingListener()]
-        if not FLAGS.sqlite_synchronous:
-            engine_args["listeners"] = [SynchronousSwitchListener()]
 
-    if 'mysql' in connection_dict.drivername:
+        _ENGINE = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
-        engine_args['listeners'] = [MySQLPingListener()]
 
-    return sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
+        try:
+            _ENGINE.connect()
+        except OperationalError, e:
+            if not is_db_connection_error(e.args[0]):
+                raise
+
+            remaining = FLAGS.sql_max_retries
+            if remaining == -1:
+                remaining = 'infinite'
+            while True:
+                msg = _('SQL connection failed. %s attempts left.')
+                LOG.warn(msg % remaining)
+                if remaining != 'infinite':
+                    remaining -= 1
+                time.sleep(FLAGS.sql_retry_interval)
+                try:
+                    _ENGINE.connect()
+                    break
+                except OperationalError, e:
+                    if (remaining != 'infinite' and remaining == 0) or \
+                       not is_db_connection_error(e.args[0]):
+                        raise
+    return _ENGINE
 
 
 def get_maker(engine, autocommit=True, expire_on_commit=False):

nova/flags.py

@@ -325,6 +325,10 @@ global_opts = [
     cfg.IntOpt('sql_idle_timeout',
                default=3600,
                help='timeout before idle sql connections are reaped'),
+    cfg.IntOpt('sql_max_retries',
+               default=10,
+               help='maximum db connection retries during startup. '
+                    '(setting -1 implies an infinite retry count)'),
     cfg.IntOpt('sql_retry_interval',
                default=10,
                help='interval between retries of opening a sql connection'),