Ensure a functional database connection
Allow retrying database connection in get_engine() at an interval. Resolves
the issue of nova components erroring at startup if a database connection is
unavailable, particularly at boot. Borrowed from a similar commit to glance,
(https://review.openstack.org/#change,5552).
This also fixes code duplication due to a half-backport of
commit 155ef7daab
Fixes Bug #959426 for nova.
Change-Id: Ifea94da8347714887c8cae02cc48288f3fa4fa7f
This commit is contained in:
parent
615087d4a3
commit
ccb93c4737
|
@ -215,6 +215,8 @@
|
|||
# sql_connection="sqlite:///$state_path/$sqlite_db"
|
||||
###### (IntOpt) timeout before idle sql connections are reaped
|
||||
# sql_idle_timeout=3600
|
||||
###### (IntOpt) maximum db connection retries during startup. (setting -1 implies an infinite retry count)
|
||||
# sql_max_retries=10
|
||||
###### (IntOpt) interval between retries of opening a sql connection
|
||||
# sql_retry_interval=10
|
||||
###### (StrOpt) the filename to use with sqlite
|
||||
|
@ -1105,4 +1107,4 @@
|
|||
###### (StrOpt) The ZFS path under which to create zvols for volumes.
|
||||
# san_zfs_volume_base="rpool/"
|
||||
|
||||
# Total option count: 466
|
||||
# Total option count: 467
|
||||
|
|
|
@ -22,7 +22,7 @@ import time
|
|||
|
||||
import sqlalchemy.interfaces
|
||||
import sqlalchemy.orm
|
||||
from sqlalchemy.exc import DisconnectionError
|
||||
from sqlalchemy.exc import DisconnectionError, OperationalError
|
||||
from sqlalchemy.pool import NullPool, StaticPool
|
||||
|
||||
import nova.exception
|
||||
|
@ -39,11 +39,11 @@ _MAKER = None
|
|||
|
||||
def get_session(autocommit=True, expire_on_commit=False):
|
||||
"""Return a SQLAlchemy session."""
|
||||
global _ENGINE, _MAKER
|
||||
global _MAKER
|
||||
|
||||
if _MAKER is None or _ENGINE is None:
|
||||
_ENGINE = get_engine()
|
||||
_MAKER = get_maker(_ENGINE, autocommit, expire_on_commit)
|
||||
if _MAKER is None:
|
||||
engine = get_engine()
|
||||
_MAKER = get_maker(engine, autocommit, expire_on_commit)
|
||||
|
||||
session = _MAKER()
|
||||
session.query = nova.exception.wrap_db_error(session.query)
|
||||
|
@ -80,6 +80,17 @@ class MySQLPingListener(object):
|
|||
raise
|
||||
|
||||
|
||||
def is_db_connection_error(args):
|
||||
"""Return True if error in connecting to db."""
|
||||
# NOTE(adam_g): This is currently MySQL specific and needs to be extended
|
||||
# to support Postgres and others.
|
||||
conn_err_codes = ('2002', '2003', '2006')
|
||||
for err_code in conn_err_codes:
|
||||
if args.find(err_code) != -1:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_engine():
|
||||
"""Return a SQLAlchemy engine."""
|
||||
global _ENGINE
|
||||
|
@ -105,21 +116,37 @@ def get_engine():
|
|||
engine_args["poolclass"] = StaticPool
|
||||
engine_args["connect_args"] = {'check_same_thread': False}
|
||||
|
||||
engine_args = {
|
||||
"pool_recycle": FLAGS.sql_idle_timeout,
|
||||
"echo": False,
|
||||
'convert_unicode': True,
|
||||
}
|
||||
if not FLAGS.sqlite_synchronous:
|
||||
engine_args["listeners"] = [SynchronousSwitchListener()]
|
||||
|
||||
if "sqlite" in connection_dict.drivername:
|
||||
engine_args["poolclass"] = sqlalchemy.pool.NullPool
|
||||
if not FLAGS.sqlite_synchronous:
|
||||
engine_args["listeners"] = [SynchronousSwitchListener()]
|
||||
if 'mysql' in connection_dict.drivername:
|
||||
engine_args['listeners'] = [MySQLPingListener()]
|
||||
|
||||
if 'mysql' in connection_dict.drivername:
|
||||
engine_args['listeners'] = [MySQLPingListener()]
|
||||
_ENGINE = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
|
||||
|
||||
return sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
|
||||
try:
|
||||
_ENGINE.connect()
|
||||
except OperationalError, e:
|
||||
if not is_db_connection_error(e.args[0]):
|
||||
raise
|
||||
|
||||
remaining = FLAGS.sql_max_retries
|
||||
if remaining == -1:
|
||||
remaining = 'infinite'
|
||||
while True:
|
||||
msg = _('SQL connection failed. %s attempts left.')
|
||||
LOG.warn(msg % remaining)
|
||||
if remaining != 'infinite':
|
||||
remaining -= 1
|
||||
time.sleep(FLAGS.sql_retry_interval)
|
||||
try:
|
||||
_ENGINE.connect()
|
||||
break
|
||||
except OperationalError, e:
|
||||
if (remaining != 'infinite' and remaining == 0) or \
|
||||
not is_db_connection_error(e.args[0]):
|
||||
raise
|
||||
return _ENGINE
|
||||
|
||||
|
||||
def get_maker(engine, autocommit=True, expire_on_commit=False):
|
||||
|
|
|
@ -325,6 +325,10 @@ global_opts = [
|
|||
cfg.IntOpt('sql_idle_timeout',
|
||||
default=3600,
|
||||
help='timeout before idle sql connections are reaped'),
|
||||
cfg.IntOpt('sql_max_retries',
|
||||
default=10,
|
||||
help='maximum db connection retries during startup. '
|
||||
'(setting -1 implies an infinite retry count)'),
|
||||
cfg.IntOpt('sql_retry_interval',
|
||||
default=10,
|
||||
help='interval between retries of opening a sql connection'),
|
||||
|
|
Loading…
Reference in New Issue