Externalize database connection pool parameters

Updating to include the pre-flight check required an update to
sqlalchemy version.

Change-Id: I3da7c4e88f31ea5c2b9f4afa433bda54b16ba8eb
This commit is contained in:
Bryan Strassner 2018-06-27 16:11:47 -05:00
parent 66da23cb20
commit e72a83f03b
6 changed files with 92 additions and 5 deletions

View File

@ -47,7 +47,25 @@
# The database for airflow (string value)
#postgresql_airflow_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/airflow
# The direcotry containing the alembic.ini file (string value)
# The SQLalchemy database connection pool size. (integer value)
#pool_size = 15
# Should DB connections be validated prior to use. (boolean value)
#pool_pre_ping = true
# How long a request for a connection should wait before one becomes available.
# (integer value)
#pool_timeout = 30
# How many connections above pool_size are allowed to be open during high
# usage. (integer value)
#pool_overflow = 10
# Time, in seconds, when a connection should be closed and re-established. -1
# for no recycling. (integer value)
#connection_recycle = -1
# The directory containing the alembic.ini file (string value)
#alembic_ini_path = /home/shipyard/shipyard

View File

@ -26,3 +26,4 @@ kubernetes>=6.0.0
git+https://git.openstack.org/openstack/airship-deckhand@177675e96fffcda9799c68bbce831424c1167020#egg=deckhand
git+https://github.com/att-comdev/drydock.git@8af92eaf29ca0dd6a129748c132ea7f6593eae83#egg=drydock_provisioner
git+https://github.com/att-comdev/armada.git@7a2ba22ab12a3f1f180b6af4085972ba44853377#egg=armada

View File

@ -47,7 +47,25 @@
# The database for airflow (string value)
#postgresql_airflow_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/airflow
# The direcotry containing the alembic.ini file (string value)
# The SQLalchemy database connection pool size. (integer value)
#pool_size = 15
# Should DB connections be validated prior to use. (boolean value)
#pool_pre_ping = true
# How long a request for a connection should wait before one becomes available.
# (integer value)
#pool_timeout = 30
# How many connections above pool_size are allowed to be open during high
# usage. (integer value)
#pool_overflow = 10
# Time, in seconds, when a connection should be closed and re-established. -1
# for no recycling. (integer value)
#connection_recycle = -1
# The directory containing the alembic.ini file (string value)
#alembic_ini_path = /home/shipyard/shipyard

View File

@ -30,7 +30,7 @@ python-dateutil==2.6.1
python-memcached==1.58
requests==2.18.4
setuptools==39.0.1
SQLAlchemy==1.1.13
SQLAlchemy==1.2.8
ulid==1.1
uwsgi==2.0.15

View File

@ -58,10 +58,38 @@ SECTIONS = [
),
help='The database for airflow'
),
cfg.IntOpt(
'pool_size',
default=15,
help='The SQLalchemy database connection pool size.'
),
cfg.BoolOpt(
'pool_pre_ping',
default=True,
help='Should DB connections be validated prior to use.'
),
cfg.IntOpt(
'pool_timeout',
default=30,
help=('How long a request for a connection should wait before '
'one becomes available.')
),
cfg.IntOpt(
'pool_overflow',
default=10,
help=('How many connections above pool_size are allowed to be '
'open during high usage.')
),
cfg.IntOpt(
'connection_recycle',
default=-1,
help=('Time, in seconds, when a connection should be closed '
'and re-established. -1 for no recycling.')
),
cfg.StrOpt(
'alembic_ini_path',
default='/home/shipyard/shipyard',
help='The direcotry containing the alembic.ini file'
help='The directory containing the alembic.ini file'
),
]
),

View File

@ -17,10 +17,12 @@ module for reused or baseclass portions of DB access
import logging
from oslo_config import cfg
import sqlalchemy
from shipyard_airflow.errors import DatabaseError
CONF = cfg.CONF
LOG = logging.getLogger(__name__)
@ -31,6 +33,14 @@ class DbAccess:
def __init__(self):
self.engine = None
# Descendent classes can override these after construciton, but before
# invoking get_engine, if there is a need to tailor the values
# for that specific database.
self.pool_size = CONF.base.pool_size
self.max_overflow = CONF.base.pool_overflow
self.pool_pre_ping = CONF.base.pool_pre_ping
self.pool_recycle = CONF.base.connection_recycle
self.pool_timeout = CONF.base.pool_timeout
def get_connection_string(self):
"""
@ -53,7 +63,19 @@ class DbAccess:
try:
connection_string = self.get_connection_string()
if connection_string is not None and self.engine is None:
self.engine = sqlalchemy.create_engine(connection_string)
LOG.info('Initializing connection <%s> with pool size: %d, '
'max overflow: %d, pool pre ping: %s, pool '
'recycle: %d, and pool timeout: %d',
connection_string, self.pool_size, self.max_overflow,
self.pool_pre_ping, self.pool_recycle,
self.pool_timeout)
self.engine = sqlalchemy.create_engine(
connection_string, pool_size=self.pool_size,
max_overflow=self.max_overflow,
pool_pre_ping=self.pool_pre_ping,
pool_recycle=self.pool_recycle,
pool_timeout=self.pool_timeout
)
if self.engine is None:
self._raise_invalid_db_config(
connection_string=connection_string