From e72a83f03b163a9b4596dd891353642fe29775c6 Mon Sep 17 00:00:00 2001 From: Bryan Strassner Date: Wed, 27 Jun 2018 16:11:47 -0500 Subject: [PATCH] Externalize database connection pool parameters Updating to include the pre-flight check required an update to sqlalchemy version. Change-Id: I3da7c4e88f31ea5c2b9f4afa433bda54b16ba8eb --- docs/source/_static/shipyard.conf.sample | 20 ++++++++++++- images/airflow/requirements.txt | 1 + .../etc/shipyard/shipyard.conf.sample | 20 ++++++++++++- src/bin/shipyard_airflow/requirements.txt | 2 +- .../shipyard_airflow/conf/config.py | 30 ++++++++++++++++++- .../shipyard_airflow/db/common_db.py | 24 ++++++++++++++- 6 files changed, 92 insertions(+), 5 deletions(-) diff --git a/docs/source/_static/shipyard.conf.sample b/docs/source/_static/shipyard.conf.sample index b8c4aaae..75d643aa 100644 --- a/docs/source/_static/shipyard.conf.sample +++ b/docs/source/_static/shipyard.conf.sample @@ -47,7 +47,25 @@ # The database for airflow (string value) #postgresql_airflow_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/airflow -# The direcotry containing the alembic.ini file (string value) +# The SQLalchemy database connection pool size. (integer value) +#pool_size = 15 + +# Should DB connections be validated prior to use. (boolean value) +#pool_pre_ping = true + +# How long a request for a connection should wait before one becomes available. +# (integer value) +#pool_timeout = 30 + +# How many connections above pool_size are allowed to be open during high +# usage. (integer value) +#pool_overflow = 10 + +# Time, in seconds, when a connection should be closed and re-established. -1 +# for no recycling. (integer value) +#connection_recycle = -1 + +# The directory containing the alembic.ini file (string value) #alembic_ini_path = /home/shipyard/shipyard diff --git a/images/airflow/requirements.txt b/images/airflow/requirements.txt index 1ba735ba..c9251ac8 100644 --- a/images/airflow/requirements.txt +++ b/images/airflow/requirements.txt @@ -26,3 +26,4 @@ kubernetes>=6.0.0 git+https://git.openstack.org/openstack/airship-deckhand@177675e96fffcda9799c68bbce831424c1167020#egg=deckhand git+https://github.com/att-comdev/drydock.git@8af92eaf29ca0dd6a129748c132ea7f6593eae83#egg=drydock_provisioner git+https://github.com/att-comdev/armada.git@7a2ba22ab12a3f1f180b6af4085972ba44853377#egg=armada + diff --git a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample index b8c4aaae..75d643aa 100644 --- a/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample +++ b/src/bin/shipyard_airflow/etc/shipyard/shipyard.conf.sample @@ -47,7 +47,25 @@ # The database for airflow (string value) #postgresql_airflow_db = postgresql+psycopg2://shipyard:changeme@postgresql.ucp:5432/airflow -# The direcotry containing the alembic.ini file (string value) +# The SQLalchemy database connection pool size. (integer value) +#pool_size = 15 + +# Should DB connections be validated prior to use. (boolean value) +#pool_pre_ping = true + +# How long a request for a connection should wait before one becomes available. +# (integer value) +#pool_timeout = 30 + +# How many connections above pool_size are allowed to be open during high +# usage. (integer value) +#pool_overflow = 10 + +# Time, in seconds, when a connection should be closed and re-established. -1 +# for no recycling. (integer value) +#connection_recycle = -1 + +# The directory containing the alembic.ini file (string value) #alembic_ini_path = /home/shipyard/shipyard diff --git a/src/bin/shipyard_airflow/requirements.txt b/src/bin/shipyard_airflow/requirements.txt index c703d0f1..2394425b 100644 --- a/src/bin/shipyard_airflow/requirements.txt +++ b/src/bin/shipyard_airflow/requirements.txt @@ -30,7 +30,7 @@ python-dateutil==2.6.1 python-memcached==1.58 requests==2.18.4 setuptools==39.0.1 -SQLAlchemy==1.1.13 +SQLAlchemy==1.2.8 ulid==1.1 uwsgi==2.0.15 diff --git a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py index 0af7d1b9..f1df08b3 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/conf/config.py @@ -58,10 +58,38 @@ SECTIONS = [ ), help='The database for airflow' ), + cfg.IntOpt( + 'pool_size', + default=15, + help='The SQLalchemy database connection pool size.' + ), + cfg.BoolOpt( + 'pool_pre_ping', + default=True, + help='Should DB connections be validated prior to use.' + ), + cfg.IntOpt( + 'pool_timeout', + default=30, + help=('How long a request for a connection should wait before ' + 'one becomes available.') + ), + cfg.IntOpt( + 'pool_overflow', + default=10, + help=('How many connections above pool_size are allowed to be ' + 'open during high usage.') + ), + cfg.IntOpt( + 'connection_recycle', + default=-1, + help=('Time, in seconds, when a connection should be closed ' + 'and re-established. -1 for no recycling.') + ), cfg.StrOpt( 'alembic_ini_path', default='/home/shipyard/shipyard', - help='The direcotry containing the alembic.ini file' + help='The directory containing the alembic.ini file' ), ] ), diff --git a/src/bin/shipyard_airflow/shipyard_airflow/db/common_db.py b/src/bin/shipyard_airflow/shipyard_airflow/db/common_db.py index 2b87ec8a..2111eaf5 100644 --- a/src/bin/shipyard_airflow/shipyard_airflow/db/common_db.py +++ b/src/bin/shipyard_airflow/shipyard_airflow/db/common_db.py @@ -17,10 +17,12 @@ module for reused or baseclass portions of DB access import logging +from oslo_config import cfg import sqlalchemy from shipyard_airflow.errors import DatabaseError +CONF = cfg.CONF LOG = logging.getLogger(__name__) @@ -31,6 +33,14 @@ class DbAccess: def __init__(self): self.engine = None + # Descendent classes can override these after construciton, but before + # invoking get_engine, if there is a need to tailor the values + # for that specific database. + self.pool_size = CONF.base.pool_size + self.max_overflow = CONF.base.pool_overflow + self.pool_pre_ping = CONF.base.pool_pre_ping + self.pool_recycle = CONF.base.connection_recycle + self.pool_timeout = CONF.base.pool_timeout def get_connection_string(self): """ @@ -53,7 +63,19 @@ class DbAccess: try: connection_string = self.get_connection_string() if connection_string is not None and self.engine is None: - self.engine = sqlalchemy.create_engine(connection_string) + LOG.info('Initializing connection <%s> with pool size: %d, ' + 'max overflow: %d, pool pre ping: %s, pool ' + 'recycle: %d, and pool timeout: %d', + connection_string, self.pool_size, self.max_overflow, + self.pool_pre_ping, self.pool_recycle, + self.pool_timeout) + self.engine = sqlalchemy.create_engine( + connection_string, pool_size=self.pool_size, + max_overflow=self.max_overflow, + pool_pre_ping=self.pool_pre_ping, + pool_recycle=self.pool_recycle, + pool_timeout=self.pool_timeout + ) if self.engine is None: self._raise_invalid_db_config( connection_string=connection_string