openstack
/
murano
Code Issues Proposed changes

Merge "Resolved sqlalchemy warning" into release-0.1

This commit is contained in:
Jenkins 2013-06-06 11:35:53 +00:00 committed by Gerrit Code Review
parent c785997025 7b4c722f6a
commit cb2564a6b1
23 changed files with 197 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
bin/murano-api
View File

@ -14,14 +14,11 @@
# License for the specific language governing permissions and limitations
# under the License.
import gettext
import os
import sys
# If ../muranoapi/__init__.py exists, add ../ to Python search path, so that
# it will override what happens to be installed in /usr/(local/)lib/python...
from muranoapi.common.service import TaskResultHandlerService
possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
os.pardir,
os.pardir))
@ -29,11 +26,11 @@ if os.path.exists(os.path.join(possible_topdir, 'muranoapi', '__init__.py')):
sys.path.insert(0, possible_topdir)
from muranoapi.common import config
from muranoapi.common.service import TaskResultHandlerService
from muranoapi.openstack.common import log
from muranoapi.openstack.common import wsgi
from muranoapi.openstack.common import service
gettext.install('muranoapi', './muranoapi/locale', unicode=1)
if __name__ == '__main__':
try:

3
etc/murano-api.conf
View File

@ -18,6 +18,9 @@ log_file = /tmp/murano-api.log
#A valid SQLAlchemy connection string for the metadata database
sql_connection = sqlite:///murano.sqlite
#A boolean that determines if the database will be automatically created
db_auto_create = True
[reports]
results_exchange = task-results
results_queue = task-results

3
muranoapi/__init__.py
View File

@ -11,3 +11,6 @@
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import gettext
gettext.install('muranoapi', './muranoapi/locale', unicode=1)

17
muranoapi/common/config.py
View File

@ -58,12 +58,27 @@ rabbit_opts = [
cfg.StrOpt('virtual_host', default='/'),
]
db_opts = [
cfg.IntOpt('sql_idle_timeout', default=3600,
help=_('Period in seconds after which SQLAlchemy should '
'reestablish its connection to the database.')),
cfg.IntOpt('sql_max_retries', default=60,
help=_('The number of times to retry a connection to the SQL'
'server.')),
cfg.IntOpt('sql_retry_interval', default=1,
help=_('The amount of time to wait (in seconds) before '
'attempting to retry the SQL connection.')),
cfg.BoolOpt('db_auto_create', default=False,
help=_('A boolean that determines if the database will be '
'automatically created.')),
]
CONF = cfg.CONF
CONF.register_opts(paste_deploy_opts, group='paste_deploy')
CONF.register_cli_opts(bind_opts)
# CONF.register_opts(bind_opts)
CONF.register_opts(reports_opts, group='reports')
CONF.register_opts(rabbit_opts, group='rabbitmq')
CONF.register_opts(db_opts)
CONF.import_opt('verbose', 'muranoapi.openstack.common.log')

181
muranoapi/db/session.py
View File

@ -20,88 +20,169 @@
"""Session management functions."""
import os
import time
import sqlalchemy
import logging
from migrate.versioning import api as versioning_api
from migrate import exceptions as versioning_exceptions
from sqlalchemy import create_engine
from sqlalchemy.engine.url import make_url
from sqlalchemy.orm import sessionmaker
from sqlalchemy.pool import NullPool
from sqlalchemy.exc import DisconnectionError
from sqlalchemy.orm import sessionmaker
from muranoapi.common.config import CONF as conf
from muranoapi.db import migrate_repo
from muranoapi.openstack.common import log as mlogging
MAKER = None
ENGINE = None
_ENGINE = None
_MAKER = None
_MAX_RETRIES = None
_RETRY_INTERVAL = None
_IDLE_TIMEOUT = None
_CONNECTION = None
sa_logger = None
log = mlogging.getLogger(__name__)
class MySQLPingListener(object):
def _ping_listener(dbapi_conn, connection_rec, connection_proxy):
"""
Ensures that MySQL connections checked out of the
pool are alive.
Borrowed from:
http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
Error codes caught:
* 2006 MySQL server has gone away
* 2013 Lost connection to MySQL server during query
* 2014 Commands out of sync; you can't run this command now
* 2045 Can't open shared memory; no answer from server (%lu)
* 2055 Lost connection to MySQL server at '%s', system error: %d
from http://dev.mysql.com/doc/refman/5.6/ru_RU/error-messages-client.html
"""
def checkout(self, dbapi_con, con_record, con_proxy):
try:
dbapi_con.cursor().execute('select 1')
except dbapi_con.OperationalError, ex:
if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
logging.warn('Got mysql server has gone away: %s', ex)
raise DisconnectionError("Database server went away")
else:
raise
try:
dbapi_conn.cursor().execute('select 1')
except dbapi_conn.OperationalError as ex:
if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
msg = 'Got mysql server has gone away: %s' % ex
log.warn(msg)
raise DisconnectionError(msg)
else:
raise
def setup_db_env():
"""
Setup configuration for database
"""
global sa_logger, _IDLE_TIMEOUT, _MAX_RETRIES, _RETRY_INTERVAL, _CONNECTION
_IDLE_TIMEOUT = conf.sql_idle_timeout
_MAX_RETRIES = conf.sql_max_retries
_RETRY_INTERVAL = conf.sql_retry_interval
_CONNECTION = conf.sql_connection
sa_logger = logging.getLogger('sqlalchemy.engine')
if conf.debug:
sa_logger.setLevel(logging.DEBUG)
def get_session(autocommit=True, expire_on_commit=False):
"""Return a SQLAlchemy session."""
global MAKER
if MAKER is None:
MAKER = sessionmaker(autocommit=autocommit,
expire_on_commit=expire_on_commit)
engine = get_engine()
MAKER.configure(bind=engine)
session = MAKER()
"""Helper method to grab session"""
global _MAKER
if not _MAKER:
get_engine()
_get_maker(autocommit, expire_on_commit)
assert _MAKER
session = _MAKER()
return session
def get_engine():
"""Return a SQLAlchemy engine."""
global ENGINE
"""May assign _ENGINE if not already assigned"""
global _ENGINE, sa_logger, _CONNECTION, _IDLE_TIMEOUT, _MAX_RETRIES,\
_RETRY_INTERVAL
connection_url = make_url(conf.sql_connection)
if ENGINE is None or not ENGINE.url == connection_url:
engine_args = {'pool_recycle': 3600,
'echo': False,
'convert_unicode': True
}
if 'sqlite' in connection_url.drivername:
engine_args['poolclass'] = NullPool
if 'mysql' in connection_url.drivername:
engine_args['listeners'] = [MySQLPingListener()]
ENGINE = create_engine(conf.sql_connection, **engine_args)
if not _ENGINE:
setup_db_env()
sync()
return ENGINE
connection_dict = sqlalchemy.engine.url.make_url(_CONNECTION)
engine_args = {
'pool_recycle': _IDLE_TIMEOUT,
'echo': False,
'convert_unicode': True}
try:
_ENGINE = sqlalchemy.create_engine(_CONNECTION, **engine_args)
if 'mysql' in connection_dict.drivername:
sqlalchemy.event.listen(_ENGINE, 'checkout', _ping_listener)
_ENGINE.connect = _wrap_db_error(_ENGINE.connect)
_ENGINE.connect()
except Exception as err:
msg = _("Error configuring registry database with supplied "
"sql_connection. Got error: %s") % err
log.error(msg)
raise
if conf.db_auto_create:
log.info(_('auto-creating DB'))
_auto_create_db()
else:
log.info(_('not auto-creating DB'))
return _ENGINE
def sync():
def _get_maker(autocommit=True, expire_on_commit=False):
"""Return a SQLAlchemy sessionmaker."""
"""May assign __MAKER if not already assigned"""
global _MAKER, _ENGINE
assert _ENGINE
if not _MAKER:
_MAKER = sessionmaker(bind=_ENGINE, autocommit=autocommit,
expire_on_commit=expire_on_commit)
return _MAKER
def _is_db_connection_error(args):
"""Return True if error in connecting to db."""
# NOTE(adam_g): This is currently MySQL specific and needs to be extended
# to support Postgres and others.
conn_err_codes = ('2002', '2003', '2006')
for err_code in conn_err_codes:
if args.find(err_code) != -1:
return True
return False
def _wrap_db_error(f):
"""Retry DB connection. Copied from nova and modified."""
def _wrap(*args, **kwargs):
try:
return f(*args, **kwargs)
except sqlalchemy.exc.OperationalError as e:
if not _is_db_connection_error(e.args[0]):
raise
remaining_attempts = _MAX_RETRIES
while True:
log.warning(_('SQL connection failed. %d attempts left.'),
remaining_attempts)
remaining_attempts -= 1
time.sleep(_RETRY_INTERVAL)
try:
return f(*args, **kwargs)
except sqlalchemy.exc.OperationalError as e:
if (remaining_attempts == 0 or
not _is_db_connection_error(e.args[0])):
raise
except sqlalchemy.exc.DBAPIError:
raise
except sqlalchemy.exc.DBAPIError:
raise
_wrap.func_name = f.func_name
return _wrap
def _auto_create_db():
repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__))
try:
versioning_api.upgrade(conf.sql_connection, repo_path)

2
muranoapi/openstack/common/eventlet_backdoor.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright (c) 2012 Openstack, LLC.
# Copyright (c) 2012 OpenStack Foundation.
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
#

2
muranoapi/openstack/common/exception.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/importutils.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

6
muranoapi/openstack/common/jsonutils.py
View File

@ -38,14 +38,10 @@ import functools
import inspect
import itertools
import json
import logging
import xmlrpclib
from muranoapi.openstack.common.gettextutils import _
from muranoapi.openstack.common import timeutils
LOG = logging.getLogger(__name__)
def to_primitive(value, convert_instances=False, convert_datetime=True,
level=0, max_depth=3):
@ -85,8 +81,6 @@ def to_primitive(value, convert_instances=False, convert_datetime=True,
return 'mock'
if level > max_depth:
LOG.error(_('Max serialization depth exceeded on object: %d %s'),
level, value)
return '?'
# The try block may not be necessary after the class check above,

2
muranoapi/openstack/common/local.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

11
muranoapi/openstack/common/log.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# Copyright 2010 United States Government as represented by the
# Administrator of the National Aeronautics and Space Administration.
# All Rights Reserved.
@ -328,7 +328,7 @@ def setup(product_name):
if CONF.log_config:
logging.config.fileConfig(CONF.log_config)
else:
_setup_logging_from_conf(product_name)
_setup_logging_from_conf()
sys.excepthook = _create_logging_excepthook(product_name)
@ -362,8 +362,8 @@ def _find_facility_from_conf():
return facility
def _setup_logging_from_conf(product_name):
log_root = getLogger(product_name).logger
def _setup_logging_from_conf():
log_root = getLogger(None).logger
for handler in log_root.handlers:
log_root.removeHandler(handler)
@ -401,7 +401,8 @@ def _setup_logging_from_conf(product_name):
if CONF.log_format:
handler.setFormatter(logging.Formatter(fmt=CONF.log_format,
datefmt=datefmt))
handler.setFormatter(LegacyFormatter(datefmt=datefmt))
else:
handler.setFormatter(LegacyFormatter(datefmt=datefmt))
if CONF.debug:
log_root.setLevel(logging.DEBUG)

2
muranoapi/openstack/common/notifier/__init__.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/notifier/api.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/notifier/log_notifier.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/notifier/no_op_notifier.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/notifier/rpc_notifier.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/notifier/rpc_notifier2.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/notifier/test_notifier.py
View File

@ -1,4 +1,4 @@
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

12
muranoapi/openstack/common/setup.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
@ -149,7 +149,7 @@ def write_git_changelog():
git_dir = _get_git_directory()
if not os.getenv('SKIP_WRITE_GIT_CHANGELOG'):
if git_dir:
git_log_cmd = 'git --git-dir=%s log --stat' % git_dir
git_log_cmd = 'git --git-dir=%s log' % git_dir
changelog = _run_shell_command(git_log_cmd)
mailmap = _parse_git_mailmap(git_dir)
with open(new_changelog, "w") as changelog_file:
@ -171,6 +171,14 @@ def generate_authors():
" log --format='%aN <%aE>' | sort -u | "
"egrep -v '" + jenkins_email + "'")
changelog = _run_shell_command(git_log_cmd)
signed_cmd = ("git log --git-dir=" + git_dir +
" | grep -i Co-authored-by: | sort -u")
signed_entries = _run_shell_command(signed_cmd)
if signed_entries:
new_entries = "\n".join(
[signed.split(":", 1)[1].strip()
for signed in signed_entries.split("\n") if signed])
changelog = "\n".join((changelog, new_entries))
mailmap = _parse_git_mailmap(git_dir)
with open(new_authors, 'w') as new_authors_fh:
new_authors_fh.write(canonicalize_emails(changelog, mailmap))

18
muranoapi/openstack/common/timeutils.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -25,18 +25,22 @@ import datetime
import iso8601
TIME_FORMAT = "%Y-%m-%dT%H:%M:%S"
PERFECT_TIME_FORMAT = "%Y-%m-%dT%H:%M:%S.%f"
# ISO 8601 extended time format with microseconds
_ISO8601_TIME_FORMAT_SUBSECOND = '%Y-%m-%dT%H:%M:%S.%f'
_ISO8601_TIME_FORMAT = '%Y-%m-%dT%H:%M:%S'
PERFECT_TIME_FORMAT = _ISO8601_TIME_FORMAT_SUBSECOND
def isotime(at=None):
def isotime(at=None, subsecond=False):
"""Stringify time in ISO 8601 format"""
if not at:
at = utcnow()
str = at.strftime(TIME_FORMAT)
st = at.strftime(_ISO8601_TIME_FORMAT
if not subsecond
else _ISO8601_TIME_FORMAT_SUBSECOND)
tz = at.tzinfo.tzname(None) if at.tzinfo else 'UTC'
str += ('Z' if tz == 'UTC' else tz)
return str
st += ('Z' if tz == 'UTC' else tz)
return st
def parse_isotime(timestr):

2
muranoapi/openstack/common/version.py
View File

@ -1,5 +1,5 @@
# Copyright 2012 OpenStack LLC
# Copyright 2012 OpenStack Foundation
# Copyright 2012-2013 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

2
muranoapi/openstack/common/wsgi.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2011 OpenStack LLC.
# Copyright 2011 OpenStack Foundation.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may

5
tools/install_venv_common.py
View File

@ -1,6 +1,6 @@
# vim: tabstop=4 shiftwidth=4 softtabstop=4
# Copyright 2013 OpenStack, LLC
# Copyright 2013 OpenStack Foundation
# Copyright 2013 IBM Corp.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
@ -185,7 +185,8 @@ class Fedora(Distro):
self.run_command(['sudo', 'yum', 'install', '-y', pkg], **kwargs)
def apply_patch(self, originalfile, patchfile):
self.run_command(['patch', originalfile, patchfile])
self.run_command(['patch', '-N', originalfile, patchfile],
check_exit_code=False)
def install_virtualenv(self):
if self.check_cmd('virtualenv'):