openstack
/
nova
Code Issues Proposed changes

Make sqlite in-memory-db usable to unittest 

Move migrate monkey patch from nova.tests.test_migrations to
nova.db.sqlalchemy.migration

Change-Id: I018e44903558cad6311fd368787583322f962d0c
This commit is contained in:
Hengqing Hu 2012-03-08 16:43:32 +08:00
parent eb42e7fcd7
commit 155ef7daab
6 changed files with 114 additions and 71 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
nova/db/sqlalchemy/migration.py
View File

@ -16,14 +16,46 @@
# License for the specific language governing permissions and limitations
# under the License.
import distutils.version as dist_version
import os
import sys
from nova.db.sqlalchemy.session import get_engine
from nova import exception
from nova import flags
import sqlalchemy
import migrate
from migrate.versioning import util as migrate_util
MIGRATE_PKG_VER = dist_version.StrictVersion(migrate.__version__)
USE_MIGRATE_PATCH = MIGRATE_PKG_VER < dist_version.StrictVersion('0.7.3')
@migrate_util.decorator
def patched_with_engine(f, *a, **kw):
url = a[0]
engine = migrate_util.construct_engine(url, **kw)
try:
kw['engine'] = engine
return f(*a, **kw)
finally:
if isinstance(engine, migrate_util.Engine) and engine is not url:
migrate_util.log.debug('Disposing SQLAlchemy engine %s', engine)
engine.dispose()
# TODO(jkoelker) When migrate 0.7.3 is released and nova depends
# on that version or higher, this can be removed
if USE_MIGRATE_PATCH:
migrate_util.with_engine = patched_with_engine
# NOTE(jkoelker) Delay importing migrate until we are patched
from migrate.versioning import api as versioning_api
from migrate.versioning.repository import Repository
try:
from migrate.versioning import exceptions as versioning_exceptions
@ -37,6 +69,8 @@ except ImportError:
FLAGS = flags.FLAGS
_REPOSITORY = None
def db_sync(version=None):
if version is not None:
@ -46,24 +80,24 @@ def db_sync(version=None):
raise exception.Error(_("version should be an integer"))
current_version = db_version()
repo_path = _find_migrate_repo()
repository = _find_migrate_repo()
if version is None or version > current_version:
return versioning_api.upgrade(FLAGS.sql_connection, repo_path, version)
return versioning_api.upgrade(get_engine(), repository, version)
else:
return versioning_api.downgrade(FLAGS.sql_connection, repo_path,
return versioning_api.downgrade(get_engine(), repository,
version)
def db_version():
repo_path = _find_migrate_repo()
repository = _find_migrate_repo()
try:
return versioning_api.db_version(FLAGS.sql_connection, repo_path)
return versioning_api.db_version(get_engine(), repository)
except versioning_exceptions.DatabaseNotControlledError:
# If we aren't version controlled we may already have the database
# in the state from before we started version control, check for that
# and set up version_control appropriately
meta = sqlalchemy.MetaData()
engine = sqlalchemy.create_engine(FLAGS.sql_connection, echo=False)
engine = get_engine()
meta.reflect(bind=engine)
try:
for table in ('auth_tokens', 'zones', 'export_devices',
@ -85,14 +119,17 @@ def db_version():
def db_version_control(version=None):
repo_path = _find_migrate_repo()
versioning_api.version_control(FLAGS.sql_connection, repo_path, version)
repository = _find_migrate_repo()
versioning_api.version_control(get_engine(), repository, version)
return version
def _find_migrate_repo():
"""Get the path for the migrate repository."""
global _REPOSITORY
path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
'migrate_repo')
assert os.path.exists(path)
return path
if _REPOSITORY is None:
_REPOSITORY = Repository(path)
return _REPOSITORY

45
nova/db/sqlalchemy/session.py
View File

@ -23,6 +23,8 @@ import time
import sqlalchemy.interfaces
import sqlalchemy.orm
from sqlalchemy.exc import DisconnectionError
from sqlalchemy.pool import NullPool, StaticPool
import time
import nova.exception
import nova.flags as flags
@ -38,11 +40,11 @@ _MAKER = None
def get_session(autocommit=True, expire_on_commit=False):
"""Return a SQLAlchemy session."""
global _ENGINE, _MAKER
global _MAKER
if _MAKER is None or _ENGINE is None:
_ENGINE = get_engine()
_MAKER = get_maker(_ENGINE, autocommit, expire_on_commit)
if _MAKER is None:
engine = get_engine()
_MAKER = get_maker(engine, autocommit, expire_on_commit)
session = _MAKER()
session.query = nova.exception.wrap_db_error(session.query)
@ -81,23 +83,32 @@ class MySQLPingListener(object):
def get_engine():
"""Return a SQLAlchemy engine."""
connection_dict = sqlalchemy.engine.url.make_url(FLAGS.sql_connection)
global _ENGINE
if _ENGINE is None:
connection_dict = sqlalchemy.engine.url.make_url(FLAGS.sql_connection)
engine_args = {
"pool_recycle": FLAGS.sql_idle_timeout,
"echo": False,
'convert_unicode': True,
}
engine_args = {
"pool_recycle": FLAGS.sql_idle_timeout,
"echo": False,
'convert_unicode': True,
}
if "sqlite" in connection_dict.drivername:
engine_args["poolclass"] = sqlalchemy.pool.NullPool
if not FLAGS.sqlite_synchronous:
engine_args["listeners"] = [SynchronousSwitchListener()]
if "sqlite" in connection_dict.drivername:
engine_args["poolclass"] = NullPool
if 'mysql' in connection_dict.drivername:
engine_args['listeners'] = [MySQLPingListener()]
if FLAGS.sql_connection == "sqlite://":
engine_args["poolclass"] = StaticPool
engine_args["connect_args"] = {'check_same_thread': False}
return sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
if not FLAGS.sqlite_synchronous:
engine_args["listeners"] = [SynchronousSwitchListener()]
if 'mysql' in connection_dict.drivername:
engine_args['listeners'] = [MySQLPingListener()]
_ENGINE = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args)
return _ENGINE
def get_maker(engine, autocommit=True, expire_on_commit=False):

4
nova/test.py
View File

@ -40,6 +40,7 @@ from nova.openstack.common import cfg
from nova import utils
from nova import service
from nova.testing.fake import rabbit
from nova.tests import reset_db
from nova.virt import fake
@ -129,8 +130,7 @@ class TestCase(unittest.TestCase):
# now that we have some required db setup for the system
# to work properly.
self.start = utils.utcnow()
shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db),
os.path.join(FLAGS.state_path, FLAGS.sqlite_db))
reset_db()
# emulate some of the mox stuff, we can't use the metaclass
# because it screws with our generators

45
nova/tests/__init__.py
View File

@ -34,25 +34,44 @@
# The code below enables nosetests to work with i18n _() blocks
import __builtin__
setattr(__builtin__, '_', lambda x: x)
import os
import shutil
from nova.db.sqlalchemy.session import get_engine
from nova import flags
FLAGS = flags.FLAGS
_DB = None
def reset_db():
if FLAGS.sql_connection == "sqlite://":
engine = get_engine()
engine.dispose()
conn = engine.connect()
conn.connection.executescript(_DB)
else:
shutil.copyfile(os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db),
os.path.join(FLAGS.state_path, FLAGS.sqlite_db))
def setup():
import mox # Fail fast if you don't have mox. Workaround for bug 810424
import os
import shutil
from nova import context
from nova import flags
from nova import db
from nova.db import migration
from nova.network import manager as network_manager
from nova.tests import fake_flags
FLAGS = flags.FLAGS
testdb = os.path.join(FLAGS.state_path, FLAGS.sqlite_db)
if os.path.exists(testdb):
return
if FLAGS.sql_connection == "sqlite://":
if migration.db_version() > 1:
return
else:
testdb = os.path.join(FLAGS.state_path, FLAGS.sqlite_db)
if os.path.exists(testdb):
return
migration.db_sync()
ctxt = context.get_admin_context()
network = network_manager.VlanManager()
@ -74,5 +93,11 @@ def setup():
for net in db.network_get_all(ctxt):
network.set_network_host(ctxt, net)
cleandb = os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db)
shutil.copyfile(testdb, cleandb)
if FLAGS.sql_connection == "sqlite://":
global _DB
engine = get_engine()
conn = engine.connect()
_DB = "".join(line for line in conn.connection.iterdump())
else:
cleandb = os.path.join(FLAGS.state_path, FLAGS.sqlite_clean_db)
shutil.copyfile(testdb, cleandb)

2
nova/tests/fake_flags.py
View File

@ -37,7 +37,7 @@ FLAGS.set_default('image_service', 'nova.image.fake.FakeImageService')
flags.DECLARE('iscsi_num_targets', 'nova.volume.driver')
FLAGS.set_default('iscsi_num_targets', 8)
FLAGS.set_default('verbose', True)
FLAGS.set_default('sqlite_db', "tests.sqlite")
FLAGS.set_default('sql_connection', "sqlite://")
FLAGS.set_default('use_ipv6', True)
FLAGS.set_default('flat_network_bridge', 'br100')
FLAGS.set_default('sqlite_synchronous', False)

34
nova/tests/test_migrations.py
View File

@ -26,50 +26,20 @@ if possible.
import ConfigParser
import commands
import distutils.version as dist_version
import os
import unittest
import urlparse
import migrate
from migrate.versioning import util as migrate_util
from migrate.versioning import repository
import sqlalchemy
import nova.db.sqlalchemy.migrate_repo
from nova.db.sqlalchemy.migration import versioning_api as migration_api
from nova import log as logging
from nova import test
LOG = logging.getLogger('nova.tests.test_migrations')
MIGRATE_PKG_VER = dist_version.StrictVersion(migrate.__version__)
USE_MIGRATE_PATCH = MIGRATE_PKG_VER < dist_version.StrictVersion('0.7.3')
@migrate_util.decorator
def patched_with_engine(f, *a, **kw):
url = a[0]
engine = migrate_util.construct_engine(url, **kw)
try:
kw['engine'] = engine
return f(*a, **kw)
finally:
if isinstance(engine, migrate_util.Engine) and engine is not url:
migrate_util.log.debug('Disposing SQLAlchemy engine %s', engine)
engine.dispose()
# TODO(jkoelker) When migrate 0.7.3 is released and nova depends
# on that version or higher, this can be removed
if USE_MIGRATE_PATCH:
migrate_util.with_engine = patched_with_engine
# NOTE(jkoelker) Delay importing migrate until we are patched
from migrate.versioning import api as migration_api
from migrate.versioning import repository
class TestMigrations(unittest.TestCase):
"""Test sqlalchemy-migrate migrations"""