Refactor tests to use Alembic to run migrations

* Functional tests now use alembic instead of sqlalchmey-migrate
  to build and destroy test database.
* All tests now use a file-based sqlite db as opposed to an in-memory
  database.

Partially-Implements: blueprint alembic-migrations
Change-Id: I77921366a05ba6f9841143af89c1f4059d8454c6
Depends-On: Ie8594ff339a13bf190aefa308f54e97ee20ecfa2
This commit is contained in:
Hemanth Makkapati 2016-11-14 17:18:34 -06:00
parent 21d431013f
commit 95c7c1b753
13 changed files with 460 additions and 1736 deletions

View File

@ -26,8 +26,6 @@ from oslo_config import cfg
from oslo_db import options as db_options
from stevedore import driver
from glance.db.sqlalchemy import api as db_api
_IMPL = None
_LOCK = threading.Lock()
@ -53,14 +51,3 @@ MIGRATE_REPO_PATH = os.path.join(
'sqlalchemy',
'migrate_repo',
)
def db_sync(version=None, init_version=0, engine=None):
"""Migrate the database to `version` or the most recent version."""
if engine is None:
engine = db_api.get_engine()
return get_backend().db_sync(engine=engine,
abs_path=MIGRATE_REPO_PATH,
version=version,
init_version=init_version)

View File

@ -20,19 +20,20 @@ from alembic import command as alembic_command
from alembic import config as alembic_config
from alembic import migration as alembic_migration
from oslo_db import exception as db_exception
from oslo_db.sqlalchemy import migration
from oslo_db.sqlalchemy import migration as sqla_migration
from glance.db import migration as db_migration
from glance.db.sqlalchemy import api as db_api
from glance.i18n import _
def get_alembic_config():
def get_alembic_config(engine=None):
"""Return a valid alembic config object"""
ini_path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
config = alembic_config.Config(os.path.abspath(ini_path))
dbconn = str(db_api.get_engine().url)
config.set_main_option('sqlalchemy.url', dbconn)
if engine is None:
engine = db_api.get_engine()
config.set_main_option('sqlalchemy.url', str(engine.url))
return config
@ -47,9 +48,9 @@ def get_current_alembic_heads():
def get_current_legacy_head():
try:
legacy_head = migration.db_version(db_api.get_engine(),
db_migration.MIGRATE_REPO_PATH,
db_migration.INIT_VERSION)
legacy_head = sqla_migration.db_version(db_api.get_engine(),
db_migration.MIGRATE_REPO_PATH,
db_migration.INIT_VERSION)
except db_exception.DbMigrationError:
legacy_head = None
return legacy_head

View File

@ -0,0 +1,48 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from oslo_db.sqlalchemy import test_base
import sqlalchemy
from glance.tests.functional.db import test_migrations
def get_indexes(table, engine):
inspector = sqlalchemy.inspect(engine)
return [idx['name'] for idx in inspector.get_indexes(table)]
class TestMitaka01Mixin(test_migrations.AlembicMigrationsMixin):
def _pre_upgrade_mitaka01(self, engine):
indexes = get_indexes('images', engine)
self.assertNotIn('created_at_image_idx', indexes)
self.assertNotIn('updated_at_image_idx', indexes)
def _check_mitaka01(self, engine, data):
indexes = get_indexes('images', engine)
self.assertIn('created_at_image_idx', indexes)
self.assertIn('updated_at_image_idx', indexes)
class TestMitaka01MySQL(TestMitaka01Mixin,
test_base.MySQLOpportunisticTestCase):
pass
class TestMitaka01PostgresSQL(TestMitaka01Mixin,
test_base.PostgreSQLOpportunisticTestCase):
pass
class TestMitaka01Sqlite(TestMitaka01Mixin, test_base.DbTestCase):
pass

View File

@ -0,0 +1,65 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
from glance.tests.functional.db import test_migrations
class TestMitaka02Mixin(test_migrations.AlembicMigrationsMixin):
def _pre_upgrade_mitaka02(self, engine):
metadef_resource_types = db_utils.get_table(engine,
'metadef_resource_types')
now = datetime.datetime.now()
db_rec1 = dict(id='9580',
name='OS::Nova::Instance',
protected=False,
created_at=now,
updated_at=now,)
db_rec2 = dict(id='9581',
name='OS::Nova::Blah',
protected=False,
created_at=now,
updated_at=now,)
db_values = (db_rec1, db_rec2)
metadef_resource_types.insert().values(db_values).execute()
def _check_mitaka02(self, engine, data):
metadef_resource_types = db_utils.get_table(engine,
'metadef_resource_types')
result = (metadef_resource_types.select()
.where(metadef_resource_types.c.name == 'OS::Nova::Instance')
.execute().fetchall())
self.assertEqual(0, len(result))
result = (metadef_resource_types.select()
.where(metadef_resource_types.c.name == 'OS::Nova::Server')
.execute().fetchall())
self.assertEqual(1, len(result))
class TestMitaka02MySQL(TestMitaka02Mixin,
test_base.MySQLOpportunisticTestCase):
pass
class TestMitaka02PostgresSQL(TestMitaka02Mixin,
test_base.PostgreSQLOpportunisticTestCase):
pass
class TestMitaka02Sqlite(TestMitaka02Mixin, test_base.DbTestCase):
pass

View File

@ -0,0 +1,142 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
from glance.tests.functional.db import test_migrations
class TestOcata01Mixin(test_migrations.AlembicMigrationsMixin):
def _pre_upgrade_ocata01(self, engine):
images = db_utils.get_table(engine, 'images')
now = datetime.datetime.now()
image_members = db_utils.get_table(engine, 'image_members')
# inserting a public image record
public_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0,
id='public_id')
images.insert().values(public_temp).execute()
# inserting a non-public image record for 'shared' visibility test
shared_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='shared_id')
images.insert().values(shared_temp).execute()
# inserting a non-public image records for 'private' visibility test
private_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='private_id_1')
images.insert().values(private_temp).execute()
private_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='private_id_2')
images.insert().values(private_temp).execute()
# adding an active as well as a deleted image member for checking
# 'shared' visibility
temp = dict(deleted=False,
created_at=now,
image_id='shared_id',
member='fake_member_452',
can_share=True,
id=45)
image_members.insert().values(temp).execute()
temp = dict(deleted=True,
created_at=now,
image_id='shared_id',
member='fake_member_453',
can_share=True,
id=453)
image_members.insert().values(temp).execute()
# adding an image member, but marking it deleted,
# for testing 'private' visibility
temp = dict(deleted=True,
created_at=now,
image_id='private_id_2',
member='fake_member_451',
can_share=True,
id=451)
image_members.insert().values(temp).execute()
# adding an active image member for the 'public' image,
# to test it remains public regardless.
temp = dict(deleted=False,
created_at=now,
image_id='public_id',
member='fake_member_450',
can_share=True,
id=450)
image_members.insert().values(temp).execute()
def _check_ocata01(self, engine, data):
# check that after migration, 'visibility' column is introduced
images = db_utils.get_table(engine, 'images')
self.assertIn('visibility', images.c)
self.assertNotIn('is_public', images.c)
# tests to identify the visibilities of images created above
rows = images.select().where(
images.c.id == 'public_id').execute().fetchall()
self.assertEqual(1, len(rows))
self.assertEqual('public', rows[0][16])
rows = images.select().where(
images.c.id == 'shared_id').execute().fetchall()
self.assertEqual(1, len(rows))
self.assertEqual('shared', rows[0][16])
rows = images.select().where(
images.c.id == 'private_id_1').execute().fetchall()
self.assertEqual(1, len(rows))
self.assertEqual('private', rows[0][16])
rows = images.select().where(
images.c.id == 'private_id_2').execute().fetchall()
self.assertEqual(1, len(rows))
self.assertEqual('private', rows[0][16])
class TestOcata01MySQL(TestOcata01Mixin, test_base.MySQLOpportunisticTestCase):
pass
class TestOcata01PostgresSQL(TestOcata01Mixin,
test_base.PostgreSQLOpportunisticTestCase):
pass
class TestOcata01Sqlite(TestOcata01Mixin, test_base.DbTestCase):
pass

View File

@ -0,0 +1,173 @@
# Copyright 2016 Rackspace
# Copyright 2016 Intel Corporation
#
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import os
from alembic import command as alembic_command
from alembic import script as alembic_script
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import test_migrations
import sqlalchemy.types as types
from glance.db.sqlalchemy import alembic_migrations
from glance.db.sqlalchemy.alembic_migrations import versions
from glance.db.sqlalchemy import models
from glance.db.sqlalchemy import models_glare
from glance.db.sqlalchemy import models_metadef
import glance.tests.utils as test_utils
class AlembicMigrationsMixin(object):
def _get_revisions(self, config):
scripts_dir = alembic_script.ScriptDirectory.from_config(config)
revisions = list(scripts_dir.walk_revisions(base='base', head='heads'))
revisions = list(reversed(revisions))
revisions = [rev.revision for rev in revisions]
return revisions
def _migrate_up(self, config, engine, revision, with_data=False):
if with_data:
data = None
pre_upgrade = getattr(self, '_pre_upgrade_%s' % revision, None)
if pre_upgrade:
data = pre_upgrade(engine)
alembic_command.upgrade(config, revision)
if with_data:
check = getattr(self, '_check_%s' % revision, None)
if check:
check(engine, data)
def test_walk_versions(self):
alembic_config = alembic_migrations.get_alembic_config(self.engine)
for revision in self._get_revisions(alembic_config):
self._migrate_up(alembic_config, self.engine, revision,
with_data=True)
class TestMysqlMigrations(test_base.MySQLOpportunisticTestCase,
AlembicMigrationsMixin):
def test_mysql_innodb_tables(self):
test_utils.db_sync(engine=self.engine)
total = self.engine.execute(
"SELECT COUNT(*) "
"FROM information_schema.TABLES "
"WHERE TABLE_SCHEMA='%s'"
% self.engine.url.database)
self.assertGreater(total.scalar(), 0, "No tables found. Wrong schema?")
noninnodb = self.engine.execute(
"SELECT count(*) "
"FROM information_schema.TABLES "
"WHERE TABLE_SCHEMA='%s' "
"AND ENGINE!='InnoDB' "
"AND TABLE_NAME!='migrate_version'"
% self.engine.url.database)
count = noninnodb.scalar()
self.assertEqual(0, count, "%d non InnoDB tables created" % count)
class TestPostgresqlMigrations(test_base.PostgreSQLOpportunisticTestCase,
AlembicMigrationsMixin):
pass
class TestSqliteMigrations(test_base.DbTestCase, AlembicMigrationsMixin):
pass
class TestMigrations(test_base.DbTestCase, test_utils.BaseTestCase):
def test_no_downgrade(self):
migrate_file = versions.__path__[0]
for parent, dirnames, filenames in os.walk(migrate_file):
for filename in filenames:
if filename.split('.')[1] == 'py':
model_name = filename.split('.')[0]
model = __import__(
'glance.db.sqlalchemy.alembic_migrations.versions.' +
model_name)
obj = getattr(getattr(getattr(getattr(getattr(
model, 'db'), 'sqlalchemy'), 'alembic_migrations'),
'versions'), model_name)
func = getattr(obj, 'downgrade', None)
self.assertIsNone(func)
class ModelsMigrationSyncMixin(object):
def get_metadata(self):
for table in models_metadef.BASE_DICT.metadata.sorted_tables:
models.BASE.metadata._add_table(table.name, table.schema, table)
for table in models_glare.BASE.metadata.sorted_tables:
models.BASE.metadata._add_table(table.name, table.schema, table)
return models.BASE.metadata
def get_engine(self):
return self.engine
def db_sync(self, engine):
test_utils.db_sync(engine=engine)
# TODO(akamyshikova): remove this method as soon as comparison with Variant
# will be implemented in oslo.db or alembic
def compare_type(self, ctxt, insp_col, meta_col, insp_type, meta_type):
if isinstance(meta_type, types.Variant):
meta_orig_type = meta_col.type
insp_orig_type = insp_col.type
meta_col.type = meta_type.impl
insp_col.type = meta_type.impl
try:
return self.compare_type(ctxt, insp_col, meta_col, insp_type,
meta_type.impl)
finally:
meta_col.type = meta_orig_type
insp_col.type = insp_orig_type
else:
ret = super(ModelsMigrationSyncMixin, self).compare_type(
ctxt, insp_col, meta_col, insp_type, meta_type)
if ret is not None:
return ret
return ctxt.impl.compare_type(insp_col, meta_col)
def include_object(self, object_, name, type_, reflected, compare_to):
if name in ['migrate_version'] and type_ == 'table':
return False
return True
class ModelsMigrationsSyncMysql(ModelsMigrationSyncMixin,
test_migrations.ModelsMigrationsSync,
test_base.MySQLOpportunisticTestCase):
pass
class ModelsMigrationsSyncPostgres(ModelsMigrationSyncMixin,
test_migrations.ModelsMigrationsSync,
test_base.PostgreSQLOpportunisticTestCase):
pass
class ModelsMigrationsSyncSqlite(ModelsMigrationSyncMixin,
test_migrations.ModelsMigrationsSync,
test_base.DbTestCase):
pass

View File

@ -21,7 +21,6 @@ from oslo_db import options
import glance.common.client
from glance.common import config
from glance.db import migration
import glance.db.sqlalchemy.api
import glance.registry.client.v1.client
from glance import tests as glance_tests
@ -171,7 +170,7 @@ class ApiTest(test_utils.BaseTestCase):
test_utils.execute('cp %s %s/tests.sqlite'
% (db_location, self.test_dir))
else:
migration.db_sync()
test_utils.db_sync()
# copy the clean db to a temp location so that it
# can be reused for future tests

View File

@ -24,7 +24,6 @@ from oslo_db import options
import glance.common.client
from glance.common import config
from glance.db import migration
import glance.db.sqlalchemy.api
import glance.registry.client.v1.client
from glance import tests as glance_tests
@ -166,7 +165,7 @@ class ApiTest(test_utils.BaseTestCase):
test_utils.execute('cp %s %s/tests.sqlite'
% (db_location, self.test_dir))
else:
migration.db_sync()
test_utils.db_sync()
# copy the clean db to a temp location so that it
# can be reused for future tests

File diff suppressed because it is too large Load Diff

View File

@ -23,6 +23,7 @@ import shutil
import socket
import subprocess
from alembic import command as alembic_command
import fixtures
from oslo_config import cfg
from oslo_config import fixture as cfg_fixture
@ -42,6 +43,7 @@ from glance.common import timeutils
from glance.common import utils
from glance.common import wsgi
from glance import context
from glance.db.sqlalchemy import alembic_migrations
from glance.db.sqlalchemy import api as db_api
from glance.db.sqlalchemy import models as db_models
@ -670,3 +672,14 @@ class HttplibWsgiAdapter(object):
response = self.req.get_response(self.app)
return FakeHTTPResponse(response.status_code, response.headers,
response.body)
def db_sync(version=None, engine=None):
"""Migrate the database to `version` or the most recent version."""
if version is None:
version = 'heads'
if engine is None:
engine = db_api.get_engine()
alembic_config = alembic_migrations.get_alembic_config(engine=engine)
alembic_command.upgrade(alembic_config, version)

View File

@ -10,6 +10,15 @@ basepython =
setenv =
VIRTUAL_ENV={envdir}
PYTHONWARNINGS=default::DeprecationWarning
# NOTE(hemanthm): The environment variable "OS_TEST_DBAPI_ADMIN_CONNECTION"
# must be set to force oslo.db tests to use a file-based sqlite database
# instead of the default in-memory database, which doesn't work well with
# alembic migrations. The file-based database pointed by the environment
# variable itself is not used for testing. Neither is it ever created. Oslo.db
# creates another file-based database for testing purposes and deletes it as a
# part of its test clean-up. Think of setting this environment variable as a
# clue for oslo.db to use file-based database.
OS_TEST_DBAPI_ADMIN_CONNECTION=sqlite:////tmp/placeholder-never-created-nor-used.db
usedevelop = True
install_command = pip install -c{env:UPPER_CONSTRAINTS_FILE:https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt} {opts} {packages}
deps = -r{toxinidir}/test-requirements.txt