Add support for sqlalchemy migration based on alembic

Implement blueprint: sqlalchemy-migration

Change-Id: Ifd0d5186299611907a388f8ab8b27e312a3cb60d
This commit is contained in:
Fei Long Wang 2016-11-09 10:09:14 +13:00
parent e306d72268
commit fb8da5d535
19 changed files with 1026 additions and 0 deletions

View File

@ -0,0 +1,3 @@
---
features:
- Add migration support for Zaqar's sqlalchemy storage driver.

View File

@ -3,6 +3,7 @@
# process, which may cause wedges in the gate later.
pbr>=1.6 # Apache-2.0
alembic>=0.8.4 # MIT
Babel>=2.3.4 # BSD
falcon>=0.1.6 # Apache-2.0
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT

View File

@ -32,6 +32,7 @@ console_scripts =
zaqar-bench = zaqar.bench.conductor:main
zaqar-server = zaqar.cmd.server:run
zaqar-gc = zaqar.cmd.gc:run
zaqar-sql-db-manage = zaqar.storage.sqlalchemy.migration.cli:main
zaqar.data.storage =
mongodb = zaqar.storage.mongodb.driver:DataDriver

View File

@ -19,6 +19,8 @@ fixtures>=3.0.0 # Apache-2.0/BSD
python-subunit>=0.0.18 # Apache-2.0/BSD
testrepository>=0.0.18 # Apache-2.0/BSD
testtools>=1.4.0 # MIT
oslo.db>=4.11.0,!=4.13.1,!=4.13.2 # Apache-2.0
testresources>=0.2.4 # Apache-2.0/BSD
# Documentation
sphinx!=1.3b1,<1.4,>=1.2.1 # BSD

View File

@ -0,0 +1,54 @@
# A generic, single database configuration.
[alembic]
# path to migration scripts
script_location = zaqar/storage/sqlalchemy/migration/alembic_migrations
# template used to generate migration files
# file_template = %%(rev)s_%%(slug)s
# max length of characters to apply to the
# "slug" field
#truncate_slug_length = 40
# set to 'true' to run the environment during
# the 'revision' command, regardless of autogenerate
# revision_environment = false
sqlalchemy.url =
# Logging configuration
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

View File

@ -0,0 +1,73 @@
<!--
Copyright 2012 New Dream Network, LLC (DreamHost)
Licensed under the Apache License, Version 2.0 (the "License"); you may
not use this file except in compliance with the License. You may obtain
a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
License for the specific language governing permissions and limitations
under the License.
-->
The migrations in `alembic_migrations/versions` contain the changes needed to migrate
between Zaqar database revisions. A migration occurs by executing a script that
details the changes needed to upgrade the database. The migration scripts
are ordered so that multiple scripts can run sequentially. The scripts are executed by
Zaqar's migration wrapper which uses the Alembic library to manage the migration. Zaqar
supports migration from Liberty or later.
You can upgrade to the latest database version via:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head
```
To check the current database version:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf current
```
To create a script to run the migration offline:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head --sql
```
To run the offline migration between specific migration versions:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade <start version>:<end version> --sql
```
Upgrade the database incrementally:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade --delta <# of revs>
```
Create new revision:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision" --autogenerate
```
Create a blank file:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision"
```
This command does not perform any migrations, it only sets the revision.
Revision may be any existing revision. Use this command carefully.
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf stamp <revision>
```
To verify that the timeline does branch, you can run this command:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf check_migration
```
If the migration path does branch, you can find the branch point via:
```
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf history
```

View File

@ -0,0 +1,96 @@
# Copyright (c) 2013 Mirantis Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Based on Neutron's migration/cli.py
from __future__ import with_statement
from logging import config as c
from alembic import context
from oslo_utils import importutils
from sqlalchemy import create_engine
from sqlalchemy import pool
from zaqar.storage.sqlalchemy import tables
importutils.try_import('zaqar.storage.sqlalchemy.tables')
# this is the Alembic Config object, which provides
# access to the values within the .ini file in use.
config = context.config
zaqar_config = config.zaqar_config
# Interpret the config file for Python logging.
# This line sets up loggers basically.
c.fileConfig(config.config_file_name)
# add your model's MetaData object here
# for 'autogenerate' support
# from myapp import mymodel
# target_metadata = mymodel.Base.metadata
target_metadata = tables.metadata
# other values from the config, defined by the needs of env.py,
# can be acquired:
# my_important_option = config.get_main_option("my_important_option")
# ... etc.
def run_migrations_offline():
"""Run migrations in 'offline' mode.
This configures the context with just a URL
and not an Engine, though an Engine is acceptable
here as well. By skipping the Engine creation
we don't even need a DBAPI to be available.
Calls to context.execute() here emit the given string to the
script output.
"""
context.configure(
url=zaqar_config['drivers:management_store:sqlalchemy'].uri)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online():
"""Run migrations in 'online' mode.
In this scenario we need to create an Engine
and associate a connection with the context.
"""
engine = create_engine(
zaqar_config['drivers:management_store:sqlalchemy'].uri,
poolclass=pool.NullPool)
connection = engine.connect()
context.configure(
connection=connection,
target_metadata=target_metadata)
try:
with context.begin_transaction():
context.run_migrations()
finally:
connection.close()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

View File

@ -0,0 +1,34 @@
# Copyright ${create_date.year} OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision}
Create Date: ${create_date}
"""
# revision identifiers, used by Alembic.
revision = ${repr(up_revision)}
down_revision = ${repr(down_revision)}
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}

View File

@ -0,0 +1,72 @@
# Copyright 2016 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Liberty release
Revision ID: 001
Revises: None
Create Date: 2015-09-13 20:46:25.783444
"""
# revision identifiers, used by Alembic.
revision = '001'
down_revision = None
from alembic import op
import sqlalchemy as sa
MYSQL_ENGINE = 'InnoDB'
MYSQL_CHARSET = 'utf8'
def upgrade():
op.create_table('queues',
sa.Column('id', sa.INTEGER, primary_key=True),
sa.Column('project', sa.String(64)),
sa.Column('name', sa.String(64)),
sa.Column('metadata', sa.LargeBinary),
sa.UniqueConstraint('project', 'name'))
op.create_table('poolgroup',
sa.Column('name', sa.String(64), primary_key=True))
op.create_table('pools',
sa.Column('name', sa.String(64), primary_key=True),
sa.Column('group', sa.String(64),
sa.ForeignKey('poolgroup.name',
ondelete='CASCADE'),
nullable=True),
sa.Column('uri', sa.String(255),
unique=True, nullable=False),
sa.Column('weight', sa.INTEGER, nullable=False),
sa.Column('options', sa.Text()))
op.create_table('flavors',
sa.Column('name', sa.String(64), primary_key=True),
sa.Column('project', sa.String(64)),
sa.Column('pool_group', sa.String(64),
sa.ForeignKey('poolgroup.name',
ondelete='CASCADE'),
nullable=False),
sa.Column('capabilities', sa.Text()))
op.create_table('catalogue',
sa.Column('pool', sa.String(64),
sa.ForeignKey('pools.name',
ondelete='CASCADE')),
sa.Column('project', sa.String(64)),
sa.Column('queue', sa.String(64), nullable=False),
sa.UniqueConstraint('project', 'queue'))

View File

@ -0,0 +1,30 @@
# Copyright 2016 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""placeholder
Revision ID: 002
Revises: 001
Create Date: 2014-04-01 21:04:47.941098
"""
# revision identifiers, used by Alembic.
revision = '002'
down_revision = '001'
def upgrade():
pass

View File

@ -0,0 +1,30 @@
# Copyright 2014 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""placeholder
Revision ID: 003
Revises: 002
Create Date: 2014-04-01 21:05:00.270366
"""
# revision identifiers, used by Alembic.
revision = '003'
down_revision = '002'
def upgrade():
pass

View File

@ -0,0 +1,30 @@
# Copyright 2014 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""placeholder
Revision ID: 004
Revises: 003
Create Date: 2014-04-01 21:04:57.627883
"""
# revision identifiers, used by Alembic.
revision = '004'
down_revision = '003'
def upgrade():
pass

View File

@ -0,0 +1,30 @@
# Copyright 2014 OpenStack Foundation.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""placeholder
Revision ID: 005
Revises: 004
Create Date: 2014-04-01 21:04:54.928605
"""
# revision identifiers, used by Alembic.
revision = '005'
down_revision = '004'
def upgrade():
pass

View File

@ -0,0 +1,118 @@
# Copyright (c) 2016 Catalyst IT Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
# implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
from alembic import command as alembic_cmd
from alembic import config as alembic_cfg
from alembic import util as alembic_u
from oslo_config import cfg
CONF = cfg.CONF
def do_alembic_command(config, cmd, *args, **kwargs):
try:
getattr(alembic_cmd, cmd)(config, *args, **kwargs)
except alembic_u.CommandError as e:
alembic_u.err(str(e))
def do_check_migration(config, _cmd):
do_alembic_command(config, 'branches')
def do_upgrade_downgrade(config, cmd):
if not CONF.command.revision and not CONF.command.delta:
raise SystemExit('You must provide a revision or relative delta')
revision = CONF.command.revision
if CONF.command.delta:
sign = '+' if CONF.command.name == 'upgrade' else '-'
revision = sign + str(CONF.command.delta)
do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
def do_stamp(config, cmd):
do_alembic_command(config, cmd,
CONF.command.revision,
sql=CONF.command.sql)
def do_revision(config, cmd):
do_alembic_command(config, cmd,
message=CONF.command.message,
autogenerate=CONF.command.autogenerate,
sql=CONF.command.sql)
def add_command_parsers(subparsers):
for name in ['current', 'history', 'branches']:
parser = subparsers.add_parser(name)
parser.set_defaults(func=do_alembic_command)
parser = subparsers.add_parser('check_migration')
parser.set_defaults(func=do_check_migration)
for name in ['upgrade', 'downgrade']:
parser = subparsers.add_parser(name)
parser.add_argument('--delta', type=int)
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision', nargs='?')
parser.set_defaults(func=do_upgrade_downgrade)
parser = subparsers.add_parser('stamp')
parser.add_argument('--sql', action='store_true')
parser.add_argument('revision')
parser.set_defaults(func=do_stamp)
parser = subparsers.add_parser('revision')
parser.add_argument('-m', '--message')
parser.add_argument('--autogenerate', action='store_true')
parser.add_argument('--sql', action='store_true')
parser.set_defaults(func=do_revision)
command_opt = cfg.SubCommandOpt('command',
title='Command',
help='Available commands',
handler=add_command_parsers)
CONF.register_cli_opt(command_opt)
sqlalchemy_opts = [cfg.StrOpt('uri',
help='The SQLAlchemy connection string to'
' use to connect to the database.',
secret=True)]
CONF.register_opts(sqlalchemy_opts,
group='drivers:management_store:sqlalchemy')
def main():
config = alembic_cfg.Config(
os.path.join(os.path.dirname(__file__), 'alembic.ini')
)
config.set_main_option('script_location',
'zaqar.storage.sqlalchemy.'
'migration:alembic_migrations')
# attach the octavia conf to the Alembic conf
config.zaqar_config = CONF
CONF(project='zaqar')
CONF.command.func(config, CONF.command.name)

View File

@ -0,0 +1,89 @@
# Copyright 2012 New Dream Network, LLC (DreamHost)
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import sys
import mock
import testscenarios
import testtools
from zaqar.storage.sqlalchemy.migration import cli
class TestCli(testtools.TestCase):
func_name = ''
exp_args = ()
exp_kwargs = {}
scenarios = [
('stamp',
dict(argv=['prog', 'stamp', 'foo'], func_name='stamp',
exp_args=('foo',), exp_kwargs={'sql': False})),
('stamp-sql',
dict(argv=['prog', 'stamp', 'foo', '--sql'], func_name='stamp',
exp_args=('foo',), exp_kwargs={'sql': True})),
('current',
dict(argv=['prog', 'current'], func_name='current',
exp_args=[], exp_kwargs=dict())),
('history',
dict(argv=['prog', 'history'], func_name='history',
exp_args=[], exp_kwargs=dict())),
('check_migration',
dict(argv=['prog', 'check_migration'], func_name='branches',
exp_args=[], exp_kwargs=dict())),
('sync_revision_autogenerate',
dict(argv=['prog', 'revision', '--autogenerate', '-m', 'message'],
func_name='revision',
exp_args=(),
exp_kwargs={
'message': 'message', 'sql': False, 'autogenerate': True})),
('sync_revision_sql',
dict(argv=['prog', 'revision', '--sql', '-m', 'message'],
func_name='revision',
exp_args=(),
exp_kwargs={
'message': 'message', 'sql': True, 'autogenerate': False})),
('upgrade-sql',
dict(argv=['prog', 'upgrade', '--sql', 'head'],
func_name='upgrade',
exp_args=('head',),
exp_kwargs={'sql': True})),
('upgrade-delta',
dict(argv=['prog', 'upgrade', '--delta', '3'],
func_name='upgrade',
exp_args=('+3',),
exp_kwargs={'sql': False}))
]
def setUp(self):
super(TestCli, self).setUp()
do_alembic_cmd_p = mock.patch.object(cli, 'do_alembic_command')
self.addCleanup(do_alembic_cmd_p.stop)
self.do_alembic_cmd = do_alembic_cmd_p.start()
self.addCleanup(cli.CONF.reset)
def test_cli(self):
with mock.patch.object(sys, 'argv', self.argv):
cli.main()
self.do_alembic_cmd.assert_has_calls(
[mock.call(
mock.ANY, self.func_name,
*self.exp_args, **self.exp_kwargs)]
)
def load_tests(loader, in_tests, pattern):
return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)

View File

@ -0,0 +1,175 @@
# Copyright 2014 OpenStack Foundation
# Copyright 2014 Mirantis Inc
# Copyright 2016 Catalyst IT Ltd.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Tests for database migrations.
For the opportunistic testing you need to set up a db named 'openstack_citest'
with user 'openstack_citest' and password 'openstack_citest' on localhost.
The test will then use that db and u/p combo to run the tests.
For postgres on Ubuntu this can be done with the following commands:
sudo -u postgres psql
postgres=# create user openstack_citest with createdb login password
'openstack_citest';
postgres=# create database openstack_citest with owner openstack_citest;
"""
import os
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
from zaqar.tests.unit.storage.sqlalchemy_migration import \
test_migrations_base as base
class ZaqarMigrationsCheckers(object):
def assertColumnExists(self, engine, table, column):
t = db_utils.get_table(engine, table)
self.assertIn(column, t.c)
def assertColumnsExist(self, engine, table, columns):
for column in columns:
self.assertColumnExists(engine, table, column)
def assertColumnType(self, engine, table, column, column_type):
t = db_utils.get_table(engine, table)
column_ref_type = str(t.c[column].type)
self.assertEqual(column_ref_type, column_type)
def assertColumnCount(self, engine, table, columns):
t = db_utils.get_table(engine, table)
self.assertEqual(len(columns), len(t.columns))
def assertColumnNotExists(self, engine, table, column):
t = db_utils.get_table(engine, table)
self.assertNotIn(column, t.c)
def assertIndexExists(self, engine, table, index):
t = db_utils.get_table(engine, table)
index_names = [idx.name for idx in t.indexes]
self.assertIn(index, index_names)
def assertIndexMembers(self, engine, table, index, members):
self.assertIndexExists(engine, table, index)
t = db_utils.get_table(engine, table)
index_columns = None
for idx in t.indexes:
if idx.name == index:
index_columns = idx.columns.keys()
break
self.assertEqual(sorted(members), sorted(index_columns))
def test_walk_versions(self):
self.walk_versions(self.engine)
def _pre_upgrade_001(self, engine):
# Anything returned from this method will be
# passed to corresponding _check_xxx method as 'data'.
pass
def _check_001(self, engine, data):
queues_columns = [
'id',
'name',
'project',
'metadata'
]
self.assertColumnsExist(
engine, 'queues', queues_columns)
self.assertColumnCount(
engine, 'queues', queues_columns)
poolgroup_columns = [
'name',
]
self.assertColumnsExist(
engine, 'poolgroup', poolgroup_columns)
self.assertColumnCount(
engine, 'poolgroup', poolgroup_columns)
pools_columns = [
'name',
'group',
'uri',
'weight',
'options',
]
self.assertColumnsExist(
engine, 'pools', pools_columns)
self.assertColumnCount(
engine, 'pools', pools_columns)
flavors_columns = [
'name',
'project',
'pool_group',
'capabilities',
]
self.assertColumnsExist(
engine, 'flavors', flavors_columns)
self.assertColumnCount(
engine, 'flavors', flavors_columns)
catalogue_columns = [
'pool',
'project',
'queue',
]
self.assertColumnsExist(
engine, 'catalogue', catalogue_columns)
self.assertColumnCount(
engine, 'catalogue', catalogue_columns)
self._data_001(engine, data)
def _data_001(self, engine, data):
datasize = 512 * 1024 # 512kB
data = os.urandom(datasize)
t = db_utils.get_table(engine, 'job_binary_internal')
engine.execute(t.insert(), data=data, id='123', name='name')
new_data = engine.execute(t.select()).fetchone().data
self.assertEqual(data, new_data)
engine.execute(t.delete())
def _check_002(self, engine, data):
# currently, 002 is just a placeholder
pass
def _check_003(self, engine, data):
# currently, 003 is just a placeholder
pass
def _check_004(self, engine, data):
# currently, 004 is just a placeholder
pass
def _check_005(self, engine, data):
# currently, 005 is just a placeholder
pass
class TestMigrationsMySQL(ZaqarMigrationsCheckers,
base.BaseWalkMigrationTestCase,
base.TestModelsMigrationsSync,
test_base.MySQLOpportunisticTestCase):
pass

View File

@ -0,0 +1,188 @@
# Copyright 2010-2011 OpenStack Foundation
# Copyright 2012-2013 IBM Corp.
# Copyright 2016 Catalyst IT Ltd.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
#
#
# Ripped off from Nova's test_migrations.py
# The only difference between Nova and this code is usage of alembic instead
# of sqlalchemy migrations.
#
# There is an ongoing work to extact similar code to oslo incubator. Once it is
# extracted we'll be able to remove this file and use oslo.
import io
import os
import sqlalchemy as sa
import alembic
from alembic import command
from alembic import config as alembic_config
from alembic import migration
from alembic import script as alembic_script
from oslo_config import cfg
from oslo_db.sqlalchemy import test_migrations as t_m
from oslo_log import log as logging
from zaqar.i18n import _LE
import zaqar.storage.sqlalchemy.migration
from zaqar.storage.sqlalchemy import tables
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
class BaseWalkMigrationTestCase(object):
ALEMBIC_CONFIG = alembic_config.Config(
os.path.join(
os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__),
'alembic.ini')
)
ALEMBIC_CONFIG.zaqar_config = CONF
def _configure(self, engine):
"""For each type of repository we should do some of configure steps.
For migrate_repo we should set under version control our database.
For alembic we should configure database settings. For this goal we
should use oslo_config and openstack.commom.db.sqlalchemy.session with
database functionality (reset default settings and session cleanup).
"""
CONF.set_override('uri', str(engine.url),
group='drivers:management_store:sqlalchemy',
enforce_type=True)
sa.cleanup()
def _alembic_command(self, alembic_command, engine, *args, **kwargs):
"""Most of alembic command return data into output.
We should redefine this setting for getting info.
"""
self.ALEMBIC_CONFIG.stdout = buf = io.StringIO()
CONF.set_override('uri', str(engine.url),
group='drivers:management_store:sqlalchemy',
enforce_type=True)
sa.cleanup()
getattr(command, alembic_command)(*args, **kwargs)
res = buf.getvalue().strip()
LOG.debug('Alembic command {command} returns: {result}'.format(
command=alembic_command, result=res))
sa.cleanup()
return res
def _get_versions(self):
"""Stores a list of versions.
Since alembic version has a random algorithm of generation
(SA-migrate has an ordered autoincrement naming) we should store
a list of versions (version for upgrade)
for successful testing of migrations in up mode.
"""
env = alembic_script.ScriptDirectory.from_config(self.ALEMBIC_CONFIG)
versions = []
for rev in env.walk_revisions():
versions.append(rev.revision)
versions.reverse()
return versions
def walk_versions(self, engine=None):
# Determine latest version script from the repo, then
# upgrade from 1 through to the latest, with no data
# in the databases. This just checks that the schema itself
# upgrades successfully.
self._configure(engine)
versions = self._get_versions()
for ver in versions:
self._migrate_up(engine, ver, with_data=True)
def _get_version_from_db(self, engine):
"""Returns latest version from db for each type of migrate repo."""
conn = engine.connect()
try:
context = migration.MigrationContext.configure(conn)
version = context.get_current_revision() or '-1'
finally:
conn.close()
return version
def _migrate(self, engine, version, cmd):
"""Base method for manipulation with migrate repo.
It will upgrade or downgrade the actual database.
"""
self._alembic_command(cmd, engine, self.ALEMBIC_CONFIG, version)
def _migrate_up(self, engine, version, with_data=False):
"""migrate up to a new version of the db.
We allow for data insertion and post checks at every
migration version with special _pre_upgrade_### and
_check_### functions in the main test.
"""
# NOTE(sdague): try block is here because it's impossible to debug
# where a failed data migration happens otherwise
check_version = version
try:
if with_data:
data = None
pre_upgrade = getattr(
self, "_pre_upgrade_%s" % check_version, None)
if pre_upgrade:
data = pre_upgrade(engine)
self._migrate(engine, version, 'upgrade')
self.assertEqual(version, self._get_version_from_db(engine))
if with_data:
check = getattr(self, "_check_%s" % check_version, None)
if check:
check(engine, data)
except Exception:
LOG.error(_LE("Failed to migrate to version {version} on engine "
"{engine}").format(version=version, engine=engine))
raise
class TestModelsMigrationsSync(t_m.ModelsMigrationsSync):
"""Class for comparison of DB migration scripts and models.
Allows to check if the DB schema obtained by applying of migration
scripts is equal to the one produced from models definitions.
"""
ALEMBIC_CONFIG = alembic_config.Config(
os.path.join(
os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__),
'alembic.ini')
)
ALEMBIC_CONFIG.zaqar_config = CONF
def get_engine(self):
return self.engine
def db_sync(self, engine):
CONF.set_override('uri', str(engine.url),
group='drivers:management_store:sqlalchemy',
enforce_type=True)
alembic.command.upgrade(self.ALEMBIC_CONFIG, 'head')
def get_metadata(self):
return tables.metadata