Merge "Add support for sqlalchemy migration based on alembic"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
5612e13b44
|
|
@ -0,0 +1,3 @@
|
|||
---
|
||||
features:
|
||||
- Add migration support for Zaqar's sqlalchemy storage driver.
|
||||
|
|
@ -3,6 +3,7 @@
|
|||
# process, which may cause wedges in the gate later.
|
||||
pbr>=1.8 # Apache-2.0
|
||||
|
||||
alembic>=0.8.4 # MIT
|
||||
Babel>=2.3.4 # BSD
|
||||
falcon>=0.1.6 # Apache-2.0
|
||||
jsonschema!=2.5.0,<3.0.0,>=2.0.0 # MIT
|
||||
|
|
|
|||
|
|
@ -32,6 +32,7 @@ console_scripts =
|
|||
zaqar-bench = zaqar.bench.conductor:main
|
||||
zaqar-server = zaqar.cmd.server:run
|
||||
zaqar-gc = zaqar.cmd.gc:run
|
||||
zaqar-sql-db-manage = zaqar.storage.sqlalchemy.migration.cli:main
|
||||
|
||||
zaqar.data.storage =
|
||||
mongodb = zaqar.storage.mongodb.driver:DataDriver
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ fixtures>=3.0.0 # Apache-2.0/BSD
|
|||
python-subunit>=0.0.18 # Apache-2.0/BSD
|
||||
testrepository>=0.0.18 # Apache-2.0/BSD
|
||||
testtools>=1.4.0 # MIT
|
||||
oslo.db>=4.11.0,!=4.13.1,!=4.13.2 # Apache-2.0
|
||||
testresources>=0.2.4 # Apache-2.0/BSD
|
||||
|
||||
# Documentation
|
||||
sphinx!=1.3b1,<1.4,>=1.2.1 # BSD
|
||||
|
|
|
|||
|
|
@ -0,0 +1,54 @@
|
|||
# A generic, single database configuration.
|
||||
|
||||
[alembic]
|
||||
# path to migration scripts
|
||||
script_location = zaqar/storage/sqlalchemy/migration/alembic_migrations
|
||||
|
||||
# template used to generate migration files
|
||||
# file_template = %%(rev)s_%%(slug)s
|
||||
|
||||
# max length of characters to apply to the
|
||||
# "slug" field
|
||||
#truncate_slug_length = 40
|
||||
|
||||
# set to 'true' to run the environment during
|
||||
# the 'revision' command, regardless of autogenerate
|
||||
# revision_environment = false
|
||||
|
||||
sqlalchemy.url =
|
||||
|
||||
|
||||
# Logging configuration
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
|
|
@ -0,0 +1,73 @@
|
|||
<!--
|
||||
Copyright 2012 New Dream Network, LLC (DreamHost)
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
not use this file except in compliance with the License. You may obtain
|
||||
a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
License for the specific language governing permissions and limitations
|
||||
under the License.
|
||||
-->
|
||||
|
||||
The migrations in `alembic_migrations/versions` contain the changes needed to migrate
|
||||
between Zaqar database revisions. A migration occurs by executing a script that
|
||||
details the changes needed to upgrade the database. The migration scripts
|
||||
are ordered so that multiple scripts can run sequentially. The scripts are executed by
|
||||
Zaqar's migration wrapper which uses the Alembic library to manage the migration. Zaqar
|
||||
supports migration from Liberty or later.
|
||||
|
||||
You can upgrade to the latest database version via:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head
|
||||
```
|
||||
|
||||
To check the current database version:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf current
|
||||
```
|
||||
|
||||
To create a script to run the migration offline:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade head --sql
|
||||
```
|
||||
|
||||
To run the offline migration between specific migration versions:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade <start version>:<end version> --sql
|
||||
```
|
||||
|
||||
Upgrade the database incrementally:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf upgrade --delta <# of revs>
|
||||
```
|
||||
|
||||
Create new revision:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision" --autogenerate
|
||||
```
|
||||
|
||||
Create a blank file:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf revision -m "description of revision"
|
||||
```
|
||||
|
||||
This command does not perform any migrations, it only sets the revision.
|
||||
Revision may be any existing revision. Use this command carefully.
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf stamp <revision>
|
||||
```
|
||||
|
||||
To verify that the timeline does branch, you can run this command:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf check_migration
|
||||
```
|
||||
|
||||
If the migration path does branch, you can find the branch point via:
|
||||
```
|
||||
$ zaqar-sql-db-manage --config-file /path/to/zaqar.conf history
|
||||
```
|
||||
|
|
@ -0,0 +1,96 @@
|
|||
# Copyright (c) 2013 Mirantis Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
# Based on Neutron's migration/cli.py
|
||||
|
||||
from __future__ import with_statement
|
||||
from logging import config as c
|
||||
|
||||
from alembic import context
|
||||
from oslo_utils import importutils
|
||||
from sqlalchemy import create_engine
|
||||
from sqlalchemy import pool
|
||||
|
||||
from zaqar.storage.sqlalchemy import tables
|
||||
|
||||
|
||||
importutils.try_import('zaqar.storage.sqlalchemy.tables')
|
||||
|
||||
# this is the Alembic Config object, which provides
|
||||
# access to the values within the .ini file in use.
|
||||
config = context.config
|
||||
zaqar_config = config.zaqar_config
|
||||
|
||||
# Interpret the config file for Python logging.
|
||||
# This line sets up loggers basically.
|
||||
c.fileConfig(config.config_file_name)
|
||||
|
||||
# add your model's MetaData object here
|
||||
# for 'autogenerate' support
|
||||
# from myapp import mymodel
|
||||
# target_metadata = mymodel.Base.metadata
|
||||
target_metadata = tables.metadata
|
||||
|
||||
# other values from the config, defined by the needs of env.py,
|
||||
# can be acquired:
|
||||
# my_important_option = config.get_main_option("my_important_option")
|
||||
# ... etc.
|
||||
|
||||
|
||||
def run_migrations_offline():
|
||||
"""Run migrations in 'offline' mode.
|
||||
|
||||
This configures the context with just a URL
|
||||
and not an Engine, though an Engine is acceptable
|
||||
here as well. By skipping the Engine creation
|
||||
we don't even need a DBAPI to be available.
|
||||
|
||||
Calls to context.execute() here emit the given string to the
|
||||
script output.
|
||||
|
||||
"""
|
||||
context.configure(
|
||||
url=zaqar_config['drivers:management_store:sqlalchemy'].uri)
|
||||
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online():
|
||||
"""Run migrations in 'online' mode.
|
||||
|
||||
In this scenario we need to create an Engine
|
||||
and associate a connection with the context.
|
||||
|
||||
"""
|
||||
engine = create_engine(
|
||||
zaqar_config['drivers:management_store:sqlalchemy'].uri,
|
||||
poolclass=pool.NullPool)
|
||||
|
||||
connection = engine.connect()
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata)
|
||||
|
||||
try:
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
finally:
|
||||
connection.close()
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
# Copyright ${create_date.year} OpenStack Foundation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision}
|
||||
Create Date: ${create_date}
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = ${repr(up_revision)}
|
||||
down_revision = ${repr(down_revision)}
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
|
||||
def upgrade():
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
# Copyright 2016 OpenStack Foundation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""Liberty release
|
||||
|
||||
Revision ID: 001
|
||||
Revises: None
|
||||
Create Date: 2015-09-13 20:46:25.783444
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '001'
|
||||
down_revision = None
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
|
||||
MYSQL_ENGINE = 'InnoDB'
|
||||
MYSQL_CHARSET = 'utf8'
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.create_table('queues',
|
||||
sa.Column('id', sa.INTEGER, primary_key=True),
|
||||
sa.Column('project', sa.String(64)),
|
||||
sa.Column('name', sa.String(64)),
|
||||
sa.Column('metadata', sa.LargeBinary),
|
||||
sa.UniqueConstraint('project', 'name'))
|
||||
|
||||
op.create_table('poolgroup',
|
||||
sa.Column('name', sa.String(64), primary_key=True))
|
||||
|
||||
op.create_table('pools',
|
||||
sa.Column('name', sa.String(64), primary_key=True),
|
||||
sa.Column('group', sa.String(64),
|
||||
sa.ForeignKey('poolgroup.name',
|
||||
ondelete='CASCADE'),
|
||||
nullable=True),
|
||||
sa.Column('uri', sa.String(255),
|
||||
unique=True, nullable=False),
|
||||
sa.Column('weight', sa.INTEGER, nullable=False),
|
||||
sa.Column('options', sa.Text()))
|
||||
|
||||
op.create_table('flavors',
|
||||
sa.Column('name', sa.String(64), primary_key=True),
|
||||
sa.Column('project', sa.String(64)),
|
||||
sa.Column('pool_group', sa.String(64),
|
||||
sa.ForeignKey('poolgroup.name',
|
||||
ondelete='CASCADE'),
|
||||
nullable=False),
|
||||
sa.Column('capabilities', sa.Text()))
|
||||
|
||||
op.create_table('catalogue',
|
||||
sa.Column('pool', sa.String(64),
|
||||
sa.ForeignKey('pools.name',
|
||||
ondelete='CASCADE')),
|
||||
sa.Column('project', sa.String(64)),
|
||||
sa.Column('queue', sa.String(64), nullable=False),
|
||||
sa.UniqueConstraint('project', 'queue'))
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# Copyright 2016 OpenStack Foundation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""placeholder
|
||||
|
||||
Revision ID: 002
|
||||
Revises: 001
|
||||
Create Date: 2014-04-01 21:04:47.941098
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '002'
|
||||
down_revision = '001'
|
||||
|
||||
|
||||
def upgrade():
|
||||
pass
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# Copyright 2014 OpenStack Foundation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""placeholder
|
||||
|
||||
Revision ID: 003
|
||||
Revises: 002
|
||||
Create Date: 2014-04-01 21:05:00.270366
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '003'
|
||||
down_revision = '002'
|
||||
|
||||
|
||||
def upgrade():
|
||||
pass
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# Copyright 2014 OpenStack Foundation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""placeholder
|
||||
|
||||
Revision ID: 004
|
||||
Revises: 003
|
||||
Create Date: 2014-04-01 21:04:57.627883
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '004'
|
||||
down_revision = '003'
|
||||
|
||||
|
||||
def upgrade():
|
||||
pass
|
||||
|
|
@ -0,0 +1,30 @@
|
|||
# Copyright 2014 OpenStack Foundation.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
"""placeholder
|
||||
|
||||
Revision ID: 005
|
||||
Revises: 004
|
||||
Create Date: 2014-04-01 21:04:54.928605
|
||||
|
||||
"""
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '005'
|
||||
down_revision = '004'
|
||||
|
||||
|
||||
def upgrade():
|
||||
pass
|
||||
|
|
@ -0,0 +1,118 @@
|
|||
# Copyright (c) 2016 Catalyst IT Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
|
||||
# implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
|
||||
from alembic import command as alembic_cmd
|
||||
from alembic import config as alembic_cfg
|
||||
from alembic import util as alembic_u
|
||||
from oslo_config import cfg
|
||||
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
def do_alembic_command(config, cmd, *args, **kwargs):
|
||||
try:
|
||||
getattr(alembic_cmd, cmd)(config, *args, **kwargs)
|
||||
except alembic_u.CommandError as e:
|
||||
alembic_u.err(str(e))
|
||||
|
||||
|
||||
def do_check_migration(config, _cmd):
|
||||
do_alembic_command(config, 'branches')
|
||||
|
||||
|
||||
def do_upgrade_downgrade(config, cmd):
|
||||
if not CONF.command.revision and not CONF.command.delta:
|
||||
raise SystemExit('You must provide a revision or relative delta')
|
||||
|
||||
revision = CONF.command.revision
|
||||
|
||||
if CONF.command.delta:
|
||||
sign = '+' if CONF.command.name == 'upgrade' else '-'
|
||||
revision = sign + str(CONF.command.delta)
|
||||
|
||||
do_alembic_command(config, cmd, revision, sql=CONF.command.sql)
|
||||
|
||||
|
||||
def do_stamp(config, cmd):
|
||||
do_alembic_command(config, cmd,
|
||||
CONF.command.revision,
|
||||
sql=CONF.command.sql)
|
||||
|
||||
|
||||
def do_revision(config, cmd):
|
||||
do_alembic_command(config, cmd,
|
||||
message=CONF.command.message,
|
||||
autogenerate=CONF.command.autogenerate,
|
||||
sql=CONF.command.sql)
|
||||
|
||||
|
||||
def add_command_parsers(subparsers):
|
||||
for name in ['current', 'history', 'branches']:
|
||||
parser = subparsers.add_parser(name)
|
||||
parser.set_defaults(func=do_alembic_command)
|
||||
|
||||
parser = subparsers.add_parser('check_migration')
|
||||
parser.set_defaults(func=do_check_migration)
|
||||
|
||||
for name in ['upgrade', 'downgrade']:
|
||||
parser = subparsers.add_parser(name)
|
||||
parser.add_argument('--delta', type=int)
|
||||
parser.add_argument('--sql', action='store_true')
|
||||
parser.add_argument('revision', nargs='?')
|
||||
parser.set_defaults(func=do_upgrade_downgrade)
|
||||
|
||||
parser = subparsers.add_parser('stamp')
|
||||
parser.add_argument('--sql', action='store_true')
|
||||
parser.add_argument('revision')
|
||||
parser.set_defaults(func=do_stamp)
|
||||
|
||||
parser = subparsers.add_parser('revision')
|
||||
parser.add_argument('-m', '--message')
|
||||
parser.add_argument('--autogenerate', action='store_true')
|
||||
parser.add_argument('--sql', action='store_true')
|
||||
parser.set_defaults(func=do_revision)
|
||||
|
||||
|
||||
command_opt = cfg.SubCommandOpt('command',
|
||||
title='Command',
|
||||
help='Available commands',
|
||||
handler=add_command_parsers)
|
||||
|
||||
CONF.register_cli_opt(command_opt)
|
||||
|
||||
sqlalchemy_opts = [cfg.StrOpt('uri',
|
||||
help='The SQLAlchemy connection string to'
|
||||
' use to connect to the database.',
|
||||
secret=True)]
|
||||
|
||||
CONF.register_opts(sqlalchemy_opts,
|
||||
group='drivers:management_store:sqlalchemy')
|
||||
|
||||
|
||||
def main():
|
||||
config = alembic_cfg.Config(
|
||||
os.path.join(os.path.dirname(__file__), 'alembic.ini')
|
||||
)
|
||||
config.set_main_option('script_location',
|
||||
'zaqar.storage.sqlalchemy.'
|
||||
'migration:alembic_migrations')
|
||||
|
||||
# attach the octavia conf to the Alembic conf
|
||||
config.zaqar_config = CONF
|
||||
|
||||
CONF(project='zaqar')
|
||||
CONF.command.func(config, CONF.command.name)
|
||||
|
|
@ -0,0 +1,89 @@
|
|||
# Copyright 2012 New Dream Network, LLC (DreamHost)
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import sys
|
||||
|
||||
import mock
|
||||
import testscenarios
|
||||
import testtools
|
||||
|
||||
from zaqar.storage.sqlalchemy.migration import cli
|
||||
|
||||
|
||||
class TestCli(testtools.TestCase):
|
||||
func_name = ''
|
||||
exp_args = ()
|
||||
exp_kwargs = {}
|
||||
|
||||
scenarios = [
|
||||
('stamp',
|
||||
dict(argv=['prog', 'stamp', 'foo'], func_name='stamp',
|
||||
exp_args=('foo',), exp_kwargs={'sql': False})),
|
||||
('stamp-sql',
|
||||
dict(argv=['prog', 'stamp', 'foo', '--sql'], func_name='stamp',
|
||||
exp_args=('foo',), exp_kwargs={'sql': True})),
|
||||
('current',
|
||||
dict(argv=['prog', 'current'], func_name='current',
|
||||
exp_args=[], exp_kwargs=dict())),
|
||||
('history',
|
||||
dict(argv=['prog', 'history'], func_name='history',
|
||||
exp_args=[], exp_kwargs=dict())),
|
||||
('check_migration',
|
||||
dict(argv=['prog', 'check_migration'], func_name='branches',
|
||||
exp_args=[], exp_kwargs=dict())),
|
||||
('sync_revision_autogenerate',
|
||||
dict(argv=['prog', 'revision', '--autogenerate', '-m', 'message'],
|
||||
func_name='revision',
|
||||
exp_args=(),
|
||||
exp_kwargs={
|
||||
'message': 'message', 'sql': False, 'autogenerate': True})),
|
||||
('sync_revision_sql',
|
||||
dict(argv=['prog', 'revision', '--sql', '-m', 'message'],
|
||||
func_name='revision',
|
||||
exp_args=(),
|
||||
exp_kwargs={
|
||||
'message': 'message', 'sql': True, 'autogenerate': False})),
|
||||
('upgrade-sql',
|
||||
dict(argv=['prog', 'upgrade', '--sql', 'head'],
|
||||
func_name='upgrade',
|
||||
exp_args=('head',),
|
||||
exp_kwargs={'sql': True})),
|
||||
|
||||
('upgrade-delta',
|
||||
dict(argv=['prog', 'upgrade', '--delta', '3'],
|
||||
func_name='upgrade',
|
||||
exp_args=('+3',),
|
||||
exp_kwargs={'sql': False}))
|
||||
]
|
||||
|
||||
def setUp(self):
|
||||
super(TestCli, self).setUp()
|
||||
do_alembic_cmd_p = mock.patch.object(cli, 'do_alembic_command')
|
||||
self.addCleanup(do_alembic_cmd_p.stop)
|
||||
self.do_alembic_cmd = do_alembic_cmd_p.start()
|
||||
self.addCleanup(cli.CONF.reset)
|
||||
|
||||
def test_cli(self):
|
||||
with mock.patch.object(sys, 'argv', self.argv):
|
||||
cli.main()
|
||||
self.do_alembic_cmd.assert_has_calls(
|
||||
[mock.call(
|
||||
mock.ANY, self.func_name,
|
||||
*self.exp_args, **self.exp_kwargs)]
|
||||
)
|
||||
|
||||
|
||||
def load_tests(loader, in_tests, pattern):
|
||||
return testscenarios.load_tests_apply_scenarios(loader, in_tests, pattern)
|
||||
|
|
@ -0,0 +1,175 @@
|
|||
# Copyright 2014 OpenStack Foundation
|
||||
# Copyright 2014 Mirantis Inc
|
||||
# Copyright 2016 Catalyst IT Ltd.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
"""
|
||||
Tests for database migrations.
|
||||
|
||||
For the opportunistic testing you need to set up a db named 'openstack_citest'
|
||||
with user 'openstack_citest' and password 'openstack_citest' on localhost.
|
||||
The test will then use that db and u/p combo to run the tests.
|
||||
|
||||
For postgres on Ubuntu this can be done with the following commands:
|
||||
|
||||
sudo -u postgres psql
|
||||
postgres=# create user openstack_citest with createdb login password
|
||||
'openstack_citest';
|
||||
postgres=# create database openstack_citest with owner openstack_citest;
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
|
||||
from oslo_db.sqlalchemy import test_base
|
||||
from oslo_db.sqlalchemy import utils as db_utils
|
||||
|
||||
from zaqar.tests.unit.storage.sqlalchemy_migration import \
|
||||
test_migrations_base as base
|
||||
|
||||
|
||||
class ZaqarMigrationsCheckers(object):
|
||||
|
||||
def assertColumnExists(self, engine, table, column):
|
||||
t = db_utils.get_table(engine, table)
|
||||
self.assertIn(column, t.c)
|
||||
|
||||
def assertColumnsExist(self, engine, table, columns):
|
||||
for column in columns:
|
||||
self.assertColumnExists(engine, table, column)
|
||||
|
||||
def assertColumnType(self, engine, table, column, column_type):
|
||||
t = db_utils.get_table(engine, table)
|
||||
column_ref_type = str(t.c[column].type)
|
||||
self.assertEqual(column_ref_type, column_type)
|
||||
|
||||
def assertColumnCount(self, engine, table, columns):
|
||||
t = db_utils.get_table(engine, table)
|
||||
self.assertEqual(len(columns), len(t.columns))
|
||||
|
||||
def assertColumnNotExists(self, engine, table, column):
|
||||
t = db_utils.get_table(engine, table)
|
||||
self.assertNotIn(column, t.c)
|
||||
|
||||
def assertIndexExists(self, engine, table, index):
|
||||
t = db_utils.get_table(engine, table)
|
||||
index_names = [idx.name for idx in t.indexes]
|
||||
self.assertIn(index, index_names)
|
||||
|
||||
def assertIndexMembers(self, engine, table, index, members):
|
||||
self.assertIndexExists(engine, table, index)
|
||||
|
||||
t = db_utils.get_table(engine, table)
|
||||
index_columns = None
|
||||
for idx in t.indexes:
|
||||
if idx.name == index:
|
||||
index_columns = idx.columns.keys()
|
||||
break
|
||||
|
||||
self.assertEqual(sorted(members), sorted(index_columns))
|
||||
|
||||
def test_walk_versions(self):
|
||||
self.walk_versions(self.engine)
|
||||
|
||||
def _pre_upgrade_001(self, engine):
|
||||
# Anything returned from this method will be
|
||||
# passed to corresponding _check_xxx method as 'data'.
|
||||
pass
|
||||
|
||||
def _check_001(self, engine, data):
|
||||
queues_columns = [
|
||||
'id',
|
||||
'name',
|
||||
'project',
|
||||
'metadata'
|
||||
]
|
||||
self.assertColumnsExist(
|
||||
engine, 'queues', queues_columns)
|
||||
self.assertColumnCount(
|
||||
engine, 'queues', queues_columns)
|
||||
|
||||
poolgroup_columns = [
|
||||
'name',
|
||||
]
|
||||
self.assertColumnsExist(
|
||||
engine, 'poolgroup', poolgroup_columns)
|
||||
self.assertColumnCount(
|
||||
engine, 'poolgroup', poolgroup_columns)
|
||||
|
||||
pools_columns = [
|
||||
'name',
|
||||
'group',
|
||||
'uri',
|
||||
'weight',
|
||||
'options',
|
||||
]
|
||||
self.assertColumnsExist(
|
||||
engine, 'pools', pools_columns)
|
||||
self.assertColumnCount(
|
||||
engine, 'pools', pools_columns)
|
||||
|
||||
flavors_columns = [
|
||||
'name',
|
||||
'project',
|
||||
'pool_group',
|
||||
'capabilities',
|
||||
]
|
||||
self.assertColumnsExist(
|
||||
engine, 'flavors', flavors_columns)
|
||||
self.assertColumnCount(
|
||||
engine, 'flavors', flavors_columns)
|
||||
|
||||
catalogue_columns = [
|
||||
'pool',
|
||||
'project',
|
||||
'queue',
|
||||
]
|
||||
self.assertColumnsExist(
|
||||
engine, 'catalogue', catalogue_columns)
|
||||
self.assertColumnCount(
|
||||
engine, 'catalogue', catalogue_columns)
|
||||
|
||||
self._data_001(engine, data)
|
||||
|
||||
def _data_001(self, engine, data):
|
||||
datasize = 512 * 1024 # 512kB
|
||||
data = os.urandom(datasize)
|
||||
t = db_utils.get_table(engine, 'job_binary_internal')
|
||||
engine.execute(t.insert(), data=data, id='123', name='name')
|
||||
new_data = engine.execute(t.select()).fetchone().data
|
||||
self.assertEqual(data, new_data)
|
||||
engine.execute(t.delete())
|
||||
|
||||
def _check_002(self, engine, data):
|
||||
# currently, 002 is just a placeholder
|
||||
pass
|
||||
|
||||
def _check_003(self, engine, data):
|
||||
# currently, 003 is just a placeholder
|
||||
pass
|
||||
|
||||
def _check_004(self, engine, data):
|
||||
# currently, 004 is just a placeholder
|
||||
pass
|
||||
|
||||
def _check_005(self, engine, data):
|
||||
# currently, 005 is just a placeholder
|
||||
pass
|
||||
|
||||
|
||||
class TestMigrationsMySQL(ZaqarMigrationsCheckers,
|
||||
base.BaseWalkMigrationTestCase,
|
||||
base.TestModelsMigrationsSync,
|
||||
test_base.MySQLOpportunisticTestCase):
|
||||
pass
|
||||
|
|
@ -0,0 +1,188 @@
|
|||
# Copyright 2010-2011 OpenStack Foundation
|
||||
# Copyright 2012-2013 IBM Corp.
|
||||
# Copyright 2016 Catalyst IT Ltd.
|
||||
# All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
#
|
||||
#
|
||||
# Ripped off from Nova's test_migrations.py
|
||||
# The only difference between Nova and this code is usage of alembic instead
|
||||
# of sqlalchemy migrations.
|
||||
#
|
||||
# There is an ongoing work to extact similar code to oslo incubator. Once it is
|
||||
# extracted we'll be able to remove this file and use oslo.
|
||||
|
||||
import io
|
||||
import os
|
||||
import sqlalchemy as sa
|
||||
|
||||
import alembic
|
||||
from alembic import command
|
||||
from alembic import config as alembic_config
|
||||
from alembic import migration
|
||||
from alembic import script as alembic_script
|
||||
from oslo_config import cfg
|
||||
from oslo_db.sqlalchemy import test_migrations as t_m
|
||||
from oslo_log import log as logging
|
||||
|
||||
from zaqar.i18n import _LE
|
||||
import zaqar.storage.sqlalchemy.migration
|
||||
from zaqar.storage.sqlalchemy import tables
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
CONF = cfg.CONF
|
||||
|
||||
|
||||
class BaseWalkMigrationTestCase(object):
|
||||
|
||||
ALEMBIC_CONFIG = alembic_config.Config(
|
||||
os.path.join(
|
||||
os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__),
|
||||
'alembic.ini')
|
||||
)
|
||||
|
||||
ALEMBIC_CONFIG.zaqar_config = CONF
|
||||
|
||||
def _configure(self, engine):
|
||||
"""For each type of repository we should do some of configure steps.
|
||||
|
||||
For migrate_repo we should set under version control our database.
|
||||
For alembic we should configure database settings. For this goal we
|
||||
should use oslo_config and openstack.commom.db.sqlalchemy.session with
|
||||
database functionality (reset default settings and session cleanup).
|
||||
"""
|
||||
CONF.set_override('uri', str(engine.url),
|
||||
group='drivers:management_store:sqlalchemy',
|
||||
enforce_type=True)
|
||||
sa.cleanup()
|
||||
|
||||
def _alembic_command(self, alembic_command, engine, *args, **kwargs):
|
||||
"""Most of alembic command return data into output.
|
||||
|
||||
We should redefine this setting for getting info.
|
||||
"""
|
||||
self.ALEMBIC_CONFIG.stdout = buf = io.StringIO()
|
||||
CONF.set_override('uri', str(engine.url),
|
||||
group='drivers:management_store:sqlalchemy',
|
||||
enforce_type=True)
|
||||
sa.cleanup()
|
||||
getattr(command, alembic_command)(*args, **kwargs)
|
||||
res = buf.getvalue().strip()
|
||||
LOG.debug('Alembic command {command} returns: {result}'.format(
|
||||
command=alembic_command, result=res))
|
||||
sa.cleanup()
|
||||
return res
|
||||
|
||||
def _get_versions(self):
|
||||
"""Stores a list of versions.
|
||||
|
||||
Since alembic version has a random algorithm of generation
|
||||
(SA-migrate has an ordered autoincrement naming) we should store
|
||||
a list of versions (version for upgrade)
|
||||
for successful testing of migrations in up mode.
|
||||
"""
|
||||
|
||||
env = alembic_script.ScriptDirectory.from_config(self.ALEMBIC_CONFIG)
|
||||
versions = []
|
||||
for rev in env.walk_revisions():
|
||||
versions.append(rev.revision)
|
||||
|
||||
versions.reverse()
|
||||
return versions
|
||||
|
||||
def walk_versions(self, engine=None):
|
||||
# Determine latest version script from the repo, then
|
||||
# upgrade from 1 through to the latest, with no data
|
||||
# in the databases. This just checks that the schema itself
|
||||
# upgrades successfully.
|
||||
|
||||
self._configure(engine)
|
||||
versions = self._get_versions()
|
||||
for ver in versions:
|
||||
self._migrate_up(engine, ver, with_data=True)
|
||||
|
||||
def _get_version_from_db(self, engine):
|
||||
"""Returns latest version from db for each type of migrate repo."""
|
||||
|
||||
conn = engine.connect()
|
||||
try:
|
||||
context = migration.MigrationContext.configure(conn)
|
||||
version = context.get_current_revision() or '-1'
|
||||
finally:
|
||||
conn.close()
|
||||
return version
|
||||
|
||||
def _migrate(self, engine, version, cmd):
|
||||
"""Base method for manipulation with migrate repo.
|
||||
|
||||
It will upgrade or downgrade the actual database.
|
||||
"""
|
||||
|
||||
self._alembic_command(cmd, engine, self.ALEMBIC_CONFIG, version)
|
||||
|
||||
def _migrate_up(self, engine, version, with_data=False):
|
||||
"""migrate up to a new version of the db.
|
||||
|
||||
We allow for data insertion and post checks at every
|
||||
migration version with special _pre_upgrade_### and
|
||||
_check_### functions in the main test.
|
||||
"""
|
||||
# NOTE(sdague): try block is here because it's impossible to debug
|
||||
# where a failed data migration happens otherwise
|
||||
check_version = version
|
||||
try:
|
||||
if with_data:
|
||||
data = None
|
||||
pre_upgrade = getattr(
|
||||
self, "_pre_upgrade_%s" % check_version, None)
|
||||
if pre_upgrade:
|
||||
data = pre_upgrade(engine)
|
||||
self._migrate(engine, version, 'upgrade')
|
||||
self.assertEqual(version, self._get_version_from_db(engine))
|
||||
if with_data:
|
||||
check = getattr(self, "_check_%s" % check_version, None)
|
||||
if check:
|
||||
check(engine, data)
|
||||
except Exception:
|
||||
LOG.error(_LE("Failed to migrate to version {version} on engine "
|
||||
"{engine}").format(version=version, engine=engine))
|
||||
raise
|
||||
|
||||
|
||||
class TestModelsMigrationsSync(t_m.ModelsMigrationsSync):
|
||||
"""Class for comparison of DB migration scripts and models.
|
||||
|
||||
Allows to check if the DB schema obtained by applying of migration
|
||||
scripts is equal to the one produced from models definitions.
|
||||
"""
|
||||
|
||||
ALEMBIC_CONFIG = alembic_config.Config(
|
||||
os.path.join(
|
||||
os.path.dirname(zaqar.storage.sqlalchemy.migration.__file__),
|
||||
'alembic.ini')
|
||||
)
|
||||
ALEMBIC_CONFIG.zaqar_config = CONF
|
||||
|
||||
def get_engine(self):
|
||||
return self.engine
|
||||
|
||||
def db_sync(self, engine):
|
||||
CONF.set_override('uri', str(engine.url),
|
||||
group='drivers:management_store:sqlalchemy',
|
||||
enforce_type=True)
|
||||
alembic.command.upgrade(self.ALEMBIC_CONFIG, 'head')
|
||||
|
||||
def get_metadata(self):
|
||||
return tables.metadata
|
||||
Loading…
Reference in New Issue