From 9030bc728511237d22c1dde476282addc7396504 Mon Sep 17 00:00:00 2001 From: Ruslan Kamaldinov Date: Thu, 29 May 2014 17:52:36 +0400 Subject: [PATCH] Added DB migrations on Alembic This commit migrates Murano database migration framework from sqlalchemy-migrate to Alembic. sqlalchemy-migrate is considered abandoned. All other OpenStack projects are in process of migration to Alembic or have already finished migration. This change doesn't preserve all the migration scripts, it merges them into a single migration script. That's what we would do even without Alembic. It's a common practice to squash migration scripts into a single one right after the release. NOTICE: CLI command to run migrations changed. Now it looks like this: $ murano-db-manage upgrade Migration path: 1. Simple solution is just to re-create your database from scratch 2. You can stamp your database with revision to make Alembic think that it already applied the first migration: $ tox -e venv -- murano-db-manage --config-file etc/murano/murano.conf stamp --revision head For usage instructions see: murano/db/migraiton/alembic_migrations/README This patch also adds opportunistic tests for migrations. These tests will run in OpenStack Infrastructure jenkins slaves. It'll use real MySQL and Postgres databases. Tests will run in a "snake walk" manner, which means that upgrade and downgrade path will be tested. Base for migration tests is copied from Nova. Please note, that at this moment we cannot use code from oslo, because it is not documented and is not tested. Once test_migrations_base appears in project 'oslo.db', we'll be able to remove our own test_migrations_base and use the one from 'oslo.db'. implements: blueprint alembic-migrations Change-Id: I5aa978f7095efc57f2d6fad81b5553e1880ad931 --- MANIFEST.in | 6 +- contrib/devstack/lib/murano | 2 +- doc/source/install/manual.rst | 2 +- murano/cmd/db_manage.py | 76 +++ murano/cmd/manage.py | 20 +- murano/db/migrate_repo/README | 4 - murano/db/migrate_repo/migrate.cfg | 20 - .../versions/001_add_initial_tables.py | 96 --- .../versions/002_add_networking_field.py | 34 - .../versions/003_add_stats_table.py | 41 -- .../versions/004_add_repository_tables.py | 155 ----- .../versions/005_add_instance_table.py | 37 -- .../versions/006_add_default_categories.py | 38 -- .../versions/007_instance_table_extended.py | 47 -- .../db/migrate_repo/versions/008_cpu_stats.py | 61 -- .../009_increase_package_name_length.py | 30 - .../010_add_unique_environment_constraint.py | 32 - .../{migrate_repo => migration}/__init__.py | 0 murano/db/migration/alembic.ini | 54 ++ murano/db/migration/alembic_migrations/README | 15 + murano/db/migration/alembic_migrations/env.py | 50 ++ .../alembic_migrations/script.py.mako | 37 ++ .../versions/001_inital_version.py | 256 ++++++++ murano/db/migration/migration.py | 79 +++ murano/db/models.py | 42 +- murano/db/session.py | 15 - .../versions => sqla}/__init__.py | 0 .../{migrate_repo/manage.py => sqla/types.py} | 19 +- murano/openstack/common/processutils.py | 267 ++++++++ murano/tests/db/__init__.py | 0 murano/tests/db/migration/__init__.py | 0 .../tests/db/migration/test_migrations.conf | 26 + murano/tests/db/migration/test_migrations.py | 74 +++ .../db/migration/test_migrations_base.py | 585 ++++++++++++++++++ requirements.txt | 3 +- setup.cfg | 1 + test-requirements.txt | 6 + 37 files changed, 1570 insertions(+), 660 deletions(-) create mode 100644 murano/cmd/db_manage.py delete mode 100644 murano/db/migrate_repo/README delete mode 100644 murano/db/migrate_repo/migrate.cfg delete mode 100644 murano/db/migrate_repo/versions/001_add_initial_tables.py delete mode 100644 murano/db/migrate_repo/versions/002_add_networking_field.py delete mode 100644 murano/db/migrate_repo/versions/003_add_stats_table.py delete mode 100644 murano/db/migrate_repo/versions/004_add_repository_tables.py delete mode 100644 murano/db/migrate_repo/versions/005_add_instance_table.py delete mode 100644 murano/db/migrate_repo/versions/006_add_default_categories.py delete mode 100644 murano/db/migrate_repo/versions/007_instance_table_extended.py delete mode 100644 murano/db/migrate_repo/versions/008_cpu_stats.py delete mode 100644 murano/db/migrate_repo/versions/009_increase_package_name_length.py delete mode 100644 murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py rename murano/db/{migrate_repo => migration}/__init__.py (100%) create mode 100644 murano/db/migration/alembic.ini create mode 100644 murano/db/migration/alembic_migrations/README create mode 100644 murano/db/migration/alembic_migrations/env.py create mode 100644 murano/db/migration/alembic_migrations/script.py.mako create mode 100644 murano/db/migration/alembic_migrations/versions/001_inital_version.py create mode 100644 murano/db/migration/migration.py rename murano/db/{migrate_repo/versions => sqla}/__init__.py (100%) rename murano/db/{migrate_repo/manage.py => sqla/types.py} (66%) create mode 100644 murano/openstack/common/processutils.py create mode 100644 murano/tests/db/__init__.py create mode 100644 murano/tests/db/migration/__init__.py create mode 100644 murano/tests/db/migration/test_migrations.conf create mode 100644 murano/tests/db/migration/test_migrations.py create mode 100644 murano/tests/db/migration/test_migrations_base.py diff --git a/MANIFEST.in b/MANIFEST.in index e17f95995..22067f620 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,8 +10,8 @@ include ChangeLog include babel.cfg include tox.ini include openstack-common.conf -include muranoapi/db/migrate_repo/README -include muranoapi/db/migrate_repo/migrate.cfg -include muranoapi/db/migrate_repo/versions/*.sql +include murano/db/migration/alembic.ini +include murano/db/migration/alembic_migrations/README +include murano/db/migration/alembic_migrations/script.py.mako recursive-include etc * global-exclude *.pyc diff --git a/contrib/devstack/lib/murano b/contrib/devstack/lib/murano index 68506a3df..b6d8e29e4 100644 --- a/contrib/devstack/lib/murano +++ b/contrib/devstack/lib/murano @@ -171,7 +171,7 @@ function init_murano() { # (re)create Murano database recreate_database murano utf8 - $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE db-sync + $MURANO_BIN_DIR/murano-db-manage --config-file $MURANO_CONF_FILE upgrade $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE import-package $MURANO_DIR/meta/io.murano } diff --git a/doc/source/install/manual.rst b/doc/source/install/manual.rst index 4df7742cc..491c34979 100644 --- a/doc/source/install/manual.rst +++ b/doc/source/install/manual.rst @@ -89,7 +89,7 @@ Installing the API service and Engine :: - $ tox -e venv -- murano-manage --config-file /etc/murano/murano-api.conf db-sync + $ tox -e venv -- murano-db-manage --config-file /etc/murano/murano-api.conf upgrade 6. Launch Murano API service: diff --git a/murano/cmd/db_manage.py b/murano/cmd/db_manage.py new file mode 100644 index 000000000..1e0422dee --- /dev/null +++ b/murano/cmd/db_manage.py @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from murano.db.migration import migration +# this forces import and registration of db related configs +from murano.db import models # noqa +from murano.openstack.common import log + +CONF = cfg.CONF + + +class DBCommand(object): + + def upgrade(self, config): + migration.upgrade(CONF.command.revision, config=config) + + def downgrade(self, config): + migration.downgrade(CONF.command.revision, config=config) + + def revision(self, config): + migration.revision(CONF.command.message, + CONF.command.autogenerate, + config=config) + + def stamp(self, config): + migration.stamp(CONF.command.revision, config=config) + + +def add_command_parsers(subparsers): + command_object = DBCommand() + + parser = subparsers.add_parser('upgrade') + parser.set_defaults(func=command_object.upgrade) + parser.add_argument('--revision', nargs='?') + + parser = subparsers.add_parser('downgrade') + parser.set_defaults(func=command_object.downgrade) + parser.add_argument('--revision', nargs='?') + + parser = subparsers.add_parser('stamp') + parser.add_argument('--revision', nargs='?') + parser.set_defaults(func=command_object.stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.set_defaults(func=command_object.revision) + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + + +def main(): + config = migration.get_alembic_config() + # attach the Murano conf to the Alembic conf + config.murano_config = CONF + + CONF(project='murano') + log.setup('murano') + CONF.command.func(config) diff --git a/murano/cmd/manage.py b/murano/cmd/manage.py index 918f1c8b9..9bc50d43d 100644 --- a/murano/cmd/manage.py +++ b/murano/cmd/manage.py @@ -13,7 +13,9 @@ # under the License. """ - CLI interface for murano management. + *** Deprecation warning *** + This file is about to be deprecated, please use python-muranoclient. + *** Deprecation warning *** """ import sys @@ -24,7 +26,6 @@ from oslo.config import cfg import murano from murano.common import consts from murano.db.catalog import api as db_catalog_api -from murano.db import session as db_session from murano.openstack.common.db import exception as db_exception from murano.openstack.common import log as logging from murano.packages import load_utils @@ -34,15 +35,6 @@ CONF = cfg.CONF LOG = logging.getLogger(__name__) -# TODO(ruhe): proper error handling -def do_db_sync(): - """ - Place a database under migration control and upgrade, - creating first if necessary. - """ - db_session.db_sync() - - class AdminContext(object): def __init__(self): self.is_admin = True @@ -124,11 +116,6 @@ def do_add_category(): def add_command_parsers(subparsers): - parser = subparsers.add_parser('db-sync') - parser.set_defaults(func=do_db_sync) - parser.add_argument('version', nargs='?') - parser.add_argument('current_version', nargs='?') - parser = subparsers.add_parser('import-package') parser.set_defaults(func=do_import_package) parser.add_argument('directory', @@ -160,6 +147,7 @@ command_opt = cfg.SubCommandOpt('command', def main(): CONF.register_cli_opt(command_opt) + try: default_config_files = cfg.find_config_files('murano', 'murano') CONF(sys.argv[1:], project='murano', prog='murano-manage', diff --git a/murano/db/migrate_repo/README b/murano/db/migrate_repo/README deleted file mode 100644 index 6218f8cac..000000000 --- a/murano/db/migrate_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ diff --git a/murano/db/migrate_repo/migrate.cfg b/murano/db/migrate_repo/migrate.cfg deleted file mode 100644 index 4fe14f42c..000000000 --- a/murano/db/migrate_repo/migrate.cfg +++ /dev/null @@ -1,20 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=Murano Migrations - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['sqlite'] -required_dbs=[] diff --git a/murano/db/migrate_repo/versions/001_add_initial_tables.py b/murano/db/migrate_repo/versions/001_add_initial_tables.py deleted file mode 100644 index eb88013a5..000000000 --- a/murano/db/migrate_repo/versions/001_add_initial_tables.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from migrate.changeset import constraint as const - -from sqlalchemy import schema -from sqlalchemy import types - - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - meta.reflect() - - environment = schema.Table( - 'environment', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('name', types.String(255), nullable=False), - schema.Column('created', types.DateTime(), nullable=False), - schema.Column('updated', types.DateTime(), nullable=False), - schema.Column('tenant_id', types.String(32), nullable=False), - schema.Column('version', types.BigInteger, nullable=False, - server_default='0'), - schema.Column('description', types.Text(), nullable=False)) - environment.create() - - session = schema.Table( - 'session', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('environment_id', types.String(32), nullable=False), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('user_id', types.String(32), nullable=False), - schema.Column('version', types.BigInteger, nullable=False, - server_default='0'), - schema.Column('description', types.Text(), nullable=True), - schema.Column('state', types.Text(), nullable=False)) - session.create() - - environment = schema.Table('environment', meta, autoload=True) - const.ForeignKeyConstraint(columns=[session.c.environment_id], - refcolumns=[environment.c.id]).create() - - deployment = schema.Table( - 'deployment', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('environment_id', types.String(32), nullable=False), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('started', types.DateTime, nullable=False), - schema.Column('description', types.Text(), nullable=True), - schema.Column('finished', types.DateTime, nullable=True)) - deployment.create() - - environment = schema.Table('environment', meta, autoload=True) - const.ForeignKeyConstraint(columns=[deployment.c.environment_id], - refcolumns=[environment.c.id]).create() - - status = schema.Table( - 'status', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('entity', types.String(10), nullable=True), - schema.Column('entity_id', types.String(32), nullable=True), - schema.Column('environment_id', types.String(32), nullable=True), - schema.Column('deployment_id', types.String(32), nullable=False), - schema.Column('text', types.Text(), nullable=False), - schema.Column('details', types.Text(), nullable=True), - schema.Column('level', types.String(32), nullable=False, - server_default='info')) - status.create() - - const.ForeignKeyConstraint(columns=[status.c.deployment_id], - refcolumns=[deployment.c.id]).create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - meta.drop_all() diff --git a/murano/db/migrate_repo/versions/002_add_networking_field.py b/murano/db/migrate_repo/versions/002_add_networking_field.py deleted file mode 100644 index 72e447a6d..000000000 --- a/murano/db/migrate_repo/versions/002_add_networking_field.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from sqlalchemy import schema -from sqlalchemy import types - - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - environment = schema.Table('environment', meta, autoload=True) - networking = schema.Column('networking', - types.Text(), - nullable=True, - default='{}') - networking.create(environment) - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - environment = schema.Table('environment', meta, autoload=True) - environment.c.networking.drop() diff --git a/murano/db/migrate_repo/versions/003_add_stats_table.py b/murano/db/migrate_repo/versions/003_add_stats_table.py deleted file mode 100644 index 4d606e401..000000000 --- a/murano/db/migrate_repo/versions/003_add_stats_table.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - stats = schema.Table( - 'apistats', - meta, - schema.Column('id', types.Integer(), primary_key=True), - schema.Column('host', types.String(80)), - schema.Column('request_count', types.BigInteger()), - schema.Column('error_count', types.BigInteger()), - schema.Column('average_response_time', types.Float()), - schema.Column('requests_per_tenant', types.Text()), - schema.Column('requests_per_second', types.Float()), - schema.Column('errors_per_second', types.Float()), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False)) - stats.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - stats = schema.Table('apistats', meta, autoload=True) - stats.drop() diff --git a/murano/db/migrate_repo/versions/004_add_repository_tables.py b/murano/db/migrate_repo/versions/004_add_repository_tables.py deleted file mode 100644 index 320a43121..000000000 --- a/murano/db/migrate_repo/versions/004_add_repository_tables.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (c) 2014 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.changeset import constraint - -from sqlalchemy.dialects.mysql import mysqldb -from sqlalchemy import schema -from sqlalchemy import types -meta = schema.MetaData() - - -class StringWithCollation(types.String): - def __init__(self, length, collation=None, **kwargs): - super(StringWithCollation, self).__init__(length, **kwargs) - self.collation = collation - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - collation = 'ascii_general_ci' \ - if isinstance(migrate_engine.dialect, mysqldb.MySQLDialect) \ - else None - package = schema.Table( - 'package', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('archive', types.LargeBinary), - schema.Column('fully_qualified_name', - StringWithCollation(512, collation=collation), - index=True, unique=True), - schema.Column('type', types.String(20)), - schema.Column('author', types.String(80)), - schema.Column('name', types.String(20)), - schema.Column('enabled', types.Boolean), - schema.Column('description', types.String(512)), - schema.Column('is_public', types.Boolean), - schema.Column('logo', types.LargeBinary), - schema.Column('owner_id', types.String(36)), - schema.Column('ui_definition', types.Text), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - package.create() - - category = schema.Table( - 'category', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('name', - types.String(80), - nullable=False, - index=True, - unique=True), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - category.create() - - package_to_category = schema.Table( - 'package_to_category', - meta, - schema.Column('package_id', types.String(32)), - schema.Column('category_id', types.String(32)) - ) - package_to_category.create() - - constraint.ForeignKeyConstraint( - columns=[package_to_category.c.package_id], - refcolumns=[package.c.id]).create() - constraint.ForeignKeyConstraint( - columns=[package_to_category.c.category_id], - refcolumns=[category.c.id]).create() - - tag = schema.Table( - 'tag', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('name', - types.String(80), - nullable=False, - index=True, - unique=True), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - tag.create() - - package_to_tag = schema.Table( - 'package_to_tag', - meta, - schema.Column('package_id', types.String(32)), - schema.Column('tag_id', types.String(32)) - ) - package_to_tag.create() - - constraint.ForeignKeyConstraint( - columns=[package_to_tag.c.package_id], - refcolumns=[package.c.id]).create() - constraint.ForeignKeyConstraint( - columns=[package_to_tag.c.tag_id], - refcolumns=[tag.c.id]).create() - class_definition = schema.Table( - 'class_definition', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('name', types.String(80), index=True), - schema.Column('package_id', types.String(32)), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - class_definition.create() - - constraint.ForeignKeyConstraint(columns=[class_definition.c.package_id], - refcolumns=[package.c.id]).create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - package_to_category = schema.Table('package_to_category', - meta, - autoload=True) - package_to_category.drop() - package_to_tag = schema.Table('package_to_tag', meta, autoload=True) - package_to_tag.drop() - class_definition = schema.Table('class_definition', meta, autoload=True) - class_definition.drop() - tag = schema.Table('tag', meta, autoload=True) - tag.drop() - category = schema.Table('category', meta, autoload=True) - category.drop() - package = schema.Table('package', meta, autoload=True) - package.drop() diff --git a/murano/db/migrate_repo/versions/005_add_instance_table.py b/murano/db/migrate_repo/versions/005_add_instance_table.py deleted file mode 100644 index fa0994950..000000000 --- a/murano/db/migrate_repo/versions/005_add_instance_table.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table( - 'instance', - meta, - schema.Column('environment_id', types.String(100), primary_key=True), - schema.Column('instance_id', types.String(100), primary_key=True), - schema.Column('instance_type', types.Integer, nullable=False), - schema.Column('created', types.Integer, nullable=False), - schema.Column('destroyed', types.Integer, nullable=True)) - table.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('instance', meta, autoload=True) - table.drop() diff --git a/murano/db/migrate_repo/versions/006_add_default_categories.py b/murano/db/migrate_repo/versions/006_add_default_categories.py deleted file mode 100644 index 562f2a4d2..000000000 --- a/murano/db/migrate_repo/versions/006_add_default_categories.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2014 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -import uuid - -from murano.common import consts -from murano.openstack.common import timeutils - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('category', meta, autoload=True) - for category in consts.CATEGORIES: - now = timeutils.utcnow() - values = {'id': uuid.uuid4().hex, 'name': category, 'updated': now, - 'created': now} - table.insert(values=values).execute() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('category', meta, autoload=True) - for category in consts.CATEGORIES: - table.delete().where(table.c.name == category).execute() diff --git a/murano/db/migrate_repo/versions/007_instance_table_extended.py b/murano/db/migrate_repo/versions/007_instance_table_extended.py deleted file mode 100644 index d80cd16d4..000000000 --- a/murano/db/migrate_repo/versions/007_instance_table_extended.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('instance', meta, autoload=True) - table.drop() - table = schema.Table( - 'instance_stats', - meta, - schema.Column('environment_id', types.String(100), primary_key=True), - schema.Column('instance_id', types.String(100), primary_key=True), - schema.Column('instance_type', types.Integer, nullable=False), - schema.Column('created', types.Integer, nullable=False), - schema.Column('destroyed', types.Integer, nullable=True), - schema.Column('type_name', types.String(512), nullable=False), - schema.Column('type_title', types.String(512)), - schema.Column('unit_count', types.Integer()), - schema.Column('tenant_id', types.String(32), nullable=False)) - table.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('instance_stats', meta, autoload=True) - table.rename('instance') - table.c.type_name.drop() - table.c.type_title.drop() - table.c.unit_count.drop() - table.c.tenant_id.drop() diff --git a/murano/db/migrate_repo/versions/008_cpu_stats.py b/murano/db/migrate_repo/versions/008_cpu_stats.py deleted file mode 100644 index ca6a4d85a..000000000 --- a/murano/db/migrate_repo/versions/008_cpu_stats.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('apistats', meta, autoload=True, extend_existing=True) - table.drop() - stats = schema.Table( - 'apistats', - meta, - schema.Column('id', types.Integer(), primary_key=True), - schema.Column('host', types.String(80)), - schema.Column('request_count', types.BigInteger()), - schema.Column('error_count', types.BigInteger()), - schema.Column('average_response_time', types.Float()), - schema.Column('requests_per_tenant', types.Text()), - schema.Column('requests_per_second', types.Float()), - schema.Column('errors_per_second', types.Float()), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('cpu_count', types.Integer()), - schema.Column('cpu_percent', types.Float()), extend_existing=True) - stats.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('apistats', meta, autoload=True, extend_existing=True) - table.drop() - stats = schema.Table( - 'apistats', - meta, - schema.Column('id', types.Integer(), primary_key=True), - schema.Column('host', types.String(80)), - schema.Column('request_count', types.BigInteger()), - schema.Column('error_count', types.BigInteger()), - schema.Column('average_response_time', types.Float()), - schema.Column('requests_per_tenant', types.Text()), - schema.Column('requests_per_second', types.Float()), - schema.Column('errors_per_second', types.Float()), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - extend_existing=True - ) - stats.create() diff --git a/murano/db/migrate_repo/versions/009_increase_package_name_length.py b/murano/db/migrate_repo/versions/009_increase_package_name_length.py deleted file mode 100644 index d15a3d929..000000000 --- a/murano/db/migrate_repo/versions/009_increase_package_name_length.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('package', meta, autoload=True) - table.c.name.alter(type=types.String(80)) - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('package', meta, autoload=True) - table.c.name.alter(type=types.String(20)) diff --git a/murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py b/murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py deleted file mode 100644 index 619e44951..000000000 --- a/murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.changeset.constraint import UniqueConstraint -from sqlalchemy import schema - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('environment', meta, autoload=True) - cons = UniqueConstraint("tenant_id", "name", table=table) - cons.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('environment', meta, autoload=True) - cons = UniqueConstraint("tenant_id", "name", table=table) - cons.drop() diff --git a/murano/db/migrate_repo/__init__.py b/murano/db/migration/__init__.py similarity index 100% rename from murano/db/migrate_repo/__init__.py rename to murano/db/migration/__init__.py diff --git a/murano/db/migration/alembic.ini b/murano/db/migration/alembic.ini new file mode 100644 index 000000000..378da37be --- /dev/null +++ b/murano/db/migration/alembic.ini @@ -0,0 +1,54 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = murano/db/migration/alembic_migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +sqlalchemy.url = + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S \ No newline at end of file diff --git a/murano/db/migration/alembic_migrations/README b/murano/db/migration/alembic_migrations/README new file mode 100644 index 000000000..f3c1411e4 --- /dev/null +++ b/murano/db/migration/alembic_migrations/README @@ -0,0 +1,15 @@ +Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation + +To create alembic migrations use: +$ murano-db-manage revision --message --autogenerate + +Stamp db with most recent migration version, without actually running migrations +$ murano-db-manage stamp --revision head + +Upgrade can be performed by: +$ murano-db-manage upgrade +$ murano-db-manage upgrade --revision head + +Downgrading db: +$ murano-db-manage downgrade +$ murano-db-manage downgrade --revision base diff --git a/murano/db/migration/alembic_migrations/env.py b/murano/db/migration/alembic_migrations/env.py new file mode 100644 index 000000000..149cdf213 --- /dev/null +++ b/murano/db/migration/alembic_migrations/env.py @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine, pool + +from murano.db import models + + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +murano_config = config.murano_config + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +target_metadata = models.BASE.metadata + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = create_engine( + murano_config.database.connection, + poolclass=pool.NullPool) + + with engine.connect() as connection: + context.configure(connection=connection, + target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +run_migrations_online() diff --git a/murano/db/migration/alembic_migrations/script.py.mako b/murano/db/migration/alembic_migrations/script.py.mako new file mode 100644 index 000000000..4a23e952c --- /dev/null +++ b/murano/db/migration/alembic_migrations/script.py.mako @@ -0,0 +1,37 @@ +# Copyright ${create_date.year} OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} \ No newline at end of file diff --git a/murano/db/migration/alembic_migrations/versions/001_inital_version.py b/murano/db/migration/alembic_migrations/versions/001_inital_version.py new file mode 100644 index 000000000..9f2376fc8 --- /dev/null +++ b/murano/db/migration/alembic_migrations/versions/001_inital_version.py @@ -0,0 +1,256 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""empty message + +Revision ID: 001 +Revises: None +Create Date: 2014-05-29 16:32:33.698760 + +""" + +# revision identifiers, used by Alembic. +revision = '001' +down_revision = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql.expression import table as sa_table +import uuid + +from murano.common import consts +from murano.db.sqla import types as st +from murano.openstack.common import timeutils + + +MYSQL_ENGINE = 'InnoDB' +MYSQL_CHARSET = 'utf8' + + +def _create_default_categories(op): + bind = op.get_bind() + table = sa_table( + 'category', + sa.Column('id', sa.String(length=36), primary_key=True), + sa.Column('created', sa.DateTime()), + sa.Column('updated', sa.DateTime()), + sa.Column('name', sa.String(length=80))) + + now = timeutils.utcnow() + + for category in consts.CATEGORIES: + values = {'id': uuid.uuid4().hex, + 'name': category, + 'updated': now, + 'created': now} + bind.execute(table.insert(values=values)) + + +def upgrade(): + op.create_table( + 'environment', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=255), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('tenant_id', sa.String(length=36), nullable=False), + sa.Column('version', sa.BigInteger(), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('networking', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('tenant_id', 'name'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'tag', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=80), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'category', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=80), nullable=False, index=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'apistats', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('host', sa.String(length=80), nullable=True), + sa.Column('request_count', sa.BigInteger(), nullable=True), + sa.Column('error_count', sa.BigInteger(), nullable=True), + sa.Column('average_response_time', sa.Float(), nullable=True), + sa.Column('requests_per_tenant', sa.Text(), nullable=True), + sa.Column('requests_per_second', sa.Float(), nullable=True), + sa.Column('errors_per_second', sa.Float(), nullable=True), + sa.Column('cpu_count', sa.Integer(), nullable=True), + sa.Column('cpu_percent', sa.Float(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'instance_stats', + sa.Column('environment_id', sa.String(length=255), nullable=False), + sa.Column('instance_id', sa.String(length=255), nullable=False), + sa.Column('instance_type', sa.Integer(), nullable=False), + sa.Column('created', sa.Integer(), nullable=False), + sa.Column('destroyed', sa.Integer(), nullable=True), + sa.Column('type_name', sa.String(length=512), nullable=False), + sa.Column('type_title', sa.String(length=512), nullable=True), + sa.Column('unit_count', sa.Integer(), nullable=True), + sa.Column('tenant_id', sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint('environment_id', 'instance_id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'package', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('archive', st.LargeBinary(), nullable=True), + sa.Column('fully_qualified_name', sa.String(length=512), + nullable=False, index=True), + sa.Column('type', sa.String(length=20), nullable=False), + sa.Column('author', sa.String(length=80), nullable=True), + sa.Column('name', sa.String(length=80), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=True), + sa.Column('description', sa.String(length=512), nullable=False), + sa.Column('is_public', sa.Boolean(), nullable=True), + sa.Column('logo', st.LargeBinary(), nullable=True), + sa.Column('owner_id', sa.String(length=36), nullable=False), + sa.Column('ui_definition', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'session', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('environment_id', sa.String(length=255), nullable=True), + sa.Column('user_id', sa.String(length=36), nullable=False), + sa.Column('state', sa.String(length=36), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('version', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['environment_id'], ['environment.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'deployment', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('started', sa.DateTime(), nullable=False), + sa.Column('finished', sa.DateTime(), nullable=True), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('environment_id', sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint(['environment_id'], ['environment.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'class_definition', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=512), nullable=False, index=True), + sa.Column('package_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'status', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('entity_id', sa.String(length=255), nullable=True), + sa.Column('entity', sa.String(length=10), nullable=True), + sa.Column('deployment_id', sa.String(length=36), nullable=True), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('level', sa.String(length=32), nullable=False), + sa.Column('details', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['deployment_id'], ['deployment.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'package_to_tag', + sa.Column('package_id', sa.String(length=36), nullable=False), + sa.Column('tag_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ondelete='CASCADE'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'package_to_category', + sa.Column('package_id', sa.String(length=36), nullable=False), + sa.Column('category_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['category_id'], + ['category.id'], + ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + _create_default_categories(op) + ### end Alembic commands ### + + +def downgrade(): + op.drop_table('status') + op.drop_table('package_to_category') + op.drop_table('class_definition') + op.drop_table('deployment') + op.drop_table('package_to_tag') + op.drop_table('session') + op.drop_table('instance_stats') + op.drop_table('package') + op.drop_table('apistats') + op.drop_table('category') + op.drop_table('tag') + op.drop_table('environment') + ### end Alembic commands ### diff --git a/murano/db/migration/migration.py b/murano/db/migration/migration.py new file mode 100644 index 000000000..c302c77c5 --- /dev/null +++ b/murano/db/migration/migration.py @@ -0,0 +1,79 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import alembic +from alembic import config as alembic_config + + +def get_alembic_config(): + path = os.path.join(os.path.dirname(__file__), 'alembic.ini') + + config = alembic_config.Config(path) + config.set_main_option('script_location', + 'murano.db.migration:alembic_migrations') + return config + + +# TODO(ruhe): implement me +#def version(config=None): +# """Current database version.""" + + +def upgrade(revision, config=None): + """Used for upgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'head' + config = config or get_alembic_config() + + alembic.command.upgrade(config, revision or 'head') + + +def downgrade(revision, config=None): + """Used for downgrading database. + + :param version: Desired database version7 + :type version: string + """ + revision = revision or 'base' + config = config or get_alembic_config() + return alembic.command.downgrade(config, revision) + + +def stamp(revision, config=None): + """Stamps database with provided revision. + Don't run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + config = config or get_alembic_config() + return alembic.command.stamp(config, revision=revision) + + +def revision(message=None, autogenerate=False, config=None): + """Creates template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + config = config or get_alembic_config() + return alembic.command.revision(config, message=message, + autogenerate=autogenerate) diff --git a/murano/db/models.py b/murano/db/models.py index 33a1cd7cf..53eac06e2 100644 --- a/murano/db/models.py +++ b/murano/db/models.py @@ -108,11 +108,11 @@ class Environment(BASE, ModificationsTrackedObject): """Represents a Environment in the metadata-store""" __tablename__ = 'environment' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(255), primary_key=True, default=uuidutils.generate_uuid) name = sa.Column(sa.String(255), nullable=False) - tenant_id = sa.Column(sa.String(32), nullable=False) + tenant_id = sa.Column(sa.String(36), nullable=False) version = sa.Column(sa.BigInteger, nullable=False, default=0) description = sa.Column(JsonBlob(), nullable=False, default={}) networking = sa.Column(JsonBlob(), nullable=True, default={}) @@ -131,10 +131,10 @@ class Environment(BASE, ModificationsTrackedObject): class Session(BASE, ModificationsTrackedObject): __tablename__ = 'session' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) - environment_id = sa.Column(sa.String(32), sa.ForeignKey('environment.id')) + environment_id = sa.Column(sa.String(255), sa.ForeignKey('environment.id')) user_id = sa.Column(sa.String(36), nullable=False) state = sa.Column(sa.String(36), nullable=False) @@ -153,13 +153,13 @@ class Session(BASE, ModificationsTrackedObject): class Deployment(BASE, ModificationsTrackedObject): __tablename__ = 'deployment' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) started = sa.Column(sa.DateTime, default=timeutils.utcnow, nullable=False) finished = sa.Column(sa.DateTime, default=None, nullable=True) description = sa.Column(JsonBlob(), nullable=False) - environment_id = sa.Column(sa.String(32), sa.ForeignKey('environment.id')) + environment_id = sa.Column(sa.String(255), sa.ForeignKey('environment.id')) statuses = sa_orm.relationship("Status", backref='deployment', cascade='save-update, merge, delete') @@ -177,12 +177,12 @@ class Deployment(BASE, ModificationsTrackedObject): class Status(BASE, ModificationsTrackedObject): __tablename__ = 'status' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) - entity_id = sa.Column(sa.String(32), nullable=True) + entity_id = sa.Column(sa.String(255), nullable=True) entity = sa.Column(sa.String(10), nullable=True) - deployment_id = sa.Column(sa.String(32), sa.ForeignKey('deployment.id')) + deployment_id = sa.Column(sa.String(36), sa.ForeignKey('deployment.id')) text = sa.Column(sa.String(), nullable=False) level = sa.Column(sa.String(32), nullable=False) details = sa.Column(sa.Text(), nullable=True) @@ -216,20 +216,20 @@ class ApiStats(BASE, ModificationsTrackedObject): package_to_category = sa.Table('package_to_category', BASE.metadata, sa.Column('package_id', - sa.String(32), + sa.String(36), sa.ForeignKey('package.id')), sa.Column('category_id', - sa.String(32), + sa.String(36), sa.ForeignKey('category.id', ondelete="RESTRICT"))) package_to_tag = sa.Table('package_to_tag', BASE.metadata, sa.Column('package_id', - sa.String(32), + sa.String(36), sa.ForeignKey('package.id')), sa.Column('tag_id', - sa.String(32), + sa.String(36), sa.ForeignKey('tag.id', ondelete="CASCADE"))) @@ -238,16 +238,16 @@ class Instance(BASE, ModelBase): __tablename__ = 'instance_stats' environment_id = sa.Column( - sa.String(100), primary_key=True, nullable=False) + sa.String(255), primary_key=True, nullable=False) instance_id = sa.Column( - sa.String(100), primary_key=True, nullable=False) + sa.String(255), primary_key=True, nullable=False) instance_type = sa.Column(sa.Integer, default=0, nullable=False) created = sa.Column(sa.Integer, nullable=False) destroyed = sa.Column(sa.Integer, nullable=True) type_name = sa.Column('type_name', sa.String(512), nullable=False) type_title = sa.Column('type_title', sa.String(512)) unit_count = sa.Column('unit_count', sa.Integer()) - tenant_id = sa.Column('tenant_id', sa.String(32), nullable=False) + tenant_id = sa.Column('tenant_id', sa.String(36), nullable=False) def to_dict(self): dictionary = super(Instance, self).to_dict() @@ -311,7 +311,7 @@ class Category(BASE, ModificationsTrackedObject): """ __tablename__ = 'category' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) name = sa.Column(sa.String(80), nullable=False, index=True, unique=True) @@ -323,7 +323,7 @@ class Tag(BASE, ModificationsTrackedObject): """ __tablename__ = 'tag' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) name = sa.Column(sa.String(80), nullable=False, unique=True) @@ -335,11 +335,11 @@ class Class(BASE, ModificationsTrackedObject): """ __tablename__ = 'class_definition' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) - name = sa.Column(sa.String(80), nullable=False, index=True) - package_id = sa.Column(sa.String(32), sa.ForeignKey('package.id')) + name = sa.Column(sa.String(512), nullable=False, index=True) + package_id = sa.Column(sa.String(36), sa.ForeignKey('package.id')) def register_models(engine): diff --git a/murano/db/session.py b/murano/db/session.py index e8f9fb158..156a0eebd 100644 --- a/murano/db/session.py +++ b/murano/db/session.py @@ -14,13 +14,7 @@ """Session management functions.""" -import os - -from migrate import exceptions as versioning_exceptions -from migrate.versioning import api as versioning_api - from murano.common import config -from murano.db import migrate_repo from murano.openstack.common.db.sqlalchemy import session as db_session from murano.openstack.common import log as logging @@ -48,12 +42,3 @@ def get_session(autocommit=True, expire_on_commit=False): def get_engine(): return _create_facade_lazily().get_engine() - - -def db_sync(): - repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__)) - try: - versioning_api.upgrade(CONF.database.connection, repo_path) - except versioning_exceptions.DatabaseNotControlledError: - versioning_api.version_control(CONF.database.connection, repo_path) - versioning_api.upgrade(CONF.database.connection, repo_path) diff --git a/murano/db/migrate_repo/versions/__init__.py b/murano/db/sqla/__init__.py similarity index 100% rename from murano/db/migrate_repo/versions/__init__.py rename to murano/db/sqla/__init__.py diff --git a/murano/db/migrate_repo/manage.py b/murano/db/sqla/types.py similarity index 66% rename from murano/db/migrate_repo/manage.py rename to murano/db/sqla/types.py index 51a855a49..c5d46e51a 100644 --- a/murano/db/migrate_repo/manage.py +++ b/murano/db/sqla/types.py @@ -1,5 +1,3 @@ -# Copyright (c) 2013 Mirantis, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,9 +10,18 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.config import cfg +import sqlalchemy as sa +from sqlalchemy.dialects import mysql -from migrate.versioning.shell import main +CONF = cfg.CONF -# This should probably be a console script entry point. -if __name__ == '__main__': - main(debug='False', repository='.') + +def _is_mysql_avail(): + return CONF.database.connection.startswith('mysql') + + +def LargeBinary(): + if _is_mysql_avail(): + return mysql.LONGBLOB + return sa.LargeBinary diff --git a/murano/openstack/common/processutils.py b/murano/openstack/common/processutils.py new file mode 100644 index 000000000..9da806c4b --- /dev/null +++ b/murano/openstack/common/processutils.py @@ -0,0 +1,267 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +System-level utilities and helper functions. +""" + +import errno +import logging as stdlib_logging +import os +import random +import shlex +import signal + +from eventlet.green import subprocess +from eventlet import greenthread +import six + +from murano.openstack.common.gettextutils import _ +from murano.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class InvalidArgumentError(Exception): + def __init__(self, message=None): + super(InvalidArgumentError, self).__init__(message) + + +class UnknownArgumentError(Exception): + def __init__(self, message=None): + super(UnknownArgumentError, self).__init__(message) + + +class ProcessExecutionError(Exception): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = _("Unexpected error while running command.") + if exit_code is None: + exit_code = '-' + message = _('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Stdout: %(stdout)r\n' + 'Stderr: %(stderr)r') % {'description': description, + 'cmd': cmd, + 'exit_code': exit_code, + 'stdout': stdout, + 'stderr': stderr} + super(ProcessExecutionError, self).__init__(message) + + +class NoRootWrapSpecified(Exception): + def __init__(self, message=None): + super(NoRootWrapSpecified, self).__init__(message) + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def execute(*cmd, **kwargs): + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. + + :param cmd: Passed to subprocess.Popen. + :type cmd: string + :param process_input: Send to opened process. + :type process_input: string + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] + :param delay_on_retry: True | False. Defaults to True. If set to True, + wait a short amount of time before retrying. + :type delay_on_retry: boolean + :param attempts: How many times to retry cmd. + :type attempts: int + :param run_as_root: True | False. Defaults to False. If set to True, + the command is prefixed by the command specified + in the root_helper kwarg. + :type run_as_root: boolean + :param root_helper: command to prefix to commands called with + run_as_root=True + :type root_helper: string + :param shell: whether or not there should be a shell used to + execute this command. Defaults to false. + :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be stdlib_logging.DEBUG or + stdlib_logging.INFO) + :returns: (stdout, stderr) from process execution + :raises: :class:`UnknownArgumentError` on + receiving unknown arguments + :raises: :class:`ProcessExecutionError` + """ + + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + delay_on_retry = kwargs.pop('delay_on_retry', True) + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) + root_helper = kwargs.pop('root_helper', '') + shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG) + + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + if kwargs: + raise UnknownArgumentError(_('Got unknown keyword args ' + 'to utils.execute: %r') % kwargs) + + if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: + if not root_helper: + raise NoRootWrapSpecified( + message=_('Command requested root, but did not ' + 'specify a root helper.')) + cmd = shlex.split(root_helper) + list(cmd) + + cmd = map(str, cmd) + + while attempts > 0: + attempts -= 1 + try: + LOG.log(loglevel, 'Running cmd (subprocess): %s', + ' '.join(cmd)) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) + result = None + for _i in six.moves.range(20): + # NOTE(russellb) 20 is an arbitrary number of retries to + # prevent any chance of looping forever here. + try: + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + except OSError as e: + if e.errno in (errno.EAGAIN, errno.EINTR): + continue + raise + break + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.log(loglevel, 'Result was %s' % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=_returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + LOG.log(loglevel, '%r failed. Retrying.', cmd) + if delay_on_retry: + greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + + +def trycmd(*args, **kwargs): + """A wrapper around execute() to more easily handle warnings and errors. + + Returns an (out, err) tuple of strings containing the output of + the command's stdout and stderr. If 'err' is not empty then the + command can be considered to have failed. + + :discard_warnings True | False. Defaults to False. If set to True, + then for succeeding commands, stderr is cleared + + """ + discard_warnings = kwargs.pop('discard_warnings', False) + + try: + out, err = execute(*args, **kwargs) + failed = False + except ProcessExecutionError as exn: + out, err = '', str(exn) + failed = True + + if not failed and discard_warnings and err: + # Handle commands that output to stderr but otherwise succeed + err = '' + + return out, err + + +def ssh_execute(ssh, cmd, process_input=None, + addl_env=None, check_exit_code=True): + LOG.debug('Running cmd (SSH): %s', cmd) + if addl_env: + raise InvalidArgumentError(_('Environment not supported over SSH')) + + if process_input: + # This is (probably) fixable if we need it... + raise InvalidArgumentError(_('process_input not supported over SSH')) + + stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) + channel = stdout_stream.channel + + # NOTE(justinsb): This seems suspicious... + # ...other SSH clients have buffering issues with this approach + stdout = stdout_stream.read() + stderr = stderr_stream.read() + stdin_stream.close() + + exit_status = channel.recv_exit_status() + + # exit_status == -1 if no exit code was returned + if exit_status != -1: + LOG.debug('Result was %s' % exit_status) + if check_exit_code and exit_status != 0: + raise ProcessExecutionError(exit_code=exit_status, + stdout=stdout, + stderr=stderr, + cmd=cmd) + + return (stdout, stderr) diff --git a/murano/tests/db/__init__.py b/murano/tests/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/murano/tests/db/migration/__init__.py b/murano/tests/db/migration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/murano/tests/db/migration/test_migrations.conf b/murano/tests/db/migration/test_migrations.conf new file mode 100644 index 000000000..b87c306bb --- /dev/null +++ b/murano/tests/db/migration/test_migrations.conf @@ -0,0 +1,26 @@ +[unit_tests] +# Set up any number of databases to test concurrently. +# The "name" used in the test is the config variable key. + +# A few tests rely on one sqlite database with 'sqlite' as the key. + +sqlite=sqlite:// +#sqlitefile=sqlite:///test_migrations_utils.db +#mysql=mysql+mysqldb://user:pass@localhost/test_migrations_utils +#postgresql=postgresql+psycopg2://user:pass@localhost/test_migrations_utils + +[migration_dbs] +# Migration DB details are listed separately as they can't be connected to +# concurrently. These databases can't be the same as above + +# Note, sqlite:// is in-memory and unique each time it is spawned. +# However file sqlite's are not unique. + +#sqlite=sqlite:// +#sqlitefile=sqlite:///test_migrations.db +#mysql=mysql+mysqldb://user:pass@localhost/test_migrations +#postgresql=postgresql+psycopg2://user:pass@localhost/test_migrations + +[walk_style] +snake_walk=yes +downgrade=yes diff --git a/murano/tests/db/migration/test_migrations.py b/murano/tests/db/migration/test_migrations.py new file mode 100644 index 000000000..6a30de07d --- /dev/null +++ b/murano/tests/db/migration/test_migrations.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +from murano.db import models # noqa +from murano.openstack.common.db.sqlalchemy import utils as db_utils +from murano.tests.db.migration import test_migrations_base as base + + +CONF = cfg.CONF + + +class TestMigrations(base.BaseWalkMigrationTestCase, base.CommonTestsMixIn): + + USER = "openstack_citest" + PASSWD = "openstack_citest" + DATABASE = "openstack_citest" + + def __init__(self, *args, **kwargs): + super(TestMigrations, self).__init__(*args, **kwargs) + + def setUp(self): + super(TestMigrations, self).setUp() + + def assertColumnExists(self, engine, table, column): + t = db_utils.get_table(engine, table) + self.assertIn(column, t.c) + + def assertColumnsExists(self, engine, table, columns): + for column in columns: + self.assertColumnExists(engine, table, column) + + def assertColumnCount(self, engine, table, columns): + t = db_utils.get_table(engine, table) + self.assertEqual(len(t.columns), len(columns)) + + def assertColumnNotExists(self, engine, table, column): + t = db_utils.get_table(engine, table) + self.assertNotIn(column, t.c) + + def assertIndexExists(self, engine, table, index): + t = db_utils.get_table(engine, table) + index_names = [idx.name for idx in t.indexes] + self.assertIn(index, index_names) + + def assertIndexMembers(self, engine, table, index, members): + self.assertIndexExists(engine, table, index) + + t = db_utils.get_table(engine, table) + index_columns = None + for idx in t.indexes: + if idx.name == index: + index_columns = idx.columns.keys() + break + + self.assertEqual(sorted(members), sorted(index_columns)) + + def _check_001(self, engine, data): + self.assertColumnExists(engine, 'category', 'id') + self.assertColumnExists(engine, 'environment', 'tenant_id') + self.assertIndexExists(engine, + 'class_definition', + 'ix_class_definition_name') diff --git a/murano/tests/db/migration/test_migrations_base.py b/murano/tests/db/migration/test_migrations_base.py new file mode 100644 index 000000000..22eadbc5f --- /dev/null +++ b/murano/tests/db/migration/test_migrations_base.py @@ -0,0 +1,585 @@ +# Copyright 2010-2011 OpenStack Foundation +# Copyright 2012-2013 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Ripped off from Nova's test_migrations.py +# The only difference between Nova and this code is usage of alembic instead +# of sqlalchemy migrations. +# +# There is an ongoing work to extact similar code to oslo incubator. Once it is +# extracted we'll be able to remove this file and use oslo. + +import ConfigParser +import io +import os + +from alembic import command +from alembic import config as alembic_config +from alembic import migration +from oslo.config import cfg +import six.moves.urllib.parse as urlparse +import sqlalchemy +import sqlalchemy.exc +import unittest2 + +import murano.db.migration +from murano.openstack.common import lockutils +from murano.openstack.common import log as logging +from murano.openstack.common import processutils + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + +synchronized = lockutils.synchronized_with_prefix('murano-') + + +def _get_connect_string(backend, user, passwd, database): + """Try to get a connection with a very specific set of values, if we get + these then we'll run the tests, otherwise they are skipped + """ + if backend == "postgres": + backend = "postgresql+psycopg2" + elif backend == "mysql": + backend = "mysql+mysqldb" + else: + raise Exception("Unrecognized backend: '%s'" % backend) + + return ("%s://%s:%s@localhost/%s" % (backend, user, passwd, database)) + + +def _is_backend_avail(backend, user, passwd, database): + try: + connect_uri = _get_connect_string(backend, user, passwd, database) + engine = sqlalchemy.create_engine(connect_uri) + connection = engine.connect() + except Exception: + # intentionally catch all to handle exceptions even if we don't + # have any backend code loaded. + return False + else: + connection.close() + engine.dispose() + return True + + +def _have_mysql(user, passwd, database): + present = os.environ.get('MURANO_MYSQL_PRESENT') + if present is None: + return _is_backend_avail('mysql', user, passwd, database) + return present.lower() in ('', 'true') + + +def _have_postgresql(user, passwd, database): + present = os.environ.get('MURANO_TEST_POSTGRESQL_PRESENT') + if present is None: + return _is_backend_avail('postgres', user, passwd, database) + return present.lower() in ('', 'true') + + +def get_mysql_connection_info(conn_pieces): + database = conn_pieces.path.strip('/') + loc_pieces = conn_pieces.netloc.split('@') + host = loc_pieces[1] + auth_pieces = loc_pieces[0].split(':') + user = auth_pieces[0] + password = "" + if len(auth_pieces) > 1: + if auth_pieces[1].strip(): + password = "-p\"%s\"" % auth_pieces[1] + + return (user, password, database, host) + + +def get_pgsql_connection_info(conn_pieces): + database = conn_pieces.path.strip('/') + loc_pieces = conn_pieces.netloc.split('@') + host = loc_pieces[1] + + auth_pieces = loc_pieces[0].split(':') + user = auth_pieces[0] + password = "" + if len(auth_pieces) > 1: + password = auth_pieces[1].strip() + + return (user, password, database, host) + + +class CommonTestsMixIn(object): + """BaseMigrationTestCase is effectively an abstract class, meant to be + derived from and not directly tested against; that's why these `test_` + methods need to be on a Mixin, so that they won't be picked up as valid + tests for BaseMigrationTestCase. + """ + def test_walk_versions(self): + for key, engine in self.engines.items(): + # We start each walk with a completely blank slate. + self._reset_database(key) + self._walk_versions(engine, self.snake_walk, self.downgrade) + + def test_mysql_opportunistically(self): + self._test_mysql_opportunistically() + + def test_mysql_connect_fail(self): + """Test that we can trigger a mysql connection failure and we fail + gracefully to ensure we don't break people without mysql + """ + if _is_backend_avail('mysql', "openstack_cifail", self.PASSWD, + self.DATABASE): + self.fail("Shouldn't have connected") + + def test_postgresql_opportunistically(self): + self._test_postgresql_opportunistically() + + def test_postgresql_connect_fail(self): + """Test that we can trigger a postgres connection failure and we fail + gracefully to ensure we don't break people without postgres + """ + if _is_backend_avail('postgres', "openstack_cifail", self.PASSWD, + self.DATABASE): + self.fail("Shouldn't have connected") + + +class BaseMigrationTestCase(unittest2.TestCase): + """Base class for testing migrations and migration utils. This sets up + and configures the databases to run tests against. + """ + + # NOTE(jhesketh): It is expected that tests clean up after themselves. + # This is necessary for concurrency to allow multiple tests to work on + # one database. + # The full migration walk tests however do call the old _reset_databases() + # to throw away whatever was there so they need to operate on their own + # database that we know isn't accessed concurrently. + # Hence, BaseWalkMigrationTestCase overwrites the engine list. + + USER = None + PASSWD = None + DATABASE = None + + TIMEOUT_SCALING_FACTOR = 2 + + def __init__(self, *args, **kwargs): + super(BaseMigrationTestCase, self).__init__(*args, **kwargs) + + self.DEFAULT_CONFIG_FILE = os.path.join( + os.path.dirname(__file__), + 'test_migrations.conf') + # Test machines can set the MURANO_TEST_MIGRATIONS_CONF variable + # to override the location of the config file for migration testing + self.CONFIG_FILE_PATH = os.environ.get( + 'MURANO_TEST_MIGRATIONS_CONF', + self.DEFAULT_CONFIG_FILE) + + self.ALEMBIC_CONFIG = alembic_config.Config( + os.path.join(os.path.dirname(murano.db.migration.__file__), + 'alembic.ini') + ) + self.ALEMBIC_CONFIG.set_main_option( + 'script_location', + 'murano.db.migration:alembic_migrations') + + self.ALEMBIC_CONFIG.murano_config = CONF + + self.snake_walk = False + self.downgrade = False + self.test_databases = {} + self.migration = None + self.migration_api = None + + def setUp(self): + super(BaseMigrationTestCase, self).setUp() + self._load_config() + + def _load_config(self): + # Load test databases from the config file. Only do this + # once. No need to re-run this on each test... + LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH) + if os.path.exists(self.CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(self.CONFIG_FILE_PATH) + config = cp.options('unit_tests') + for key in config: + self.test_databases[key] = cp.get('unit_tests', key) + self.snake_walk = cp.getboolean('walk_style', 'snake_walk') + self.downgrade = cp.getboolean('walk_style', 'downgrade') + + except ConfigParser.ParsingError as e: + self.fail("Failed to read test_migrations.conf config " + "file. Got error: %s" % e) + else: + self.fail("Failed to find test_migrations.conf config " + "file.") + + self.engines = {} + for key, value in self.test_databases.items(): + self.engines[key] = sqlalchemy.create_engine(value) + + # NOTE(jhesketh): We only need to make sure the databases are created + # not necessarily clean of tables. + self._create_databases() + + def execute_cmd(self, cmd=None): + out, err = processutils.trycmd(cmd, shell=True, discard_warnings=True) + output = out or err + LOG.debug(output) + self.assertEqual('', err, + "Failed to run: %s\n%s" % (cmd, output)) + + @synchronized('pgadmin', external=True, lock_path='/tmp') + def _reset_pg(self, conn_pieces): + (user, password, database, host) = \ + get_pgsql_connection_info(conn_pieces) + os.environ['PGPASSWORD'] = password + os.environ['PGUSER'] = user + # note(boris-42): We must create and drop database, we can't + # drop database which we have connected to, so for such + # operations there is a special database template1. + sqlcmd = ("psql -w -U %(user)s -h %(host)s -c" + " '%(sql)s' -d template1") + sqldict = {'user': user, 'host': host} + + sqldict['sql'] = ("drop database if exists %s;") % database + droptable = sqlcmd % sqldict + self.execute_cmd(droptable) + + sqldict['sql'] = ("create database %s;") % database + createtable = sqlcmd % sqldict + self.execute_cmd(createtable) + + os.unsetenv('PGPASSWORD') + os.unsetenv('PGUSER') + + @synchronized('mysql', external=True, lock_path='/tmp') + def _reset_mysql(self, conn_pieces): + # We can execute the MySQL client to destroy and re-create + # the MYSQL database, which is easier and less error-prone + # than using SQLAlchemy to do this via MetaData...trust me. + (user, password, database, host) = \ + get_mysql_connection_info(conn_pieces) + sql = ("drop database if exists %(database)s; " + "create database %(database)s;" % {'database': database}) + cmd = ("mysql -u \"%(user)s\" %(password)s -h %(host)s -e \"%(sql)s\"" + % {'user': user, 'password': password, + 'host': host, 'sql': sql}) + self.execute_cmd(cmd) + + @synchronized('sqlite', external=True, lock_path='/tmp') + def _reset_sqlite(self, conn_pieces): + # We can just delete the SQLite database, which is + # the easiest and cleanest solution + db_path = conn_pieces.path.strip('/') + if os.path.exists(db_path): + os.unlink(db_path) + # No need to recreate the SQLite DB. SQLite will + # create it for us if it's not there... + + def _create_databases(self): + """Create all configured databases as needed.""" + for key, engine in self.engines.items(): + self._create_database(key) + + def _create_database(self, key): + """Create database if it doesn't exist.""" + conn_string = self.test_databases[key] + conn_pieces = urlparse.urlparse(conn_string) + + if conn_string.startswith('mysql'): + (user, password, database, host) = \ + get_mysql_connection_info(conn_pieces) + sql = "create database if not exists %s;" % database + cmd = ("mysql -u \"%(user)s\" %(password)s -h %(host)s " + "-e \"%(sql)s\"" % {'user': user, 'password': password, + 'host': host, 'sql': sql}) + self.execute_cmd(cmd) + elif conn_string.startswith('postgresql'): + (user, password, database, host) = \ + get_pgsql_connection_info(conn_pieces) + os.environ['PGPASSWORD'] = password + os.environ['PGUSER'] = user + + sqlcmd = ("psql -w -U %(user)s -h %(host)s -c" + " '%(sql)s' -d template1") + + sql = ("create database if not exists %s;") % database + createtable = sqlcmd % {'user': user, 'host': host, 'sql': sql} + # 0 means databases is created + # 256 means it already exists (which is fine) + # otherwise raise an error + out, err = processutils.trycmd(createtable, shell=True, + check_exit_code=[0, 256], + discard_warnings=True) + output = out or err + if err != '': + self.fail("Failed to run: %s\n%s" % (createtable, output)) + + os.unsetenv('PGPASSWORD') + os.unsetenv('PGUSER') + + def _reset_databases(self): + """Reset all configured databases.""" + for key, engine in self.engines.items(): + self._reset_database(key) + + def _reset_database(self, key): + """Reset specific database.""" + engine = self.engines[key] + conn_string = self.test_databases[key] + conn_pieces = urlparse.urlparse(conn_string) + engine.dispose() + if conn_string.startswith('sqlite'): + self._reset_sqlite(conn_pieces) + elif conn_string.startswith('mysql'): + self._reset_mysql(conn_pieces) + elif conn_string.startswith('postgresql'): + self._reset_pg(conn_pieces) + + +class BaseWalkMigrationTestCase(BaseMigrationTestCase): + """BaseWalkMigrationTestCase loads in an alternative set of databases for + testing against. This is necessary as the default databases can run tests + concurrently without interfering with itself. It is expected that + databases listed under [migraiton_dbs] in the configuration are only being + accessed by one test at a time. Currently only test_walk_versions accesses + the databases (and is the only method that calls _reset_database() which + is clearly problematic for concurrency). + """ + + def _load_config(self): + # Load test databases from the config file. Only do this + # once. No need to re-run this on each test... + LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH) + if os.path.exists(self.CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(self.CONFIG_FILE_PATH) + config = cp.options('migration_dbs') + for key in config: + self.test_databases[key] = cp.get('migration_dbs', key) + self.snake_walk = cp.getboolean('walk_style', 'snake_walk') + self.downgrade = cp.getboolean('walk_style', 'downgrade') + except ConfigParser.ParsingError as e: + self.fail("Failed to read test_migrations.conf config " + "file. Got error: %s" % e) + else: + self.fail("Failed to find test_migrations.conf config " + "file.") + + self.engines = {} + for key, value in self.test_databases.items(): + self.engines[key] = sqlalchemy.create_engine(value) + + self._create_databases() + + def _configure(self, engine): + """For each type of repository we should do some of configure steps. + For migrate_repo we should set under version control our database. + For alembic we should configure database settings. For this goal we + should use oslo.config and openstack.commom.db.sqlalchemy.session with + database functionality (reset default settings and session cleanup). + """ + CONF.set_override('connection', str(engine.url), group='database') + #session.cleanup() + + def _test_mysql_opportunistically(self): + # Test that table creation on mysql only builds InnoDB tables + if not _have_mysql(self.USER, self.PASSWD, self.DATABASE): + self.skipTest("mysql not available") + # add this to the global lists to make reset work with it, it's removed + # automatically in tearDown so no need to clean it up here. + connect_string = _get_connect_string( + "mysql", self.USER, self.PASSWD, self.DATABASE) + (user, password, database, host) = \ + get_mysql_connection_info(urlparse.urlparse(connect_string)) + engine = sqlalchemy.create_engine(connect_string) + self.engines[database] = engine + self.test_databases[database] = connect_string + + # build a fully populated mysql database with all the tables + self._reset_database(database) + self._walk_versions(engine, self.snake_walk, self.downgrade) + + connection = engine.connect() + # sanity check + total = connection.execute("SELECT count(*) " + "from information_schema.TABLES " + "where TABLE_SCHEMA='%(database)s'" % + {'database': database}) + self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?") + + connection.close() + + del(self.engines[database]) + del(self.test_databases[database]) + + def _test_postgresql_opportunistically(self): + # Test postgresql database migration walk + if not _have_postgresql(self.USER, self.PASSWD, self.DATABASE): + self.skipTest("postgresql not available") + # add this to the global lists to make reset work with it, it's removed + # automatically in tearDown so no need to clean it up here. + connect_string = _get_connect_string( + "postgres", self.USER, self.PASSWD, self.DATABASE) + engine = sqlalchemy.create_engine(connect_string) + (user, password, database, host) = \ + get_mysql_connection_info(urlparse.urlparse(connect_string)) + self.engines[database] = engine + self.test_databases[database] = connect_string + + # build a fully populated postgresql database with all the tables + self._reset_database(database) + self._walk_versions(engine, self.snake_walk, self.downgrade) + del(self.engines[database]) + del(self.test_databases[database]) + + def _alembic_command(self, alembic_command, engine, *args, **kwargs): + """Most of alembic command return data into output. + We should redefine this setting for getting info. + """ + self.ALEMBIC_CONFIG.stdout = buf = io.StringIO() + CONF.set_override('connection', str(engine.url), group='database') + #session.cleanup() + getattr(command, alembic_command)(*args, **kwargs) + res = buf.getvalue().strip() + LOG.debug('Alembic command `%s` returns: %s' % (alembic_command, res)) + #session.cleanup() + return res + + def _get_alembic_versions(self, engine): + """For support of full testing of migrations + we should have an opportunity to run command step by step for each + version in repo. This method returns list of alembic_versions by + historical order. + """ + full_history = self._alembic_command('history', + engine, self.ALEMBIC_CONFIG) + # The piece of output data with version can looked as: + # 'Rev: 17738166b91 (head)' or 'Rev: 43b1a023dfaa' + alembic_history = [r.split(' ')[1] for r in full_history.split("\n") + if r.startswith("Rev")] + alembic_history.reverse() + return alembic_history + + def _up_and_down_versions(self, engine): + """Since alembic version has a random algorithm of generation + (SA-migrate has an ordered autoincrement naming) we should store + a tuple of versions (version for upgrade and version for downgrade) + for successful testing of migrations in up>down>up mode. + """ + versions = self._get_alembic_versions(engine) + return zip(versions, ['-1'] + versions) + + def _walk_versions(self, engine=None, snake_walk=False, + downgrade=True): + # Determine latest version script from the repo, then + # upgrade from 1 through to the latest, with no data + # in the databases. This just checks that the schema itself + # upgrades successfully. + + self._configure(engine) + up_and_down_versions = self._up_and_down_versions(engine) + for ver_up, ver_down in up_and_down_versions: + # upgrade -> downgrade -> upgrade + self._migrate_up(engine, ver_up, with_data=True) + if snake_walk: + downgraded = self._migrate_down(engine, + ver_down, + with_data=True, + next_version=ver_up) + if downgraded: + self._migrate_up(engine, ver_up) + + if downgrade: + # Now walk it back down to 0 from the latest, testing + # the downgrade paths. + up_and_down_versions.reverse() + for ver_up, ver_down in up_and_down_versions: + # downgrade -> upgrade -> downgrade + downgraded = self._migrate_down(engine, + ver_down, next_version=ver_up) + + if snake_walk and downgraded: + self._migrate_up(engine, ver_up) + self._migrate_down(engine, ver_down, next_version=ver_up) + + def _get_version_from_db(self, engine): + """For each type of migrate repo latest version from db + will be returned. + """ + conn = engine.connect() + try: + context = migration.MigrationContext.configure(conn) + version = context.get_current_revision() or '-1' + finally: + conn.close() + return version + + def _migrate(self, engine, version, cmd): + """Base method for manipulation with migrate repo. + It will upgrade or downgrade the actual database. + """ + + self._alembic_command(cmd, engine, self.ALEMBIC_CONFIG, version) + + def _migrate_down(self, engine, version, with_data=False, + next_version=None): + try: + self._migrate(engine, version, 'downgrade') + except NotImplementedError: + # NOTE(sirp): some migrations, namely release-level + # migrations, don't support a downgrade. + return False + self.assertEqual(version, self._get_version_from_db(engine)) + + # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target' + # version). So if we have any downgrade checks, they need to be run for + # the previous (higher numbered) migration. + if with_data: + post_downgrade = getattr( + self, "_post_downgrade_%s" % next_version, None) + if post_downgrade: + post_downgrade(engine) + + return True + + def _migrate_up(self, engine, version, with_data=False): + """migrate up to a new version of the db. + + We allow for data insertion and post checks at every + migration version with special _pre_upgrade_### and + _check_### functions in the main test. + """ + # NOTE(sdague): try block is here because it's impossible to debug + # where a failed data migration happens otherwise + check_version = version + try: + if with_data: + data = None + pre_upgrade = getattr( + self, "_pre_upgrade_%s" % check_version, None) + if pre_upgrade: + data = pre_upgrade(engine) + self._migrate(engine, version, 'upgrade') + self.assertEqual(version, self._get_version_from_db(engine)) + if with_data: + check = getattr(self, "_check_%s" % check_version, None) + if check: + check(engine, data) + except Exception: + LOG.error("Failed to migrate to version %s on engine %s" % + (version, engine)) + raise diff --git a/requirements.txt b/requirements.txt index c39981158..b6d89f758 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ pbr>=0.6,<1.0 Babel>=1.3 SQLAlchemy>=0.7.8,<=0.9.99 +alembic>=0.4.1 anyjson>=0.3.3 eventlet>=0.13.0 PasteDeploy>=1.5.0 @@ -9,7 +10,6 @@ WebOb>=1.2.3 wsgiref>=0.1.2 argparse ordereddict -sqlalchemy-migrate>=0.8.2,!=0.8.4 kombu>=2.4.8 lockfile>=0.8 pycrypto>=2.6 @@ -34,4 +34,3 @@ oslo.messaging>=1.3.0a9 # not listed in global requirements yaql>=0.2.2,<0.3 python-muranoclient>=0.5.2 - diff --git a/setup.cfg b/setup.cfg index a06195315..2da1b1dec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ console_scripts = murano-api = murano.cmd.api:main murano-engine = murano.cmd.engine:main murano-manage = murano.cmd.manage:main + murano-db-manage = murano.cmd.db_manage:main [build_sphinx] all_files = 1 diff --git a/test-requirements.txt b/test-requirements.txt index d7c7fc7b0..d9c34c61e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,10 +4,16 @@ coverage>=3.6 discover fixtures>=0.3.14 mock>=1.0 +posix_ipc +sqlalchemy-migrate>=0.8.2,!=0.8.4 testrepository>=0.0.18 testscenarios>=0.4 testtools>=0.9.34 unittest2 +# Some of the tests use real MySQL and Postgres databases +MySQL-python +psycopg2 + # doc build requirements sphinx>=1.1.2,<1.2