diff --git a/MANIFEST.in b/MANIFEST.in index e17f95995..22067f620 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -10,8 +10,8 @@ include ChangeLog include babel.cfg include tox.ini include openstack-common.conf -include muranoapi/db/migrate_repo/README -include muranoapi/db/migrate_repo/migrate.cfg -include muranoapi/db/migrate_repo/versions/*.sql +include murano/db/migration/alembic.ini +include murano/db/migration/alembic_migrations/README +include murano/db/migration/alembic_migrations/script.py.mako recursive-include etc * global-exclude *.pyc diff --git a/contrib/devstack/lib/murano b/contrib/devstack/lib/murano index 68506a3df..b6d8e29e4 100644 --- a/contrib/devstack/lib/murano +++ b/contrib/devstack/lib/murano @@ -171,7 +171,7 @@ function init_murano() { # (re)create Murano database recreate_database murano utf8 - $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE db-sync + $MURANO_BIN_DIR/murano-db-manage --config-file $MURANO_CONF_FILE upgrade $MURANO_BIN_DIR/murano-manage --config-file $MURANO_CONF_FILE import-package $MURANO_DIR/meta/io.murano } diff --git a/doc/source/install/manual.rst b/doc/source/install/manual.rst index 4df7742cc..491c34979 100644 --- a/doc/source/install/manual.rst +++ b/doc/source/install/manual.rst @@ -89,7 +89,7 @@ Installing the API service and Engine :: - $ tox -e venv -- murano-manage --config-file /etc/murano/murano-api.conf db-sync + $ tox -e venv -- murano-db-manage --config-file /etc/murano/murano-api.conf upgrade 6. Launch Murano API service: diff --git a/murano/cmd/db_manage.py b/murano/cmd/db_manage.py new file mode 100644 index 000000000..1e0422dee --- /dev/null +++ b/murano/cmd/db_manage.py @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from murano.db.migration import migration +# this forces import and registration of db related configs +from murano.db import models # noqa +from murano.openstack.common import log + +CONF = cfg.CONF + + +class DBCommand(object): + + def upgrade(self, config): + migration.upgrade(CONF.command.revision, config=config) + + def downgrade(self, config): + migration.downgrade(CONF.command.revision, config=config) + + def revision(self, config): + migration.revision(CONF.command.message, + CONF.command.autogenerate, + config=config) + + def stamp(self, config): + migration.stamp(CONF.command.revision, config=config) + + +def add_command_parsers(subparsers): + command_object = DBCommand() + + parser = subparsers.add_parser('upgrade') + parser.set_defaults(func=command_object.upgrade) + parser.add_argument('--revision', nargs='?') + + parser = subparsers.add_parser('downgrade') + parser.set_defaults(func=command_object.downgrade) + parser.add_argument('--revision', nargs='?') + + parser = subparsers.add_parser('stamp') + parser.add_argument('--revision', nargs='?') + parser.set_defaults(func=command_object.stamp) + + parser = subparsers.add_parser('revision') + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.set_defaults(func=command_object.revision) + + +command_opt = cfg.SubCommandOpt('command', + title='Command', + help='Available commands', + handler=add_command_parsers) + +CONF.register_cli_opt(command_opt) + + +def main(): + config = migration.get_alembic_config() + # attach the Murano conf to the Alembic conf + config.murano_config = CONF + + CONF(project='murano') + log.setup('murano') + CONF.command.func(config) diff --git a/murano/cmd/manage.py b/murano/cmd/manage.py index 918f1c8b9..9bc50d43d 100644 --- a/murano/cmd/manage.py +++ b/murano/cmd/manage.py @@ -13,7 +13,9 @@ # under the License. """ - CLI interface for murano management. + *** Deprecation warning *** + This file is about to be deprecated, please use python-muranoclient. + *** Deprecation warning *** """ import sys @@ -24,7 +26,6 @@ from oslo.config import cfg import murano from murano.common import consts from murano.db.catalog import api as db_catalog_api -from murano.db import session as db_session from murano.openstack.common.db import exception as db_exception from murano.openstack.common import log as logging from murano.packages import load_utils @@ -34,15 +35,6 @@ CONF = cfg.CONF LOG = logging.getLogger(__name__) -# TODO(ruhe): proper error handling -def do_db_sync(): - """ - Place a database under migration control and upgrade, - creating first if necessary. - """ - db_session.db_sync() - - class AdminContext(object): def __init__(self): self.is_admin = True @@ -124,11 +116,6 @@ def do_add_category(): def add_command_parsers(subparsers): - parser = subparsers.add_parser('db-sync') - parser.set_defaults(func=do_db_sync) - parser.add_argument('version', nargs='?') - parser.add_argument('current_version', nargs='?') - parser = subparsers.add_parser('import-package') parser.set_defaults(func=do_import_package) parser.add_argument('directory', @@ -160,6 +147,7 @@ command_opt = cfg.SubCommandOpt('command', def main(): CONF.register_cli_opt(command_opt) + try: default_config_files = cfg.find_config_files('murano', 'murano') CONF(sys.argv[1:], project='murano', prog='murano-manage', diff --git a/murano/db/migrate_repo/README b/murano/db/migrate_repo/README deleted file mode 100644 index 6218f8cac..000000000 --- a/murano/db/migrate_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -http://code.google.com/p/sqlalchemy-migrate/ diff --git a/murano/db/migrate_repo/migrate.cfg b/murano/db/migrate_repo/migrate.cfg deleted file mode 100644 index 4fe14f42c..000000000 --- a/murano/db/migrate_repo/migrate.cfg +++ /dev/null @@ -1,20 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=Murano Migrations - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['sqlite'] -required_dbs=[] diff --git a/murano/db/migrate_repo/versions/001_add_initial_tables.py b/murano/db/migrate_repo/versions/001_add_initial_tables.py deleted file mode 100644 index eb88013a5..000000000 --- a/murano/db/migrate_repo/versions/001_add_initial_tables.py +++ /dev/null @@ -1,96 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from migrate.changeset import constraint as const - -from sqlalchemy import schema -from sqlalchemy import types - - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - meta.reflect() - - environment = schema.Table( - 'environment', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('name', types.String(255), nullable=False), - schema.Column('created', types.DateTime(), nullable=False), - schema.Column('updated', types.DateTime(), nullable=False), - schema.Column('tenant_id', types.String(32), nullable=False), - schema.Column('version', types.BigInteger, nullable=False, - server_default='0'), - schema.Column('description', types.Text(), nullable=False)) - environment.create() - - session = schema.Table( - 'session', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('environment_id', types.String(32), nullable=False), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('user_id', types.String(32), nullable=False), - schema.Column('version', types.BigInteger, nullable=False, - server_default='0'), - schema.Column('description', types.Text(), nullable=True), - schema.Column('state', types.Text(), nullable=False)) - session.create() - - environment = schema.Table('environment', meta, autoload=True) - const.ForeignKeyConstraint(columns=[session.c.environment_id], - refcolumns=[environment.c.id]).create() - - deployment = schema.Table( - 'deployment', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('environment_id', types.String(32), nullable=False), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('started', types.DateTime, nullable=False), - schema.Column('description', types.Text(), nullable=True), - schema.Column('finished', types.DateTime, nullable=True)) - deployment.create() - - environment = schema.Table('environment', meta, autoload=True) - const.ForeignKeyConstraint(columns=[deployment.c.environment_id], - refcolumns=[environment.c.id]).create() - - status = schema.Table( - 'status', - meta, - schema.Column('id', types.String(32), primary_key=True), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('entity', types.String(10), nullable=True), - schema.Column('entity_id', types.String(32), nullable=True), - schema.Column('environment_id', types.String(32), nullable=True), - schema.Column('deployment_id', types.String(32), nullable=False), - schema.Column('text', types.Text(), nullable=False), - schema.Column('details', types.Text(), nullable=True), - schema.Column('level', types.String(32), nullable=False, - server_default='info')) - status.create() - - const.ForeignKeyConstraint(columns=[status.c.deployment_id], - refcolumns=[deployment.c.id]).create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - meta.drop_all() diff --git a/murano/db/migrate_repo/versions/002_add_networking_field.py b/murano/db/migrate_repo/versions/002_add_networking_field.py deleted file mode 100644 index 72e447a6d..000000000 --- a/murano/db/migrate_repo/versions/002_add_networking_field.py +++ /dev/null @@ -1,34 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from sqlalchemy import schema -from sqlalchemy import types - - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - environment = schema.Table('environment', meta, autoload=True) - networking = schema.Column('networking', - types.Text(), - nullable=True, - default='{}') - networking.create(environment) - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - environment = schema.Table('environment', meta, autoload=True) - environment.c.networking.drop() diff --git a/murano/db/migrate_repo/versions/003_add_stats_table.py b/murano/db/migrate_repo/versions/003_add_stats_table.py deleted file mode 100644 index 4d606e401..000000000 --- a/murano/db/migrate_repo/versions/003_add_stats_table.py +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - stats = schema.Table( - 'apistats', - meta, - schema.Column('id', types.Integer(), primary_key=True), - schema.Column('host', types.String(80)), - schema.Column('request_count', types.BigInteger()), - schema.Column('error_count', types.BigInteger()), - schema.Column('average_response_time', types.Float()), - schema.Column('requests_per_tenant', types.Text()), - schema.Column('requests_per_second', types.Float()), - schema.Column('errors_per_second', types.Float()), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False)) - stats.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - stats = schema.Table('apistats', meta, autoload=True) - stats.drop() diff --git a/murano/db/migrate_repo/versions/004_add_repository_tables.py b/murano/db/migrate_repo/versions/004_add_repository_tables.py deleted file mode 100644 index 320a43121..000000000 --- a/murano/db/migrate_repo/versions/004_add_repository_tables.py +++ /dev/null @@ -1,155 +0,0 @@ -# Copyright (c) 2014 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.changeset import constraint - -from sqlalchemy.dialects.mysql import mysqldb -from sqlalchemy import schema -from sqlalchemy import types -meta = schema.MetaData() - - -class StringWithCollation(types.String): - def __init__(self, length, collation=None, **kwargs): - super(StringWithCollation, self).__init__(length, **kwargs) - self.collation = collation - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - collation = 'ascii_general_ci' \ - if isinstance(migrate_engine.dialect, mysqldb.MySQLDialect) \ - else None - package = schema.Table( - 'package', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('archive', types.LargeBinary), - schema.Column('fully_qualified_name', - StringWithCollation(512, collation=collation), - index=True, unique=True), - schema.Column('type', types.String(20)), - schema.Column('author', types.String(80)), - schema.Column('name', types.String(20)), - schema.Column('enabled', types.Boolean), - schema.Column('description', types.String(512)), - schema.Column('is_public', types.Boolean), - schema.Column('logo', types.LargeBinary), - schema.Column('owner_id', types.String(36)), - schema.Column('ui_definition', types.Text), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - package.create() - - category = schema.Table( - 'category', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('name', - types.String(80), - nullable=False, - index=True, - unique=True), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - category.create() - - package_to_category = schema.Table( - 'package_to_category', - meta, - schema.Column('package_id', types.String(32)), - schema.Column('category_id', types.String(32)) - ) - package_to_category.create() - - constraint.ForeignKeyConstraint( - columns=[package_to_category.c.package_id], - refcolumns=[package.c.id]).create() - constraint.ForeignKeyConstraint( - columns=[package_to_category.c.category_id], - refcolumns=[category.c.id]).create() - - tag = schema.Table( - 'tag', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('name', - types.String(80), - nullable=False, - index=True, - unique=True), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - tag.create() - - package_to_tag = schema.Table( - 'package_to_tag', - meta, - schema.Column('package_id', types.String(32)), - schema.Column('tag_id', types.String(32)) - ) - package_to_tag.create() - - constraint.ForeignKeyConstraint( - columns=[package_to_tag.c.package_id], - refcolumns=[package.c.id]).create() - constraint.ForeignKeyConstraint( - columns=[package_to_tag.c.tag_id], - refcolumns=[tag.c.id]).create() - class_definition = schema.Table( - 'class_definition', - meta, - schema.Column('id', - types.String(32), - primary_key=True, - nullable=False), - schema.Column('name', types.String(80), index=True), - schema.Column('package_id', types.String(32)), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - ) - class_definition.create() - - constraint.ForeignKeyConstraint(columns=[class_definition.c.package_id], - refcolumns=[package.c.id]).create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - package_to_category = schema.Table('package_to_category', - meta, - autoload=True) - package_to_category.drop() - package_to_tag = schema.Table('package_to_tag', meta, autoload=True) - package_to_tag.drop() - class_definition = schema.Table('class_definition', meta, autoload=True) - class_definition.drop() - tag = schema.Table('tag', meta, autoload=True) - tag.drop() - category = schema.Table('category', meta, autoload=True) - category.drop() - package = schema.Table('package', meta, autoload=True) - package.drop() diff --git a/murano/db/migrate_repo/versions/005_add_instance_table.py b/murano/db/migrate_repo/versions/005_add_instance_table.py deleted file mode 100644 index fa0994950..000000000 --- a/murano/db/migrate_repo/versions/005_add_instance_table.py +++ /dev/null @@ -1,37 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table( - 'instance', - meta, - schema.Column('environment_id', types.String(100), primary_key=True), - schema.Column('instance_id', types.String(100), primary_key=True), - schema.Column('instance_type', types.Integer, nullable=False), - schema.Column('created', types.Integer, nullable=False), - schema.Column('destroyed', types.Integer, nullable=True)) - table.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('instance', meta, autoload=True) - table.drop() diff --git a/murano/db/migrate_repo/versions/006_add_default_categories.py b/murano/db/migrate_repo/versions/006_add_default_categories.py deleted file mode 100644 index 562f2a4d2..000000000 --- a/murano/db/migrate_repo/versions/006_add_default_categories.py +++ /dev/null @@ -1,38 +0,0 @@ -# Copyright (c) 2014 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -import uuid - -from murano.common import consts -from murano.openstack.common import timeutils - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('category', meta, autoload=True) - for category in consts.CATEGORIES: - now = timeutils.utcnow() - values = {'id': uuid.uuid4().hex, 'name': category, 'updated': now, - 'created': now} - table.insert(values=values).execute() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('category', meta, autoload=True) - for category in consts.CATEGORIES: - table.delete().where(table.c.name == category).execute() diff --git a/murano/db/migrate_repo/versions/007_instance_table_extended.py b/murano/db/migrate_repo/versions/007_instance_table_extended.py deleted file mode 100644 index d80cd16d4..000000000 --- a/murano/db/migrate_repo/versions/007_instance_table_extended.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('instance', meta, autoload=True) - table.drop() - table = schema.Table( - 'instance_stats', - meta, - schema.Column('environment_id', types.String(100), primary_key=True), - schema.Column('instance_id', types.String(100), primary_key=True), - schema.Column('instance_type', types.Integer, nullable=False), - schema.Column('created', types.Integer, nullable=False), - schema.Column('destroyed', types.Integer, nullable=True), - schema.Column('type_name', types.String(512), nullable=False), - schema.Column('type_title', types.String(512)), - schema.Column('unit_count', types.Integer()), - schema.Column('tenant_id', types.String(32), nullable=False)) - table.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('instance_stats', meta, autoload=True) - table.rename('instance') - table.c.type_name.drop() - table.c.type_title.drop() - table.c.unit_count.drop() - table.c.tenant_id.drop() diff --git a/murano/db/migrate_repo/versions/008_cpu_stats.py b/murano/db/migrate_repo/versions/008_cpu_stats.py deleted file mode 100644 index ca6a4d85a..000000000 --- a/murano/db/migrate_repo/versions/008_cpu_stats.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright (c) 2013 Mirantis, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('apistats', meta, autoload=True, extend_existing=True) - table.drop() - stats = schema.Table( - 'apistats', - meta, - schema.Column('id', types.Integer(), primary_key=True), - schema.Column('host', types.String(80)), - schema.Column('request_count', types.BigInteger()), - schema.Column('error_count', types.BigInteger()), - schema.Column('average_response_time', types.Float()), - schema.Column('requests_per_tenant', types.Text()), - schema.Column('requests_per_second', types.Float()), - schema.Column('errors_per_second', types.Float()), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - schema.Column('cpu_count', types.Integer()), - schema.Column('cpu_percent', types.Float()), extend_existing=True) - stats.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('apistats', meta, autoload=True, extend_existing=True) - table.drop() - stats = schema.Table( - 'apistats', - meta, - schema.Column('id', types.Integer(), primary_key=True), - schema.Column('host', types.String(80)), - schema.Column('request_count', types.BigInteger()), - schema.Column('error_count', types.BigInteger()), - schema.Column('average_response_time', types.Float()), - schema.Column('requests_per_tenant', types.Text()), - schema.Column('requests_per_second', types.Float()), - schema.Column('errors_per_second', types.Float()), - schema.Column('created', types.DateTime, nullable=False), - schema.Column('updated', types.DateTime, nullable=False), - extend_existing=True - ) - stats.create() diff --git a/murano/db/migrate_repo/versions/009_increase_package_name_length.py b/murano/db/migrate_repo/versions/009_increase_package_name_length.py deleted file mode 100644 index d15a3d929..000000000 --- a/murano/db/migrate_repo/versions/009_increase_package_name_length.py +++ /dev/null @@ -1,30 +0,0 @@ -# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from sqlalchemy import schema -from sqlalchemy import types - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('package', meta, autoload=True) - table.c.name.alter(type=types.String(80)) - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('package', meta, autoload=True) - table.c.name.alter(type=types.String(20)) diff --git a/murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py b/murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py deleted file mode 100644 index 619e44951..000000000 --- a/murano/db/migrate_repo/versions/010_add_unique_environment_constraint.py +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright (c) 2014 Hewlett-Packard Development Company, L.P. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.changeset.constraint import UniqueConstraint -from sqlalchemy import schema - -meta = schema.MetaData() - - -def upgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('environment', meta, autoload=True) - cons = UniqueConstraint("tenant_id", "name", table=table) - cons.create() - - -def downgrade(migrate_engine): - meta.bind = migrate_engine - table = schema.Table('environment', meta, autoload=True) - cons = UniqueConstraint("tenant_id", "name", table=table) - cons.drop() diff --git a/murano/db/migrate_repo/__init__.py b/murano/db/migration/__init__.py similarity index 100% rename from murano/db/migrate_repo/__init__.py rename to murano/db/migration/__init__.py diff --git a/murano/db/migration/alembic.ini b/murano/db/migration/alembic.ini new file mode 100644 index 000000000..378da37be --- /dev/null +++ b/murano/db/migration/alembic.ini @@ -0,0 +1,54 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = murano/db/migration/alembic_migrations + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +sqlalchemy.url = + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S \ No newline at end of file diff --git a/murano/db/migration/alembic_migrations/README b/murano/db/migration/alembic_migrations/README new file mode 100644 index 000000000..f3c1411e4 --- /dev/null +++ b/murano/db/migration/alembic_migrations/README @@ -0,0 +1,15 @@ +Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation + +To create alembic migrations use: +$ murano-db-manage revision --message --autogenerate + +Stamp db with most recent migration version, without actually running migrations +$ murano-db-manage stamp --revision head + +Upgrade can be performed by: +$ murano-db-manage upgrade +$ murano-db-manage upgrade --revision head + +Downgrading db: +$ murano-db-manage downgrade +$ murano-db-manage downgrade --revision base diff --git a/murano/db/migration/alembic_migrations/env.py b/murano/db/migration/alembic_migrations/env.py new file mode 100644 index 000000000..149cdf213 --- /dev/null +++ b/murano/db/migration/alembic_migrations/env.py @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# from logging.config import fileConfig + +from alembic import context +from sqlalchemy import create_engine, pool + +from murano.db import models + + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config +murano_config = config.murano_config + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +target_metadata = models.BASE.metadata + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = create_engine( + murano_config.database.connection, + poolclass=pool.NullPool) + + with engine.connect() as connection: + context.configure(connection=connection, + target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +run_migrations_online() diff --git a/murano/db/migration/alembic_migrations/script.py.mako b/murano/db/migration/alembic_migrations/script.py.mako new file mode 100644 index 000000000..4a23e952c --- /dev/null +++ b/murano/db/migration/alembic_migrations/script.py.mako @@ -0,0 +1,37 @@ +# Copyright ${create_date.year} OpenStack Foundation. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} + + +def downgrade(): + ${downgrades if downgrades else "pass"} \ No newline at end of file diff --git a/murano/db/migration/alembic_migrations/versions/001_inital_version.py b/murano/db/migration/alembic_migrations/versions/001_inital_version.py new file mode 100644 index 000000000..9f2376fc8 --- /dev/null +++ b/murano/db/migration/alembic_migrations/versions/001_inital_version.py @@ -0,0 +1,256 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""empty message + +Revision ID: 001 +Revises: None +Create Date: 2014-05-29 16:32:33.698760 + +""" + +# revision identifiers, used by Alembic. +revision = '001' +down_revision = None + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.sql.expression import table as sa_table +import uuid + +from murano.common import consts +from murano.db.sqla import types as st +from murano.openstack.common import timeutils + + +MYSQL_ENGINE = 'InnoDB' +MYSQL_CHARSET = 'utf8' + + +def _create_default_categories(op): + bind = op.get_bind() + table = sa_table( + 'category', + sa.Column('id', sa.String(length=36), primary_key=True), + sa.Column('created', sa.DateTime()), + sa.Column('updated', sa.DateTime()), + sa.Column('name', sa.String(length=80))) + + now = timeutils.utcnow() + + for category in consts.CATEGORIES: + values = {'id': uuid.uuid4().hex, + 'name': category, + 'updated': now, + 'created': now} + bind.execute(table.insert(values=values)) + + +def upgrade(): + op.create_table( + 'environment', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=255), nullable=False), + sa.Column('name', sa.String(length=255), nullable=False), + sa.Column('tenant_id', sa.String(length=36), nullable=False), + sa.Column('version', sa.BigInteger(), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('networking', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('tenant_id', 'name'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'tag', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=80), nullable=False), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('name'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'category', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=80), nullable=False, index=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'apistats', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('host', sa.String(length=80), nullable=True), + sa.Column('request_count', sa.BigInteger(), nullable=True), + sa.Column('error_count', sa.BigInteger(), nullable=True), + sa.Column('average_response_time', sa.Float(), nullable=True), + sa.Column('requests_per_tenant', sa.Text(), nullable=True), + sa.Column('requests_per_second', sa.Float(), nullable=True), + sa.Column('errors_per_second', sa.Float(), nullable=True), + sa.Column('cpu_count', sa.Integer(), nullable=True), + sa.Column('cpu_percent', sa.Float(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'instance_stats', + sa.Column('environment_id', sa.String(length=255), nullable=False), + sa.Column('instance_id', sa.String(length=255), nullable=False), + sa.Column('instance_type', sa.Integer(), nullable=False), + sa.Column('created', sa.Integer(), nullable=False), + sa.Column('destroyed', sa.Integer(), nullable=True), + sa.Column('type_name', sa.String(length=512), nullable=False), + sa.Column('type_title', sa.String(length=512), nullable=True), + sa.Column('unit_count', sa.Integer(), nullable=True), + sa.Column('tenant_id', sa.String(length=36), nullable=False), + sa.PrimaryKeyConstraint('environment_id', 'instance_id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'package', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('archive', st.LargeBinary(), nullable=True), + sa.Column('fully_qualified_name', sa.String(length=512), + nullable=False, index=True), + sa.Column('type', sa.String(length=20), nullable=False), + sa.Column('author', sa.String(length=80), nullable=True), + sa.Column('name', sa.String(length=80), nullable=False), + sa.Column('enabled', sa.Boolean(), nullable=True), + sa.Column('description', sa.String(length=512), nullable=False), + sa.Column('is_public', sa.Boolean(), nullable=True), + sa.Column('logo', st.LargeBinary(), nullable=True), + sa.Column('owner_id', sa.String(length=36), nullable=False), + sa.Column('ui_definition', sa.Text(), nullable=True), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'session', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('environment_id', sa.String(length=255), nullable=True), + sa.Column('user_id', sa.String(length=36), nullable=False), + sa.Column('state', sa.String(length=36), nullable=False), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('version', sa.BigInteger(), nullable=False), + sa.ForeignKeyConstraint(['environment_id'], ['environment.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'deployment', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('started', sa.DateTime(), nullable=False), + sa.Column('finished', sa.DateTime(), nullable=True), + sa.Column('description', sa.Text(), nullable=False), + sa.Column('environment_id', sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint(['environment_id'], ['environment.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'class_definition', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('name', sa.String(length=512), nullable=False, index=True), + sa.Column('package_id', sa.String(length=36), nullable=True), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'status', + sa.Column('created', sa.DateTime(), nullable=False), + sa.Column('updated', sa.DateTime(), nullable=False), + sa.Column('id', sa.String(length=36), nullable=False), + sa.Column('entity_id', sa.String(length=255), nullable=True), + sa.Column('entity', sa.String(length=10), nullable=True), + sa.Column('deployment_id', sa.String(length=36), nullable=True), + sa.Column('text', sa.Text(), nullable=False), + sa.Column('level', sa.String(length=32), nullable=False), + sa.Column('details', sa.Text(), nullable=True), + sa.ForeignKeyConstraint(['deployment_id'], ['deployment.id'], ), + sa.PrimaryKeyConstraint('id'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'package_to_tag', + sa.Column('package_id', sa.String(length=36), nullable=False), + sa.Column('tag_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], ondelete='CASCADE'), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + op.create_table( + 'package_to_category', + sa.Column('package_id', sa.String(length=36), nullable=False), + sa.Column('category_id', sa.String(length=36), nullable=False), + sa.ForeignKeyConstraint(['category_id'], + ['category.id'], + ondelete='RESTRICT'), + sa.ForeignKeyConstraint(['package_id'], ['package.id'], ), + mysql_engine=MYSQL_ENGINE, + mysql_charset=MYSQL_CHARSET + ) + + _create_default_categories(op) + ### end Alembic commands ### + + +def downgrade(): + op.drop_table('status') + op.drop_table('package_to_category') + op.drop_table('class_definition') + op.drop_table('deployment') + op.drop_table('package_to_tag') + op.drop_table('session') + op.drop_table('instance_stats') + op.drop_table('package') + op.drop_table('apistats') + op.drop_table('category') + op.drop_table('tag') + op.drop_table('environment') + ### end Alembic commands ### diff --git a/murano/db/migration/migration.py b/murano/db/migration/migration.py new file mode 100644 index 000000000..c302c77c5 --- /dev/null +++ b/murano/db/migration/migration.py @@ -0,0 +1,79 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import alembic +from alembic import config as alembic_config + + +def get_alembic_config(): + path = os.path.join(os.path.dirname(__file__), 'alembic.ini') + + config = alembic_config.Config(path) + config.set_main_option('script_location', + 'murano.db.migration:alembic_migrations') + return config + + +# TODO(ruhe): implement me +#def version(config=None): +# """Current database version.""" + + +def upgrade(revision, config=None): + """Used for upgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'head' + config = config or get_alembic_config() + + alembic.command.upgrade(config, revision or 'head') + + +def downgrade(revision, config=None): + """Used for downgrading database. + + :param version: Desired database version7 + :type version: string + """ + revision = revision or 'base' + config = config or get_alembic_config() + return alembic.command.downgrade(config, revision) + + +def stamp(revision, config=None): + """Stamps database with provided revision. + Don't run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + config = config or get_alembic_config() + return alembic.command.stamp(config, revision=revision) + + +def revision(message=None, autogenerate=False, config=None): + """Creates template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + config = config or get_alembic_config() + return alembic.command.revision(config, message=message, + autogenerate=autogenerate) diff --git a/murano/db/models.py b/murano/db/models.py index 33a1cd7cf..53eac06e2 100644 --- a/murano/db/models.py +++ b/murano/db/models.py @@ -108,11 +108,11 @@ class Environment(BASE, ModificationsTrackedObject): """Represents a Environment in the metadata-store""" __tablename__ = 'environment' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(255), primary_key=True, default=uuidutils.generate_uuid) name = sa.Column(sa.String(255), nullable=False) - tenant_id = sa.Column(sa.String(32), nullable=False) + tenant_id = sa.Column(sa.String(36), nullable=False) version = sa.Column(sa.BigInteger, nullable=False, default=0) description = sa.Column(JsonBlob(), nullable=False, default={}) networking = sa.Column(JsonBlob(), nullable=True, default={}) @@ -131,10 +131,10 @@ class Environment(BASE, ModificationsTrackedObject): class Session(BASE, ModificationsTrackedObject): __tablename__ = 'session' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) - environment_id = sa.Column(sa.String(32), sa.ForeignKey('environment.id')) + environment_id = sa.Column(sa.String(255), sa.ForeignKey('environment.id')) user_id = sa.Column(sa.String(36), nullable=False) state = sa.Column(sa.String(36), nullable=False) @@ -153,13 +153,13 @@ class Session(BASE, ModificationsTrackedObject): class Deployment(BASE, ModificationsTrackedObject): __tablename__ = 'deployment' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) started = sa.Column(sa.DateTime, default=timeutils.utcnow, nullable=False) finished = sa.Column(sa.DateTime, default=None, nullable=True) description = sa.Column(JsonBlob(), nullable=False) - environment_id = sa.Column(sa.String(32), sa.ForeignKey('environment.id')) + environment_id = sa.Column(sa.String(255), sa.ForeignKey('environment.id')) statuses = sa_orm.relationship("Status", backref='deployment', cascade='save-update, merge, delete') @@ -177,12 +177,12 @@ class Deployment(BASE, ModificationsTrackedObject): class Status(BASE, ModificationsTrackedObject): __tablename__ = 'status' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) - entity_id = sa.Column(sa.String(32), nullable=True) + entity_id = sa.Column(sa.String(255), nullable=True) entity = sa.Column(sa.String(10), nullable=True) - deployment_id = sa.Column(sa.String(32), sa.ForeignKey('deployment.id')) + deployment_id = sa.Column(sa.String(36), sa.ForeignKey('deployment.id')) text = sa.Column(sa.String(), nullable=False) level = sa.Column(sa.String(32), nullable=False) details = sa.Column(sa.Text(), nullable=True) @@ -216,20 +216,20 @@ class ApiStats(BASE, ModificationsTrackedObject): package_to_category = sa.Table('package_to_category', BASE.metadata, sa.Column('package_id', - sa.String(32), + sa.String(36), sa.ForeignKey('package.id')), sa.Column('category_id', - sa.String(32), + sa.String(36), sa.ForeignKey('category.id', ondelete="RESTRICT"))) package_to_tag = sa.Table('package_to_tag', BASE.metadata, sa.Column('package_id', - sa.String(32), + sa.String(36), sa.ForeignKey('package.id')), sa.Column('tag_id', - sa.String(32), + sa.String(36), sa.ForeignKey('tag.id', ondelete="CASCADE"))) @@ -238,16 +238,16 @@ class Instance(BASE, ModelBase): __tablename__ = 'instance_stats' environment_id = sa.Column( - sa.String(100), primary_key=True, nullable=False) + sa.String(255), primary_key=True, nullable=False) instance_id = sa.Column( - sa.String(100), primary_key=True, nullable=False) + sa.String(255), primary_key=True, nullable=False) instance_type = sa.Column(sa.Integer, default=0, nullable=False) created = sa.Column(sa.Integer, nullable=False) destroyed = sa.Column(sa.Integer, nullable=True) type_name = sa.Column('type_name', sa.String(512), nullable=False) type_title = sa.Column('type_title', sa.String(512)) unit_count = sa.Column('unit_count', sa.Integer()) - tenant_id = sa.Column('tenant_id', sa.String(32), nullable=False) + tenant_id = sa.Column('tenant_id', sa.String(36), nullable=False) def to_dict(self): dictionary = super(Instance, self).to_dict() @@ -311,7 +311,7 @@ class Category(BASE, ModificationsTrackedObject): """ __tablename__ = 'category' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) name = sa.Column(sa.String(80), nullable=False, index=True, unique=True) @@ -323,7 +323,7 @@ class Tag(BASE, ModificationsTrackedObject): """ __tablename__ = 'tag' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) name = sa.Column(sa.String(80), nullable=False, unique=True) @@ -335,11 +335,11 @@ class Class(BASE, ModificationsTrackedObject): """ __tablename__ = 'class_definition' - id = sa.Column(sa.String(32), + id = sa.Column(sa.String(36), primary_key=True, default=uuidutils.generate_uuid) - name = sa.Column(sa.String(80), nullable=False, index=True) - package_id = sa.Column(sa.String(32), sa.ForeignKey('package.id')) + name = sa.Column(sa.String(512), nullable=False, index=True) + package_id = sa.Column(sa.String(36), sa.ForeignKey('package.id')) def register_models(engine): diff --git a/murano/db/session.py b/murano/db/session.py index e8f9fb158..156a0eebd 100644 --- a/murano/db/session.py +++ b/murano/db/session.py @@ -14,13 +14,7 @@ """Session management functions.""" -import os - -from migrate import exceptions as versioning_exceptions -from migrate.versioning import api as versioning_api - from murano.common import config -from murano.db import migrate_repo from murano.openstack.common.db.sqlalchemy import session as db_session from murano.openstack.common import log as logging @@ -48,12 +42,3 @@ def get_session(autocommit=True, expire_on_commit=False): def get_engine(): return _create_facade_lazily().get_engine() - - -def db_sync(): - repo_path = os.path.abspath(os.path.dirname(migrate_repo.__file__)) - try: - versioning_api.upgrade(CONF.database.connection, repo_path) - except versioning_exceptions.DatabaseNotControlledError: - versioning_api.version_control(CONF.database.connection, repo_path) - versioning_api.upgrade(CONF.database.connection, repo_path) diff --git a/murano/db/migrate_repo/versions/__init__.py b/murano/db/sqla/__init__.py similarity index 100% rename from murano/db/migrate_repo/versions/__init__.py rename to murano/db/sqla/__init__.py diff --git a/murano/db/migrate_repo/manage.py b/murano/db/sqla/types.py similarity index 66% rename from murano/db/migrate_repo/manage.py rename to murano/db/sqla/types.py index 51a855a49..c5d46e51a 100644 --- a/murano/db/migrate_repo/manage.py +++ b/murano/db/sqla/types.py @@ -1,5 +1,3 @@ -# Copyright (c) 2013 Mirantis, Inc. -# # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain # a copy of the License at @@ -12,9 +10,18 @@ # License for the specific language governing permissions and limitations # under the License. +from oslo.config import cfg +import sqlalchemy as sa +from sqlalchemy.dialects import mysql -from migrate.versioning.shell import main +CONF = cfg.CONF -# This should probably be a console script entry point. -if __name__ == '__main__': - main(debug='False', repository='.') + +def _is_mysql_avail(): + return CONF.database.connection.startswith('mysql') + + +def LargeBinary(): + if _is_mysql_avail(): + return mysql.LONGBLOB + return sa.LargeBinary diff --git a/murano/openstack/common/processutils.py b/murano/openstack/common/processutils.py new file mode 100644 index 000000000..9da806c4b --- /dev/null +++ b/murano/openstack/common/processutils.py @@ -0,0 +1,267 @@ +# Copyright 2011 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +System-level utilities and helper functions. +""" + +import errno +import logging as stdlib_logging +import os +import random +import shlex +import signal + +from eventlet.green import subprocess +from eventlet import greenthread +import six + +from murano.openstack.common.gettextutils import _ +from murano.openstack.common import log as logging + + +LOG = logging.getLogger(__name__) + + +class InvalidArgumentError(Exception): + def __init__(self, message=None): + super(InvalidArgumentError, self).__init__(message) + + +class UnknownArgumentError(Exception): + def __init__(self, message=None): + super(UnknownArgumentError, self).__init__(message) + + +class ProcessExecutionError(Exception): + def __init__(self, stdout=None, stderr=None, exit_code=None, cmd=None, + description=None): + self.exit_code = exit_code + self.stderr = stderr + self.stdout = stdout + self.cmd = cmd + self.description = description + + if description is None: + description = _("Unexpected error while running command.") + if exit_code is None: + exit_code = '-' + message = _('%(description)s\n' + 'Command: %(cmd)s\n' + 'Exit code: %(exit_code)s\n' + 'Stdout: %(stdout)r\n' + 'Stderr: %(stderr)r') % {'description': description, + 'cmd': cmd, + 'exit_code': exit_code, + 'stdout': stdout, + 'stderr': stderr} + super(ProcessExecutionError, self).__init__(message) + + +class NoRootWrapSpecified(Exception): + def __init__(self, message=None): + super(NoRootWrapSpecified, self).__init__(message) + + +def _subprocess_setup(): + # Python installs a SIGPIPE handler by default. This is usually not what + # non-Python subprocesses expect. + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + + +def execute(*cmd, **kwargs): + """Helper method to shell out and execute a command through subprocess. + + Allows optional retry. + + :param cmd: Passed to subprocess.Popen. + :type cmd: string + :param process_input: Send to opened process. + :type process_input: string + :param check_exit_code: Single bool, int, or list of allowed exit + codes. Defaults to [0]. Raise + :class:`ProcessExecutionError` unless + program exits with one of these code. + :type check_exit_code: boolean, int, or [int] + :param delay_on_retry: True | False. Defaults to True. If set to True, + wait a short amount of time before retrying. + :type delay_on_retry: boolean + :param attempts: How many times to retry cmd. + :type attempts: int + :param run_as_root: True | False. Defaults to False. If set to True, + the command is prefixed by the command specified + in the root_helper kwarg. + :type run_as_root: boolean + :param root_helper: command to prefix to commands called with + run_as_root=True + :type root_helper: string + :param shell: whether or not there should be a shell used to + execute this command. Defaults to false. + :type shell: boolean + :param loglevel: log level for execute commands. + :type loglevel: int. (Should be stdlib_logging.DEBUG or + stdlib_logging.INFO) + :returns: (stdout, stderr) from process execution + :raises: :class:`UnknownArgumentError` on + receiving unknown arguments + :raises: :class:`ProcessExecutionError` + """ + + process_input = kwargs.pop('process_input', None) + check_exit_code = kwargs.pop('check_exit_code', [0]) + ignore_exit_code = False + delay_on_retry = kwargs.pop('delay_on_retry', True) + attempts = kwargs.pop('attempts', 1) + run_as_root = kwargs.pop('run_as_root', False) + root_helper = kwargs.pop('root_helper', '') + shell = kwargs.pop('shell', False) + loglevel = kwargs.pop('loglevel', stdlib_logging.DEBUG) + + if isinstance(check_exit_code, bool): + ignore_exit_code = not check_exit_code + check_exit_code = [0] + elif isinstance(check_exit_code, int): + check_exit_code = [check_exit_code] + + if kwargs: + raise UnknownArgumentError(_('Got unknown keyword args ' + 'to utils.execute: %r') % kwargs) + + if run_as_root and hasattr(os, 'geteuid') and os.geteuid() != 0: + if not root_helper: + raise NoRootWrapSpecified( + message=_('Command requested root, but did not ' + 'specify a root helper.')) + cmd = shlex.split(root_helper) + list(cmd) + + cmd = map(str, cmd) + + while attempts > 0: + attempts -= 1 + try: + LOG.log(loglevel, 'Running cmd (subprocess): %s', + ' '.join(cmd)) + _PIPE = subprocess.PIPE # pylint: disable=E1101 + + if os.name == 'nt': + preexec_fn = None + close_fds = False + else: + preexec_fn = _subprocess_setup + close_fds = True + + obj = subprocess.Popen(cmd, + stdin=_PIPE, + stdout=_PIPE, + stderr=_PIPE, + close_fds=close_fds, + preexec_fn=preexec_fn, + shell=shell) + result = None + for _i in six.moves.range(20): + # NOTE(russellb) 20 is an arbitrary number of retries to + # prevent any chance of looping forever here. + try: + if process_input is not None: + result = obj.communicate(process_input) + else: + result = obj.communicate() + except OSError as e: + if e.errno in (errno.EAGAIN, errno.EINTR): + continue + raise + break + obj.stdin.close() # pylint: disable=E1101 + _returncode = obj.returncode # pylint: disable=E1101 + LOG.log(loglevel, 'Result was %s' % _returncode) + if not ignore_exit_code and _returncode not in check_exit_code: + (stdout, stderr) = result + raise ProcessExecutionError(exit_code=_returncode, + stdout=stdout, + stderr=stderr, + cmd=' '.join(cmd)) + return result + except ProcessExecutionError: + if not attempts: + raise + else: + LOG.log(loglevel, '%r failed. Retrying.', cmd) + if delay_on_retry: + greenthread.sleep(random.randint(20, 200) / 100.0) + finally: + # NOTE(termie): this appears to be necessary to let the subprocess + # call clean something up in between calls, without + # it two execute calls in a row hangs the second one + greenthread.sleep(0) + + +def trycmd(*args, **kwargs): + """A wrapper around execute() to more easily handle warnings and errors. + + Returns an (out, err) tuple of strings containing the output of + the command's stdout and stderr. If 'err' is not empty then the + command can be considered to have failed. + + :discard_warnings True | False. Defaults to False. If set to True, + then for succeeding commands, stderr is cleared + + """ + discard_warnings = kwargs.pop('discard_warnings', False) + + try: + out, err = execute(*args, **kwargs) + failed = False + except ProcessExecutionError as exn: + out, err = '', str(exn) + failed = True + + if not failed and discard_warnings and err: + # Handle commands that output to stderr but otherwise succeed + err = '' + + return out, err + + +def ssh_execute(ssh, cmd, process_input=None, + addl_env=None, check_exit_code=True): + LOG.debug('Running cmd (SSH): %s', cmd) + if addl_env: + raise InvalidArgumentError(_('Environment not supported over SSH')) + + if process_input: + # This is (probably) fixable if we need it... + raise InvalidArgumentError(_('process_input not supported over SSH')) + + stdin_stream, stdout_stream, stderr_stream = ssh.exec_command(cmd) + channel = stdout_stream.channel + + # NOTE(justinsb): This seems suspicious... + # ...other SSH clients have buffering issues with this approach + stdout = stdout_stream.read() + stderr = stderr_stream.read() + stdin_stream.close() + + exit_status = channel.recv_exit_status() + + # exit_status == -1 if no exit code was returned + if exit_status != -1: + LOG.debug('Result was %s' % exit_status) + if check_exit_code and exit_status != 0: + raise ProcessExecutionError(exit_code=exit_status, + stdout=stdout, + stderr=stderr, + cmd=cmd) + + return (stdout, stderr) diff --git a/murano/tests/db/__init__.py b/murano/tests/db/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/murano/tests/db/migration/__init__.py b/murano/tests/db/migration/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/murano/tests/db/migration/test_migrations.conf b/murano/tests/db/migration/test_migrations.conf new file mode 100644 index 000000000..b87c306bb --- /dev/null +++ b/murano/tests/db/migration/test_migrations.conf @@ -0,0 +1,26 @@ +[unit_tests] +# Set up any number of databases to test concurrently. +# The "name" used in the test is the config variable key. + +# A few tests rely on one sqlite database with 'sqlite' as the key. + +sqlite=sqlite:// +#sqlitefile=sqlite:///test_migrations_utils.db +#mysql=mysql+mysqldb://user:pass@localhost/test_migrations_utils +#postgresql=postgresql+psycopg2://user:pass@localhost/test_migrations_utils + +[migration_dbs] +# Migration DB details are listed separately as they can't be connected to +# concurrently. These databases can't be the same as above + +# Note, sqlite:// is in-memory and unique each time it is spawned. +# However file sqlite's are not unique. + +#sqlite=sqlite:// +#sqlitefile=sqlite:///test_migrations.db +#mysql=mysql+mysqldb://user:pass@localhost/test_migrations +#postgresql=postgresql+psycopg2://user:pass@localhost/test_migrations + +[walk_style] +snake_walk=yes +downgrade=yes diff --git a/murano/tests/db/migration/test_migrations.py b/murano/tests/db/migration/test_migrations.py new file mode 100644 index 000000000..6a30de07d --- /dev/null +++ b/murano/tests/db/migration/test_migrations.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from oslo.config import cfg + +from murano.db import models # noqa +from murano.openstack.common.db.sqlalchemy import utils as db_utils +from murano.tests.db.migration import test_migrations_base as base + + +CONF = cfg.CONF + + +class TestMigrations(base.BaseWalkMigrationTestCase, base.CommonTestsMixIn): + + USER = "openstack_citest" + PASSWD = "openstack_citest" + DATABASE = "openstack_citest" + + def __init__(self, *args, **kwargs): + super(TestMigrations, self).__init__(*args, **kwargs) + + def setUp(self): + super(TestMigrations, self).setUp() + + def assertColumnExists(self, engine, table, column): + t = db_utils.get_table(engine, table) + self.assertIn(column, t.c) + + def assertColumnsExists(self, engine, table, columns): + for column in columns: + self.assertColumnExists(engine, table, column) + + def assertColumnCount(self, engine, table, columns): + t = db_utils.get_table(engine, table) + self.assertEqual(len(t.columns), len(columns)) + + def assertColumnNotExists(self, engine, table, column): + t = db_utils.get_table(engine, table) + self.assertNotIn(column, t.c) + + def assertIndexExists(self, engine, table, index): + t = db_utils.get_table(engine, table) + index_names = [idx.name for idx in t.indexes] + self.assertIn(index, index_names) + + def assertIndexMembers(self, engine, table, index, members): + self.assertIndexExists(engine, table, index) + + t = db_utils.get_table(engine, table) + index_columns = None + for idx in t.indexes: + if idx.name == index: + index_columns = idx.columns.keys() + break + + self.assertEqual(sorted(members), sorted(index_columns)) + + def _check_001(self, engine, data): + self.assertColumnExists(engine, 'category', 'id') + self.assertColumnExists(engine, 'environment', 'tenant_id') + self.assertIndexExists(engine, + 'class_definition', + 'ix_class_definition_name') diff --git a/murano/tests/db/migration/test_migrations_base.py b/murano/tests/db/migration/test_migrations_base.py new file mode 100644 index 000000000..22eadbc5f --- /dev/null +++ b/murano/tests/db/migration/test_migrations_base.py @@ -0,0 +1,585 @@ +# Copyright 2010-2011 OpenStack Foundation +# Copyright 2012-2013 IBM Corp. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# +# +# Ripped off from Nova's test_migrations.py +# The only difference between Nova and this code is usage of alembic instead +# of sqlalchemy migrations. +# +# There is an ongoing work to extact similar code to oslo incubator. Once it is +# extracted we'll be able to remove this file and use oslo. + +import ConfigParser +import io +import os + +from alembic import command +from alembic import config as alembic_config +from alembic import migration +from oslo.config import cfg +import six.moves.urllib.parse as urlparse +import sqlalchemy +import sqlalchemy.exc +import unittest2 + +import murano.db.migration +from murano.openstack.common import lockutils +from murano.openstack.common import log as logging +from murano.openstack.common import processutils + + +LOG = logging.getLogger(__name__) +CONF = cfg.CONF + +synchronized = lockutils.synchronized_with_prefix('murano-') + + +def _get_connect_string(backend, user, passwd, database): + """Try to get a connection with a very specific set of values, if we get + these then we'll run the tests, otherwise they are skipped + """ + if backend == "postgres": + backend = "postgresql+psycopg2" + elif backend == "mysql": + backend = "mysql+mysqldb" + else: + raise Exception("Unrecognized backend: '%s'" % backend) + + return ("%s://%s:%s@localhost/%s" % (backend, user, passwd, database)) + + +def _is_backend_avail(backend, user, passwd, database): + try: + connect_uri = _get_connect_string(backend, user, passwd, database) + engine = sqlalchemy.create_engine(connect_uri) + connection = engine.connect() + except Exception: + # intentionally catch all to handle exceptions even if we don't + # have any backend code loaded. + return False + else: + connection.close() + engine.dispose() + return True + + +def _have_mysql(user, passwd, database): + present = os.environ.get('MURANO_MYSQL_PRESENT') + if present is None: + return _is_backend_avail('mysql', user, passwd, database) + return present.lower() in ('', 'true') + + +def _have_postgresql(user, passwd, database): + present = os.environ.get('MURANO_TEST_POSTGRESQL_PRESENT') + if present is None: + return _is_backend_avail('postgres', user, passwd, database) + return present.lower() in ('', 'true') + + +def get_mysql_connection_info(conn_pieces): + database = conn_pieces.path.strip('/') + loc_pieces = conn_pieces.netloc.split('@') + host = loc_pieces[1] + auth_pieces = loc_pieces[0].split(':') + user = auth_pieces[0] + password = "" + if len(auth_pieces) > 1: + if auth_pieces[1].strip(): + password = "-p\"%s\"" % auth_pieces[1] + + return (user, password, database, host) + + +def get_pgsql_connection_info(conn_pieces): + database = conn_pieces.path.strip('/') + loc_pieces = conn_pieces.netloc.split('@') + host = loc_pieces[1] + + auth_pieces = loc_pieces[0].split(':') + user = auth_pieces[0] + password = "" + if len(auth_pieces) > 1: + password = auth_pieces[1].strip() + + return (user, password, database, host) + + +class CommonTestsMixIn(object): + """BaseMigrationTestCase is effectively an abstract class, meant to be + derived from and not directly tested against; that's why these `test_` + methods need to be on a Mixin, so that they won't be picked up as valid + tests for BaseMigrationTestCase. + """ + def test_walk_versions(self): + for key, engine in self.engines.items(): + # We start each walk with a completely blank slate. + self._reset_database(key) + self._walk_versions(engine, self.snake_walk, self.downgrade) + + def test_mysql_opportunistically(self): + self._test_mysql_opportunistically() + + def test_mysql_connect_fail(self): + """Test that we can trigger a mysql connection failure and we fail + gracefully to ensure we don't break people without mysql + """ + if _is_backend_avail('mysql', "openstack_cifail", self.PASSWD, + self.DATABASE): + self.fail("Shouldn't have connected") + + def test_postgresql_opportunistically(self): + self._test_postgresql_opportunistically() + + def test_postgresql_connect_fail(self): + """Test that we can trigger a postgres connection failure and we fail + gracefully to ensure we don't break people without postgres + """ + if _is_backend_avail('postgres', "openstack_cifail", self.PASSWD, + self.DATABASE): + self.fail("Shouldn't have connected") + + +class BaseMigrationTestCase(unittest2.TestCase): + """Base class for testing migrations and migration utils. This sets up + and configures the databases to run tests against. + """ + + # NOTE(jhesketh): It is expected that tests clean up after themselves. + # This is necessary for concurrency to allow multiple tests to work on + # one database. + # The full migration walk tests however do call the old _reset_databases() + # to throw away whatever was there so they need to operate on their own + # database that we know isn't accessed concurrently. + # Hence, BaseWalkMigrationTestCase overwrites the engine list. + + USER = None + PASSWD = None + DATABASE = None + + TIMEOUT_SCALING_FACTOR = 2 + + def __init__(self, *args, **kwargs): + super(BaseMigrationTestCase, self).__init__(*args, **kwargs) + + self.DEFAULT_CONFIG_FILE = os.path.join( + os.path.dirname(__file__), + 'test_migrations.conf') + # Test machines can set the MURANO_TEST_MIGRATIONS_CONF variable + # to override the location of the config file for migration testing + self.CONFIG_FILE_PATH = os.environ.get( + 'MURANO_TEST_MIGRATIONS_CONF', + self.DEFAULT_CONFIG_FILE) + + self.ALEMBIC_CONFIG = alembic_config.Config( + os.path.join(os.path.dirname(murano.db.migration.__file__), + 'alembic.ini') + ) + self.ALEMBIC_CONFIG.set_main_option( + 'script_location', + 'murano.db.migration:alembic_migrations') + + self.ALEMBIC_CONFIG.murano_config = CONF + + self.snake_walk = False + self.downgrade = False + self.test_databases = {} + self.migration = None + self.migration_api = None + + def setUp(self): + super(BaseMigrationTestCase, self).setUp() + self._load_config() + + def _load_config(self): + # Load test databases from the config file. Only do this + # once. No need to re-run this on each test... + LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH) + if os.path.exists(self.CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(self.CONFIG_FILE_PATH) + config = cp.options('unit_tests') + for key in config: + self.test_databases[key] = cp.get('unit_tests', key) + self.snake_walk = cp.getboolean('walk_style', 'snake_walk') + self.downgrade = cp.getboolean('walk_style', 'downgrade') + + except ConfigParser.ParsingError as e: + self.fail("Failed to read test_migrations.conf config " + "file. Got error: %s" % e) + else: + self.fail("Failed to find test_migrations.conf config " + "file.") + + self.engines = {} + for key, value in self.test_databases.items(): + self.engines[key] = sqlalchemy.create_engine(value) + + # NOTE(jhesketh): We only need to make sure the databases are created + # not necessarily clean of tables. + self._create_databases() + + def execute_cmd(self, cmd=None): + out, err = processutils.trycmd(cmd, shell=True, discard_warnings=True) + output = out or err + LOG.debug(output) + self.assertEqual('', err, + "Failed to run: %s\n%s" % (cmd, output)) + + @synchronized('pgadmin', external=True, lock_path='/tmp') + def _reset_pg(self, conn_pieces): + (user, password, database, host) = \ + get_pgsql_connection_info(conn_pieces) + os.environ['PGPASSWORD'] = password + os.environ['PGUSER'] = user + # note(boris-42): We must create and drop database, we can't + # drop database which we have connected to, so for such + # operations there is a special database template1. + sqlcmd = ("psql -w -U %(user)s -h %(host)s -c" + " '%(sql)s' -d template1") + sqldict = {'user': user, 'host': host} + + sqldict['sql'] = ("drop database if exists %s;") % database + droptable = sqlcmd % sqldict + self.execute_cmd(droptable) + + sqldict['sql'] = ("create database %s;") % database + createtable = sqlcmd % sqldict + self.execute_cmd(createtable) + + os.unsetenv('PGPASSWORD') + os.unsetenv('PGUSER') + + @synchronized('mysql', external=True, lock_path='/tmp') + def _reset_mysql(self, conn_pieces): + # We can execute the MySQL client to destroy and re-create + # the MYSQL database, which is easier and less error-prone + # than using SQLAlchemy to do this via MetaData...trust me. + (user, password, database, host) = \ + get_mysql_connection_info(conn_pieces) + sql = ("drop database if exists %(database)s; " + "create database %(database)s;" % {'database': database}) + cmd = ("mysql -u \"%(user)s\" %(password)s -h %(host)s -e \"%(sql)s\"" + % {'user': user, 'password': password, + 'host': host, 'sql': sql}) + self.execute_cmd(cmd) + + @synchronized('sqlite', external=True, lock_path='/tmp') + def _reset_sqlite(self, conn_pieces): + # We can just delete the SQLite database, which is + # the easiest and cleanest solution + db_path = conn_pieces.path.strip('/') + if os.path.exists(db_path): + os.unlink(db_path) + # No need to recreate the SQLite DB. SQLite will + # create it for us if it's not there... + + def _create_databases(self): + """Create all configured databases as needed.""" + for key, engine in self.engines.items(): + self._create_database(key) + + def _create_database(self, key): + """Create database if it doesn't exist.""" + conn_string = self.test_databases[key] + conn_pieces = urlparse.urlparse(conn_string) + + if conn_string.startswith('mysql'): + (user, password, database, host) = \ + get_mysql_connection_info(conn_pieces) + sql = "create database if not exists %s;" % database + cmd = ("mysql -u \"%(user)s\" %(password)s -h %(host)s " + "-e \"%(sql)s\"" % {'user': user, 'password': password, + 'host': host, 'sql': sql}) + self.execute_cmd(cmd) + elif conn_string.startswith('postgresql'): + (user, password, database, host) = \ + get_pgsql_connection_info(conn_pieces) + os.environ['PGPASSWORD'] = password + os.environ['PGUSER'] = user + + sqlcmd = ("psql -w -U %(user)s -h %(host)s -c" + " '%(sql)s' -d template1") + + sql = ("create database if not exists %s;") % database + createtable = sqlcmd % {'user': user, 'host': host, 'sql': sql} + # 0 means databases is created + # 256 means it already exists (which is fine) + # otherwise raise an error + out, err = processutils.trycmd(createtable, shell=True, + check_exit_code=[0, 256], + discard_warnings=True) + output = out or err + if err != '': + self.fail("Failed to run: %s\n%s" % (createtable, output)) + + os.unsetenv('PGPASSWORD') + os.unsetenv('PGUSER') + + def _reset_databases(self): + """Reset all configured databases.""" + for key, engine in self.engines.items(): + self._reset_database(key) + + def _reset_database(self, key): + """Reset specific database.""" + engine = self.engines[key] + conn_string = self.test_databases[key] + conn_pieces = urlparse.urlparse(conn_string) + engine.dispose() + if conn_string.startswith('sqlite'): + self._reset_sqlite(conn_pieces) + elif conn_string.startswith('mysql'): + self._reset_mysql(conn_pieces) + elif conn_string.startswith('postgresql'): + self._reset_pg(conn_pieces) + + +class BaseWalkMigrationTestCase(BaseMigrationTestCase): + """BaseWalkMigrationTestCase loads in an alternative set of databases for + testing against. This is necessary as the default databases can run tests + concurrently without interfering with itself. It is expected that + databases listed under [migraiton_dbs] in the configuration are only being + accessed by one test at a time. Currently only test_walk_versions accesses + the databases (and is the only method that calls _reset_database() which + is clearly problematic for concurrency). + """ + + def _load_config(self): + # Load test databases from the config file. Only do this + # once. No need to re-run this on each test... + LOG.debug('config_path is %s' % self.CONFIG_FILE_PATH) + if os.path.exists(self.CONFIG_FILE_PATH): + cp = ConfigParser.RawConfigParser() + try: + cp.read(self.CONFIG_FILE_PATH) + config = cp.options('migration_dbs') + for key in config: + self.test_databases[key] = cp.get('migration_dbs', key) + self.snake_walk = cp.getboolean('walk_style', 'snake_walk') + self.downgrade = cp.getboolean('walk_style', 'downgrade') + except ConfigParser.ParsingError as e: + self.fail("Failed to read test_migrations.conf config " + "file. Got error: %s" % e) + else: + self.fail("Failed to find test_migrations.conf config " + "file.") + + self.engines = {} + for key, value in self.test_databases.items(): + self.engines[key] = sqlalchemy.create_engine(value) + + self._create_databases() + + def _configure(self, engine): + """For each type of repository we should do some of configure steps. + For migrate_repo we should set under version control our database. + For alembic we should configure database settings. For this goal we + should use oslo.config and openstack.commom.db.sqlalchemy.session with + database functionality (reset default settings and session cleanup). + """ + CONF.set_override('connection', str(engine.url), group='database') + #session.cleanup() + + def _test_mysql_opportunistically(self): + # Test that table creation on mysql only builds InnoDB tables + if not _have_mysql(self.USER, self.PASSWD, self.DATABASE): + self.skipTest("mysql not available") + # add this to the global lists to make reset work with it, it's removed + # automatically in tearDown so no need to clean it up here. + connect_string = _get_connect_string( + "mysql", self.USER, self.PASSWD, self.DATABASE) + (user, password, database, host) = \ + get_mysql_connection_info(urlparse.urlparse(connect_string)) + engine = sqlalchemy.create_engine(connect_string) + self.engines[database] = engine + self.test_databases[database] = connect_string + + # build a fully populated mysql database with all the tables + self._reset_database(database) + self._walk_versions(engine, self.snake_walk, self.downgrade) + + connection = engine.connect() + # sanity check + total = connection.execute("SELECT count(*) " + "from information_schema.TABLES " + "where TABLE_SCHEMA='%(database)s'" % + {'database': database}) + self.assertTrue(total.scalar() > 0, "No tables found. Wrong schema?") + + connection.close() + + del(self.engines[database]) + del(self.test_databases[database]) + + def _test_postgresql_opportunistically(self): + # Test postgresql database migration walk + if not _have_postgresql(self.USER, self.PASSWD, self.DATABASE): + self.skipTest("postgresql not available") + # add this to the global lists to make reset work with it, it's removed + # automatically in tearDown so no need to clean it up here. + connect_string = _get_connect_string( + "postgres", self.USER, self.PASSWD, self.DATABASE) + engine = sqlalchemy.create_engine(connect_string) + (user, password, database, host) = \ + get_mysql_connection_info(urlparse.urlparse(connect_string)) + self.engines[database] = engine + self.test_databases[database] = connect_string + + # build a fully populated postgresql database with all the tables + self._reset_database(database) + self._walk_versions(engine, self.snake_walk, self.downgrade) + del(self.engines[database]) + del(self.test_databases[database]) + + def _alembic_command(self, alembic_command, engine, *args, **kwargs): + """Most of alembic command return data into output. + We should redefine this setting for getting info. + """ + self.ALEMBIC_CONFIG.stdout = buf = io.StringIO() + CONF.set_override('connection', str(engine.url), group='database') + #session.cleanup() + getattr(command, alembic_command)(*args, **kwargs) + res = buf.getvalue().strip() + LOG.debug('Alembic command `%s` returns: %s' % (alembic_command, res)) + #session.cleanup() + return res + + def _get_alembic_versions(self, engine): + """For support of full testing of migrations + we should have an opportunity to run command step by step for each + version in repo. This method returns list of alembic_versions by + historical order. + """ + full_history = self._alembic_command('history', + engine, self.ALEMBIC_CONFIG) + # The piece of output data with version can looked as: + # 'Rev: 17738166b91 (head)' or 'Rev: 43b1a023dfaa' + alembic_history = [r.split(' ')[1] for r in full_history.split("\n") + if r.startswith("Rev")] + alembic_history.reverse() + return alembic_history + + def _up_and_down_versions(self, engine): + """Since alembic version has a random algorithm of generation + (SA-migrate has an ordered autoincrement naming) we should store + a tuple of versions (version for upgrade and version for downgrade) + for successful testing of migrations in up>down>up mode. + """ + versions = self._get_alembic_versions(engine) + return zip(versions, ['-1'] + versions) + + def _walk_versions(self, engine=None, snake_walk=False, + downgrade=True): + # Determine latest version script from the repo, then + # upgrade from 1 through to the latest, with no data + # in the databases. This just checks that the schema itself + # upgrades successfully. + + self._configure(engine) + up_and_down_versions = self._up_and_down_versions(engine) + for ver_up, ver_down in up_and_down_versions: + # upgrade -> downgrade -> upgrade + self._migrate_up(engine, ver_up, with_data=True) + if snake_walk: + downgraded = self._migrate_down(engine, + ver_down, + with_data=True, + next_version=ver_up) + if downgraded: + self._migrate_up(engine, ver_up) + + if downgrade: + # Now walk it back down to 0 from the latest, testing + # the downgrade paths. + up_and_down_versions.reverse() + for ver_up, ver_down in up_and_down_versions: + # downgrade -> upgrade -> downgrade + downgraded = self._migrate_down(engine, + ver_down, next_version=ver_up) + + if snake_walk and downgraded: + self._migrate_up(engine, ver_up) + self._migrate_down(engine, ver_down, next_version=ver_up) + + def _get_version_from_db(self, engine): + """For each type of migrate repo latest version from db + will be returned. + """ + conn = engine.connect() + try: + context = migration.MigrationContext.configure(conn) + version = context.get_current_revision() or '-1' + finally: + conn.close() + return version + + def _migrate(self, engine, version, cmd): + """Base method for manipulation with migrate repo. + It will upgrade or downgrade the actual database. + """ + + self._alembic_command(cmd, engine, self.ALEMBIC_CONFIG, version) + + def _migrate_down(self, engine, version, with_data=False, + next_version=None): + try: + self._migrate(engine, version, 'downgrade') + except NotImplementedError: + # NOTE(sirp): some migrations, namely release-level + # migrations, don't support a downgrade. + return False + self.assertEqual(version, self._get_version_from_db(engine)) + + # NOTE(sirp): `version` is what we're downgrading to (i.e. the 'target' + # version). So if we have any downgrade checks, they need to be run for + # the previous (higher numbered) migration. + if with_data: + post_downgrade = getattr( + self, "_post_downgrade_%s" % next_version, None) + if post_downgrade: + post_downgrade(engine) + + return True + + def _migrate_up(self, engine, version, with_data=False): + """migrate up to a new version of the db. + + We allow for data insertion and post checks at every + migration version with special _pre_upgrade_### and + _check_### functions in the main test. + """ + # NOTE(sdague): try block is here because it's impossible to debug + # where a failed data migration happens otherwise + check_version = version + try: + if with_data: + data = None + pre_upgrade = getattr( + self, "_pre_upgrade_%s" % check_version, None) + if pre_upgrade: + data = pre_upgrade(engine) + self._migrate(engine, version, 'upgrade') + self.assertEqual(version, self._get_version_from_db(engine)) + if with_data: + check = getattr(self, "_check_%s" % check_version, None) + if check: + check(engine, data) + except Exception: + LOG.error("Failed to migrate to version %s on engine %s" % + (version, engine)) + raise diff --git a/requirements.txt b/requirements.txt index c39981158..b6d89f758 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ pbr>=0.6,<1.0 Babel>=1.3 SQLAlchemy>=0.7.8,<=0.9.99 +alembic>=0.4.1 anyjson>=0.3.3 eventlet>=0.13.0 PasteDeploy>=1.5.0 @@ -9,7 +10,6 @@ WebOb>=1.2.3 wsgiref>=0.1.2 argparse ordereddict -sqlalchemy-migrate>=0.8.2,!=0.8.4 kombu>=2.4.8 lockfile>=0.8 pycrypto>=2.6 @@ -34,4 +34,3 @@ oslo.messaging>=1.3.0a9 # not listed in global requirements yaql>=0.2.2,<0.3 python-muranoclient>=0.5.2 - diff --git a/setup.cfg b/setup.cfg index a06195315..2da1b1dec 100644 --- a/setup.cfg +++ b/setup.cfg @@ -45,6 +45,7 @@ console_scripts = murano-api = murano.cmd.api:main murano-engine = murano.cmd.engine:main murano-manage = murano.cmd.manage:main + murano-db-manage = murano.cmd.db_manage:main [build_sphinx] all_files = 1 diff --git a/test-requirements.txt b/test-requirements.txt index d7c7fc7b0..d9c34c61e 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -4,10 +4,16 @@ coverage>=3.6 discover fixtures>=0.3.14 mock>=1.0 +posix_ipc +sqlalchemy-migrate>=0.8.2,!=0.8.4 testrepository>=0.0.18 testscenarios>=0.4 testtools>=0.9.34 unittest2 +# Some of the tests use real MySQL and Postgres databases +MySQL-python +psycopg2 + # doc build requirements sphinx>=1.1.2,<1.2