From 2968b2213ed978e980f6c8b11398c993766b395f Mon Sep 17 00:00:00 2001 From: Oleksii Chuprykov Date: Mon, 8 Sep 2014 16:22:50 +0300 Subject: [PATCH] Add sync check for models_metadef Correct migrations to satisfy models_metadef. Add nullable=False for json_schema in metadef_object and metadef_property according to changes in Change-Id: I2c574210f8d62c77a438afab83ff80f3e5bd2fe7 Set updated_at nullable=True due to problems with setting default value NOW() for datetime type for mysql (the same as in models.py). Closes-Bug: #1365436 Change-Id: I5399923621913724914499aad9148a9410faa4ef --- ...9_add_changes_to_satisfy_models_metadef.py | 295 ++++++++++++++++++ glance/db/sqlalchemy/models_metadef.py | 6 +- glance/tests/unit/test_migrations.py | 138 +++++++- 3 files changed, 426 insertions(+), 13 deletions(-) create mode 100644 glance/db/sqlalchemy/migrate_repo/versions/039_add_changes_to_satisfy_models_metadef.py diff --git a/glance/db/sqlalchemy/migrate_repo/versions/039_add_changes_to_satisfy_models_metadef.py b/glance/db/sqlalchemy/migrate_repo/versions/039_add_changes_to_satisfy_models_metadef.py new file mode 100644 index 0000000000..45ea0f0854 --- /dev/null +++ b/glance/db/sqlalchemy/migrate_repo/versions/039_add_changes_to_satisfy_models_metadef.py @@ -0,0 +1,295 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import migrate +import sqlalchemy +from sqlalchemy import inspect +from sqlalchemy import (Table, Index, UniqueConstraint) +from sqlalchemy.schema import (AddConstraint, DropConstraint) + + +def upgrade(migrate_engine): + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + inspector = inspect(migrate_engine) + + metadef_namespaces = Table('metadef_namespaces', meta, autoload=True) + metadef_properties = Table('metadef_properties', meta, autoload=True) + metadef_objects = Table('metadef_objects', meta, autoload=True) + metadef_ns_res_types = Table('metadef_namespace_resource_types', + meta, autoload=True) + metadef_resource_types = Table('metadef_resource_types', meta, + autoload=True) + metadef_tags = Table('metadef_tags', meta, autoload=True) + + Index('ix_namespaces_namespace', metadef_namespaces.c.namespace).drop() + + Index('ix_objects_namespace_id_name', metadef_objects.c.namespace_id, + metadef_objects.c.name).drop() + + Index('ix_metadef_properties_namespace_id_name', + metadef_properties.c.namespace_id, + metadef_properties.c.name).drop() + + fkc = migrate.ForeignKeyConstraint([metadef_tags.c.namespace_id], + [metadef_namespaces.c.id]) + fkc.create() + + # `migrate` module removes unique constraint after adding + # foreign key to the table in sqlite. + # The reason of this issue is that it isn't possible to add fkc to + # existing table in sqlite. Instead of this we should recreate the table + # with needed fkc in the declaration. Migrate package provide us with such + # possibility, but unfortunately it recreates the table without + # constraints. Create unique constraint manually. + if migrate_engine.name == 'sqlite' and len( + inspector.get_unique_constraints('metadef_tags')) == 0: + uc = migrate.UniqueConstraint(metadef_tags.c.namespace_id, + metadef_tags.c.name) + uc.create() + + Index('ix_tags_namespace_id_name', metadef_tags.c.namespace_id, + metadef_tags.c.name).drop() + + Index('ix_metadef_tags_name', metadef_tags.c.name).create() + + Index('ix_metadef_tags_namespace_id', metadef_tags.c.namespace_id, + metadef_tags.c.name).create() + + if migrate_engine.name == 'mysql': + # We need to drop some foreign keys first because unique constraints + # that we want to delete depend on them. So drop the fk and recreate + # it again after unique constraint deletion. + fkc = migrate.ForeignKeyConstraint([metadef_properties.c.namespace_id], + [metadef_namespaces.c.id], + name='metadef_properties_ibfk_1') + fkc.drop() + constraint = UniqueConstraint(metadef_properties.c.namespace_id, + metadef_properties.c.name, + name='namespace_id') + migrate_engine.execute(DropConstraint(constraint)) + fkc.create() + + fkc = migrate.ForeignKeyConstraint([metadef_objects.c.namespace_id], + [metadef_namespaces.c.id], + name='metadef_objects_ibfk_1') + fkc.drop() + constraint = UniqueConstraint(metadef_objects.c.namespace_id, + metadef_objects.c.name, + name='namespace_id') + migrate_engine.execute(DropConstraint(constraint)) + fkc.create() + + constraint = UniqueConstraint(metadef_ns_res_types.c.resource_type_id, + metadef_ns_res_types.c.namespace_id, + name='resource_type_id') + migrate_engine.execute(DropConstraint(constraint)) + + constraint = UniqueConstraint(metadef_namespaces.c.namespace, + name='namespace') + migrate_engine.execute(DropConstraint(constraint)) + + constraint = UniqueConstraint(metadef_resource_types.c.name, + name='name') + migrate_engine.execute(DropConstraint(constraint)) + + if migrate_engine.name == 'postgresql': + met_obj_index_name = ( + inspector.get_unique_constraints('metadef_objects')[0]['name']) + constraint = UniqueConstraint( + metadef_objects.c.namespace_id, + metadef_objects.c.name, + name=met_obj_index_name) + migrate_engine.execute(DropConstraint(constraint)) + + met_prop_index_name = ( + inspector.get_unique_constraints('metadef_properties')[0]['name']) + constraint = UniqueConstraint( + metadef_properties.c.namespace_id, + metadef_properties.c.name, + name=met_prop_index_name) + migrate_engine.execute(DropConstraint(constraint)) + + metadef_namespaces_name = ( + inspector.get_unique_constraints( + 'metadef_namespaces')[0]['name']) + constraint = UniqueConstraint( + metadef_namespaces.c.namespace, + name=metadef_namespaces_name) + migrate_engine.execute(DropConstraint(constraint)) + + metadef_resource_types_name = (inspector.get_unique_constraints( + 'metadef_resource_types')[0]['name']) + constraint = UniqueConstraint( + metadef_resource_types.c.name, + name=metadef_resource_types_name) + migrate_engine.execute(DropConstraint(constraint)) + + constraint = UniqueConstraint( + metadef_tags.c.namespace_id, + metadef_tags.c.name, + name='metadef_tags_namespace_id_name_key') + migrate_engine.execute(DropConstraint(constraint)) + + Index('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.c.namespace_id).create() + + Index('ix_metadef_namespaces_namespace', + metadef_namespaces.c.namespace).create() + + Index('ix_metadef_namespaces_owner', metadef_namespaces.c.owner).create() + + Index('ix_metadef_objects_name', metadef_objects.c.name).create() + + Index('ix_metadef_objects_namespace_id', + metadef_objects.c.namespace_id).create() + + Index('ix_metadef_properties_name', metadef_properties.c.name).create() + + Index('ix_metadef_properties_namespace_id', + metadef_properties.c.namespace_id).create() + + +def downgrade(migrate_engine): + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + + metadef_namespaces = Table('metadef_namespaces', meta, autoload=True) + metadef_properties = Table('metadef_properties', meta, autoload=True) + metadef_objects = Table('metadef_objects', meta, autoload=True) + metadef_ns_res_types = Table('metadef_namespace_resource_types', + meta, autoload=True) + metadef_resource_types = Table('metadef_resource_types', meta, + autoload=True) + metadef_tags = Table('metadef_tags', meta, autoload=True) + + Index('ix_namespaces_namespace', metadef_namespaces.c.namespace).create() + + Index('ix_objects_namespace_id_name', metadef_objects.c.namespace_id, + metadef_objects.c.name).create() + + Index('ix_metadef_properties_namespace_id_name', + metadef_properties.c.namespace_id, + metadef_properties.c.name).create() + + Index('ix_metadef_tags_name', metadef_tags.c.name).drop() + + Index('ix_metadef_tags_namespace_id', metadef_tags.c.namespace_id, + metadef_tags.c.name).drop() + + if migrate_engine.name != 'sqlite': + fkc = migrate.ForeignKeyConstraint([metadef_tags.c.namespace_id], + [metadef_namespaces.c.id]) + fkc.drop() + + Index('ix_tags_namespace_id_name', metadef_tags.c.namespace_id, + metadef_tags.c.name).create() + else: + # NOTE(ochuprykov): fkc can't be dropped via `migrate` in sqlite,so it + # is necessary to recreate table manually and populate it with data + temp = Table('temp_', meta, *( + [c.copy() for c in metadef_tags.columns])) + temp.create() + migrate_engine.execute('insert into temp_ select * from metadef_tags') + metadef_tags.drop() + migrate_engine.execute('alter table temp_ rename to metadef_tags') + + # Refresh old metadata for this table + meta = sqlalchemy.MetaData() + meta.bind = migrate_engine + metadef_tags = Table('metadef_tags', meta, autoload=True) + + Index('ix_tags_namespace_id_name', metadef_tags.c.namespace_id, + metadef_tags.c.name).create() + + uc = migrate.UniqueConstraint(metadef_tags.c.namespace_id, + metadef_tags.c.name) + uc.create() + + if migrate_engine.name == 'mysql': + constraint = UniqueConstraint(metadef_properties.c.namespace_id, + metadef_properties.c.name, + name='namespace_id') + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint(metadef_objects.c.namespace_id, + metadef_objects.c.name, + name='namespace_id') + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint(metadef_ns_res_types.c.resource_type_id, + metadef_ns_res_types.c.namespace_id, + name='resource_type_id') + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint(metadef_namespaces.c.namespace, + name='namespace') + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint(metadef_resource_types.c.name, + name='name') + migrate_engine.execute(AddConstraint(constraint)) + + if migrate_engine.name == 'postgresql': + constraint = UniqueConstraint( + metadef_objects.c.namespace_id, + metadef_objects.c.name) + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint( + metadef_properties.c.namespace_id, + metadef_properties.c.name) + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint( + metadef_namespaces.c.namespace) + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint( + metadef_resource_types.c.name) + migrate_engine.execute(AddConstraint(constraint)) + + constraint = UniqueConstraint( + metadef_tags.c.namespace_id, + metadef_tags.c.name, + name='metadef_tags_namespace_id_name_key') + migrate_engine.execute(AddConstraint(constraint)) + + if migrate_engine.name == 'mysql': + fkc = migrate.ForeignKeyConstraint( + [metadef_ns_res_types.c.resource_type_id], + [metadef_namespaces.c.id], + name='metadef_namespace_resource_types_ibfk_2') + fkc.drop() + + Index('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.c.namespace_id).drop() + + fkc.create() + else: + Index('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.c.namespace_id).drop() + + Index('ix_metadef_namespaces_namespace', + metadef_namespaces.c.namespace).drop() + + Index('ix_metadef_namespaces_owner', metadef_namespaces.c.owner).drop() + + Index('ix_metadef_objects_name', metadef_objects.c.name).drop() + + Index('ix_metadef_objects_namespace_id', + metadef_objects.c.namespace_id).drop() + + Index('ix_metadef_properties_name', metadef_properties.c.name).drop() + + Index('ix_metadef_properties_namespace_id', + metadef_properties.c.namespace_id).drop() diff --git a/glance/db/sqlalchemy/models_metadef.py b/glance/db/sqlalchemy/models_metadef.py index 81432de45a..5af42ec599 100644 --- a/glance/db/sqlalchemy/models_metadef.py +++ b/glance/db/sqlalchemy/models_metadef.py @@ -59,7 +59,7 @@ class GlanceMetadefBase(models.TimestampMixin): # required and make changes in oslo (if required) or # in glance (if not). updated_at = Column(DateTime, default=lambda: timeutils.utcnow(), - nullable=False, onupdate=lambda: timeutils.utcnow()) + nullable=True, onupdate=lambda: timeutils.utcnow()) class MetadefNamespace(BASE_DICT, GlanceMetadefBase): @@ -89,7 +89,7 @@ class MetadefObject(BASE_DICT, GlanceMetadefBase): name = Column(String(80), nullable=False) description = Column(Text()) required = Column(Text()) - json_schema = Column(JSONEncodedDict(), default={}) + json_schema = Column(JSONEncodedDict(), default={}, nullable=False) class MetadefProperty(BASE_DICT, GlanceMetadefBase): @@ -103,7 +103,7 @@ class MetadefProperty(BASE_DICT, GlanceMetadefBase): namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'), nullable=False) name = Column(String(80), nullable=False) - json_schema = Column(JSONEncodedDict(), default={}) + json_schema = Column(JSONEncodedDict(), default={}, nullable=False) class MetadefNamespaceResourceType(BASE_DICT, GlanceMetadefBase): diff --git a/glance/tests/unit/test_migrations.py b/glance/tests/unit/test_migrations.py index 98dc3c6eb5..7936af9a7c 100644 --- a/glance/tests/unit/test_migrations.py +++ b/glance/tests/unit/test_migrations.py @@ -40,6 +40,7 @@ from oslo.db.sqlalchemy import utils as db_utils from oslo.serialization import jsonutils from oslo.utils import timeutils import sqlalchemy +from sqlalchemy import inspect from glance.common import crypt from glance.common import exception @@ -48,6 +49,7 @@ from glance.db import migration from glance.db.sqlalchemy import migrate_repo from glance.db.sqlalchemy.migrate_repo.schema import from_migration_import from glance.db.sqlalchemy import models +from glance.db.sqlalchemy import models_metadef from glance import i18n _ = i18n._ @@ -1358,6 +1360,129 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin): self.assertRaises(sqlalchemy.exc.NoSuchTableError, db_utils.get_table, engine, 'metadef_tags') + def _check_039(self, engine, data): + meta = sqlalchemy.MetaData() + meta.bind = engine + + metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta, + autoload=True) + metadef_properties = sqlalchemy.Table('metadef_properties', meta, + autoload=True) + metadef_objects = sqlalchemy.Table('metadef_objects', meta, + autoload=True) + metadef_ns_res_types = sqlalchemy.Table( + 'metadef_namespace_resource_types', + meta, autoload=True) + metadef_resource_types = sqlalchemy.Table('metadef_resource_types', + meta, autoload=True) + + tables = [metadef_namespaces, metadef_properties, metadef_objects, + metadef_ns_res_types, metadef_resource_types] + + for table in tables: + for index_name in ['ix_namespaces_namespace', + 'ix_objects_namespace_id_name', + 'ix_metadef_properties_namespace_id_name']: + self.assertFalse(index_exist(index_name, table.name, engine)) + for uc_name in ['resource_type_id', 'namespace', 'name', + 'namespace_id', + 'metadef_objects_namespace_id_name_key', + 'metadef_properties_namespace_id_name_key']: + self.assertFalse(unique_constraint_exist(uc_name, table.name, + engine)) + + self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.name, engine)) + + self.assertTrue(index_exist('ix_metadef_namespaces_namespace', + metadef_namespaces.name, engine)) + + self.assertTrue(index_exist('ix_metadef_namespaces_owner', + metadef_namespaces.name, engine)) + + self.assertTrue(index_exist('ix_metadef_objects_name', + metadef_objects.name, engine)) + + self.assertTrue(index_exist('ix_metadef_objects_namespace_id', + metadef_objects.name, engine)) + + self.assertTrue(index_exist('ix_metadef_properties_name', + metadef_properties.name, engine)) + + self.assertTrue(index_exist('ix_metadef_properties_namespace_id', + metadef_properties.name, engine)) + + def _post_downgrade_039(self, engine): + meta = sqlalchemy.MetaData() + meta.bind = engine + + metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta, + autoload=True) + metadef_properties = sqlalchemy.Table('metadef_properties', meta, + autoload=True) + metadef_objects = sqlalchemy.Table('metadef_objects', meta, + autoload=True) + metadef_ns_res_types = sqlalchemy.Table( + 'metadef_namespace_resource_types', + meta, autoload=True) + metadef_resource_types = sqlalchemy.Table('metadef_resource_types', + meta, autoload=True) + + self.assertFalse(index_exist('ix_metadef_ns_res_types_namespace_id', + metadef_ns_res_types.name, engine)) + + self.assertFalse(index_exist('ix_metadef_namespaces_namespace', + metadef_namespaces.name, engine)) + + self.assertFalse(index_exist('ix_metadef_namespaces_owner', + metadef_namespaces.name, engine)) + + self.assertFalse(index_exist('ix_metadef_objects_name', + metadef_objects.name, engine)) + + self.assertFalse(index_exist('ix_metadef_objects_namespace_id', + metadef_objects.name, engine)) + + self.assertFalse(index_exist('ix_metadef_properties_name', + metadef_properties.name, engine)) + + self.assertFalse(index_exist('ix_metadef_properties_namespace_id', + metadef_properties.name, engine)) + + self.assertTrue(index_exist('ix_namespaces_namespace', + metadef_namespaces.name, engine)) + + self.assertTrue(index_exist('ix_objects_namespace_id_name', + metadef_objects.name, engine)) + + self.assertTrue(index_exist('ix_metadef_properties_namespace_id_name', + metadef_properties.name, engine)) + + if engine.name == 'postgresql': + inspector = inspect(engine) + + self.assertEqual(1, len(inspector.get_unique_constraints( + 'metadef_objects'))) + + self.assertEqual(1, len(inspector.get_unique_constraints( + 'metadef_properties'))) + + if engine.name == 'mysql': + self.assertTrue(unique_constraint_exist( + 'namespace_id', metadef_properties.name, engine)) + + self.assertTrue(unique_constraint_exist( + 'namespace_id', metadef_objects.name, engine)) + + self.assertTrue(unique_constraint_exist( + 'resource_type_id', metadef_ns_res_types.name, engine)) + + self.assertTrue(unique_constraint_exist( + 'namespace', metadef_namespaces.name, engine)) + + self.assertTrue(unique_constraint_exist( + 'name', metadef_resource_types.name, engine)) + class TestMysqlMigrations(test_base.MySQLOpportunisticTestCase, MigrationsMixin): @@ -1397,6 +1522,8 @@ class TestSqliteMigrations(test_base.DbTestCase, class ModelsMigrationSyncMixin(object): def get_metadata(self): + for table in models_metadef.BASE_DICT.metadata.sorted_tables: + models.BASE.metadata._add_table(table.name, table.schema, table) return models.BASE.metadata def get_engine(self): @@ -1406,16 +1533,7 @@ class ModelsMigrationSyncMixin(object): migration.db_sync(engine=engine) def include_object(self, object_, name, type_, reflected, compare_to): - # We need to exclude tables that aren't in models.py and table that - # contains migrate version - - # TODO(ochuprykov): We need to include this tables back after merge of - # models.py and models_metadef.py - # (except 'migrate_version') - if name in ['migrate_version', 'metadef_objects', 'metadef_namespaces', - 'metadef_properties', 'metadef_resource_types', - 'metadef_tags', - 'metadef_namespace_resource_types'] and type_ == 'table': + if name in ['migrate_version'] and type_ == 'table': return False return True