Merge "Add sync check for models_metadef"

This commit is contained in:
Jenkins 2015-03-18 17:41:34 +00:00 committed by Gerrit Code Review
commit 86ea22a2db
3 changed files with 426 additions and 13 deletions

View File

@ -0,0 +1,295 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import migrate
import sqlalchemy
from sqlalchemy import inspect
from sqlalchemy import (Table, Index, UniqueConstraint)
from sqlalchemy.schema import (AddConstraint, DropConstraint)
def upgrade(migrate_engine):
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
inspector = inspect(migrate_engine)
metadef_namespaces = Table('metadef_namespaces', meta, autoload=True)
metadef_properties = Table('metadef_properties', meta, autoload=True)
metadef_objects = Table('metadef_objects', meta, autoload=True)
metadef_ns_res_types = Table('metadef_namespace_resource_types',
meta, autoload=True)
metadef_resource_types = Table('metadef_resource_types', meta,
autoload=True)
metadef_tags = Table('metadef_tags', meta, autoload=True)
Index('ix_namespaces_namespace', metadef_namespaces.c.namespace).drop()
Index('ix_objects_namespace_id_name', metadef_objects.c.namespace_id,
metadef_objects.c.name).drop()
Index('ix_metadef_properties_namespace_id_name',
metadef_properties.c.namespace_id,
metadef_properties.c.name).drop()
fkc = migrate.ForeignKeyConstraint([metadef_tags.c.namespace_id],
[metadef_namespaces.c.id])
fkc.create()
# `migrate` module removes unique constraint after adding
# foreign key to the table in sqlite.
# The reason of this issue is that it isn't possible to add fkc to
# existing table in sqlite. Instead of this we should recreate the table
# with needed fkc in the declaration. Migrate package provide us with such
# possibility, but unfortunately it recreates the table without
# constraints. Create unique constraint manually.
if migrate_engine.name == 'sqlite' and len(
inspector.get_unique_constraints('metadef_tags')) == 0:
uc = migrate.UniqueConstraint(metadef_tags.c.namespace_id,
metadef_tags.c.name)
uc.create()
Index('ix_tags_namespace_id_name', metadef_tags.c.namespace_id,
metadef_tags.c.name).drop()
Index('ix_metadef_tags_name', metadef_tags.c.name).create()
Index('ix_metadef_tags_namespace_id', metadef_tags.c.namespace_id,
metadef_tags.c.name).create()
if migrate_engine.name == 'mysql':
# We need to drop some foreign keys first because unique constraints
# that we want to delete depend on them. So drop the fk and recreate
# it again after unique constraint deletion.
fkc = migrate.ForeignKeyConstraint([metadef_properties.c.namespace_id],
[metadef_namespaces.c.id],
name='metadef_properties_ibfk_1')
fkc.drop()
constraint = UniqueConstraint(metadef_properties.c.namespace_id,
metadef_properties.c.name,
name='namespace_id')
migrate_engine.execute(DropConstraint(constraint))
fkc.create()
fkc = migrate.ForeignKeyConstraint([metadef_objects.c.namespace_id],
[metadef_namespaces.c.id],
name='metadef_objects_ibfk_1')
fkc.drop()
constraint = UniqueConstraint(metadef_objects.c.namespace_id,
metadef_objects.c.name,
name='namespace_id')
migrate_engine.execute(DropConstraint(constraint))
fkc.create()
constraint = UniqueConstraint(metadef_ns_res_types.c.resource_type_id,
metadef_ns_res_types.c.namespace_id,
name='resource_type_id')
migrate_engine.execute(DropConstraint(constraint))
constraint = UniqueConstraint(metadef_namespaces.c.namespace,
name='namespace')
migrate_engine.execute(DropConstraint(constraint))
constraint = UniqueConstraint(metadef_resource_types.c.name,
name='name')
migrate_engine.execute(DropConstraint(constraint))
if migrate_engine.name == 'postgresql':
met_obj_index_name = (
inspector.get_unique_constraints('metadef_objects')[0]['name'])
constraint = UniqueConstraint(
metadef_objects.c.namespace_id,
metadef_objects.c.name,
name=met_obj_index_name)
migrate_engine.execute(DropConstraint(constraint))
met_prop_index_name = (
inspector.get_unique_constraints('metadef_properties')[0]['name'])
constraint = UniqueConstraint(
metadef_properties.c.namespace_id,
metadef_properties.c.name,
name=met_prop_index_name)
migrate_engine.execute(DropConstraint(constraint))
metadef_namespaces_name = (
inspector.get_unique_constraints(
'metadef_namespaces')[0]['name'])
constraint = UniqueConstraint(
metadef_namespaces.c.namespace,
name=metadef_namespaces_name)
migrate_engine.execute(DropConstraint(constraint))
metadef_resource_types_name = (inspector.get_unique_constraints(
'metadef_resource_types')[0]['name'])
constraint = UniqueConstraint(
metadef_resource_types.c.name,
name=metadef_resource_types_name)
migrate_engine.execute(DropConstraint(constraint))
constraint = UniqueConstraint(
metadef_tags.c.namespace_id,
metadef_tags.c.name,
name='metadef_tags_namespace_id_name_key')
migrate_engine.execute(DropConstraint(constraint))
Index('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.c.namespace_id).create()
Index('ix_metadef_namespaces_namespace',
metadef_namespaces.c.namespace).create()
Index('ix_metadef_namespaces_owner', metadef_namespaces.c.owner).create()
Index('ix_metadef_objects_name', metadef_objects.c.name).create()
Index('ix_metadef_objects_namespace_id',
metadef_objects.c.namespace_id).create()
Index('ix_metadef_properties_name', metadef_properties.c.name).create()
Index('ix_metadef_properties_namespace_id',
metadef_properties.c.namespace_id).create()
def downgrade(migrate_engine):
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
metadef_namespaces = Table('metadef_namespaces', meta, autoload=True)
metadef_properties = Table('metadef_properties', meta, autoload=True)
metadef_objects = Table('metadef_objects', meta, autoload=True)
metadef_ns_res_types = Table('metadef_namespace_resource_types',
meta, autoload=True)
metadef_resource_types = Table('metadef_resource_types', meta,
autoload=True)
metadef_tags = Table('metadef_tags', meta, autoload=True)
Index('ix_namespaces_namespace', metadef_namespaces.c.namespace).create()
Index('ix_objects_namespace_id_name', metadef_objects.c.namespace_id,
metadef_objects.c.name).create()
Index('ix_metadef_properties_namespace_id_name',
metadef_properties.c.namespace_id,
metadef_properties.c.name).create()
Index('ix_metadef_tags_name', metadef_tags.c.name).drop()
Index('ix_metadef_tags_namespace_id', metadef_tags.c.namespace_id,
metadef_tags.c.name).drop()
if migrate_engine.name != 'sqlite':
fkc = migrate.ForeignKeyConstraint([metadef_tags.c.namespace_id],
[metadef_namespaces.c.id])
fkc.drop()
Index('ix_tags_namespace_id_name', metadef_tags.c.namespace_id,
metadef_tags.c.name).create()
else:
# NOTE(ochuprykov): fkc can't be dropped via `migrate` in sqlite,so it
# is necessary to recreate table manually and populate it with data
temp = Table('temp_', meta, *(
[c.copy() for c in metadef_tags.columns]))
temp.create()
migrate_engine.execute('insert into temp_ select * from metadef_tags')
metadef_tags.drop()
migrate_engine.execute('alter table temp_ rename to metadef_tags')
# Refresh old metadata for this table
meta = sqlalchemy.MetaData()
meta.bind = migrate_engine
metadef_tags = Table('metadef_tags', meta, autoload=True)
Index('ix_tags_namespace_id_name', metadef_tags.c.namespace_id,
metadef_tags.c.name).create()
uc = migrate.UniqueConstraint(metadef_tags.c.namespace_id,
metadef_tags.c.name)
uc.create()
if migrate_engine.name == 'mysql':
constraint = UniqueConstraint(metadef_properties.c.namespace_id,
metadef_properties.c.name,
name='namespace_id')
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(metadef_objects.c.namespace_id,
metadef_objects.c.name,
name='namespace_id')
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(metadef_ns_res_types.c.resource_type_id,
metadef_ns_res_types.c.namespace_id,
name='resource_type_id')
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(metadef_namespaces.c.namespace,
name='namespace')
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(metadef_resource_types.c.name,
name='name')
migrate_engine.execute(AddConstraint(constraint))
if migrate_engine.name == 'postgresql':
constraint = UniqueConstraint(
metadef_objects.c.namespace_id,
metadef_objects.c.name)
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(
metadef_properties.c.namespace_id,
metadef_properties.c.name)
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(
metadef_namespaces.c.namespace)
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(
metadef_resource_types.c.name)
migrate_engine.execute(AddConstraint(constraint))
constraint = UniqueConstraint(
metadef_tags.c.namespace_id,
metadef_tags.c.name,
name='metadef_tags_namespace_id_name_key')
migrate_engine.execute(AddConstraint(constraint))
if migrate_engine.name == 'mysql':
fkc = migrate.ForeignKeyConstraint(
[metadef_ns_res_types.c.resource_type_id],
[metadef_namespaces.c.id],
name='metadef_namespace_resource_types_ibfk_2')
fkc.drop()
Index('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.c.namespace_id).drop()
fkc.create()
else:
Index('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.c.namespace_id).drop()
Index('ix_metadef_namespaces_namespace',
metadef_namespaces.c.namespace).drop()
Index('ix_metadef_namespaces_owner', metadef_namespaces.c.owner).drop()
Index('ix_metadef_objects_name', metadef_objects.c.name).drop()
Index('ix_metadef_objects_namespace_id',
metadef_objects.c.namespace_id).drop()
Index('ix_metadef_properties_name', metadef_properties.c.name).drop()
Index('ix_metadef_properties_namespace_id',
metadef_properties.c.namespace_id).drop()

View File

@ -59,7 +59,7 @@ class GlanceMetadefBase(models.TimestampMixin):
# required and make changes in oslo (if required) or
# in glance (if not).
updated_at = Column(DateTime, default=lambda: timeutils.utcnow(),
nullable=False, onupdate=lambda: timeutils.utcnow())
nullable=True, onupdate=lambda: timeutils.utcnow())
class MetadefNamespace(BASE_DICT, GlanceMetadefBase):
@ -89,7 +89,7 @@ class MetadefObject(BASE_DICT, GlanceMetadefBase):
name = Column(String(80), nullable=False)
description = Column(Text())
required = Column(Text())
json_schema = Column(JSONEncodedDict(), default={})
json_schema = Column(JSONEncodedDict(), default={}, nullable=False)
class MetadefProperty(BASE_DICT, GlanceMetadefBase):
@ -103,7 +103,7 @@ class MetadefProperty(BASE_DICT, GlanceMetadefBase):
namespace_id = Column(Integer(), ForeignKey('metadef_namespaces.id'),
nullable=False)
name = Column(String(80), nullable=False)
json_schema = Column(JSONEncodedDict(), default={})
json_schema = Column(JSONEncodedDict(), default={}, nullable=False)
class MetadefNamespaceResourceType(BASE_DICT, GlanceMetadefBase):

View File

@ -42,6 +42,7 @@ from oslo_utils import timeutils
# NOTE(jokke): simplified transition to py3, behaves like py2 xrange
from six.moves import range
import sqlalchemy
from sqlalchemy import inspect
from glance.common import crypt
from glance.common import exception
@ -50,6 +51,7 @@ from glance.db import migration
from glance.db.sqlalchemy import migrate_repo
from glance.db.sqlalchemy.migrate_repo.schema import from_migration_import
from glance.db.sqlalchemy import models
from glance.db.sqlalchemy import models_metadef
from glance import i18n
_ = i18n._
@ -1360,6 +1362,129 @@ class MigrationsMixin(test_migrations.WalkVersionsMixin):
self.assertRaises(sqlalchemy.exc.NoSuchTableError,
db_utils.get_table, engine, 'metadef_tags')
def _check_039(self, engine, data):
meta = sqlalchemy.MetaData()
meta.bind = engine
metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta,
autoload=True)
metadef_properties = sqlalchemy.Table('metadef_properties', meta,
autoload=True)
metadef_objects = sqlalchemy.Table('metadef_objects', meta,
autoload=True)
metadef_ns_res_types = sqlalchemy.Table(
'metadef_namespace_resource_types',
meta, autoload=True)
metadef_resource_types = sqlalchemy.Table('metadef_resource_types',
meta, autoload=True)
tables = [metadef_namespaces, metadef_properties, metadef_objects,
metadef_ns_res_types, metadef_resource_types]
for table in tables:
for index_name in ['ix_namespaces_namespace',
'ix_objects_namespace_id_name',
'ix_metadef_properties_namespace_id_name']:
self.assertFalse(index_exist(index_name, table.name, engine))
for uc_name in ['resource_type_id', 'namespace', 'name',
'namespace_id',
'metadef_objects_namespace_id_name_key',
'metadef_properties_namespace_id_name_key']:
self.assertFalse(unique_constraint_exist(uc_name, table.name,
engine))
self.assertTrue(index_exist('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.name, engine))
self.assertTrue(index_exist('ix_metadef_namespaces_namespace',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_metadef_namespaces_owner',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_metadef_objects_name',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_objects_namespace_id',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_name',
metadef_properties.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_namespace_id',
metadef_properties.name, engine))
def _post_downgrade_039(self, engine):
meta = sqlalchemy.MetaData()
meta.bind = engine
metadef_namespaces = sqlalchemy.Table('metadef_namespaces', meta,
autoload=True)
metadef_properties = sqlalchemy.Table('metadef_properties', meta,
autoload=True)
metadef_objects = sqlalchemy.Table('metadef_objects', meta,
autoload=True)
metadef_ns_res_types = sqlalchemy.Table(
'metadef_namespace_resource_types',
meta, autoload=True)
metadef_resource_types = sqlalchemy.Table('metadef_resource_types',
meta, autoload=True)
self.assertFalse(index_exist('ix_metadef_ns_res_types_namespace_id',
metadef_ns_res_types.name, engine))
self.assertFalse(index_exist('ix_metadef_namespaces_namespace',
metadef_namespaces.name, engine))
self.assertFalse(index_exist('ix_metadef_namespaces_owner',
metadef_namespaces.name, engine))
self.assertFalse(index_exist('ix_metadef_objects_name',
metadef_objects.name, engine))
self.assertFalse(index_exist('ix_metadef_objects_namespace_id',
metadef_objects.name, engine))
self.assertFalse(index_exist('ix_metadef_properties_name',
metadef_properties.name, engine))
self.assertFalse(index_exist('ix_metadef_properties_namespace_id',
metadef_properties.name, engine))
self.assertTrue(index_exist('ix_namespaces_namespace',
metadef_namespaces.name, engine))
self.assertTrue(index_exist('ix_objects_namespace_id_name',
metadef_objects.name, engine))
self.assertTrue(index_exist('ix_metadef_properties_namespace_id_name',
metadef_properties.name, engine))
if engine.name == 'postgresql':
inspector = inspect(engine)
self.assertEqual(1, len(inspector.get_unique_constraints(
'metadef_objects')))
self.assertEqual(1, len(inspector.get_unique_constraints(
'metadef_properties')))
if engine.name == 'mysql':
self.assertTrue(unique_constraint_exist(
'namespace_id', metadef_properties.name, engine))
self.assertTrue(unique_constraint_exist(
'namespace_id', metadef_objects.name, engine))
self.assertTrue(unique_constraint_exist(
'resource_type_id', metadef_ns_res_types.name, engine))
self.assertTrue(unique_constraint_exist(
'namespace', metadef_namespaces.name, engine))
self.assertTrue(unique_constraint_exist(
'name', metadef_resource_types.name, engine))
class TestMysqlMigrations(test_base.MySQLOpportunisticTestCase,
MigrationsMixin):
@ -1399,6 +1524,8 @@ class TestSqliteMigrations(test_base.DbTestCase,
class ModelsMigrationSyncMixin(object):
def get_metadata(self):
for table in models_metadef.BASE_DICT.metadata.sorted_tables:
models.BASE.metadata._add_table(table.name, table.schema, table)
return models.BASE.metadata
def get_engine(self):
@ -1408,16 +1535,7 @@ class ModelsMigrationSyncMixin(object):
migration.db_sync(engine=engine)
def include_object(self, object_, name, type_, reflected, compare_to):
# We need to exclude tables that aren't in models.py and table that
# contains migrate version
# TODO(ochuprykov): We need to include this tables back after merge of
# models.py and models_metadef.py
# (except 'migrate_version')
if name in ['migrate_version', 'metadef_objects', 'metadef_namespaces',
'metadef_properties', 'metadef_resource_types',
'metadef_tags',
'metadef_namespace_resource_types'] and type_ == 'table':
if name in ['migrate_version'] and type_ == 'table':
return False
return True