From 503421d3d492cad1bd110ec72870f05d80fe021c Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Thu, 23 Dec 2021 19:09:37 +0000 Subject: [PATCH] sql: Remove legacy 'migrate_repo' migration repo This is now folded into the initial migration of the 'expand_repo' repository. Previously, this was a dummy migration. We simply move things across and remove any code that was trying to work with the older repo since it's no longer necessary. A release note is added, even though it's not really necessary since nothing will change for users. It's more of a heads up that things are afoot. Change-Id: I59882d88fe593ec1ae37415b2157584f7f3c85f8 Signed-off-by: Stephen Finucane --- doc/source/conf.py | 26 +- .../contributor/api_change_tutorial.rst | 13 + .../contributor/database-migrations.rst | 10 +- doc/source/contributor/testing-keystone.rst | 6 +- .../001_data_initial_null_migration.py | 40 +- .../001_expand_initial_null_migration.py | 728 +++++++++++++++- keystone/common/sql/migrate_repo/README | 4 - keystone/common/sql/migrate_repo/__init__.py | 0 keystone/common/sql/migrate_repo/manage.py | 18 - keystone/common/sql/migrate_repo/migrate.cfg | 25 - .../sql/migrate_repo/versions/109_newton.py | 776 ------------------ .../sql/migrate_repo/versions/__init__.py | 0 keystone/common/sql/upgrades.py | 42 +- .../tests/unit/test_sql_banned_operations.py | 86 +- keystone/tests/unit/test_sql_upgrade.py | 149 +--- ...ve-legacy-migrations-647f60019c8dd9e8.yaml | 7 + 16 files changed, 867 insertions(+), 1063 deletions(-) delete mode 100644 keystone/common/sql/migrate_repo/README delete mode 100644 keystone/common/sql/migrate_repo/__init__.py delete mode 100644 keystone/common/sql/migrate_repo/manage.py delete mode 100644 keystone/common/sql/migrate_repo/migrate.cfg delete mode 100644 keystone/common/sql/migrate_repo/versions/109_newton.py delete mode 100644 keystone/common/sql/migrate_repo/versions/__init__.py create mode 100644 releasenotes/notes/remove-legacy-migrations-647f60019c8dd9e8.yaml diff --git a/doc/source/conf.py b/doc/source/conf.py index 45cd82f0e8..41a2456324 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -32,18 +32,20 @@ # Add any Sphinx extension module names here, as strings. They can be # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom # ones. -extensions = ['sphinx.ext.coverage', - 'sphinx.ext.viewcode', - 'oslo_config.sphinxconfiggen', - 'oslo_config.sphinxext', - 'oslo_policy.sphinxpolicygen', - 'openstackdocstheme', - 'oslo_policy.sphinxext', - 'sphinxcontrib.apidoc', - 'sphinxcontrib.seqdiag', - 'sphinx_feature_classification.support_matrix', - 'sphinxcontrib.blockdiag' - ] +extensions = [ + 'sphinx.ext.coverage', + 'sphinx.ext.viewcode', + 'sphinx.ext.todo', + 'oslo_config.sphinxconfiggen', + 'oslo_config.sphinxext', + 'oslo_policy.sphinxpolicygen', + 'openstackdocstheme', + 'oslo_policy.sphinxext', + 'sphinxcontrib.apidoc', + 'sphinxcontrib.seqdiag', + 'sphinx_feature_classification.support_matrix', + 'sphinxcontrib.blockdiag' +] blockdiag_html_image_format = 'SVG' diff --git a/doc/source/contributor/api_change_tutorial.rst b/doc/source/contributor/api_change_tutorial.rst index 90ef8a33b9..dc63de4331 100644 --- a/doc/source/contributor/api_change_tutorial.rst +++ b/doc/source/contributor/api_change_tutorial.rst @@ -87,6 +87,19 @@ files, respectively (currently only the SQL driver is supported). Changing the SQL Model and Driver ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +.. note:: + + The below guidance is out-of-date and refers to the legacy ``migrate_repo`` + migration repository, which was removed in 21.0.0 (Yoga). Nowadays, for a + change like this, you would create an additive or "expand" migration in the + ``expand_repo`` repository along with null migrations in the + ``contract_repo`` and ``data_migration_repo`` repositories. For more + information, refer to :doc:`/contributor/database-migrations`. + +.. todo:: + + Update this section to reflect the new migration model. + First, you need to change the role model to include the description attribute. Go to `keystone/assignment/role_backends/sql.py` and update it like:: diff --git a/doc/source/contributor/database-migrations.rst b/doc/source/contributor/database-migrations.rst index 09b1e8e3d7..f8e8a7a5e3 100644 --- a/doc/source/contributor/database-migrations.rst +++ b/doc/source/contributor/database-migrations.rst @@ -43,14 +43,6 @@ do in a specific phase, then include a no-op migration to simply ``pass`` (in fact the ``001`` migration in each of these repositories is a no-op migration, so that can be used as a template). -.. NOTE:: - - Since rolling upgrade support was added part way through the Newton cycle, - some migrations had already been added to the legacy repository - (``keystone/common/sql/migrate_repo``). This repository is now closed and - no new migrations should be added (except for backporting of previous - placeholders). - In order to support rolling upgrades, where two releases of keystone briefly operate side-by-side using the same database without downtime, each phase of the migration must adhere to following constraints: @@ -79,7 +71,7 @@ Data Migration phase: No schema changes are allowed. Contract phase: - Only contractive schema changes are allowed, such as dropping or altering + Only destructive schema changes are allowed, such as dropping or altering columns, tables, indices, and triggers. Data insertion, modification, and removal is not allowed. diff --git a/doc/source/contributor/testing-keystone.rst b/doc/source/contributor/testing-keystone.rst index 4f4bbd2267..893d67e2a4 100644 --- a/doc/source/contributor/testing-keystone.rst +++ b/doc/source/contributor/testing-keystone.rst @@ -151,9 +151,9 @@ version control: .. code-block:: bash - $ python keystone/common/sql/migrate_repo/manage.py test \ - --url=sqlite:///test.db \ - --repository=keystone/common/sql/migrate_repo/ + $ python keystone/common/sql/expand_repo/manage.py test \ + --url=sqlite:///test.db \ + --repository=keystone/common/sql/expand_repo/ This command references to a SQLite database (test.db) to be used. Depending on the migration, this command alone does not make assertions as to the integrity diff --git a/keystone/common/sql/data_migration_repo/versions/001_data_initial_null_migration.py b/keystone/common/sql/data_migration_repo/versions/001_data_initial_null_migration.py index 1cd34e6171..d05b151b85 100644 --- a/keystone/common/sql/data_migration_repo/versions/001_data_initial_null_migration.py +++ b/keystone/common/sql/data_migration_repo/versions/001_data_initial_null_migration.py @@ -13,6 +13,44 @@ # A null initial migration to open this repo. Do not re-use replace this with # a real migration, add additional ones in subsequent version scripts. +import sqlalchemy as sql +import sqlalchemy.orm + +NULL_DOMAIN_ID = '<>' + def upgrade(migrate_engine): - pass + + def _generate_root_domain_project(): + # Generate a project that will act as a root for all domains, in order + # for use to be able to use a FK constraint on domain_id. Projects + # acting as a domain will not reference this as their parent_id, just + # as domain_id. + # + # This special project is filtered out by the driver, so is never + # visible to the manager or API. + + project_ref = { + 'id': NULL_DOMAIN_ID, + 'name': NULL_DOMAIN_ID, + 'enabled': False, + 'description': '', + 'domain_id': NULL_DOMAIN_ID, + 'is_domain': True, + 'parent_id': None, + 'extra': '{}', + } + return project_ref + + meta = sql.MetaData() + meta.bind = migrate_engine + session = sql.orm.sessionmaker(bind=migrate_engine)() + + project = sql.Table('project', meta, autoload=True) + + root_domain_project = _generate_root_domain_project() + new_entry = project.insert().values(**root_domain_project) + session.execute(new_entry) + session.commit() + + session.close() diff --git a/keystone/common/sql/expand_repo/versions/001_expand_initial_null_migration.py b/keystone/common/sql/expand_repo/versions/001_expand_initial_null_migration.py index 1cd34e6171..b12cb8556e 100644 --- a/keystone/common/sql/expand_repo/versions/001_expand_initial_null_migration.py +++ b/keystone/common/sql/expand_repo/versions/001_expand_initial_null_migration.py @@ -10,9 +10,731 @@ # License for the specific language governing permissions and limitations # under the License. -# A null initial migration to open this repo. Do not re-use replace this with -# a real migration, add additional ones in subsequent version scripts. + +import migrate +from oslo_log import log +import sqlalchemy as sql + +from keystone.assignment.backends import sql as assignment_sql +from keystone.common import sql as ks_sql +import keystone.conf +from keystone.identity.mapping_backends import mapping as mapping_backend + +CONF = keystone.conf.CONF +LOG = log.getLogger(__name__) def upgrade(migrate_engine): - pass + meta = sql.MetaData() + meta.bind = migrate_engine + + if migrate_engine.name == 'mysql': + # In Folsom we explicitly converted migrate_version to UTF8. + migrate_engine.execute( + 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8' + ) + # Set default DB charset to UTF8. + migrate_engine.execute( + 'ALTER DATABASE %s DEFAULT CHARACTER SET utf8' + % migrate_engine.url.database + ) + + access_token = sql.Table( + 'access_token', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('access_secret', sql.String(64), nullable=False), + sql.Column( + 'authorizing_user_id', sql.String(64), nullable=False, index=True + ), + sql.Column('project_id', sql.String(64), nullable=False), + sql.Column('role_ids', sql.Text(), nullable=False), + sql.Column( + 'consumer_id', + sql.String(64), + sql.ForeignKey('consumer.id'), + nullable=False, + index=True, + ), + sql.Column('expires_at', sql.String(64), nullable=True), + ) + + consumer = sql.Table( + 'consumer', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('description', sql.String(64), nullable=True), + sql.Column('secret', sql.String(64), nullable=False), + sql.Column('extra', sql.Text(), nullable=False), + ) + + credential = sql.Table( + 'credential', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('user_id', sql.String(length=64), nullable=False), + sql.Column('project_id', sql.String(length=64)), + sql.Column('blob', ks_sql.JsonBlob, nullable=False), + sql.Column('type', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + endpoint = sql.Table( + 'endpoint', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('legacy_endpoint_id', sql.String(length=64)), + sql.Column('interface', sql.String(length=8), nullable=False), + sql.Column('service_id', sql.String(length=64), nullable=False), + sql.Column('url', sql.Text, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column( + 'enabled', + sql.Boolean, + nullable=False, + default=True, + server_default='1', + ), + sql.Column('region_id', sql.String(length=255), nullable=True), + # NOTE(stevemar): The index was named 'service_id' in + # 050_fk_consistent_indexes.py and needs to be preserved + sql.Index('service_id', 'service_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + endpoint_group = sql.Table( + 'endpoint_group', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('name', sql.String(255), nullable=False), + sql.Column('description', sql.Text, nullable=True), + sql.Column('filters', sql.Text(), nullable=False), + ) + + federated_user = sql.Table( + 'federated_user', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column( + 'user_id', + sql.String(64), + sql.ForeignKey('user.id', ondelete='CASCADE'), + nullable=False, + ), + sql.Column( + 'idp_id', + sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), + nullable=False, + ), + sql.Column('protocol_id', sql.String(64), nullable=False), + sql.Column('unique_id', sql.String(255), nullable=False), + sql.Column('display_name', sql.String(255), nullable=True), + sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + federation_protocol = sql.Table( + 'federation_protocol', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column( + 'idp_id', + sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), + primary_key=True, + ), + sql.Column('mapping_id', sql.String(64), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + group = sql.Table( + 'group', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('domain_id', sql.String(length=64), nullable=False), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('description', sql.Text), + sql.Column('extra', ks_sql.JsonBlob.impl), + migrate.UniqueConstraint( + 'domain_id', + 'name', + name='ixu_group_name_domain_id', + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + identity_provider = sql.Table( + 'identity_provider', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('enabled', sql.Boolean, nullable=False), + sql.Column('description', sql.Text(), nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + idp_remote_ids = sql.Table( + 'idp_remote_ids', + meta, + sql.Column( + 'idp_id', + sql.String(64), + sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), + ), + sql.Column('remote_id', sql.String(255), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + implied_role = sql.Table( + 'implied_role', + meta, + sql.Column('prior_role_id', sql.String(length=64), primary_key=True), + sql.Column('implied_role_id', sql.String(length=64), primary_key=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + local_user = sql.Table( + 'local_user', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column( + 'user_id', + sql.String(64), + sql.ForeignKey('user.id', ondelete='CASCADE'), + nullable=False, + unique=True, + ), + sql.Column('domain_id', sql.String(64), nullable=False), + sql.Column('name', sql.String(255), nullable=False), + sql.Column('failed_auth_count', sql.Integer, nullable=True), + sql.Column('failed_auth_at', sql.DateTime(), nullable=True), + sql.UniqueConstraint('domain_id', 'name'), + ) + + mapping = sql.Table( + 'mapping', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('rules', sql.Text(), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + password = sql.Table( + 'password', + meta, + sql.Column('id', sql.Integer, primary_key=True, nullable=False), + sql.Column( + 'local_user_id', + sql.Integer, + sql.ForeignKey(local_user.c.id, ondelete='CASCADE'), + nullable=False, + ), + sql.Column('password', sql.String(128), nullable=True), + sql.Column('created_at', sql.DateTime(), nullable=True), + sql.Column('expires_at', sql.DateTime(), nullable=True), + sql.Column( + 'self_service', + sql.Boolean, + nullable=False, + server_default='0', + default=False, + ), + ) + + policy = sql.Table( + 'policy', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('type', sql.String(length=255), nullable=False), + sql.Column('blob', ks_sql.JsonBlob, nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + policy_association = sql.Table( + 'policy_association', + meta, + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('policy_id', sql.String(64), nullable=False), + sql.Column('endpoint_id', sql.String(64), nullable=True), + sql.Column('service_id', sql.String(64), nullable=True), + sql.Column('region_id', sql.String(64), nullable=True), + sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + project = sql.Table( + 'project', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=64), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('description', sql.Text), + sql.Column('enabled', sql.Boolean), + sql.Column('domain_id', sql.String(length=64), nullable=False), + sql.Column('parent_id', sql.String(64), nullable=True), + sql.Column( + 'is_domain', + sql.Boolean, + nullable=False, + server_default='0', + default=False, + ), + migrate.UniqueConstraint( + 'domain_id', + 'name', + name='ixu_project_name_domain_id', + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + project_endpoint = sql.Table( + 'project_endpoint', + meta, + sql.Column( + 'endpoint_id', sql.String(64), primary_key=True, nullable=False + ), + sql.Column( + 'project_id', sql.String(64), primary_key=True, nullable=False + ), + ) + + project_endpoint_group = sql.Table( + 'project_endpoint_group', + meta, + sql.Column( + 'endpoint_group_id', + sql.String(64), + sql.ForeignKey('endpoint_group.id'), + nullable=False, + ), + sql.Column('project_id', sql.String(64), nullable=False), + sql.PrimaryKeyConstraint('endpoint_group_id', 'project_id'), + ) + + config_register = sql.Table( + 'config_register', + meta, + sql.Column('type', sql.String(64), primary_key=True), + sql.Column('domain_id', sql.String(64), nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + request_token = sql.Table( + 'request_token', + meta, + sql.Column('id', sql.String(64), primary_key=True, nullable=False), + sql.Column('request_secret', sql.String(64), nullable=False), + sql.Column('verifier', sql.String(64), nullable=True), + sql.Column('authorizing_user_id', sql.String(64), nullable=True), + sql.Column('requested_project_id', sql.String(64), nullable=False), + sql.Column('role_ids', sql.Text(), nullable=True), + sql.Column( + 'consumer_id', + sql.String(64), + sql.ForeignKey('consumer.id'), + nullable=False, + index=True, + ), + sql.Column('expires_at', sql.String(64), nullable=True), + ) + + revocation_event = sql.Table( + 'revocation_event', + meta, + sql.Column('id', sql.Integer, primary_key=True), + sql.Column('domain_id', sql.String(64)), + sql.Column('project_id', sql.String(64)), + sql.Column('user_id', sql.String(64)), + sql.Column('role_id', sql.String(64)), + sql.Column('trust_id', sql.String(64)), + sql.Column('consumer_id', sql.String(64)), + sql.Column('access_token_id', sql.String(64)), + sql.Column('issued_before', sql.DateTime(), nullable=False), + sql.Column('expires_at', sql.DateTime()), + sql.Column('revoked_at', sql.DateTime(), nullable=False), + sql.Column('audit_id', sql.String(32), nullable=True), + sql.Column('audit_chain_id', sql.String(32), nullable=True), + # NOTE(stephenfin): The '_new' suffix here is due to migration 095, + # which changed the 'id' column from String(64) to Integer. It did this + # by creating a 'revocation_event_new' table and populating it with + # data from the 'revocation_event' table before deleting the + # 'revocation_event' table and renaming the 'revocation_event_new' + # table to 'revocation_event'. Because the 'revoked_at' column had + # 'index=True', sqlalchemy automatically generated the index name as + # 'ix_{table}_{column}'. However, when intitially created, '{table}' + # was 'revocation_event_new' so the index got that name. We may wish to + # rename this eventually. + sql.Index('ix_revocation_event_new_revoked_at', 'revoked_at'), + ) + + role = sql.Table( + 'role', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('name', sql.String(length=255), nullable=False), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column( + 'domain_id', + sql.String(64), + nullable=False, + server_default='<>', + ), + migrate.UniqueConstraint( + 'name', + 'domain_id', + name='ixu_role_name_domain_id', + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + service = sql.Table( + 'service', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('type', sql.String(length=255)), + sql.Column( + 'enabled', + sql.Boolean, + nullable=False, + default=True, + server_default='1', + ), + sql.Column('extra', ks_sql.JsonBlob.impl), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + service_provider = sql.Table( + 'service_provider', + meta, + sql.Column('auth_url', sql.String(256), nullable=False), + sql.Column('id', sql.String(64), primary_key=True), + sql.Column('enabled', sql.Boolean, nullable=False), + sql.Column('description', sql.Text(), nullable=True), + sql.Column('sp_url', sql.String(256), nullable=False), + sql.Column( + 'relay_state_prefix', + sql.String(256), + nullable=False, + server_default=CONF.saml.relay_state_prefix, + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + token = sql.Table( + 'token', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('expires', sql.DateTime, default=None), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('valid', sql.Boolean, default=True, nullable=False), + sql.Column('trust_id', sql.String(length=64)), + sql.Column('user_id', sql.String(length=64)), + sql.Index('ix_token_expires', 'expires'), + sql.Index( + 'ix_token_expires_valid', 'expires', 'valid' + ), + sql.Index('ix_token_user_id', 'user_id'), + sql.Index('ix_token_trust_id', 'trust_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + trust = sql.Table( + 'trust', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('trustor_user_id', sql.String(length=64), nullable=False), + sql.Column('trustee_user_id', sql.String(length=64), nullable=False), + sql.Column('project_id', sql.String(length=64)), + sql.Column('impersonation', sql.Boolean, nullable=False), + sql.Column('deleted_at', sql.DateTime), + sql.Column('expires_at', sql.DateTime), + sql.Column('remaining_uses', sql.Integer, nullable=True), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.UniqueConstraint( + 'trustor_user_id', + 'trustee_user_id', + 'project_id', + 'impersonation', + 'expires_at', + name='duplicate_trust_constraint', + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + trust_role = sql.Table( + 'trust_role', + meta, + sql.Column( + 'trust_id', sql.String(length=64), primary_key=True, nullable=False + ), + sql.Column( + 'role_id', sql.String(length=64), primary_key=True, nullable=False + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + user = sql.Table( + 'user', + meta, + sql.Column('id', sql.String(length=64), primary_key=True), + sql.Column('extra', ks_sql.JsonBlob.impl), + sql.Column('enabled', sql.Boolean), + sql.Column('default_project_id', sql.String(length=64)), + sql.Column('created_at', sql.DateTime(), nullable=True), + sql.Column('last_active_at', sql.Date(), nullable=True), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + nonlocal_user = sql.Table( + 'nonlocal_user', + meta, + sql.Column('domain_id', sql.String(64), primary_key=True), + sql.Column('name', sql.String(255), primary_key=True), + sql.Column( + 'user_id', + sql.String(64), + sql.ForeignKey(user.c.id, ondelete='CASCADE'), + nullable=False, + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + user_group_membership = sql.Table( + 'user_group_membership', + meta, + sql.Column('user_id', sql.String(length=64), primary_key=True), + sql.Column('group_id', sql.String(length=64), primary_key=True), + # NOTE(stevemar): The index was named 'group_id' in + # 050_fk_consistent_indexes.py and needs to be preserved + sql.Index('group_id', 'group_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + region = sql.Table( + 'region', + meta, + sql.Column('id', sql.String(255), primary_key=True), + sql.Column('description', sql.String(255), nullable=False), + sql.Column('parent_region_id', sql.String(255), nullable=True), + sql.Column('extra', sql.Text()), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + assignment = sql.Table( + 'assignment', + meta, + sql.Column( + 'type', + sql.Enum( + assignment_sql.AssignmentType.USER_PROJECT, + assignment_sql.AssignmentType.GROUP_PROJECT, + assignment_sql.AssignmentType.USER_DOMAIN, + assignment_sql.AssignmentType.GROUP_DOMAIN, + name='type', + ), + nullable=False, + ), + sql.Column('actor_id', sql.String(64), nullable=False), + sql.Column('target_id', sql.String(64), nullable=False), + sql.Column('role_id', sql.String(64), nullable=False), + sql.Column('inherited', sql.Boolean, default=False, nullable=False), + sql.PrimaryKeyConstraint( + 'type', + 'actor_id', + 'target_id', + 'role_id', + 'inherited', + ), + sql.Index('ix_actor_id', 'actor_id'), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + id_mapping = sql.Table( + 'id_mapping', + meta, + sql.Column('public_id', sql.String(64), primary_key=True), + sql.Column('domain_id', sql.String(64), nullable=False), + sql.Column('local_id', sql.String(64), nullable=False), + sql.Column( + 'entity_type', + sql.Enum( + mapping_backend.EntityType.USER, + mapping_backend.EntityType.GROUP, + name='entity_type', + ), + nullable=False, + ), + migrate.UniqueConstraint( + 'domain_id', + 'local_id', + 'entity_type', + name='domain_id', + ), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + whitelisted_config = sql.Table( + 'whitelisted_config', + meta, + sql.Column('domain_id', sql.String(64), primary_key=True), + sql.Column('group', sql.String(255), primary_key=True), + sql.Column('option', sql.String(255), primary_key=True), + sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + sensitive_config = sql.Table( + 'sensitive_config', + meta, + sql.Column('domain_id', sql.String(64), primary_key=True), + sql.Column('group', sql.String(255), primary_key=True), + sql.Column('option', sql.String(255), primary_key=True), + sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), + mysql_engine='InnoDB', + mysql_charset='utf8', + ) + + # create all tables + tables = [ + credential, + endpoint, + group, + policy, + project, + role, + service, + token, + trust, + trust_role, + user, + user_group_membership, + region, + assignment, + id_mapping, + whitelisted_config, + sensitive_config, + config_register, + policy_association, + identity_provider, + federation_protocol, + mapping, + service_provider, + idp_remote_ids, + consumer, + request_token, + access_token, + revocation_event, + project_endpoint, + endpoint_group, + project_endpoint_group, + implied_role, + local_user, + password, + federated_user, + nonlocal_user, + ] + + for table in tables: + try: + table.create() + except Exception: + LOG.exception('Exception while creating table: %r', table) + raise + + fkeys = [ + { + 'columns': [endpoint.c.service_id], + 'references': [service.c.id], + }, + { + 'columns': [user_group_membership.c.group_id], + 'references': [group.c.id], + 'name': 'fk_user_group_membership_group_id', + }, + { + 'columns': [user_group_membership.c.user_id], + 'references': [user.c.id], + 'name': 'fk_user_group_membership_user_id', + }, + { + 'columns': [project.c.domain_id], + 'references': [project.c.id], + }, + { + 'columns': [endpoint.c.region_id], + 'references': [region.c.id], + 'name': 'fk_endpoint_region_id', + }, + { + 'columns': [project.c.parent_id], + 'references': [project.c.id], + 'name': 'project_parent_id_fkey', + }, + { + 'columns': [implied_role.c.prior_role_id], + 'references': [role.c.id], + 'ondelete': 'CASCADE', + }, + { + 'columns': [implied_role.c.implied_role_id], + 'references': [role.c.id], + 'ondelete': 'CASCADE', + }, + { + 'columns': [ + federated_user.c.protocol_id, + federated_user.c.idp_id, + ], + 'references': [ + federation_protocol.c.id, + federation_protocol.c.idp_id, + ], + }, + ] + + if migrate_engine.name == 'sqlite': + # NOTE(stevemar): We need to keep this FK constraint due to 073, but + # only for sqlite, once we collapse 073 we can remove this constraint + fkeys.append( + { + 'columns': [assignment.c.role_id], + 'references': [role.c.id], + 'name': 'fk_assignment_role_id', + }, + ) + + for fkey in fkeys: + migrate.ForeignKeyConstraint( + columns=fkey['columns'], + refcolumns=fkey['references'], + name=fkey.get('name'), + ondelete=fkey.get('ondelete'), + ).create() diff --git a/keystone/common/sql/migrate_repo/README b/keystone/common/sql/migrate_repo/README deleted file mode 100644 index 1311171041..0000000000 --- a/keystone/common/sql/migrate_repo/README +++ /dev/null @@ -1,4 +0,0 @@ -This is a database migration repository. - -More information at -https://opendev.org/openstack/sqlalchemy-migrate diff --git a/keystone/common/sql/migrate_repo/__init__.py b/keystone/common/sql/migrate_repo/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/keystone/common/sql/migrate_repo/manage.py b/keystone/common/sql/migrate_repo/manage.py deleted file mode 100644 index 41cba1adbd..0000000000 --- a/keystone/common/sql/migrate_repo/manage.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(debug='False') diff --git a/keystone/common/sql/migrate_repo/migrate.cfg b/keystone/common/sql/migrate_repo/migrate.cfg deleted file mode 100644 index db531bb413..0000000000 --- a/keystone/common/sql/migrate_repo/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=keystone - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/keystone/common/sql/migrate_repo/versions/109_newton.py b/keystone/common/sql/migrate_repo/versions/109_newton.py deleted file mode 100644 index 460e70959a..0000000000 --- a/keystone/common/sql/migrate_repo/versions/109_newton.py +++ /dev/null @@ -1,776 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - - -import migrate -from oslo_log import log -import sqlalchemy as sql - -from keystone.assignment.backends import sql as assignment_sql -from keystone.common import sql as ks_sql -import keystone.conf -from keystone.identity.mapping_backends import mapping as mapping_backend - -CONF = keystone.conf.CONF -LOG = log.getLogger(__name__) - -NULL_DOMAIN_ID = '<>' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - meta.bind = migrate_engine - - if migrate_engine.name == 'mysql': - # In Folsom we explicitly converted migrate_version to UTF8. - migrate_engine.execute( - 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8' - ) - # Set default DB charset to UTF8. - migrate_engine.execute( - 'ALTER DATABASE %s DEFAULT CHARACTER SET utf8' - % migrate_engine.url.database - ) - - access_token = sql.Table( - 'access_token', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('access_secret', sql.String(64), nullable=False), - sql.Column( - 'authorizing_user_id', sql.String(64), nullable=False, index=True - ), - sql.Column('project_id', sql.String(64), nullable=False), - sql.Column('role_ids', sql.Text(), nullable=False), - sql.Column( - 'consumer_id', - sql.String(64), - sql.ForeignKey('consumer.id'), - nullable=False, - index=True, - ), - sql.Column('expires_at', sql.String(64), nullable=True), - ) - - consumer = sql.Table( - 'consumer', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('description', sql.String(64), nullable=True), - sql.Column('secret', sql.String(64), nullable=False), - sql.Column('extra', sql.Text(), nullable=False), - ) - - credential = sql.Table( - 'credential', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('user_id', sql.String(length=64), nullable=False), - sql.Column('project_id', sql.String(length=64)), - sql.Column('blob', ks_sql.JsonBlob, nullable=False), - sql.Column('type', sql.String(length=255), nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - endpoint = sql.Table( - 'endpoint', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('legacy_endpoint_id', sql.String(length=64)), - sql.Column('interface', sql.String(length=8), nullable=False), - sql.Column('service_id', sql.String(length=64), nullable=False), - sql.Column('url', sql.Text, nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column( - 'enabled', - sql.Boolean, - nullable=False, - default=True, - server_default='1', - ), - sql.Column('region_id', sql.String(length=255), nullable=True), - # NOTE(stevemar): The index was named 'service_id' in - # 050_fk_consistent_indexes.py and needs to be preserved - sql.Index('service_id', 'service_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - endpoint_group = sql.Table( - 'endpoint_group', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(255), nullable=False), - sql.Column('description', sql.Text, nullable=True), - sql.Column('filters', sql.Text(), nullable=False), - ) - - federated_user = sql.Table( - 'federated_user', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id', ondelete='CASCADE'), - nullable=False, - ), - sql.Column( - 'idp_id', - sql.String(64), - sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), - nullable=False, - ), - sql.Column('protocol_id', sql.String(64), nullable=False), - sql.Column('unique_id', sql.String(255), nullable=False), - sql.Column('display_name', sql.String(255), nullable=True), - sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - federation_protocol = sql.Table( - 'federation_protocol', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column( - 'idp_id', - sql.String(64), - sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), - primary_key=True, - ), - sql.Column('mapping_id', sql.String(64), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - group = sql.Table( - 'group', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('domain_id', sql.String(length=64), nullable=False), - sql.Column('name', sql.String(length=64), nullable=False), - sql.Column('description', sql.Text), - sql.Column('extra', ks_sql.JsonBlob.impl), - migrate.UniqueConstraint( - 'domain_id', - 'name', - name='ixu_group_name_domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - identity_provider = sql.Table( - 'identity_provider', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('enabled', sql.Boolean, nullable=False), - sql.Column('description', sql.Text(), nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - idp_remote_ids = sql.Table( - 'idp_remote_ids', - meta, - sql.Column( - 'idp_id', - sql.String(64), - sql.ForeignKey('identity_provider.id', ondelete='CASCADE'), - ), - sql.Column('remote_id', sql.String(255), primary_key=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - implied_role = sql.Table( - 'implied_role', - meta, - sql.Column('prior_role_id', sql.String(length=64), primary_key=True), - sql.Column('implied_role_id', sql.String(length=64), primary_key=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - local_user = sql.Table( - 'local_user', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey('user.id', ondelete='CASCADE'), - nullable=False, - unique=True, - ), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.Column('name', sql.String(255), nullable=False), - sql.Column('failed_auth_count', sql.Integer, nullable=True), - sql.Column('failed_auth_at', sql.DateTime(), nullable=True), - sql.UniqueConstraint('domain_id', 'name'), - ) - - mapping = sql.Table( - 'mapping', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('rules', sql.Text(), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - password = sql.Table( - 'password', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column( - 'local_user_id', - sql.Integer, - sql.ForeignKey(local_user.c.id, ondelete='CASCADE'), - nullable=False, - ), - sql.Column('password', sql.String(128), nullable=True), - sql.Column('created_at', sql.DateTime(), nullable=True), - sql.Column('expires_at', sql.DateTime(), nullable=True), - sql.Column( - 'self_service', - sql.Boolean, - nullable=False, - server_default='0', - default=False, - ), - ) - - policy = sql.Table( - 'policy', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('type', sql.String(length=255), nullable=False), - sql.Column('blob', ks_sql.JsonBlob, nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - policy_association = sql.Table( - 'policy_association', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('policy_id', sql.String(64), nullable=False), - sql.Column('endpoint_id', sql.String(64), nullable=True), - sql.Column('service_id', sql.String(64), nullable=True), - sql.Column('region_id', sql.String(64), nullable=True), - sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - project = sql.Table( - 'project', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('name', sql.String(length=64), nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('description', sql.Text), - sql.Column('enabled', sql.Boolean), - sql.Column('domain_id', sql.String(length=64), nullable=False), - sql.Column('parent_id', sql.String(64), nullable=True), - sql.Column( - 'is_domain', - sql.Boolean, - nullable=False, - server_default='0', - default=False, - ), - migrate.UniqueConstraint( - 'domain_id', - 'name', - name='ixu_project_name_domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - project_endpoint = sql.Table( - 'project_endpoint', - meta, - sql.Column( - 'endpoint_id', sql.String(64), primary_key=True, nullable=False - ), - sql.Column( - 'project_id', sql.String(64), primary_key=True, nullable=False - ), - ) - - project_endpoint_group = sql.Table( - 'project_endpoint_group', - meta, - sql.Column( - 'endpoint_group_id', - sql.String(64), - sql.ForeignKey('endpoint_group.id'), - nullable=False, - ), - sql.Column('project_id', sql.String(64), nullable=False), - sql.PrimaryKeyConstraint('endpoint_group_id', 'project_id'), - ) - - config_register = sql.Table( - 'config_register', - meta, - sql.Column('type', sql.String(64), primary_key=True), - sql.Column('domain_id', sql.String(64), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - request_token = sql.Table( - 'request_token', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('request_secret', sql.String(64), nullable=False), - sql.Column('verifier', sql.String(64), nullable=True), - sql.Column('authorizing_user_id', sql.String(64), nullable=True), - sql.Column('requested_project_id', sql.String(64), nullable=False), - sql.Column('role_ids', sql.Text(), nullable=True), - sql.Column( - 'consumer_id', - sql.String(64), - sql.ForeignKey('consumer.id'), - nullable=False, - index=True, - ), - sql.Column('expires_at', sql.String(64), nullable=True), - ) - - revocation_event = sql.Table( - 'revocation_event', - meta, - sql.Column('id', sql.Integer, primary_key=True), - sql.Column('domain_id', sql.String(64)), - sql.Column('project_id', sql.String(64)), - sql.Column('user_id', sql.String(64)), - sql.Column('role_id', sql.String(64)), - sql.Column('trust_id', sql.String(64)), - sql.Column('consumer_id', sql.String(64)), - sql.Column('access_token_id', sql.String(64)), - sql.Column('issued_before', sql.DateTime(), nullable=False), - sql.Column('expires_at', sql.DateTime()), - sql.Column('revoked_at', sql.DateTime(), nullable=False), - sql.Column('audit_id', sql.String(32), nullable=True), - sql.Column('audit_chain_id', sql.String(32), nullable=True), - # NOTE(stephenfin): The '_new' suffix here is due to migration 095, - # which changed the 'id' column from String(64) to Integer. It did this - # by creating a 'revocation_event_new' table and populating it with - # data from the 'revocation_event' table before deleting the - # 'revocation_event' table and renaming the 'revocation_event_new' - # table to 'revocation_event'. Because the 'revoked_at' column had - # 'index=True', sqlalchemy automatically generated the index name as - # 'ix_{table}_{column}'. However, when intitially created, '{table}' - # was 'revocation_event_new' so the index got that name. We may wish to - # rename this eventually. - sql.Index('ix_revocation_event_new_revoked_at', 'revoked_at'), - ) - - role = sql.Table( - 'role', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('name', sql.String(length=255), nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column( - 'domain_id', - sql.String(64), - nullable=False, - server_default='<>', - ), - migrate.UniqueConstraint( - 'name', - 'domain_id', - name='ixu_role_name_domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - service = sql.Table( - 'service', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('type', sql.String(length=255)), - sql.Column( - 'enabled', - sql.Boolean, - nullable=False, - default=True, - server_default='1', - ), - sql.Column('extra', ks_sql.JsonBlob.impl), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - service_provider = sql.Table( - 'service_provider', - meta, - sql.Column('auth_url', sql.String(256), nullable=False), - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('enabled', sql.Boolean, nullable=False), - sql.Column('description', sql.Text(), nullable=True), - sql.Column('sp_url', sql.String(256), nullable=False), - sql.Column( - 'relay_state_prefix', - sql.String(256), - nullable=False, - server_default=CONF.saml.relay_state_prefix, - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - token = sql.Table( - 'token', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('expires', sql.DateTime, default=None), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('valid', sql.Boolean, default=True, nullable=False), - sql.Column('trust_id', sql.String(length=64)), - sql.Column('user_id', sql.String(length=64)), - sql.Index('ix_token_expires', 'expires'), - sql.Index( - 'ix_token_expires_valid', 'expires', 'valid' - ), - sql.Index('ix_token_user_id', 'user_id'), - sql.Index('ix_token_trust_id', 'trust_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - trust = sql.Table( - 'trust', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('trustor_user_id', sql.String(length=64), nullable=False), - sql.Column('trustee_user_id', sql.String(length=64), nullable=False), - sql.Column('project_id', sql.String(length=64)), - sql.Column('impersonation', sql.Boolean, nullable=False), - sql.Column('deleted_at', sql.DateTime), - sql.Column('expires_at', sql.DateTime), - sql.Column('remaining_uses', sql.Integer, nullable=True), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.UniqueConstraint( - 'trustor_user_id', - 'trustee_user_id', - 'project_id', - 'impersonation', - 'expires_at', - name='duplicate_trust_constraint', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - trust_role = sql.Table( - 'trust_role', - meta, - sql.Column( - 'trust_id', sql.String(length=64), primary_key=True, nullable=False - ), - sql.Column( - 'role_id', sql.String(length=64), primary_key=True, nullable=False - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - user = sql.Table( - 'user', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('enabled', sql.Boolean), - sql.Column('default_project_id', sql.String(length=64)), - sql.Column('created_at', sql.DateTime(), nullable=True), - sql.Column('last_active_at', sql.Date(), nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - nonlocal_user = sql.Table( - 'nonlocal_user', - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(255), primary_key=True), - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey(user.c.id, ondelete='CASCADE'), - nullable=False, - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - user_group_membership = sql.Table( - 'user_group_membership', - meta, - sql.Column('user_id', sql.String(length=64), primary_key=True), - sql.Column('group_id', sql.String(length=64), primary_key=True), - # NOTE(stevemar): The index was named 'group_id' in - # 050_fk_consistent_indexes.py and needs to be preserved - sql.Index('group_id', 'group_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - region = sql.Table( - 'region', - meta, - sql.Column('id', sql.String(255), primary_key=True), - sql.Column('description', sql.String(255), nullable=False), - sql.Column('parent_region_id', sql.String(255), nullable=True), - sql.Column('extra', sql.Text()), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - assignment = sql.Table( - 'assignment', - meta, - sql.Column( - 'type', - sql.Enum( - assignment_sql.AssignmentType.USER_PROJECT, - assignment_sql.AssignmentType.GROUP_PROJECT, - assignment_sql.AssignmentType.USER_DOMAIN, - assignment_sql.AssignmentType.GROUP_DOMAIN, - name='type', - ), - nullable=False, - ), - sql.Column('actor_id', sql.String(64), nullable=False), - sql.Column('target_id', sql.String(64), nullable=False), - sql.Column('role_id', sql.String(64), nullable=False), - sql.Column('inherited', sql.Boolean, default=False, nullable=False), - sql.PrimaryKeyConstraint( - 'type', - 'actor_id', - 'target_id', - 'role_id', - 'inherited', - ), - sql.Index('ix_actor_id', 'actor_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - id_mapping = sql.Table( - 'id_mapping', - meta, - sql.Column('public_id', sql.String(64), primary_key=True), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.Column('local_id', sql.String(64), nullable=False), - sql.Column( - 'entity_type', - sql.Enum( - mapping_backend.EntityType.USER, - mapping_backend.EntityType.GROUP, - name='entity_type', - ), - nullable=False, - ), - migrate.UniqueConstraint( - 'domain_id', - 'local_id', - 'entity_type', - name='domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - whitelisted_config = sql.Table( - 'whitelisted_config', - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('group', sql.String(255), primary_key=True), - sql.Column('option', sql.String(255), primary_key=True), - sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - sensitive_config = sql.Table( - 'sensitive_config', - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('group', sql.String(255), primary_key=True), - sql.Column('option', sql.String(255), primary_key=True), - sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - # create all tables - tables = [ - credential, - endpoint, - group, - policy, - project, - role, - service, - token, - trust, - trust_role, - user, - user_group_membership, - region, - assignment, - id_mapping, - whitelisted_config, - sensitive_config, - config_register, - policy_association, - identity_provider, - federation_protocol, - mapping, - service_provider, - idp_remote_ids, - consumer, - request_token, - access_token, - revocation_event, - project_endpoint, - endpoint_group, - project_endpoint_group, - implied_role, - local_user, - password, - federated_user, - nonlocal_user, - ] - - for table in tables: - try: - table.create() - except Exception: - LOG.exception('Exception while creating table: %r', table) - raise - - fkeys = [ - { - 'columns': [endpoint.c.service_id], - 'references': [service.c.id], - }, - { - 'columns': [user_group_membership.c.group_id], - 'references': [group.c.id], - 'name': 'fk_user_group_membership_group_id', - }, - { - 'columns': [user_group_membership.c.user_id], - 'references': [user.c.id], - 'name': 'fk_user_group_membership_user_id', - }, - { - 'columns': [project.c.domain_id], - 'references': [project.c.id], - }, - { - 'columns': [endpoint.c.region_id], - 'references': [region.c.id], - 'name': 'fk_endpoint_region_id', - }, - { - 'columns': [project.c.parent_id], - 'references': [project.c.id], - 'name': 'project_parent_id_fkey', - }, - { - 'columns': [implied_role.c.prior_role_id], - 'references': [role.c.id], - 'ondelete': 'CASCADE', - }, - { - 'columns': [implied_role.c.implied_role_id], - 'references': [role.c.id], - 'ondelete': 'CASCADE', - }, - { - 'columns': [ - federated_user.c.protocol_id, - federated_user.c.idp_id, - ], - 'references': [ - federation_protocol.c.id, - federation_protocol.c.idp_id, - ], - }, - ] - - if migrate_engine.name == 'sqlite': - # NOTE(stevemar): We need to keep this FK constraint due to 073, but - # only for sqlite, once we collapse 073 we can remove this constraint - fkeys.append( - { - 'columns': [assignment.c.role_id], - 'references': [role.c.id], - 'name': 'fk_assignment_role_id', - }, - ) - - for fkey in fkeys: - migrate.ForeignKeyConstraint( - columns=fkey['columns'], - refcolumns=fkey['references'], - name=fkey.get('name'), - ondelete=fkey.get('ondelete'), - ).create() - - # data generation - - def _generate_root_domain_project(): - # Generate a project that will act as a root for all domains, in order - # for use to be able to use a FK constraint on domain_id. Projects - # acting as a domain will not reference this as their parent_id, just - # as domain_id. - # - # This special project is filtered out by the driver, so is never - # visible to the manager or API. - - project_ref = { - 'id': NULL_DOMAIN_ID, - 'name': NULL_DOMAIN_ID, - 'enabled': False, - 'description': '', - 'domain_id': NULL_DOMAIN_ID, - 'is_domain': True, - 'parent_id': None, - 'extra': '{}', - } - return project_ref - - meta = sql.MetaData() - meta.bind = migrate_engine - session = sql.orm.sessionmaker(bind=migrate_engine)() - - root_domain_project = _generate_root_domain_project() - new_entry = project.insert().values(**root_domain_project) - session.execute(new_entry) - session.commit() - - session.close() diff --git a/keystone/common/sql/migrate_repo/versions/__init__.py b/keystone/common/sql/migrate_repo/versions/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/keystone/common/sql/upgrades.py b/keystone/common/sql/upgrades.py index 8bfe453cf1..fda8942dc5 100644 --- a/keystone/common/sql/upgrades.py +++ b/keystone/common/sql/upgrades.py @@ -30,7 +30,6 @@ from keystone.i18n import _ USE_TRIGGERS = True -LEGACY_REPO = 'migrate_repo' EXPAND_REPO = 'expand_repo' DATA_MIGRATION_REPO = 'data_migration_repo' CONTRACT_REPO = 'contract_repo' @@ -136,16 +135,6 @@ def find_repo(repo_name): return path -def _sync_common_repo(version): - abs_path = find_repo(LEGACY_REPO) - init_version = get_init_version() - with sql.session_for_write() as session: - engine = session.get_bind() - _assert_not_schema_downgrade(version=version) - migration.db_sync(engine, abs_path, version=version, - init_version=init_version, sanity_check=False) - - def _sync_repo(repo_name): abs_path = find_repo(repo_name) with sql.session_for_write() as session: @@ -170,7 +159,7 @@ def get_init_version(abs_path=None): :return: initial version number or None, if DB is empty. """ if abs_path is None: - abs_path = find_repo(LEGACY_REPO) + abs_path = find_repo(EXPAND_REPO) repo = migrate.versioning.repository.Repository(abs_path) @@ -221,14 +210,14 @@ def offline_sync_database_to_version(version=None): USE_TRIGGERS = False if version: - _sync_common_repo(version) - else: - expand_schema() - migrate_data() - contract_schema() + raise Exception('Specifying a version is no longer supported') + + expand_schema() + migrate_data() + contract_schema() -def get_db_version(repo=LEGACY_REPO): +def get_db_version(repo=EXPAND_REPO): with sql.session_for_read() as session: repo = find_repo(repo) return migration.db_version( @@ -254,19 +243,7 @@ def validate_upgrade_order(repo_name, target_repo_version=None): db_sync_order = {DATA_MIGRATION_REPO: EXPAND_REPO, CONTRACT_REPO: DATA_MIGRATION_REPO} - if repo_name == LEGACY_REPO: - return - # If expand is being run, we validate that Legacy repo is at the maximum - # version before running the additional schema expansions. - elif repo_name == EXPAND_REPO: - abs_path = find_repo(LEGACY_REPO) - repo = migrate.versioning.repository.Repository(abs_path) - if int(repo.latest) != get_db_version(): - raise db_exception.DBMigrationError( - 'Your Legacy repo version is not up to date. Please refer to ' - 'https://docs.openstack.org/keystone/latest/admin/' - 'identity-upgrading.html ' - 'to see the proper steps for rolling upgrades.') + if repo_name == EXPAND_REPO: return # find the latest version that the current command will upgrade to if there @@ -295,9 +272,6 @@ def expand_schema(): keystone node is migrated to the latest release. """ - # Make sure all the legacy migrations are run before we run any new - # expand migrations. - _sync_common_repo(version=None) validate_upgrade_order(EXPAND_REPO) _sync_repo(repo_name=EXPAND_REPO) diff --git a/keystone/tests/unit/test_sql_banned_operations.py b/keystone/tests/unit/test_sql_banned_operations.py index 2207140753..3b26a69068 100644 --- a/keystone/tests/unit/test_sql_banned_operations.py +++ b/keystone/tests/unit/test_sql_banned_operations.py @@ -28,7 +28,6 @@ import testtools from keystone.common.sql import contract_repo from keystone.common.sql import data_migration_repo from keystone.common.sql import expand_repo -from keystone.common.sql import migrate_repo from keystone.common.sql import upgrades @@ -39,9 +38,8 @@ class DBOperationNotAllowed(Exception): class BannedDBSchemaOperations(fixtures.Fixture): """Ban some operations for migrations.""" - def __init__(self, banned_ops=None, - migration_repo=migrate_repo.__file__): - super(BannedDBSchemaOperations, self).__init__() + def __init__(self, banned_ops, migration_repo): + super().__init__() self._banned_ops = banned_ops or {} self._migration_repo = migration_repo @@ -54,7 +52,7 @@ class BannedDBSchemaOperations(fixtures.Fixture): resource_op, repo_name)) def setUp(self): - super(BannedDBSchemaOperations, self).setUp() + super().setUp() explode_lambda = { 'Table.create': lambda *a, **k: self._explode( 'Table.create', self._migration_repo), @@ -91,7 +89,9 @@ class TestBannedDBSchemaOperations(testtools.TestCase): """Test column operations raise DBOperationNotAllowed.""" column = sqlalchemy.Column() with BannedDBSchemaOperations( - banned_ops={'Column': ['create', 'alter', 'drop']}): + banned_ops={'Column': ['create', 'alter', 'drop']}, + migration_repo=expand_repo.__file__, + ): self.assertRaises(DBOperationNotAllowed, column.drop) self.assertRaises(DBOperationNotAllowed, column.alter) self.assertRaises(DBOperationNotAllowed, column.create) @@ -100,8 +100,10 @@ class TestBannedDBSchemaOperations(testtools.TestCase): """Test table operations raise DBOperationNotAllowed.""" table = sqlalchemy.Table() with BannedDBSchemaOperations( - banned_ops={'Table': ['create', 'alter', 'drop', - 'insert', 'update', 'delete']}): + banned_ops={'Table': ['create', 'alter', 'drop', + 'insert', 'update', 'delete']}, + migration_repo=expand_repo.__file__, + ): self.assertRaises(DBOperationNotAllowed, table.drop) self.assertRaises(DBOperationNotAllowed, table.alter) self.assertRaises(DBOperationNotAllowed, table.create) @@ -113,29 +115,14 @@ class TestBannedDBSchemaOperations(testtools.TestCase): class KeystoneMigrationsCheckers(test_migrations.WalkVersionsMixin): """Walk over and test all sqlalchemy-migrate migrations.""" - # NOTE(xek): We start requiring things be additive in Newton, so - # ignore all migrations before the first version in Newton. - migrate_file = migrate_repo.__file__ - first_version = 101 - # NOTE(henry-nash): We don't ban data modification in the legacy repo, - # since there are already migrations that do this for Newton (and these - # do not cause us issues, or are already worked around). - banned_ops = {'Table': ['alter', 'drop'], - 'Column': ['alter', 'drop']} + migrate_file = None + first_version = 1 + # A mapping of entity (Table, Column, ...) to operation + banned_ops = {} exceptions = [ # NOTE(xek): Reviewers: DO NOT ALLOW THINGS TO BE ADDED HERE UNLESS # JUSTIFICATION CAN BE PROVIDED AS TO WHY THIS WILL NOT CAUSE # PROBLEMS FOR ROLLING UPGRADES. - - # Migration 102 drops the domain table in the Newton release. All - # code that referenced the domain table was removed in the Mitaka - # release, hence this migration will not cause problems when - # running a mixture of Mitaka and Newton versions of keystone. - 102, - - # Migration 106 simply allows the password column to be nullable. - # This change would not impact a rolling upgrade. - 106 ] @property @@ -190,8 +177,7 @@ class KeystoneMigrationsCheckers(test_migrations.WalkVersionsMixin): else: banned_ops = None with BannedDBSchemaOperations(banned_ops, self.migrate_file): - super(KeystoneMigrationsCheckers, - self).migrate_up(version, with_data) + super().migrate_up(version, with_data) snake_walk = False downgrade = False @@ -200,43 +186,7 @@ class KeystoneMigrationsCheckers(test_migrations.WalkVersionsMixin): self.walk_versions(self.snake_walk, self.downgrade) -class TestKeystoneMigrationsMySQL( - KeystoneMigrationsCheckers, - db_fixtures.OpportunisticDBTestMixin, - test_base.BaseTestCase): - FIXTURE = db_fixtures.MySQLOpportunisticFixture - - def setUp(self): - super(TestKeystoneMigrationsMySQL, self).setUp() - self.engine = enginefacade.writer.get_engine() - self.sessionmaker = enginefacade.writer.get_sessionmaker() - - -class TestKeystoneMigrationsPostgreSQL( - KeystoneMigrationsCheckers, - db_fixtures.OpportunisticDBTestMixin, - test_base.BaseTestCase): - FIXTURE = db_fixtures.PostgresqlOpportunisticFixture - - def setUp(self): - super(TestKeystoneMigrationsPostgreSQL, self).setUp() - self.engine = enginefacade.writer.get_engine() - self.sessionmaker = enginefacade.writer.get_sessionmaker() - - -class TestKeystoneMigrationsSQLite( - KeystoneMigrationsCheckers, - db_fixtures.OpportunisticDBTestMixin, - test_base.BaseTestCase): - - def setUp(self): - super(TestKeystoneMigrationsSQLite, self).setUp() - self.engine = enginefacade.writer.get_engine() - self.sessionmaker = enginefacade.writer.get_sessionmaker() - - -class TestKeystoneExpandSchemaMigrations( - KeystoneMigrationsCheckers): +class TestKeystoneExpandSchemaMigrations(KeystoneMigrationsCheckers): migrate_file = expand_repo.__file__ first_version = 1 @@ -285,7 +235,6 @@ class TestKeystoneExpandSchemaMigrationsMySQL( super(TestKeystoneExpandSchemaMigrationsMySQL, self).setUp() self.engine = enginefacade.writer.get_engine() self.sessionmaker = enginefacade.writer.get_sessionmaker() - self.migrate_fully(migrate_repo.__file__) class TestKeystoneExpandSchemaMigrationsPostgreSQL( @@ -298,7 +247,6 @@ class TestKeystoneExpandSchemaMigrationsPostgreSQL( super(TestKeystoneExpandSchemaMigrationsPostgreSQL, self).setUp() self.engine = enginefacade.writer.get_engine() self.sessionmaker = enginefacade.writer.get_sessionmaker() - self.migrate_fully(migrate_repo.__file__) class TestKeystoneDataMigrations( @@ -326,7 +274,6 @@ class TestKeystoneDataMigrations( def setUp(self): super(TestKeystoneDataMigrations, self).setUp() - self.migrate_fully(migrate_repo.__file__) self.migrate_fully(expand_repo.__file__) @@ -387,7 +334,6 @@ class TestKeystoneContractSchemaMigrations( def setUp(self): super(TestKeystoneContractSchemaMigrations, self).setUp() - self.migrate_fully(migrate_repo.__file__) self.migrate_fully(expand_repo.__file__) self.migrate_fully(data_migration_repo.__file__) diff --git a/keystone/tests/unit/test_sql_upgrade.py b/keystone/tests/unit/test_sql_upgrade.py index b37b80d1c8..fd038105e4 100644 --- a/keystone/tests/unit/test_sql_upgrade.py +++ b/keystone/tests/unit/test_sql_upgrade.py @@ -194,7 +194,6 @@ INITIAL_TABLE_STRUCTURE = { ], } -LEGACY_REPO = 'migrate_repo' EXPAND_REPO = 'expand_repo' DATA_MIGRATION_REPO = 'data_migration_repo' CONTRACT_REPO = 'contract_repo' @@ -222,7 +221,7 @@ class SqlUpgradeGetInitVersionTests(unit.TestCase): # first invocation of repo. Cannot match the full path because it is # based on where the test is run. param = repo.call_args_list[0][0][0] - self.assertTrue(param.endswith('/sql/' + LEGACY_REPO)) + self.assertTrue(param.endswith('/sql/' + EXPAND_REPO)) @mock.patch.object(repository, 'Repository') def test_get_init_version_with_path_initial_version_0(self, repo): @@ -235,7 +234,7 @@ class SqlUpgradeGetInitVersionTests(unit.TestCase): # os.path.isdir() is called by `find_repo()`. Mock it to avoid # an exception. with mock.patch('os.path.isdir', return_value=True): - path = '/keystone/' + LEGACY_REPO + '/' + path = '/keystone/' + EXPAND_REPO + '/' # since 0 is the smallest version expect None version = upgrades.get_init_version(abs_path=path) @@ -253,16 +252,18 @@ class SqlUpgradeGetInitVersionTests(unit.TestCase): # os.path.isdir() is called by `find_repo()`. Mock it to avoid # an exception. with mock.patch('os.path.isdir', return_value=True): - path = '/keystone/' + LEGACY_REPO + '/' + path = '/keystone/' + EXPAND_REPO + '/' version = upgrades.get_init_version(abs_path=path) self.assertEqual(initial_version, version) -class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin, - test_base.BaseTestCase): +class MigrateBase( + db_fixtures.OpportunisticDBTestMixin, + test_base.BaseTestCase, +): def setUp(self): - super(SqlMigrateBase, self).setUp() + super().setUp() self.engine = enginefacade.writer.get_engine() self.sessionmaker = enginefacade.writer.get_sessionmaker() @@ -284,15 +285,12 @@ class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin, self.addCleanup(sql.cleanup) self.repos = { - LEGACY_REPO: upgrades.Repository(self.engine, LEGACY_REPO), EXPAND_REPO: upgrades.Repository(self.engine, EXPAND_REPO), DATA_MIGRATION_REPO: upgrades.Repository( - self.engine, DATA_MIGRATION_REPO), - CONTRACT_REPO: upgrades.Repository(self.engine, CONTRACT_REPO)} - - def upgrade(self, *args, **kwargs): - """Upgrade the legacy migration repository.""" - self.repos[LEGACY_REPO].upgrade(*args, **kwargs) + self.engine, DATA_MIGRATION_REPO, + ), + CONTRACT_REPO: upgrades.Repository(self.engine, CONTRACT_REPO), + } def expand(self, *args, **kwargs): """Expand database schema.""" @@ -403,21 +401,18 @@ class SqlMigrateBase(db_fixtures.OpportunisticDBTestMixin, return False -class SqlLegacyRepoUpgradeTests(SqlMigrateBase): +class ExpandSchemaUpgradeTests(MigrateBase): + + def test_start_version_db_init_version(self): + self.assertEqual( + self.repos[EXPAND_REPO].min_version, + self.repos[EXPAND_REPO].version) def test_blank_db_to_start(self): self.assertTableDoesNotExist('user') - def test_start_version_db_init_version(self): - self.assertEqual( - self.repos[LEGACY_REPO].min_version, - self.repos[LEGACY_REPO].version, - 'DB is not at version %s' % ( - self.repos[LEGACY_REPO].min_version) - ) - def test_upgrade_add_initial_tables(self): - self.upgrade(self.repos[LEGACY_REPO].min_version + 1) + self.expand(1) self.check_initial_table_structure() def check_initial_table_structure(self): @@ -425,45 +420,24 @@ class SqlLegacyRepoUpgradeTests(SqlMigrateBase): self.assertTableColumns(table, INITIAL_TABLE_STRUCTURE[table]) -class MySQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests): - FIXTURE = db_fixtures.MySQLOpportunisticFixture - - -class PostgreSQLOpportunisticUpgradeTestCase(SqlLegacyRepoUpgradeTests): - FIXTURE = db_fixtures.PostgresqlOpportunisticFixture - - -class SqlExpandSchemaUpgradeTests(SqlMigrateBase): - - def setUp(self): - # Make sure the main repo is fully upgraded for this release since the - # expand phase is only run after such an upgrade - super(SqlExpandSchemaUpgradeTests, self).setUp() - self.upgrade() - - def test_start_version_db_init_version(self): - self.assertEqual( - self.repos[EXPAND_REPO].min_version, - self.repos[EXPAND_REPO].version) - - class MySQLOpportunisticExpandSchemaUpgradeTestCase( - SqlExpandSchemaUpgradeTests): + ExpandSchemaUpgradeTests, +): FIXTURE = db_fixtures.MySQLOpportunisticFixture class PostgreSQLOpportunisticExpandSchemaUpgradeTestCase( - SqlExpandSchemaUpgradeTests): + ExpandSchemaUpgradeTests, +): FIXTURE = db_fixtures.PostgresqlOpportunisticFixture -class SqlDataMigrationUpgradeTests(SqlMigrateBase): +class DataMigrationUpgradeTests(MigrateBase): def setUp(self): - # Make sure the legacy and expand repos are fully upgraded, since the - # data migration phase is only run after these are upgraded - super(SqlDataMigrationUpgradeTests, self).setUp() - self.upgrade() + # Make sure the expand repo is fully upgraded, since the data migration + # phase is only run after this is upgraded + super().setUp() self.expand() def test_start_version_db_init_version(self): @@ -473,22 +447,24 @@ class SqlDataMigrationUpgradeTests(SqlMigrateBase): class MySQLOpportunisticDataMigrationUpgradeTestCase( - SqlDataMigrationUpgradeTests): + DataMigrationUpgradeTests, +): FIXTURE = db_fixtures.MySQLOpportunisticFixture class PostgreSQLOpportunisticDataMigrationUpgradeTestCase( - SqlDataMigrationUpgradeTests): + DataMigrationUpgradeTests, +): FIXTURE = db_fixtures.PostgresqlOpportunisticFixture -class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase): +class ContractSchemaUpgradeTests(MigrateBase, unit.TestCase): def setUp(self): - # Make sure the legacy, expand and data migration repos are fully + # Make sure the expand and data migration repos are fully # upgraded, since the contract phase is only run after these are # upgraded. - super(SqlContractSchemaUpgradeTests, self).setUp() + super().setUp() self.useFixture( ksfixtures.KeyRepository( self.config_fixture, @@ -496,7 +472,6 @@ class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase): credential_fernet.MAX_ACTIVE_KEYS ) ) - self.upgrade() self.expand() self.migrate() @@ -507,53 +482,18 @@ class SqlContractSchemaUpgradeTests(SqlMigrateBase, unit.TestCase): class MySQLOpportunisticContractSchemaUpgradeTestCase( - SqlContractSchemaUpgradeTests): + ContractSchemaUpgradeTests, +): FIXTURE = db_fixtures.MySQLOpportunisticFixture class PostgreSQLOpportunisticContractSchemaUpgradeTestCase( - SqlContractSchemaUpgradeTests): + ContractSchemaUpgradeTests, +): FIXTURE = db_fixtures.PostgresqlOpportunisticFixture -class VersionTests(SqlMigrateBase): - def test_core_initial(self): - """Get the version before migrated, it's the initial DB version.""" - self.assertEqual( - self.repos[LEGACY_REPO].min_version, - self.repos[LEGACY_REPO].version) - - def test_core_max(self): - """When get the version after upgrading, it's the new version.""" - self.upgrade() - self.assertEqual( - self.repos[LEGACY_REPO].max_version, - self.repos[LEGACY_REPO].version) - - def test_assert_not_schema_downgrade(self): - self.upgrade() - self.assertRaises( - db_exception.DBMigrationError, - upgrades._sync_common_repo, - self.repos[LEGACY_REPO].max_version - 1) - - def test_these_are_not_the_migrations_you_are_looking_for(self): - """Keystone has shifted to rolling upgrades. - - New database migrations should no longer land in the legacy migration - repository. Instead, new database migrations should be divided into - three discrete steps: schema expansion, data migration, and schema - contraction. These migrations live in a new set of database migration - repositories, called ``expand_repo``, ``data_migration_repo``, and - ``contract_repo``. - - For more information, see "Database Migrations" here: - - https://docs.openstack.org/keystone/latest/contributor/database-migrations.html - - """ - # Note to reviewers: this version number should never change. - self.assertEqual(109, self.repos[LEGACY_REPO].max_version) +class VersionTests(MigrateBase): def test_migrate_repos_stay_in_lockstep(self): """Rolling upgrade repositories should always stay in lockstep. @@ -618,7 +558,7 @@ class VersionTests(SqlMigrateBase): self.assertRegex(file_name, pattern, msg) -class MigrationValidation(SqlMigrateBase, unit.TestCase): +class MigrationValidation(MigrateBase, unit.TestCase): """Test validation of database between database phases.""" def _set_db_sync_command_versions(self): @@ -630,7 +570,6 @@ class MigrationValidation(SqlMigrateBase, unit.TestCase): self.assertEqual(upgrades.get_db_version('contract_repo'), 1) def test_running_db_sync_migrate_ahead_of_expand_fails(self): - self.upgrade() self._set_db_sync_command_versions() self.assertRaises( db_exception.DBMigrationError, @@ -639,7 +578,6 @@ class MigrationValidation(SqlMigrateBase, unit.TestCase): "You are attempting to upgrade migrate ahead of expand") def test_running_db_sync_contract_ahead_of_migrate_fails(self): - self.upgrade() self._set_db_sync_command_versions() self.assertRaises( db_exception.DBMigrationError, @@ -648,14 +586,9 @@ class MigrationValidation(SqlMigrateBase, unit.TestCase): "You are attempting to upgrade contract ahead of migrate") -class FullMigration(SqlMigrateBase, unit.TestCase): +class FullMigration(MigrateBase, unit.TestCase): """Test complete orchestration between all database phases.""" - def setUp(self): - super(FullMigration, self).setUp() - # Upgrade the legacy repository - self.upgrade() - def test_db_sync_check(self): checker = cli.DbSync() latest_version = self.repos[EXPAND_REPO].max_version diff --git a/releasenotes/notes/remove-legacy-migrations-647f60019c8dd9e8.yaml b/releasenotes/notes/remove-legacy-migrations-647f60019c8dd9e8.yaml new file mode 100644 index 0000000000..11f33886be --- /dev/null +++ b/releasenotes/notes/remove-legacy-migrations-647f60019c8dd9e8.yaml @@ -0,0 +1,7 @@ +--- +upgrade: + - | + The legacy migrations that existed before the split into separate expand + schema, contract schema, and data migration migration have now been + removed. These have been deprecated since 10.0.0 (Newton). This should + have no user-facing impact.