From 8c9462f6fa65fb615af4d4bfe8afae0d0143a644 Mon Sep 17 00:00:00 2001 From: Stephen Finucane Date: Tue, 28 Feb 2023 13:17:31 +0000 Subject: [PATCH] db: Remove legacy migrations sqlalchemy-migrate does not (and will not) support sqlalchemy 2.0. We need to drop these migrations to ensure we can upgrade our sqlalchemy version. Change-Id: I31ba9e4f129a7cc28744e814b5fd28eb284ae3de Signed-off-by: Stephen Finucane --- keystone/cmd/cli.py | 8 - .../common/sql/legacy_migrations/__init__.py | 0 .../contract_repo/README.rst | 13 - .../contract_repo/__init__.py | 0 .../legacy_migrations/contract_repo/manage.py | 18 - .../contract_repo/migrate.cfg | 25 - .../073_contract_initial_migration.py | 18 - .../contract_repo/versions/074_placeholder.py | 18 - .../contract_repo/versions/075_placeholder.py | 18 - .../contract_repo/versions/076_placeholder.py | 18 - .../contract_repo/versions/077_placeholder.py | 18 - .../contract_repo/versions/078_placeholder.py | 18 - .../079_contract_update_local_id_limit.py | 18 - .../contract_repo/versions/__init__.py | 0 .../data_migration_repo/README.rst | 13 - .../data_migration_repo/__init__.py | 0 .../data_migration_repo/manage.py | 18 - .../data_migration_repo/migrate.cfg | 25 - .../versions/073_migrate_initial_migration.py | 59 - .../versions/074_placeholder.py | 18 - .../versions/075_placeholder.py | 18 - .../versions/076_placeholder.py | 18 - .../versions/077_placeholder.py | 18 - .../versions/078_placeholder.py | 18 - .../079_migrate_update_local_id_limit.py | 18 - .../data_migration_repo/versions/__init__.py | 0 .../legacy_migrations/expand_repo/README.rst | 13 - .../legacy_migrations/expand_repo/__init__.py | 15 - .../legacy_migrations/expand_repo/manage.py | 18 - .../legacy_migrations/expand_repo/migrate.cfg | 25 - .../versions/073_expand_initial_migration.py | 1190 ----------------- .../expand_repo/versions/074_placeholder.py | 18 - .../expand_repo/versions/075_placeholder.py | 18 - .../expand_repo/versions/076_placeholder.py | 18 - .../expand_repo/versions/077_placeholder.py | 18 - .../expand_repo/versions/078_placeholder.py | 18 - .../079_expand_update_local_id_limit.py | 28 - .../expand_repo/versions/__init__.py | 15 - keystone/common/sql/upgrades.py | 79 +- .../tests/unit/common/sql/test_upgrades.py | 47 - keystone/tests/unit/test_sql_upgrade.py | 6 - ...e-sqlalchemy-migrate-a4fa47685c7e28c6.yaml | 5 + requirements.txt | 1 - tools/generate-schemas | 134 -- 44 files changed, 8 insertions(+), 2071 deletions(-) delete mode 100644 keystone/common/sql/legacy_migrations/__init__.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/README.rst delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/__init__.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/manage.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/migrate.cfg delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/073_contract_initial_migration.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/074_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/075_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/076_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/077_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/078_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/079_contract_update_local_id_limit.py delete mode 100644 keystone/common/sql/legacy_migrations/contract_repo/versions/__init__.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/README.rst delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/__init__.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/manage.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/migrate.cfg delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/073_migrate_initial_migration.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/074_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/075_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/076_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/077_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/078_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/079_migrate_update_local_id_limit.py delete mode 100644 keystone/common/sql/legacy_migrations/data_migration_repo/versions/__init__.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/README.rst delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/__init__.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/manage.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/migrate.cfg delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/073_expand_initial_migration.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/074_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/075_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/076_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/077_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/078_placeholder.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/079_expand_update_local_id_limit.py delete mode 100644 keystone/common/sql/legacy_migrations/expand_repo/versions/__init__.py create mode 100644 releasenotes/notes/remove-sqlalchemy-migrate-a4fa47685c7e28c6.yaml delete mode 100755 tools/generate-schemas diff --git a/keystone/cmd/cli.py b/keystone/cmd/cli.py index ad65b26229..f74ae6a441 100644 --- a/keystone/cmd/cli.py +++ b/keystone/cmd/cli.py @@ -286,14 +286,6 @@ class DbSync(BaseApp): ) return 2 - if isinstance(expand_version, int): - # we're still using sqlalchemy-migrate - LOG.info( - 'Your database is currently using legacy version control. ' - 'Your first step is to run `keystone-manage db_sync --expand`.' - ) - return 2 - try: contract_version = upgrades.get_db_version(branch='contract') except db_exception.DBMigrationError: diff --git a/keystone/common/sql/legacy_migrations/__init__.py b/keystone/common/sql/legacy_migrations/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/keystone/common/sql/legacy_migrations/contract_repo/README.rst b/keystone/common/sql/legacy_migrations/contract_repo/README.rst deleted file mode 100644 index 6ecb178cca..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -Contract repo migrations -======================== - -.. warning:: - - This repo is deprecated and will be removed in a future release. All new - migrations should be alembic-based and placed in - ``keystone/common/sql/migrations``. - -Contract-style or destructive migrations for the database. - -This is a database migration repository. More information at -https://opendev.org/x/sqlalchemy-migrate diff --git a/keystone/common/sql/legacy_migrations/contract_repo/__init__.py b/keystone/common/sql/legacy_migrations/contract_repo/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/keystone/common/sql/legacy_migrations/contract_repo/manage.py b/keystone/common/sql/legacy_migrations/contract_repo/manage.py deleted file mode 100644 index 41cba1adbd..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/manage.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(debug='False') diff --git a/keystone/common/sql/legacy_migrations/contract_repo/migrate.cfg b/keystone/common/sql/legacy_migrations/contract_repo/migrate.cfg deleted file mode 100644 index fd50aa5462..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=keystone_contract - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/073_contract_initial_migration.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/073_contract_initial_migration.py deleted file mode 100644 index 1cd34e6171..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/versions/073_contract_initial_migration.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# A null initial migration to open this repo. Do not re-use replace this with -# a real migration, add additional ones in subsequent version scripts. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/074_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/074_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/versions/074_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/075_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/075_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/versions/075_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/076_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/076_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/versions/076_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/077_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/077_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/versions/077_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/078_placeholder.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/078_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/versions/078_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/079_contract_update_local_id_limit.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/079_contract_update_local_id_limit.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/contract_repo/versions/079_contract_update_local_id_limit.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/contract_repo/versions/__init__.py b/keystone/common/sql/legacy_migrations/contract_repo/versions/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/README.rst b/keystone/common/sql/legacy_migrations/data_migration_repo/README.rst deleted file mode 100644 index 0b4202f9f0..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -Data migration repo migrations -============================== - -.. warning:: - - This repo is deprecated and will be removed in a future release. All new - migrations should be alembic-based and placed in - ``keystone/common/sql/migrations``. - -Data migrations for the database. - -This is a database migration repository. More information at -https://opendev.org/x/sqlalchemy-migrate diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/__init__.py b/keystone/common/sql/legacy_migrations/data_migration_repo/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/manage.py b/keystone/common/sql/legacy_migrations/data_migration_repo/manage.py deleted file mode 100644 index 41cba1adbd..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/manage.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(debug='False') diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/migrate.cfg b/keystone/common/sql/legacy_migrations/data_migration_repo/migrate.cfg deleted file mode 100644 index 97f8e1d0e6..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=keystone_data_migrate - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/073_migrate_initial_migration.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/073_migrate_initial_migration.py deleted file mode 100644 index e7f552a19b..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/073_migrate_initial_migration.py +++ /dev/null @@ -1,59 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# A null initial migration to open this repo. Do not re-use replace this with -# a real migration, add additional ones in subsequent version scripts. - -import sqlalchemy as sql -import sqlalchemy.orm - -NULL_DOMAIN_ID = '<>' - - -def upgrade(migrate_engine): - - def _generate_root_domain_project(): - # Generate a project that will act as a root for all domains, in order - # for use to be able to use a FK constraint on domain_id. Projects - # acting as a domain will not reference this as their parent_id, just - # as domain_id. - # - # This special project is filtered out by the driver, so is never - # visible to the manager or API. - - project_ref = { - 'id': NULL_DOMAIN_ID, - 'name': NULL_DOMAIN_ID, - 'enabled': False, - 'description': '', - 'domain_id': NULL_DOMAIN_ID, - 'is_domain': True, - 'parent_id': None, - 'extra': '{}', - } - return project_ref - - meta = sql.MetaData() - # NOTE(stephenfin): This is not compatible with SQLAlchemy 2.0 but neither - # is sqlalchemy-migrate which requires this. We'll remove these migrations - # when dropping SQLAlchemy < 2.x support - meta.bind = migrate_engine - session = sql.orm.sessionmaker(bind=migrate_engine)() - - project = sql.Table('project', meta, autoload_with=migrate_engine) - - root_domain_project = _generate_root_domain_project() - new_entry = project.insert().values(**root_domain_project) - session.execute(new_entry) - session.commit() - - session.close() diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/074_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/074_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/074_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/075_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/075_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/075_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/076_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/076_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/076_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/077_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/077_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/077_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/078_placeholder.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/078_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/078_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/079_migrate_update_local_id_limit.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/079_migrate_update_local_id_limit.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/079_migrate_update_local_id_limit.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/data_migration_repo/versions/__init__.py b/keystone/common/sql/legacy_migrations/data_migration_repo/versions/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/keystone/common/sql/legacy_migrations/expand_repo/README.rst b/keystone/common/sql/legacy_migrations/expand_repo/README.rst deleted file mode 100644 index 6019e2d6c7..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/README.rst +++ /dev/null @@ -1,13 +0,0 @@ -Expand repo migrations -====================== - -.. warning:: - - This repo is deprecated and will be removed in a future release. All new - migrations should be alembic-based and placed in - ``keystone/common/sql/migrations``. - -Expand-style or additive migrations for the database. - -This is a database migration repository. More information at -https://opendev.org/x/sqlalchemy-migrate diff --git a/keystone/common/sql/legacy_migrations/expand_repo/__init__.py b/keystone/common/sql/legacy_migrations/expand_repo/__init__.py deleted file mode 100644 index 84e0fb83bf..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from keystone.common.sql.core import * # noqa diff --git a/keystone/common/sql/legacy_migrations/expand_repo/manage.py b/keystone/common/sql/legacy_migrations/expand_repo/manage.py deleted file mode 100644 index 41cba1adbd..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/manage.py +++ /dev/null @@ -1,18 +0,0 @@ -#!/usr/bin/env python - -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from migrate.versioning.shell import main - -if __name__ == '__main__': - main(debug='False') diff --git a/keystone/common/sql/legacy_migrations/expand_repo/migrate.cfg b/keystone/common/sql/legacy_migrations/expand_repo/migrate.cfg deleted file mode 100644 index 74a33e330b..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/migrate.cfg +++ /dev/null @@ -1,25 +0,0 @@ -[db_settings] -# Used to identify which repository this database is versioned under. -# You can use the name of your project. -repository_id=keystone_expand - -# The name of the database table used to track the schema version. -# This name shouldn't already be used by your project. -# If this is changed once a database is under version control, you'll need to -# change the table name in each database too. -version_table=migrate_version - -# When committing a change script, Migrate will attempt to generate the -# sql for all supported databases; normally, if one of them fails - probably -# because you don't have that database installed - it is ignored and the -# commit continues, perhaps ending successfully. -# Databases in this list MUST compile successfully during a commit, or the -# entire commit will fail. List the databases your application will actually -# be using to ensure your updates to that database work properly. -# This must be a list; example: ['postgres','sqlite'] -required_dbs=[] - -# When creating new change scripts, Migrate will stamp the new script with -# a version number. By default this is latest_version + 1. You can set this -# to 'true' to tell Migrate to use the UTC timestamp instead. -use_timestamp_numbering=False diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/073_expand_initial_migration.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/073_expand_initial_migration.py deleted file mode 100644 index 9fcf26b670..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/073_expand_initial_migration.py +++ /dev/null @@ -1,1190 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import datetime -import textwrap - -import migrate -from oslo_log import log -import sqlalchemy as sql - -from keystone.assignment.backends import sql as assignment_sql -from keystone.common import sql as ks_sql -import keystone.conf -from keystone.identity.mapping_backends import mapping as mapping_backend - -CONF = keystone.conf.CONF -LOG = log.getLogger(__name__) - -# FIXME(stephenfin): Remove this as soon as we're done reworking the -# migrations. Until then, this is necessary to allow us to use the native -# sqlalchemy-migrate tooling (which won't register opts). Alternatively, maybe -# the server default *shouldn't* rely on a (changeable) config option value? -try: - service_provider_relay_state_prefix_default = CONF.saml.relay_state_prefix -except Exception: - service_provider_relay_state_prefix_default = 'ss:mem:' - - -def upgrade(migrate_engine): - meta = sql.MetaData() - - # NOTE(stephenfin): This is not compatible with SQLAlchemy 2.0 but neither - # is sqlalchemy-migrate which requires this. We'll remove these migrations - # when dropping SQLAlchemy < 2.x support - meta.bind = migrate_engine - - if migrate_engine.name == 'mysql': - # In Folsom we explicitly converted migrate_version to UTF8. - with migrate_engine.connect() as conn: - conn.exec_driver_sql( - 'ALTER TABLE migrate_version CONVERT TO CHARACTER SET utf8' - ) - # Set default DB charset to UTF8. - conn.exec_driver_sql( - 'ALTER DATABASE %s DEFAULT CHARACTER SET utf8' - % migrate_engine.url.database - ) - - application_credential = sql.Table( - 'application_credential', - meta, - sql.Column( - 'internal_id', sql.Integer, primary_key=True, nullable=False - ), - sql.Column('id', sql.String(length=64), nullable=False), - sql.Column('name', sql.String(length=255), nullable=False), - sql.Column('secret_hash', sql.String(length=255), nullable=False), - sql.Column('description', sql.Text), - sql.Column('user_id', sql.String(length=64), nullable=False), - sql.Column('project_id', sql.String(64), nullable=True), - sql.Column('expires_at', ks_sql.DateTimeInt()), - sql.Column('system', sql.String(64), nullable=True), - sql.Column('unrestricted', sql.Boolean), - sql.UniqueConstraint( - 'user_id', 'name', name='duplicate_app_cred_constraint' - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - assignment = sql.Table( - 'assignment', - meta, - sql.Column( - 'type', - sql.Enum( - assignment_sql.AssignmentType.USER_PROJECT, - assignment_sql.AssignmentType.GROUP_PROJECT, - assignment_sql.AssignmentType.USER_DOMAIN, - assignment_sql.AssignmentType.GROUP_DOMAIN, - name='type', - ), - nullable=False, - ), - sql.Column('actor_id', sql.String(64), nullable=False), - sql.Column('target_id', sql.String(64), nullable=False), - sql.Column('role_id', sql.String(64), nullable=False), - sql.Column('inherited', sql.Boolean, default=False, nullable=False), - sql.PrimaryKeyConstraint( - 'type', - 'actor_id', - 'target_id', - 'role_id', - 'inherited', - ), - sql.Index('ix_actor_id', 'actor_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - access_rule = sql.Table( - 'access_rule', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column('service', sql.String(64)), - sql.Column('path', sql.String(128)), - sql.Column('method', sql.String(16)), - sql.Column('external_id', sql.String(64)), - sql.Column('user_id', sql.String(64)), - sql.UniqueConstraint( - 'external_id', - name='access_rule_external_id_key', - ), - sql.UniqueConstraint( - 'user_id', - 'service', - 'path', - 'method', - name='duplicate_access_rule_for_user_constraint', - ), - sql.Index('user_id', 'user_id'), - sql.Index('external_id', 'external_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - config_register = sql.Table( - 'config_register', - meta, - sql.Column('type', sql.String(64), primary_key=True), - sql.Column('domain_id', sql.String(64), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - consumer = sql.Table( - 'consumer', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('description', sql.String(64), nullable=True), - sql.Column('secret', sql.String(64), nullable=False), - sql.Column('extra', sql.Text(), nullable=False), - ) - - credential = sql.Table( - 'credential', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('user_id', sql.String(length=64), nullable=False), - sql.Column('project_id', sql.String(length=64)), - sql.Column('type', sql.String(length=255), nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('key_hash', sql.String(64), nullable=False), - sql.Column( - 'encrypted_blob', - ks_sql.Text, - nullable=False, - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - group = sql.Table( - 'group', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('domain_id', sql.String(length=64), nullable=False), - sql.Column('name', sql.String(length=64), nullable=False), - sql.Column('description', sql.Text), - sql.Column('extra', ks_sql.JsonBlob.impl), - migrate.UniqueConstraint( - 'domain_id', - 'name', - name='ixu_group_name_domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - id_mapping = sql.Table( - 'id_mapping', - meta, - sql.Column('public_id', sql.String(64), primary_key=True), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.Column('local_id', sql.String(64), nullable=False), - sql.Column( - 'entity_type', - sql.Enum( - mapping_backend.EntityType.USER, - mapping_backend.EntityType.GROUP, - name='entity_type', - ), - nullable=False, - ), - migrate.UniqueConstraint( - 'domain_id', - 'local_id', - 'entity_type', - name='domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - identity_provider = sql.Table( - 'identity_provider', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('enabled', sql.Boolean, nullable=False), - sql.Column('description', sql.Text(), nullable=True), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.Column('authorization_ttl', sql.Integer, nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - idp_remote_ids = sql.Table( - 'idp_remote_ids', - meta, - sql.Column( - 'idp_id', - sql.String(64), - sql.ForeignKey(identity_provider.c.id, ondelete='CASCADE'), - ), - sql.Column('remote_id', sql.String(255), primary_key=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - mapping = sql.Table( - 'mapping', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('rules', sql.Text(), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - policy = sql.Table( - 'policy', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('type', sql.String(length=255), nullable=False), - sql.Column('blob', ks_sql.JsonBlob, nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - policy_association = sql.Table( - 'policy_association', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('policy_id', sql.String(64), nullable=False), - sql.Column('endpoint_id', sql.String(64), nullable=True), - sql.Column('service_id', sql.String(64), nullable=True), - sql.Column('region_id', sql.String(64), nullable=True), - sql.UniqueConstraint('endpoint_id', 'service_id', 'region_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - project = sql.Table( - 'project', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('name', sql.String(length=64), nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('description', sql.Text), - sql.Column('enabled', sql.Boolean), - sql.Column( - 'domain_id', - sql.String(length=64), - sql.ForeignKey( - 'project.id', - name='project_domain_id_fkey', - ), - nullable=False, - ), - sql.Column( - 'parent_id', - sql.String(64), - sql.ForeignKey( - 'project.id', - name='project_parent_id_fkey', - ), - nullable=True, - ), - sql.Column( - 'is_domain', - sql.Boolean, - nullable=False, - server_default='0', - default=False, - ), - migrate.UniqueConstraint( - 'domain_id', - 'name', - name='ixu_project_name_domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - project_endpoint = sql.Table( - 'project_endpoint', - meta, - sql.Column( - 'endpoint_id', sql.String(64), primary_key=True, nullable=False - ), - sql.Column( - 'project_id', sql.String(64), primary_key=True, nullable=False - ), - ) - - project_option = sql.Table( - 'project_option', - meta, - sql.Column( - 'project_id', - sql.String(64), - sql.ForeignKey(project.c.id, ondelete='CASCADE'), - nullable=False, - primary_key=True, - ), - sql.Column( - 'option_id', sql.String(4), nullable=False, primary_key=True - ), - sql.Column('option_value', ks_sql.JsonBlob, nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - # NOTE(lamt) To allow tag name to be case sensitive for MySQL, the 'name' - # column needs to use collation, which is incompatible with Postgresql. - # Using unicode to mirror nova's server tag: - # https://github.com/openstack/nova/blob/master/nova/db/sqlalchemy/models.py - project_tag = sql.Table( - 'project_tag', - meta, - sql.Column( - 'project_id', - sql.String(64), - sql.ForeignKey(project.c.id, ondelete='CASCADE'), - nullable=False, - primary_key=True, - ), - sql.Column('name', sql.Unicode(255), nullable=False, primary_key=True), - sql.UniqueConstraint('project_id', 'name'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - region = sql.Table( - 'region', - meta, - sql.Column('id', sql.String(255), primary_key=True), - sql.Column('description', sql.String(255), nullable=False), - sql.Column('parent_region_id', sql.String(255), nullable=True), - sql.Column('extra', sql.Text()), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - registered_limit = sql.Table( - 'registered_limit', - meta, - sql.Column('id', sql.String(length=64), nullable=False), - sql.Column('service_id', sql.String(255)), - sql.Column('region_id', sql.String(64), nullable=True), - sql.Column('resource_name', sql.String(255)), - sql.Column('default_limit', sql.Integer, nullable=False), - sql.Column('description', sql.Text), - sql.Column('internal_id', sql.Integer, primary_key=True), - # NOTE(stephenfin): Name chosen to preserve backwards compatibility - # with names used for primary key unique constraints - sql.UniqueConstraint('id', name='registered_limit_id_key'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - request_token = sql.Table( - 'request_token', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('request_secret', sql.String(64), nullable=False), - sql.Column('verifier', sql.String(64), nullable=True), - sql.Column('authorizing_user_id', sql.String(64), nullable=True), - sql.Column('requested_project_id', sql.String(64), nullable=False), - sql.Column('role_ids', sql.Text(), nullable=True), - sql.Column( - 'consumer_id', - sql.String(64), - sql.ForeignKey(consumer.c.id), - nullable=False, - index=True, - ), - sql.Column('expires_at', sql.String(64), nullable=True), - ) - - revocation_event = sql.Table( - 'revocation_event', - meta, - sql.Column('id', sql.Integer, primary_key=True), - sql.Column('domain_id', sql.String(64)), - sql.Column('project_id', sql.String(64)), - sql.Column('user_id', sql.String(64)), - sql.Column('role_id', sql.String(64)), - sql.Column('trust_id', sql.String(64)), - sql.Column('consumer_id', sql.String(64)), - sql.Column('access_token_id', sql.String(64)), - sql.Column('issued_before', sql.DateTime(), nullable=False), - sql.Column('expires_at', sql.DateTime()), - sql.Column('revoked_at', sql.DateTime(), nullable=False), - sql.Column('audit_id', sql.String(32), nullable=True), - sql.Column('audit_chain_id', sql.String(32), nullable=True), - # NOTE(stephenfin): The '_new' suffix here is due to migration 095, - # which changed the 'id' column from String(64) to Integer. It did this - # by creating a 'revocation_event_new' table and populating it with - # data from the 'revocation_event' table before deleting the - # 'revocation_event' table and renaming the 'revocation_event_new' - # table to 'revocation_event'. Because the 'revoked_at' column had - # 'index=True', sqlalchemy automatically generated the index name as - # 'ix_{table}_{column}'. However, when intitially created, '{table}' - # was 'revocation_event_new' so the index got that name. We may wish to - # rename this eventually. - sql.Index('ix_revocation_event_new_revoked_at', 'revoked_at'), - sql.Index('ix_revocation_event_issued_before', 'issued_before'), - sql.Index( - 'ix_revocation_event_project_id_issued_before', - 'project_id', - 'issued_before', - ), - sql.Index( - 'ix_revocation_event_user_id_issued_before', - 'user_id', - 'issued_before', - ), - sql.Index( - 'ix_revocation_event_audit_id_issued_before', - 'audit_id', - 'issued_before', - ), - ) - - role = sql.Table( - 'role', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('name', sql.String(length=255), nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column( - 'domain_id', - sql.String(64), - nullable=False, - server_default='<>', - ), - sql.Column('description', sql.String(255), nullable=True), - migrate.UniqueConstraint( - 'name', - 'domain_id', - name='ixu_role_name_domain_id', - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - role_option = sql.Table( - 'role_option', - meta, - sql.Column( - 'role_id', - sql.String(64), - sql.ForeignKey(role.c.id, ondelete='CASCADE'), - nullable=False, - primary_key=True, - ), - sql.Column( - 'option_id', sql.String(4), nullable=False, primary_key=True - ), - sql.Column('option_value', ks_sql.JsonBlob, nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - sensitive_config = sql.Table( - 'sensitive_config', - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('group', sql.String(255), primary_key=True), - sql.Column('option', sql.String(255), primary_key=True), - sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - service = sql.Table( - 'service', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('type', sql.String(length=255)), - sql.Column( - 'enabled', - sql.Boolean, - nullable=False, - default=True, - server_default='1', - ), - sql.Column('extra', ks_sql.JsonBlob.impl), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - service_provider = sql.Table( - 'service_provider', - meta, - sql.Column('auth_url', sql.String(256), nullable=False), - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('enabled', sql.Boolean, nullable=False), - sql.Column('description', sql.Text(), nullable=True), - sql.Column('sp_url', sql.String(256), nullable=False), - sql.Column( - 'relay_state_prefix', - sql.String(256), - nullable=False, - server_default=service_provider_relay_state_prefix_default, - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - system_assignment = sql.Table( - 'system_assignment', - meta, - sql.Column('type', sql.String(64), nullable=False), - sql.Column('actor_id', sql.String(64), nullable=False), - sql.Column('target_id', sql.String(64), nullable=False), - sql.Column('role_id', sql.String(64), nullable=False), - sql.Column('inherited', sql.Boolean, default=False, nullable=False), - sql.PrimaryKeyConstraint( - 'type', 'actor_id', 'target_id', 'role_id', 'inherited' - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - token = sql.Table( - 'token', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('expires', sql.DateTime, default=None), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('valid', sql.Boolean, default=True, nullable=False), - sql.Column('trust_id', sql.String(length=64)), - sql.Column('user_id', sql.String(length=64)), - sql.Index('ix_token_expires', 'expires'), - sql.Index('ix_token_expires_valid', 'expires', 'valid'), - sql.Index('ix_token_user_id', 'user_id'), - sql.Index('ix_token_trust_id', 'trust_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - trust = sql.Table( - 'trust', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('trustor_user_id', sql.String(length=64), nullable=False), - sql.Column('trustee_user_id', sql.String(length=64), nullable=False), - sql.Column('project_id', sql.String(length=64)), - sql.Column('impersonation', sql.Boolean, nullable=False), - sql.Column('deleted_at', sql.DateTime), - sql.Column('expires_at', sql.DateTime), - sql.Column('remaining_uses', sql.Integer, nullable=True), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('expires_at_int', ks_sql.DateTimeInt()), - sql.UniqueConstraint( - 'trustor_user_id', - 'trustee_user_id', - 'project_id', - 'impersonation', - 'expires_at', - 'expires_at_int', - name='duplicate_trust_constraint_expanded', - ), - sql.Column( - 'redelegated_trust_id', - sql.String(64), - nullable=True, - ), - sql.Column( - 'redelegation_count', - sql.Integer, - nullable=True, - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - trust_role = sql.Table( - 'trust_role', - meta, - sql.Column( - 'trust_id', sql.String(length=64), primary_key=True, nullable=False - ), - sql.Column( - 'role_id', sql.String(length=64), primary_key=True, nullable=False - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - user = sql.Table( - 'user', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column('enabled', sql.Boolean), - sql.Column('default_project_id', sql.String(length=64)), - sql.Column('created_at', sql.DateTime(), nullable=True), - sql.Column('last_active_at', sql.Date(), nullable=True), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.UniqueConstraint('id', 'domain_id', name='ixu_user_id_domain_id'), - sql.Index('ix_default_project_id', 'default_project_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - user_group_membership = sql.Table( - 'user_group_membership', - meta, - sql.Column( - 'user_id', - sql.String(length=64), - sql.ForeignKey( - user.c.id, - name='fk_user_group_membership_user_id', - ), - primary_key=True, - ), - sql.Column( - 'group_id', - sql.String(length=64), - sql.ForeignKey( - group.c.id, - name='fk_user_group_membership_group_id', - ), - primary_key=True, - ), - # NOTE(stevemar): The index was named 'group_id' in - # 050_fk_consistent_indexes.py and needs to be preserved - sql.Index('group_id', 'group_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - user_option = sql.Table( - 'user_option', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey(user.c.id, ondelete='CASCADE'), - nullable=False, - primary_key=True, - ), - sql.Column( - 'option_id', sql.String(4), nullable=False, primary_key=True - ), - sql.Column('option_value', ks_sql.JsonBlob, nullable=True), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - whitelisted_config = sql.Table( - 'whitelisted_config', - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('group', sql.String(255), primary_key=True), - sql.Column('option', sql.String(255), primary_key=True), - sql.Column('value', ks_sql.JsonBlob.impl, nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - access_token = sql.Table( - 'access_token', - meta, - sql.Column('id', sql.String(64), primary_key=True, nullable=False), - sql.Column('access_secret', sql.String(64), nullable=False), - sql.Column( - 'authorizing_user_id', sql.String(64), nullable=False, index=True - ), - sql.Column('project_id', sql.String(64), nullable=False), - sql.Column('role_ids', sql.Text(), nullable=False), - sql.Column( - 'consumer_id', - sql.String(64), - sql.ForeignKey(consumer.c.id), - nullable=False, - index=True, - ), - sql.Column('expires_at', sql.String(64), nullable=True), - ) - - application_credential_role = sql.Table( - 'application_credential_role', - meta, - sql.Column( - 'application_credential_id', - sql.Integer, - sql.ForeignKey( - application_credential.c.internal_id, ondelete='CASCADE' - ), - primary_key=True, - nullable=False, - ), - sql.Column( - 'role_id', sql.String(length=64), primary_key=True, nullable=False - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - application_credential_access_rule = sql.Table( - 'application_credential_access_rule', - meta, - sql.Column( - 'application_credential_id', - sql.Integer, - sql.ForeignKey( - application_credential.c.internal_id, ondelete='CASCADE' - ), - primary_key=True, - nullable=False, - ), - sql.Column( - 'access_rule_id', - sql.Integer, - sql.ForeignKey(access_rule.c.id, ondelete='CASCADE'), - primary_key=True, - nullable=False, - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - endpoint = sql.Table( - 'endpoint', - meta, - sql.Column('id', sql.String(length=64), primary_key=True), - sql.Column('legacy_endpoint_id', sql.String(length=64)), - sql.Column('interface', sql.String(length=8), nullable=False), - sql.Column( - 'service_id', - sql.String(length=64), - sql.ForeignKey( - service.c.id, - name='endpoint_service_id_fkey', - ), - nullable=False, - ), - sql.Column('url', sql.Text, nullable=False), - sql.Column('extra', ks_sql.JsonBlob.impl), - sql.Column( - 'enabled', - sql.Boolean, - nullable=False, - default=True, - server_default='1', - ), - sql.Column( - 'region_id', - sql.String(length=255), - sql.ForeignKey( - region.c.id, - name='fk_endpoint_region_id', - ), - nullable=True, - ), - # NOTE(stevemar): The index was named 'service_id' in - # 050_fk_consistent_indexes.py and needs to be preserved - sql.Index('service_id', 'service_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - endpoint_group = sql.Table( - 'endpoint_group', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(255), nullable=False), - sql.Column('description', sql.Text, nullable=True), - sql.Column('filters', sql.Text(), nullable=False), - ) - - expiring_user_group_membership = sql.Table( - 'expiring_user_group_membership', - meta, - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey(user.c.id), - primary_key=True, - ), - sql.Column( - 'group_id', - sql.String(64), - sql.ForeignKey(group.c.id), - primary_key=True, - ), - sql.Column( - 'idp_id', - sql.String(64), - sql.ForeignKey(identity_provider.c.id, ondelete='CASCADE'), - primary_key=True, - ), - sql.Column('last_verified', sql.DateTime(), nullable=False), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - federation_protocol = sql.Table( - 'federation_protocol', - meta, - sql.Column('id', sql.String(64), primary_key=True), - sql.Column( - 'idp_id', - sql.String(64), - sql.ForeignKey(identity_provider.c.id, ondelete='CASCADE'), - primary_key=True, - ), - sql.Column('mapping_id', sql.String(64), nullable=False), - sql.Column('remote_id_attribute', sql.String(64)), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - implied_role = sql.Table( - 'implied_role', - meta, - sql.Column( - 'prior_role_id', - sql.String(length=64), - sql.ForeignKey( - role.c.id, - name='implied_role_prior_role_id_fkey', - ondelete='CASCADE', - ), - primary_key=True, - ), - sql.Column( - 'implied_role_id', - sql.String(length=64), - sql.ForeignKey( - role.c.id, - name='implied_role_implied_role_id_fkey', - ondelete='CASCADE', - ), - primary_key=True, - ), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - limit = sql.Table( - 'limit', - meta, - sql.Column('id', sql.String(length=64), nullable=False), - sql.Column('project_id', sql.String(64), nullable=True), - sql.Column('resource_limit', sql.Integer, nullable=False), - sql.Column('description', sql.Text), - sql.Column('internal_id', sql.Integer, primary_key=True), - # FIXME(stephenfin): This should have a foreign key constraint on - # registered_limit.id, but sqlalchemy-migrate clearly didn't handle - # creating a column with embedded FK info as was attempted in 048 - sql.Column( - 'registered_limit_id', - sql.String(64), - ), - sql.Column('domain_id', sql.String(64), nullable=True), - # NOTE(stephenfin): Name chosen to preserve backwards compatibility - # with names used for primary key unique constraints - sql.UniqueConstraint('id', name='limit_id_key'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - local_user = sql.Table( - 'local_user', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column( - 'user_id', - sql.String(64), - nullable=False, - unique=True, - ), - sql.Column('domain_id', sql.String(64), nullable=False), - sql.Column('name', sql.String(255), nullable=False), - sql.Column('failed_auth_count', sql.Integer, nullable=True), - sql.Column('failed_auth_at', sql.DateTime(), nullable=True), - sql.ForeignKeyConstraint( - ['user_id', 'domain_id'], - [user.c.id, user.c.domain_id], - name='local_user_user_id_fkey', - onupdate='CASCADE', - ondelete='CASCADE', - ), - sql.UniqueConstraint('domain_id', 'name'), - ) - - nonlocal_user = sql.Table( - 'nonlocal_user', - meta, - sql.Column('domain_id', sql.String(64), primary_key=True), - sql.Column('name', sql.String(255), primary_key=True), - sql.Column( - 'user_id', - sql.String(64), - nullable=False, - ), - sql.ForeignKeyConstraint( - ['user_id', 'domain_id'], - [user.c.id, user.c.domain_id], - name='nonlocal_user_user_id_fkey', - onupdate='CASCADE', - ondelete='CASCADE', - ), - sql.UniqueConstraint('user_id', name='ixu_nonlocal_user_user_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - password = sql.Table( - 'password', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column( - 'local_user_id', - sql.Integer, - sql.ForeignKey(local_user.c.id, ondelete='CASCADE'), - nullable=False, - ), - sql.Column('expires_at', sql.DateTime(), nullable=True), - sql.Column( - 'self_service', - sql.Boolean, - nullable=False, - server_default='0', - default=False, - ), - # NOTE(notmorgan): To support the full range of scrypt and pbkfd - # password hash lengths, this should be closer to varchar(1500) instead - # of varchar(255). - sql.Column('password_hash', sql.String(255), nullable=True), - sql.Column( - 'created_at_int', - ks_sql.DateTimeInt(), - nullable=False, - default=0, - server_default='0', - ), - sql.Column('expires_at_int', ks_sql.DateTimeInt(), nullable=True), - sql.Column( - 'created_at', - sql.DateTime(), - nullable=False, - default=datetime.datetime.utcnow, - ), - ) - - project_endpoint_group = sql.Table( - 'project_endpoint_group', - meta, - sql.Column( - 'endpoint_group_id', - sql.String(64), - sql.ForeignKey(endpoint_group.c.id), - nullable=False, - ), - sql.Column('project_id', sql.String(64), nullable=False), - sql.PrimaryKeyConstraint('endpoint_group_id', 'project_id'), - ) - - federated_user = sql.Table( - 'federated_user', - meta, - sql.Column('id', sql.Integer, primary_key=True, nullable=False), - sql.Column( - 'user_id', - sql.String(64), - sql.ForeignKey(user.c.id, ondelete='CASCADE'), - nullable=False, - ), - sql.Column( - 'idp_id', - sql.String(64), - sql.ForeignKey(identity_provider.c.id, ondelete='CASCADE'), - nullable=False, - ), - sql.Column('protocol_id', sql.String(64), nullable=False), - sql.Column('unique_id', sql.String(255), nullable=False), - sql.Column('display_name', sql.String(255), nullable=True), - sql.ForeignKeyConstraint( - ['protocol_id', 'idp_id'], - [federation_protocol.c.id, federation_protocol.c.idp_id], - name='federated_user_protocol_id_fkey', - ondelete='CASCADE', - ), - sql.UniqueConstraint('idp_id', 'protocol_id', 'unique_id'), - mysql_engine='InnoDB', - mysql_charset='utf8', - ) - - # create all tables - tables = [ - access_rule, - application_credential, - assignment, - config_register, - consumer, - credential, - group, - id_mapping, - identity_provider, - idp_remote_ids, - mapping, - policy, - policy_association, - project, - project_endpoint, - project_option, - project_tag, - region, - registered_limit, - request_token, - revocation_event, - role, - role_option, - sensitive_config, - service, - service_provider, - system_assignment, - token, - trust, - trust_role, - user, - user_group_membership, - user_option, - whitelisted_config, - - access_token, - application_credential_access_rule, - application_credential_role, - endpoint, - endpoint_group, - expiring_user_group_membership, - federation_protocol, - implied_role, - limit, - local_user, - nonlocal_user, - password, - project_endpoint_group, - - federated_user, - ] - - for table in tables: - try: - table.create() - except Exception: - LOG.exception('Exception while creating table: %r', table) - raise - - fkeys = [] - - if migrate_engine.name == 'sqlite': - # NOTE(stevemar): We need to keep this FK constraint due to 073, but - # only for sqlite, once we collapse 073 we can remove this constraint - fkeys.append( - { - 'columns': [assignment.c.role_id], - 'references': [role.c.id], - 'name': 'fk_assignment_role_id', - }, - ) - - for fkey in fkeys: - migrate.ForeignKeyConstraint( - columns=fkey['columns'], - refcolumns=fkey['references'], - name=fkey.get('name'), - ondelete=fkey.get('ondelete'), - onupdate=fkey.get('onupdate'), - ).create() - - # TODO(stephenfin): Remove these procedures in a future contract migration - - if migrate_engine.name == 'postgresql': - error_message = ( - 'Credential migration in progress. Cannot perform ' - 'writes to credential table.' - ) - credential_update_trigger = textwrap.dedent(f""" - CREATE OR REPLACE FUNCTION keystone_read_only_update() - RETURNS trigger AS - $BODY$ - BEGIN - IF NEW.encrypted_blob IS NULL THEN - RAISE EXCEPTION '{error_message}'; - END IF; - IF NEW.encrypted_blob IS NOT NULL AND OLD.blob IS NULL THEN - RAISE EXCEPTION '{error_message}'; - END IF; - RETURN NEW; - END - $BODY$ LANGUAGE plpgsql; - """) - - error_message = ( - 'Identity provider migration in progress. Cannot ' - 'insert new rows into the identity_provider table at ' - 'this time.' - ) - identity_provider_insert_trigger = textwrap.dedent(f""" - CREATE OR REPLACE FUNCTION keystone_read_only_insert() - RETURNS trigger AS - $BODY$ - BEGIN - RAISE EXCEPTION '{error_message}'; - END - $BODY$ LANGUAGE plpgsql; - """) - - federated_user_insert_trigger = textwrap.dedent(""" - CREATE OR REPLACE FUNCTION update_federated_user_domain_id() - RETURNS trigger AS - $BODY$ - BEGIN - UPDATE "user" SET domain_id = ( - SELECT domain_id FROM identity_provider WHERE id = NEW.idp_id) - WHERE id = NEW.user_id and domain_id IS NULL; - RETURN NULL; - END - $BODY$ LANGUAGE plpgsql; - """) - - local_user_insert_trigger = textwrap.dedent(""" - CREATE OR REPLACE FUNCTION update_user_domain_id() - RETURNS trigger AS - $BODY$ - BEGIN - UPDATE "user" SET domain_id = NEW.domain_id - WHERE id = NEW.user_id; - RETURN NULL; - END - $BODY$ LANGUAGE plpgsql; - """) - - with migrate_engine.connect() as conn: - conn.exec_driver_sql(credential_update_trigger) - conn.exec_driver_sql(identity_provider_insert_trigger) - conn.exec_driver_sql(federated_user_insert_trigger) - conn.exec_driver_sql(local_user_insert_trigger) - - # FIXME(stephenfin): Remove these indexes. They're left over from attempts - # to remove foreign key constraints in past migrations. Apparently - # sqlalchemy-migrate didn't do the job fully and left behind indexes - if migrate_engine.name == 'mysql': - sql.Index('region_id', registered_limit.c.region_id).create() - - # FIXME(stephenfin): This should be dropped when we add the FK - # constraint to this column - sql.Index('registered_limit_id', limit.c.registered_limit_id).create() - - # FIXME(stephenfin): These are leftover from when we removed a FK - # constraint and should probable be dropped - sql.Index('domain_id', identity_provider.c.domain_id).create() - sql.Index('domain_id', user.c.domain_id).create() diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/074_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/074_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/074_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/075_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/075_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/075_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/076_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/076_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/076_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/077_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/077_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/077_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/078_placeholder.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/078_placeholder.py deleted file mode 100644 index 2b09cbc99a..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/078_placeholder.py +++ /dev/null @@ -1,18 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -# This is a placeholder for Ussuri backports. Do not use this number for new -# Victoria work. New Victoria work starts after all the placeholders. - - -def upgrade(migrate_engine): - pass diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/079_expand_update_local_id_limit.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/079_expand_update_local_id_limit.py deleted file mode 100644 index 433de09a1f..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/079_expand_update_local_id_limit.py +++ /dev/null @@ -1,28 +0,0 @@ -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import sqlalchemy as sql - - -def upgrade(migrate_engine): - - meta = sql.MetaData() - - # NOTE(stephenfin): This is not compatible with SQLAlchemy 2.0 but neither - # is sqlalchemy-migrate which requires this. We'll remove these migrations - # when dropping SQLAlchemy < 2.x support - meta.bind = migrate_engine - - id_mapping_table = sql.Table( - 'id_mapping', meta, autoload_with=migrate_engine, - ) - id_mapping_table.c.local_id.alter(type=sql.String(255)) diff --git a/keystone/common/sql/legacy_migrations/expand_repo/versions/__init__.py b/keystone/common/sql/legacy_migrations/expand_repo/versions/__init__.py deleted file mode 100644 index 84e0fb83bf..0000000000 --- a/keystone/common/sql/legacy_migrations/expand_repo/versions/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -# Copyright 2012 OpenStack Foundation -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -from keystone.common.sql.core import * # noqa diff --git a/keystone/common/sql/upgrades.py b/keystone/common/sql/upgrades.py index a075716e97..a12d2f54f8 100644 --- a/keystone/common/sql/upgrades.py +++ b/keystone/common/sql/upgrades.py @@ -20,9 +20,6 @@ from alembic import command as alembic_api from alembic import config as alembic_config from alembic import migration as alembic_migration from alembic import script as alembic_script -from migrate import exceptions as migrate_exceptions -from migrate.versioning import api as migrate_api -from migrate.versioning import repository as migrate_repository from oslo_db import exception as db_exception from oslo_log import log as logging @@ -33,7 +30,6 @@ CONF = keystone.conf.CONF LOG = logging.getLogger(__name__) ALEMBIC_INIT_VERSION = '27e647c0fad4' -MIGRATE_INIT_VERSION = 72 EXPAND_BRANCH = 'expand' DATA_MIGRATION_BRANCH = 'data_migration' @@ -50,25 +46,6 @@ VERSIONS_PATH = os.path.join( ) -def _find_migrate_repo(branch): - """Get the project's change script repository. - - :param branch: Name of the repository "branch" to be used; this will be - transformed to repository path. - :returns: An instance of ``migrate.versioning.repository.Repository`` - """ - abs_path = os.path.abspath( - os.path.join( - os.path.dirname(sql.__file__), - 'legacy_migrations', - f'{branch}_repo', - ) - ) - if not os.path.exists(abs_path): - raise db_exception.DBMigrationError("Path %s not found" % abs_path) - return migrate_repository.Repository(abs_path) - - def _find_alembic_conf(): """Get the project's alembic configuration. @@ -140,45 +117,12 @@ def get_current_heads(): return heads -def _is_database_under_migrate_control(engine): - # if any of the repos is present, they're all present (in theory, at least) - repository = _find_migrate_repo('expand') - try: - migrate_api.db_version(engine, repository) - return True - except migrate_exceptions.DatabaseNotControlledError: - return False - - def _is_database_under_alembic_control(engine): with engine.connect() as conn: context = alembic_migration.MigrationContext.configure(conn) return bool(context.get_current_heads()) -def _init_alembic_on_legacy_database(engine, config): - """Init alembic in an existing environment with sqlalchemy-migrate.""" - LOG.info( - 'The database is still under sqlalchemy-migrate control; ' - 'applying any remaining sqlalchemy-migrate-based migrations ' - 'and fake applying the initial alembic migration' - ) - - # bring all repos up to date; note that we're relying on the fact that - # there aren't any "real" contract migrations left (since the great squash - # of migrations in yoga) so we're really only applying the expand side of - # '079_expand_update_local_id_limit' and the rest are for completeness' - # sake - for branch in (EXPAND_BRANCH, DATA_MIGRATION_BRANCH, CONTRACT_BRANCH): - repository = _find_migrate_repo(branch or 'expand') - migrate_api.upgrade(engine, repository) - - # re-use the connection rather than creating a new one - with engine.begin() as connection: - config.attributes['connection'] = connection - alembic_api.stamp(config, ALEMBIC_INIT_VERSION) - - def _upgrade_alembic(engine, config, branch): revision = 'heads' if branch: @@ -209,17 +153,10 @@ def get_db_version(branch=EXPAND_BRANCH, *, engine=None): engine_url = str(engine.url).replace('%', '%%') config.set_main_option('sqlalchemy.url', str(engine_url)) - migrate_version = None - if _is_database_under_migrate_control(engine): - repository = _find_migrate_repo(branch) - migrate_version = migrate_api.db_version(engine, repository) + # we use '.get' since the particular branch might not have been created + alembic_version = _get_current_heads(engine, config).get(branch) - alembic_version = None - if _is_database_under_alembic_control(engine): - # we use '.get' since the particular branch might not have been created - alembic_version = _get_current_heads(engine, config).get(branch) - - return alembic_version or migrate_version + return alembic_version def _db_sync(branch=None, *, engine=None): @@ -241,16 +178,6 @@ def _db_sync(branch=None, *, engine=None): engine_url = str(engine.url).replace('%', '%%') config.set_main_option('sqlalchemy.url', str(engine_url)) - # if we're in a deployment where sqlalchemy-migrate is already present, - # then apply all the updates for that and fake apply the initial - # alembic migration; if we're not then 'upgrade' will take care of - # everything this should be a one-time operation - if ( - not _is_database_under_alembic_control(engine) and - _is_database_under_migrate_control(engine) - ): - _init_alembic_on_legacy_database(engine, config) - _upgrade_alembic(engine, config, branch) diff --git a/keystone/tests/unit/common/sql/test_upgrades.py b/keystone/tests/unit/common/sql/test_upgrades.py index bb53cbd23d..0f313000a7 100644 --- a/keystone/tests/unit/common/sql/test_upgrades.py +++ b/keystone/tests/unit/common/sql/test_upgrades.py @@ -21,7 +21,6 @@ test will then use that DB and username/password combo to run the tests. """ import fixtures -from migrate.versioning import api as migrate_api from oslo_db import options as db_options from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import test_fixtures @@ -292,49 +291,3 @@ class TestModelsSyncPostgreSQL( base.BaseTestCase, ): FIXTURE = test_fixtures.PostgresqlOpportunisticFixture - - -class KeystoneModelsMigrationsLegacySync(KeystoneModelsMigrationsSync): - """Test that the models match the database after old migrations are run.""" - - def db_sync(self, engine): - # the 'upgrades._db_sync' method will not use the legacy - # sqlalchemy-migrate-based migration flow unless the database is - # already controlled with sqlalchemy-migrate, so we need to manually - # enable version controlling with this tool to test this code path - for branch in ( - upgrades.EXPAND_BRANCH, - upgrades.DATA_MIGRATION_BRANCH, - upgrades.CONTRACT_BRANCH, - ): - repository = upgrades._find_migrate_repo(branch) - migrate_api.version_control( - engine, repository, upgrades.MIGRATE_INIT_VERSION) - - # now we can apply migrations as expected and the legacy path will be - # followed - super().db_sync(engine) - - -class TestModelsLegacySyncSQLite( - KeystoneModelsMigrationsLegacySync, - test_fixtures.OpportunisticDBTestMixin, - base.BaseTestCase, -): - pass - - -class TestModelsLegacySyncMySQL( - KeystoneModelsMigrationsLegacySync, - test_fixtures.OpportunisticDBTestMixin, - base.BaseTestCase, -): - FIXTURE = test_fixtures.MySQLOpportunisticFixture - - -class TestModelsLegacySyncPostgreSQL( - KeystoneModelsMigrationsLegacySync, - test_fixtures.OpportunisticDBTestMixin, - base.BaseTestCase, -): - FIXTURE = test_fixtures.PostgresqlOpportunisticFixture diff --git a/keystone/tests/unit/test_sql_upgrade.py b/keystone/tests/unit/test_sql_upgrade.py index a5ef2bd68a..fa751d4a0e 100644 --- a/keystone/tests/unit/test_sql_upgrade.py +++ b/keystone/tests/unit/test_sql_upgrade.py @@ -40,7 +40,6 @@ For further information, see `oslo.db documentation """ import fixtures -from migrate.versioning import script from oslo_db import options as db_options from oslo_db.sqlalchemy import enginefacade from oslo_db.sqlalchemy import test_fixtures as db_fixtures @@ -239,11 +238,6 @@ class MigrateBase( self.engine = enginefacade.writer.get_engine() self.sessionmaker = enginefacade.writer.get_sessionmaker() - # NOTE(dstanek): Clear out sqlalchemy-migrate's script cache to allow - # us to have multiple repos (expand, migrate, contract) where the - # modules have the same name (001_awesome.py). - self.addCleanup(script.PythonScript.clear) - db_options.set_defaults(CONF, connection=self.engine.url) # Override keystone's context manager to be oslo.db's global context diff --git a/releasenotes/notes/remove-sqlalchemy-migrate-a4fa47685c7e28c6.yaml b/releasenotes/notes/remove-sqlalchemy-migrate-a4fa47685c7e28c6.yaml new file mode 100644 index 0000000000..6ecb466215 --- /dev/null +++ b/releasenotes/notes/remove-sqlalchemy-migrate-a4fa47685c7e28c6.yaml @@ -0,0 +1,5 @@ +--- +upgrade: + - | + The legacy ``sqlalchemy-migrate`` migrations, which have been deprecated + since Zed, have been removed. There should be no end-user impact. diff --git a/requirements.txt b/requirements.txt index a109f3d294..0c68696cb5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,6 @@ Flask!=0.11,>=1.0.2 # BSD Flask-RESTful>=0.3.5 # BSD cryptography>=2.7 # BSD/Apache-2.0 SQLAlchemy>=1.4.0 # MIT -sqlalchemy-migrate>=0.13.0 # Apache-2.0 stevedore>=1.20.0 # Apache-2.0 passlib>=1.7.0 # BSD python-keystoneclient>=3.8.0 # Apache-2.0 diff --git a/tools/generate-schemas b/tools/generate-schemas deleted file mode 100755 index 59fbb226c6..0000000000 --- a/tools/generate-schemas +++ /dev/null @@ -1,134 +0,0 @@ -#!/usr/bin/env bash -# -# Script to generate schemas for the various versions. -# -# Some setup is required, similar to the opportunistic tests. -# -# MySQL -> -# -# $ mysql -uroot -# MariaDB [(none)]> CREATE DATABASE keystone -# MariaDB [(none)]> GRANT ALL PRIVILEGES ON keystone.* TO 'keystone'@'localhost' IDENTIFIED BY 'password'; -# MariaDB [(none)]> quit; -# -# Postgres -> -# -# $ sudo -u postgres psql -# postgres=# create user keystone with createdb login password 'password'; -# postgres=# create database keystone with owner keystone; -# postgres=# quit; -# -# Note that you may also have to configure 'pg_hba.conf' to use password-based -# auth instead of "ident", if you haven't done so already. You can locate this -# with 'locate pg_hba.conf'. More details at -# https://ubuntu.com/server/docs/databases-postgresql - -set -o xtrace -set -e - -source .tox/py38/bin/activate - -INIT_VERSION=$(ls -1 keystone/common/sql/legacy_migrations/expand_repo/versions/ | head -1 | awk -F_ '{print $1}' | sed 's/^0*//') -INIT_VERSION=$(($INIT_VERSION-1)) - -echo "Detected init version of $INIT_VERSION" - -mkdir -p /tmp/keystone-schemas -rm -f "/tmp/keystone-schemas/$INIT_VERSION-*.sql" - -# -# functions -# - -function sync () { - DB_URL=$1 - - python keystone/common/sql/legacy_migrations/expand_repo/manage.py version_control \ - --database "$DB_URL" \ - --version "$INIT_VERSION" \ - --repository keystone/common/sql/legacy_migrations/expand_repo/ - python keystone/common/sql/legacy_migrations/data_migration_repo/manage.py version_control \ - --database "$DB_URL" \ - --version "$INIT_VERSION" \ - --repository keystone/common/sql/legacy_migrations/data_migration_repo/ - python keystone/common/sql/legacy_migrations/contract_repo/manage.py version_control \ - --database "$DB_URL" \ - --version "$INIT_VERSION" \ - --repository keystone/common/sql/legacy_migrations/contract_repo/ - - python keystone/common/sql/legacy_migrations/expand_repo/manage.py upgrade \ - --database "$DB_URL" \ - --repository keystone/common/sql/legacy_migrations/expand_repo/ - python keystone/common/sql/legacy_migrations/data_migration_repo/manage.py upgrade \ - --database "$DB_URL" \ - --repository keystone/common/sql/legacy_migrations/data_migration_repo/ - python keystone/common/sql/legacy_migrations/contract_repo/manage.py upgrade \ - --database "$DB_URL" \ - --repository keystone/common/sql/legacy_migrations/contract_repo/ -} - -# -# sqlite -# - -# cleanup from previous runs - -rm -f /tmp/keystone.db - -# sync schema - -sync 'sqlite:////tmp/keystone.db' - -# dump the schema - -sqlite3 /tmp/keystone.db << EOF -.output "/tmp/keystone-schemas/${INIT_VERSION}-sqlite.sql" -.schema -.quit -EOF - -rm -f /tmp/keystone.db - -# -# mysql -# - -# cleanup from previous runs - -mysql -u keystone -ppassword << EOF -DROP DATABASE IF EXISTS keystone; -CREATE DATABASE keystone; -EOF - -# sync schema - -sync 'mysql+pymysql://keystone:password@localhost/keystone' - -# dump the schema - -mysqldump --no-data --skip-comments -u keystone -ppassword \ - keystone > "/tmp/keystone-schemas/${INIT_VERSION}-mysql.sql" - -mysql -u keystone -ppassword << EOF -DROP DATABASE IF EXISTS keystone; -EOF - -# -# postgres -# - -# cleanup from previous runs - -sudo -u postgres dropdb --if-exists keystone -sudo -u postgres createdb --owner=keystone keystone - -# sync to initial version - -sync 'postgresql://keystone:password@localhost/keystone' - -# dump the schema - -pg_dump postgresql://keystone:password@localhost/keystone \ - --schema-only > "/tmp/keystone-schemas/${INIT_VERSION}-postgres.sql" - -sudo -u postgres dropdb --if-exists keystone