db: Fix charset updates in migrations

We are failing py311 jobs with our migration checks;
and the reason is that we have some 'implied' sql defaults
that might be inconsistent with the server versions - one
of them is the charset. We recently [1] fixed a bunch of
un-annotated tables with explicit charset declarations.
This is a continuation of that attempt.

Unfortunately, we can't change history. On existing
deployments, changes to these migrations will be ignored,
and must be done manually if there are any inconsistencies
to be fixed.

While we're at it, this commit also cleans up separate
steps to set the charset and engine to pre-created
tables.

[1] I7cd6fa0cc8e054af112493746e753fef2024000f
Closes-Bug: #2053211
Change-Id: I16345f63a279549e81cbde7b60fed29521d48a02
Signed-off-by: Goutham Pacha Ravi <gouthampravi@gmail.com>

manila/db/migrations/alembic/versions/162a3e673105_manila_init.py

@@ -26,7 +26,7 @@ down_revision = None
 
 from alembic import op
 from oslo_log import log
-from sqlalchemy import Boolean, Column, DateTime, ForeignKey
+from sqlalchemy import Boolean, Column, DateTime, ForeignKeyConstraint
 from sqlalchemy import Integer, MetaData, String, Table, UniqueConstraint
 
 LOG = log.getLogger(__name__)
@@ -129,10 +129,7 @@ def upgrade():
                       unicode_error=None,
                       _warn_on_bytestring=False),
                nullable=False),
-        Column('usage_id',
-               Integer(),
-               ForeignKey('quota_usages.id'),
-               nullable=False),
+        Column('usage_id', Integer(), nullable=False),
         Column('project_id',
                String(length=255, convert_unicode=True,
                       unicode_error=None,
@@ -144,6 +141,7 @@ def upgrade():
                       _warn_on_bytestring=False)),
         Column('delta', Integer(), nullable=False),
         Column('expire', DateTime(timezone=False)),
+        ForeignKeyConstraint(['usage_id'], ['quota_usages.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8',
     )
@@ -183,17 +181,13 @@ def upgrade():
         Column('display_name', String(length=255)),
         Column('display_description', String(length=255)),
         Column('snapshot_id', String(length=36)),
-        Column('share_network_id',
-               String(length=36),
-               ForeignKey('share_networks.id'),
-               nullable=True),
-        Column('share_server_id',
-               String(length=36),
-               ForeignKey('share_servers.id'),
-               nullable=True),
+        Column('share_network_id', String(length=36), nullable=True),
+        Column('share_server_id', String(length=36), nullable=True),
         Column('share_proto', String(255)),
         Column('export_location', String(255)),
         Column('volume_type_id', String(length=36)),
+        ForeignKeyConstraint(['share_network_id'], ['share_networks.id']),
+        ForeignKeyConstraint(['share_server_id'], ['share_servers.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8'
     )
@@ -205,11 +199,11 @@ def upgrade():
         Column('deleted_at', DateTime),
         Column('deleted', String(length=36), default='False'),
         Column('id', String(length=36), primary_key=True, nullable=False),
-        Column('share_id', String(36), ForeignKey('shares.id'),
-               nullable=False),
+        Column('share_id', String(36), nullable=False),
         Column('access_type', String(255)),
         Column('access_to', String(255)),
         Column('state', String(255)),
+        ForeignKeyConstraint(['share_id'], ['shares.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8'
     )
@@ -223,8 +217,7 @@ def upgrade():
         Column('id', String(length=36), primary_key=True, nullable=False),
         Column('user_id', String(length=255)),
         Column('project_id', String(length=255)),
-        Column('share_id', String(36), ForeignKey('shares.id'),
-               nullable=False),
+        Column('share_id', String(36), nullable=False),
         Column('size', Integer),
         Column('status', String(length=255)),
         Column('progress', String(length=255)),
@@ -233,6 +226,7 @@ def upgrade():
         Column('share_size', Integer),
         Column('share_proto', String(length=255)),
         Column('export_location', String(255)),
+        ForeignKeyConstraint(['share_id'], ['shares.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8'
     )
@@ -244,10 +238,10 @@ def upgrade():
         Column('deleted_at', DateTime),
         Column('deleted', Integer, default=0),
         Column('id', Integer, primary_key=True, nullable=False),
-        Column('share_id', String(length=36), ForeignKey('shares.id'),
-               nullable=False),
+        Column('share_id', String(length=36), nullable=False),
         Column('key', String(length=255), nullable=False),
         Column('value', String(length=1023), nullable=False),
+        ForeignKeyConstraint(['share_id'], ['shares.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8'
     )
@@ -301,10 +295,10 @@ def upgrade():
         Column('deleted_at', DateTime),
         Column('deleted', String(length=36), default='False'),
         Column('id', String(length=36), primary_key=True, nullable=False),
-        Column('share_network_id', String(length=36),
-               ForeignKey('share_networks.id'), nullable=True),
+        Column('share_network_id', String(length=36), nullable=True),
         Column('host', String(length=255), nullable=True),
         Column('status', String(length=32)),
+        ForeignKeyConstraint(['share_network_id'], ['share_networks.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8',
     )
@@ -316,11 +310,10 @@ def upgrade():
         Column('deleted_at', DateTime),
         Column('deleted', String(length=36), default=0),
         Column('id', Integer, primary_key=True, nullable=False),
-        Column('share_server_id', String(length=36),
-               ForeignKey('share_servers.id'),
-               nullable=False),
+        Column('share_server_id', String(length=36), nullable=False),
         Column('key', String(length=255), nullable=False),
         Column('value', String(length=1023), nullable=False),
+        ForeignKeyConstraint(['share_server_id'], ['share_servers.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8'
     )
@@ -334,9 +327,9 @@ def upgrade():
         Column('id', String(length=36), primary_key=True, nullable=False),
         Column('ip_address', String(length=64), nullable=True),
         Column('mac_address', String(length=32), nullable=True),
-        Column('share_server_id', String(length=36),
-               ForeignKey('share_servers.id'), nullable=False),
+        Column('share_server_id', String(length=36), nullable=False),
         Column('status', String(length=32)),
+        ForeignKeyConstraint(['share_server_id'], ['share_servers.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8',
     )
@@ -348,10 +341,11 @@ def upgrade():
         Column('deleted_at', DateTime),
         Column('deleted', Integer, default=0),
         Column('id', Integer, primary_key=True, nullable=False),
-        Column('share_network_id', String(length=36),
-               ForeignKey('share_networks.id'), nullable=False),
-        Column('security_service_id', String(length=36),
-               ForeignKey('security_services.id'), nullable=False),
+        Column('share_network_id', String(length=36), nullable=False),
+        Column('security_service_id', String(length=36), nullable=False),
+        ForeignKeyConstraint(['share_network_id'], ['share_networks.id']),
+        ForeignKeyConstraint(['security_service_id'],
+                             ['security_services.id']),
         mysql_engine='InnoDB',
         mysql_charset='utf8',
     )
@@ -365,7 +359,8 @@ def upgrade():
         Column('id', String(length=36), primary_key=True, nullable=False),
         Column('name', String(length=255)),
         UniqueConstraint('name', 'deleted', name='vt_name_uc'),
-        mysql_engine='InnoDB'
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
     )
 
     volume_type_extra_specs = Table(
@@ -375,11 +370,12 @@ def upgrade():
         Column('deleted_at', DateTime),
         Column('deleted', Boolean),
         Column('id', Integer, primary_key=True, nullable=False),
-        Column('volume_type_id', String(length=36),
-               ForeignKey('volume_types.id'), nullable=False),
+        Column('volume_type_id', String(length=36), nullable=False),
         Column('key', String(length=255)),
         Column('value', String(length=255)),
-        mysql_engine='InnoDB'
+        ForeignKeyConstraint(['volume_type_id'], ['volume_types.id']),
+        mysql_engine='InnoDB',
+        mysql_charset='utf8'
     )
 
     # create all tables
@@ -400,25 +396,6 @@ def upgrade():
                 LOG.exception('Exception while creating table.')
                 raise
 
-    if migrate_engine.name == "mysql":
-        tables = ["quotas", "services", "quota_classes", "quota_usages",
-                  "reservations", "project_user_quotas", "share_access_map",
-                  "share_snapshots", "share_metadata", "security_services",
-                  "share_networks", "network_allocations", "shares",
-                  "share_servers",
-                  "share_network_security_service_association", "volume_types",
-                  "volume_type_extra_specs", "share_server_backend_details"]
-
-        migrate_engine.execute("SET foreign_key_checks = 0")
-        for table in tables:
-            migrate_engine.execute(
-                "ALTER TABLE %s CONVERT TO CHARACTER SET utf8" % table)
-        migrate_engine.execute("SET foreign_key_checks = 1")
-        migrate_engine.execute(
-            "ALTER DATABASE %s DEFAULT CHARACTER SET utf8" %
-            migrate_engine.url.database)
-        migrate_engine.execute("ALTER TABLE %s Engine=InnoDB" % table)
-
 
 def downgrade():
     raise NotImplementedError('Downgrade from initial Manila install is not'

manila/db/migrations/alembic/versions/38e632621e5a_change_volume_type_to_share_type.py

@@ -80,9 +80,10 @@ def downgrade():
         sa.Column('deleted', sa.Boolean),
         sa.Column('id', sa.Integer, primary_key=True, nullable=False),
         sa.Column('volume_type_id', sa.String(length=36),
-                  sa.ForeignKey('share_types.id'), nullable=False),
+                  nullable=False),
         sa.Column('key', sa.String(length=255)),
         sa.Column('value', sa.String(length=255)),
+        mysql_charset='utf8',
         mysql_engine='InnoDB')
 
     LOG.info("Migrating share_type_extra_specs to "
@@ -97,6 +98,12 @@ def downgrade():
     op.create_unique_constraint('vt_name_uc', 'share_types',
                                 ['name', 'deleted'])
     op.rename_table("share_types", "volume_types")
+    op.create_foreign_key(
+        "volume_type_extra_specs_ibfk_1",
+        "volume_type_extra_specs",
+        "volume_types",
+        ["volume_type_id"], ["id"]
+    )
 
     LOG.info("Renaming column name shares.share_type_id to "
              "shares.volume_type.id")

manila/db/migrations/alembic/versions/3a482171410f_add_drivers_private_data_table.py

@@ -50,6 +50,7 @@ def upgrade():
                        nullable=False, primary_key=True),
             sql.Column('value', sql.String(1023), nullable=False),
             mysql_engine='InnoDB',
+            mysql_charset='utf8',
         )
     except Exception:
         LOG.error("Table |%s| not created!",

manila/db/migrations/alembic/versions/478c445d8d3e_add_security_service_update_control_fields.py

@@ -51,6 +51,7 @@ def upgrade():
                       nullable=False, primary_key=True),
             sa.Column('value', sa.String(1023), nullable=False),
             mysql_engine='InnoDB',
+            mysql_charset='utf8',
         )
         op.add_column(
             SHARE_SERVERS_TABLE,

manila/db/migrations/alembic/versions/4a482571410f_add_backends_info_table.py

@@ -47,6 +47,7 @@ def upgrade():
             sql.Column('info_hash', sql.String(255),
                        nullable=False),
             mysql_engine='InnoDB',
+            mysql_charset='utf8',
         )
     except Exception:
         LOG.error("Table |%s| not created!",

manila/db/migrations/alembic/versions/e8ea58723178_remove_host_from_driver_private_data.py

@@ -80,6 +80,7 @@ def downgrade():
                    nullable=False, primary_key=True),
         sql.Column('value', sql.String(1023), nullable=False),
         mysql_engine='InnoDB',
+        mysql_charset='utf8',
     )
 
     LOG.info("Copying data from %(from_table)s to the migration "