Add new tables for cache operations

Added expand script and model to define `cached_images` and
`node_reference` database tables.

Related blueprint centralized-cache-db

Change-Id: I9991a4e945bd4d231e942096445fd5374ff7ff2a

glance/db/migration.py

@@ -29,5 +29,5 @@ db_options.set_defaults(cfg.CONF)
 # Migration-related constants
 EXPAND_BRANCH = 'expand'
 CONTRACT_BRANCH = 'contract'
-CURRENT_RELEASE = '2023_2'
+CURRENT_RELEASE = '2024_1'
 ALEMBIC_INIT_VERSION = 'liberty'

glance/db/sqlalchemy/alembic_migrations/data_migrations/2024_1_migrate01_empty.py

@@ -0,0 +1,26 @@
+# Copyright (C) 2023 RedHat Inc.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+def has_migrations(engine):
+    """Returns true if at least one data row can be migrated."""
+
+    return False
+
+
+def migrate(engine):
+    """Return the number of rows migrated."""
+
+    return 0

glance/db/sqlalchemy/alembic_migrations/versions/2024_1_contract01_empty.py

@@ -0,0 +1,25 @@
+# Copyright (C) 2023 RedHat Inc
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+
+# revision identifiers, used by Alembic.
+revision = '2024_1_contract01'
+down_revision = '2023_1_contract01'
+branch_labels = None
+depends_on = '2024_1_expand01'
+
+
+def upgrade():
+    pass

glance/db/sqlalchemy/alembic_migrations/versions/2024_1_expand01_add_cache_tables.py

@@ -0,0 +1,82 @@
+# Copyright (C) 2023 RedHat Inc
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""adds cache_node_reference and cached_images table(s)
+
+Revision ID: 2024_1_expand01
+Revises: 2023_1_expand01
+Create Date: 2023-10-31 11:55:16.657499
+
+"""
+
+from alembic import op
+from sqlalchemy.schema import (
+    Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint)
+
+from glance.db.sqlalchemy.schema import (
+    Integer, BigInteger, DateTime, String)  # noqa
+
+# revision identifiers, used by Alembic.
+revision = '2024_1_expand01'
+down_revision = '2023_1_expand01'
+branch_labels = None
+depends_on = None
+
+
+def _add_node_reference_table():
+    op.create_table('node_reference',
+                    Column('node_reference_id',
+                           BigInteger().with_variant(Integer, 'sqlite'),
+                           nullable=False,
+                           autoincrement=True),
+                    Column('node_reference_url', String(length=255),
+                           nullable=False),
+                    PrimaryKeyConstraint('node_reference_id'),
+                    UniqueConstraint(
+                        'node_reference_url',
+                        name='uq_node_reference_node_reference_url'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+
+def _add_cached_images_table():
+    op.create_table('cached_images',
+                    Column('id', BigInteger().with_variant(Integer, 'sqlite'),
+                           autoincrement=True,
+                           nullable=False),
+                    Column('image_id', String(length=36), nullable=False),
+                    Column('last_accessed', DateTime(), nullable=False),
+                    Column('last_modified', DateTime(), nullable=False),
+                    Column('size', BigInteger(), nullable=False),
+                    Column('hits', Integer(), nullable=False),
+                    Column('checksum', String(length=32), nullable=True),
+                    Column('node_reference_id',
+                           BigInteger().with_variant(Integer, 'sqlite'),
+                           nullable=False),
+                    PrimaryKeyConstraint('id'),
+                    ForeignKeyConstraint(
+                        ['node_reference_id'],
+                        ['node_reference.node_reference_id'], ),
+                    UniqueConstraint(
+                        'image_id',
+                        'node_reference_id',
+                        name='ix_cached_images_image_id_node_reference_id'),
+                    mysql_engine='InnoDB',
+                    mysql_charset='utf8',
+                    extend_existing=True)
+
+
+def upgrade():
+    _add_node_reference_table()
+    _add_cached_images_table()

glance/db/sqlalchemy/models.py

@@ -258,6 +258,43 @@ class TaskInfo(BASE, models.ModelBase):
     message = Column(Text)
 
 
+class NodeReference(BASE, models.ModelBase):
+    """Represents node info in the datastore"""
+    __tablename__ = 'node_reference'
+    __table_args__ = (UniqueConstraint(
+        'node_reference_url',
+        name='uq_node_reference_node_reference_url'),)
+
+    node_reference_id = Column(BigInteger().with_variant(Integer, 'sqlite'),
+                               primary_key=True,
+                               nullable=False, autoincrement=True)
+    node_reference_url = Column(String(length=255),
+                                nullable=False)
+
+
+class CachedImages(BASE, models.ModelBase):
+    """Represents an image tag in the datastore."""
+    __tablename__ = 'cached_images'
+    __table_args__ = (UniqueConstraint(
+        'image_id',
+        'node_reference_id',
+        name='ix_cached_images_image_id_node_reference_id'),)
+
+    id = Column(BigInteger().with_variant(Integer, 'sqlite'),
+                primary_key=True, autoincrement=True,
+                nullable=False)
+    image_id = Column(String(36), nullable=False)
+    last_accessed = Column(DateTime, nullable=False)
+    last_modified = Column(DateTime, nullable=False)
+    size = Column(BigInteger(), nullable=False)
+    hits = Column(Integer, nullable=False)
+    checksum = Column(String(32), nullable=True)
+    node_reference_id = Column(
+        BigInteger().with_variant(Integer, 'sqlite'),
+        ForeignKey('node_reference.node_reference_id'),
+        nullable=False)
+
+
 def register_models(engine):
     """Create database tables for all models with the given engine."""
     models = (Image, ImageProperty, ImageMember)

glance/tests/functional/db/migrations/test_2024_1_expand01.py

@@ -0,0 +1,67 @@
+#    Copyright (c) 2023 RedHat, Inc.
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_db.sqlalchemy import test_fixtures
+from oslo_db.sqlalchemy import utils as db_utils
+import sqlalchemy
+
+from glance.tests.functional.db import test_migrations
+import glance.tests.utils as test_utils
+
+
+class Test2024_1Expand01Mixin(test_migrations.AlembicMigrationsMixin):
+
+    def _get_revisions(self, config):
+        return test_migrations.AlembicMigrationsMixin._get_revisions(
+            self, config, head='2024_1_expand01')
+
+    def _pre_upgrade_2024_1_expand01(self, engine):
+        self.assertRaises(sqlalchemy.exc.NoSuchTableError,
+                          db_utils.get_table, engine, 'node_reference')
+        self.assertRaises(sqlalchemy.exc.NoSuchTableError,
+                          db_utils.get_table, engine, 'cached_images')
+
+    def _check_2024_1_expand01(self, engine, data):
+        # check that after migration, 'node_reference' and 'cached_images'
+        # tables are created with expected columns and indexes
+        node_reference = db_utils.get_table(engine, 'node_reference')
+        self.assertIn('node_reference_id', node_reference.c)
+        self.assertIn('node_reference_url', node_reference.c)
+        self.assertTrue(db_utils.index_exists(
+            engine, 'node_reference',
+            'uq_node_reference_node_reference_url'),
+            'Index %s on table %s does not exist' %
+            ('uq_node_reference_node_reference_url', 'node_reference'))
+
+        cached_images = db_utils.get_table(engine, 'cached_images')
+        self.assertIn('id', cached_images.c)
+        self.assertIn('image_id', cached_images.c)
+        self.assertIn('last_accessed', cached_images.c)
+        self.assertIn('last_modified', cached_images.c)
+        self.assertIn('size', cached_images.c)
+        self.assertIn('hits', cached_images.c)
+        self.assertIn('checksum', cached_images.c)
+        self.assertIn('node_reference_id', cached_images.c)
+        self.assertTrue(db_utils.index_exists(
+            engine, 'cached_images',
+            'ix_cached_images_image_id_node_reference_id'),
+            'Index %s on table %s does not exist' %
+            ('ix_cached_images_image_id_node_reference_id', 'cached_images'))
+
+
+class Test2024_1Expand01MySQL(
+    Test2024_1Expand01Mixin,
+    test_fixtures.OpportunisticDBTestMixin,
+    test_utils.BaseTestCase,
+):
+    FIXTURE = test_fixtures.MySQLOpportunisticFixture