summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHemanth Makkapati <hemanth.makkapati@rackspace.com>2016-10-05 22:18:42 -0500
committerHemanth Makkapati <hemanth.makkapati@rackspace.com>2017-01-31 22:42:43 -0600
commit21d431013f6ad8a9f4a2afc34b66f67ff0d628eb (patch)
treede4dd04e162284537246bcbada290641c9d1a87d
parenta77ca91133949559860a364f38237906571655bb (diff)
Port Glance Migrations to Alembic
This change proposes the use of Alembic to manage Glance migrations. * Introduce new directory ``alembic_migrations`` under ``glance/db/sqlalchemy``. This directory is the home for all glance migrations henceforth. All the migration scripts reside under ``versions`` directory. * All the migrations up to Liberty are consolidated into one migration called ``liberty_initial`` as those migrations are not supported any more. Mitaka migrations are retained but under a different naming convention. * All the glance manage db commands are changed appropriately. They now use alembic to perform operations such as ``version``, ``upgrade``, ``sync`` and ``version_control``. * The database versions are not numerical any more. They are the revision ID of the last migration applied on the database. Since we don't support migrations before Mitaka, the Liberty version ``42`` will now appear as ``liberty``. Migration ``43`` and ``44`` in Mitaka appear as ``mitaka01`` and ``mitaka02`` respectively. * When one performs a ``sync`` or ``upgrade`` command, the database is first stamped with an equivalent alembic version before upgrading. * The older migration scripts are retained so that users can correlate with the new migrations. Also, it is probably safe to retain them until the alembic migrations become stable. Similarly, the ``migrate_version`` table is not removed yet. Partially-Implements: blueprint alembic-migrations Change-Id: Ie8594ff339a13bf190aefa308f54e97ee20ecfa2 Co-Authored-By: Alexander Bashmakov <alexander.bashmakov@intel.com> Depends-On: I1596499529af249bc48dfe859bbd31e90c48a5e0
Notes
Notes (review): Code-Review+1: Dharini Chandrasekar <dharini.chandrasekar@intel.com> Code-Review+2: Steve Lewis (stevelle) <steve.lewis@rackspace.com> Code-Review+2: Ian Cordasco <sigmavirus24@gmail.com> Code-Review+2: Nikhil Komawar <nik.komawar@gmail.com> Workflow+1: Nikhil Komawar <nik.komawar@gmail.com> Code-Review+2: Brian Rosmaita <brian.rosmaita@rackspace.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Wed, 01 Feb 2017 19:00:04 +0000 Reviewed-on: https://review.openstack.org/382958 Project: openstack/glance Branch: refs/heads/master
-rw-r--r--doc/source/db.rst5
-rw-r--r--doc/source/man/glancemanage.rst10
-rw-r--r--glance/cmd/manage.py77
-rw-r--r--glance/db/migration.py1
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/README1
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/__init__.py99
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py224
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/add_images_tables.py201
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py171
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py66
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/alembic.ini69
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/env.py92
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/migrate.cfg20
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/script.py.mako20
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py40
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py47
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py42
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py72
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql162
-rw-r--r--glance/tests/unit/test_manage.py112
-rw-r--r--requirements.txt2
21 files changed, 1424 insertions, 109 deletions
diff --git a/doc/source/db.rst b/doc/source/db.rst
index a57052a..770565e 100644
--- a/doc/source/db.rst
+++ b/doc/source/db.rst
@@ -29,9 +29,10 @@ The commands should be executed as a subcommand of 'db':
29Sync the Database 29Sync the Database
30----------------- 30-----------------
31 31
32 glance-manage db sync <version> <current_version> 32 glance-manage db sync <VERSION>
33 33
34Place a database under migration control and upgrade, creating it first if necessary. 34Place an existing database under migration control and upgrade it to the
35specified VERSION.
35 36
36 37
37Determining the Database Version 38Determining the Database Version
diff --git a/doc/source/man/glancemanage.rst b/doc/source/man/glancemanage.rst
index 621bfca..0ecd289 100644
--- a/doc/source/man/glancemanage.rst
+++ b/doc/source/man/glancemanage.rst
@@ -53,9 +53,9 @@ COMMANDS
53 **db_version_control** 53 **db_version_control**
54 Place the database under migration control. 54 Place the database under migration control.
55 55
56 **db_sync <VERSION> <CURRENT_VERSION>** 56 **db_sync <VERSION>**
57 Place a database under migration control and upgrade, creating 57 Place an existing database under migration control and upgrade it to
58 it first if necessary. 58 the specified VERSION.
59 59
60 **db_export_metadefs [PATH | PREFIX]** 60 **db_export_metadefs [PATH | PREFIX]**
61 Export the metadata definitions into json format. By default the 61 Export the metadata definitions into json format. By default the
@@ -80,10 +80,6 @@ OPTIONS
80 80
81 .. include:: general_options.rst 81 .. include:: general_options.rst
82 82
83 **--sql_connection=CONN_STRING**
84 A proper SQLAlchemy connection string as described
85 `here <http://www.sqlalchemy.org/docs/05/reference/sqlalchemy/connections.html?highlight=engine#sqlalchemy.create_engine>`_
86
87.. include:: footer.rst 83.. include:: footer.rst
88 84
89CONFIGURATION 85CONFIGURATION
diff --git a/glance/cmd/manage.py b/glance/cmd/manage.py
index 16e08c5..aa2ec04 100644
--- a/glance/cmd/manage.py
+++ b/glance/cmd/manage.py
@@ -39,8 +39,9 @@ possible_topdir = os.path.normpath(os.path.join(os.path.abspath(sys.argv[0]),
39if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')): 39if os.path.exists(os.path.join(possible_topdir, 'glance', '__init__.py')):
40 sys.path.insert(0, possible_topdir) 40 sys.path.insert(0, possible_topdir)
41 41
42from alembic import command as alembic_command
43
42from oslo_config import cfg 44from oslo_config import cfg
43from oslo_db.sqlalchemy import migration
44from oslo_log import log as logging 45from oslo_log import log as logging
45from oslo_utils import encodeutils 46from oslo_utils import encodeutils
46import six 47import six
@@ -49,6 +50,7 @@ from glance.common import config
49from glance.common import exception 50from glance.common import exception
50from glance import context 51from glance import context
51from glance.db import migration as db_migration 52from glance.db import migration as db_migration
53from glance.db.sqlalchemy import alembic_migrations
52from glance.db.sqlalchemy import api as db_api 54from glance.db.sqlalchemy import api as db_api
53from glance.db.sqlalchemy import metadata 55from glance.db.sqlalchemy import metadata
54from glance.i18n import _ 56from glance.i18n import _
@@ -73,39 +75,56 @@ class DbCommands(object):
73 75
74 def version(self): 76 def version(self):
75 """Print database's current migration level""" 77 """Print database's current migration level"""
76 print(migration.db_version(db_api.get_engine(), 78 current_heads = alembic_migrations.get_current_alembic_heads()
77 db_migration.MIGRATE_REPO_PATH, 79 if current_heads:
78 db_migration.INIT_VERSION)) 80 # Migrations are managed by alembic
81 for head in current_heads:
82 print(head)
83 else:
84 # Migrations are managed by legacy versioning scheme
85 print(_('Database is either not under migration control or under '
86 'legacy migration control, please run '
87 '"glance-manage db sync" to place the database under '
88 'alembic migration control.'))
79 89
80 @args('--version', metavar='<version>', help='Database version') 90 @args('--version', metavar='<version>', help='Database version')
81 def upgrade(self, version=None): 91 def upgrade(self, version='heads'):
82 """Upgrade the database's migration level""" 92 """Upgrade the database's migration level"""
83 migration.db_sync(db_api.get_engine(), 93 self.sync(version)
84 db_migration.MIGRATE_REPO_PATH,
85 version)
86 94
87 @args('--version', metavar='<version>', help='Database version') 95 @args('--version', metavar='<version>', help='Database version')
88 def version_control(self, version=None): 96 def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION):
89 """Place a database under migration control""" 97 """Place a database under migration control"""
90 migration.db_version_control(db_api.get_engine(), 98
91 db_migration.MIGRATE_REPO_PATH, 99 if version is None:
92 version) 100 version = db_migration.ALEMBIC_INIT_VERSION
101
102 a_config = alembic_migrations.get_alembic_config()
103 alembic_command.stamp(a_config, version)
104 print(_("Placed database under migration control at "
105 "revision:"), version)
93 106
94 @args('--version', metavar='<version>', help='Database version') 107 @args('--version', metavar='<version>', help='Database version')
95 @args('--current_version', metavar='<version>', 108 def sync(self, version='heads'):
96 help='Current Database version')
97 def sync(self, version=None, current_version=None):
98 """ 109 """
99 Place a database under migration control and upgrade it, 110 Place an existing database under migration control and upgrade it.
100 creating first if necessary.
101 """ 111 """
102 if current_version not in (None, 'None'): 112 if version is None:
103 migration.db_version_control(db_api.get_engine(), 113 version = 'heads'
104 db_migration.MIGRATE_REPO_PATH, 114
105 version=current_version) 115 alembic_migrations.place_database_under_alembic_control()
106 migration.db_sync(db_api.get_engine(), 116
107 db_migration.MIGRATE_REPO_PATH, 117 a_config = alembic_migrations.get_alembic_config()
108 version) 118 alembic_command.upgrade(a_config, version)
119 heads = alembic_migrations.get_current_alembic_heads()
120 if heads is None:
121 raise Exception("Database sync failed")
122 revs = ", ".join(heads)
123 if version is 'heads':
124 print(_("Upgraded database, current revision(s):"), revs)
125 else:
126 print(_('Upgraded database to: %(v)s, current revision(s): %(r)s')
127 % {'v': version, 'r': revs})
109 128
110 @args('--path', metavar='<path>', help='Path to the directory or file ' 129 @args('--path', metavar='<path>', help='Path to the directory or file '
111 'where json metadata is stored') 130 'where json metadata is stored')
@@ -179,15 +198,14 @@ class DbLegacyCommands(object):
179 def version(self): 198 def version(self):
180 self.command_object.version() 199 self.command_object.version()
181 200
182 def upgrade(self, version=None): 201 def upgrade(self, version='heads'):
183 self.command_object.upgrade(CONF.command.version) 202 self.command_object.upgrade(CONF.command.version)
184 203
185 def version_control(self, version=None): 204 def version_control(self, version=db_migration.ALEMBIC_INIT_VERSION):
186 self.command_object.version_control(CONF.command.version) 205 self.command_object.version_control(CONF.command.version)
187 206
188 def sync(self, version=None, current_version=None): 207 def sync(self, version='heads'):
189 self.command_object.sync(CONF.command.version, 208 self.command_object.sync(CONF.command.version)
190 CONF.command.current_version)
191 209
192 def load_metadefs(self, path=None, merge=False, 210 def load_metadefs(self, path=None, merge=False,
193 prefer_new=False, overwrite=False): 211 prefer_new=False, overwrite=False):
@@ -224,7 +242,6 @@ def add_legacy_command_parsers(command_object, subparsers):
224 parser = subparsers.add_parser('db_sync') 242 parser = subparsers.add_parser('db_sync')
225 parser.set_defaults(action_fn=legacy_command_object.sync) 243 parser.set_defaults(action_fn=legacy_command_object.sync)
226 parser.add_argument('version', nargs='?') 244 parser.add_argument('version', nargs='?')
227 parser.add_argument('current_version', nargs='?')
228 parser.set_defaults(action='db_sync') 245 parser.set_defaults(action='db_sync')
229 246
230 parser = subparsers.add_parser('db_load_metadefs') 247 parser = subparsers.add_parser('db_load_metadefs')
diff --git a/glance/db/migration.py b/glance/db/migration.py
index 3709f06..a35591d 100644
--- a/glance/db/migration.py
+++ b/glance/db/migration.py
@@ -45,6 +45,7 @@ def get_backend():
45 cfg.CONF.database.backend).driver 45 cfg.CONF.database.backend).driver
46 return _IMPL 46 return _IMPL
47 47
48ALEMBIC_INIT_VERSION = 'liberty'
48INIT_VERSION = 0 49INIT_VERSION = 0
49 50
50MIGRATE_REPO_PATH = os.path.join( 51MIGRATE_REPO_PATH = os.path.join(
diff --git a/glance/db/sqlalchemy/alembic_migrations/README b/glance/db/sqlalchemy/alembic_migrations/README
new file mode 100644
index 0000000..2500aa1
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/README
@@ -0,0 +1 @@
Generic single-database configuration.
diff --git a/glance/db/sqlalchemy/alembic_migrations/__init__.py b/glance/db/sqlalchemy/alembic_migrations/__init__.py
new file mode 100644
index 0000000..b1a0499
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/__init__.py
@@ -0,0 +1,99 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16import os
17import sys
18
19from alembic import command as alembic_command
20from alembic import config as alembic_config
21from alembic import migration as alembic_migration
22from oslo_db import exception as db_exception
23from oslo_db.sqlalchemy import migration
24
25from glance.db import migration as db_migration
26from glance.db.sqlalchemy import api as db_api
27from glance.i18n import _
28
29
30def get_alembic_config():
31 """Return a valid alembic config object"""
32 ini_path = os.path.join(os.path.dirname(__file__), 'alembic.ini')
33 config = alembic_config.Config(os.path.abspath(ini_path))
34 dbconn = str(db_api.get_engine().url)
35 config.set_main_option('sqlalchemy.url', dbconn)
36 return config
37
38
39def get_current_alembic_heads():
40 """Return current heads (if any) from the alembic migration table"""
41 engine = db_api.get_engine()
42 with engine.connect() as conn:
43 context = alembic_migration.MigrationContext.configure(conn)
44 heads = context.get_current_heads()
45 return heads
46
47
48def get_current_legacy_head():
49 try:
50 legacy_head = migration.db_version(db_api.get_engine(),
51 db_migration.MIGRATE_REPO_PATH,
52 db_migration.INIT_VERSION)
53 except db_exception.DbMigrationError:
54 legacy_head = None
55 return legacy_head
56
57
58def is_database_under_alembic_control():
59 if get_current_alembic_heads():
60 return True
61 return False
62
63
64def is_database_under_migrate_control():
65 if get_current_legacy_head():
66 return True
67 return False
68
69
70def place_database_under_alembic_control():
71 a_config = get_alembic_config()
72
73 if not is_database_under_migrate_control():
74 return
75
76 if not is_database_under_alembic_control():
77 print(_("Database is currently not under Alembic's migration "
78 "control."))
79 head = get_current_legacy_head()
80 if head == 42:
81 alembic_version = 'liberty'
82 elif head == 43:
83 alembic_version = 'mitaka01'
84 elif head == 44:
85 alembic_version = 'mitaka02'
86 elif head == 45:
87 alembic_version = 'ocata01'
88 elif head in range(1, 42):
89 print("Legacy head: ", head)
90 sys.exit(_("The current database version is not supported any "
91 "more. Please upgrade to Liberty release first."))
92 else:
93 sys.exit(_("Unable to place database under Alembic's migration "
94 "control. Unknown database state, can't proceed "
95 "further."))
96
97 print(_("Placing database under Alembic's migration control at "
98 "revision:"), alembic_version)
99 alembic_command.stamp(a_config, alembic_version)
diff --git a/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py
new file mode 100644
index 0000000..6d965f6
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/add_artifacts_tables.py
@@ -0,0 +1,224 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16from alembic import op
17from sqlalchemy.schema import (
18 Column, PrimaryKeyConstraint, ForeignKeyConstraint)
19
20from glance.db.sqlalchemy.migrate_repo.schema import (
21 Boolean, DateTime, Integer, BigInteger, String, Text, Numeric) # noqa
22
23
24def _add_artifacts_table():
25 op.create_table('artifacts',
26 Column('id', String(length=36), nullable=False),
27 Column('name', String(length=255), nullable=False),
28 Column('type_name', String(length=255), nullable=False),
29 Column('type_version_prefix',
30 BigInteger(),
31 nullable=False),
32 Column('type_version_suffix',
33 String(length=255),
34 nullable=True),
35 Column('type_version_meta',
36 String(length=255),
37 nullable=True),
38 Column('version_prefix', BigInteger(), nullable=False),
39 Column('version_suffix',
40 String(length=255),
41 nullable=True),
42 Column('version_meta', String(length=255), nullable=True),
43 Column('description', Text(), nullable=True),
44 Column('visibility', String(length=32), nullable=False),
45 Column('state', String(length=32), nullable=False),
46 Column('owner', String(length=255), nullable=False),
47 Column('created_at', DateTime(), nullable=False),
48 Column('updated_at', DateTime(), nullable=False),
49 Column('deleted_at', DateTime(), nullable=True),
50 Column('published_at', DateTime(), nullable=True),
51 PrimaryKeyConstraint('id'),
52 mysql_engine='InnoDB',
53 mysql_charset='utf8',
54 extend_existing=True)
55
56 op.create_index('ix_artifact_name_and_version',
57 'artifacts',
58 ['name', 'version_prefix', 'version_suffix'],
59 unique=False)
60 op.create_index('ix_artifact_owner', 'artifacts', ['owner'], unique=False)
61 op.create_index('ix_artifact_state', 'artifacts', ['state'], unique=False)
62 op.create_index('ix_artifact_type',
63 'artifacts',
64 ['type_name',
65 'type_version_prefix',
66 'type_version_suffix'],
67 unique=False)
68 op.create_index('ix_artifact_visibility',
69 'artifacts',
70 ['visibility'],
71 unique=False)
72
73
74def _add_artifact_blobs_table():
75 op.create_table('artifact_blobs',
76 Column('id', String(length=36), nullable=False),
77 Column('artifact_id', String(length=36), nullable=False),
78 Column('size', BigInteger(), nullable=False),
79 Column('checksum', String(length=32), nullable=True),
80 Column('name', String(length=255), nullable=False),
81 Column('item_key', String(length=329), nullable=True),
82 Column('position', Integer(), nullable=True),
83 Column('created_at', DateTime(), nullable=False),
84 Column('updated_at', DateTime(), nullable=False),
85 ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
86 PrimaryKeyConstraint('id'),
87 mysql_engine='InnoDB',
88 mysql_charset='utf8',
89 extend_existing=True)
90
91 op.create_index('ix_artifact_blobs_artifact_id',
92 'artifact_blobs',
93 ['artifact_id'],
94 unique=False)
95 op.create_index('ix_artifact_blobs_name',
96 'artifact_blobs',
97 ['name'],
98 unique=False)
99
100
101def _add_artifact_dependencies_table():
102 op.create_table('artifact_dependencies',
103 Column('id', String(length=36), nullable=False),
104 Column('artifact_source',
105 String(length=36),
106 nullable=False),
107 Column('artifact_dest', String(length=36), nullable=False),
108 Column('artifact_origin',
109 String(length=36),
110 nullable=False),
111 Column('is_direct', Boolean(), nullable=False),
112 Column('position', Integer(), nullable=True),
113 Column('name', String(length=36), nullable=True),
114 Column('created_at', DateTime(), nullable=False),
115 Column('updated_at', DateTime(), nullable=False),
116 ForeignKeyConstraint(['artifact_dest'],
117 ['artifacts.id'], ),
118 ForeignKeyConstraint(['artifact_origin'],
119 ['artifacts.id'], ),
120 ForeignKeyConstraint(['artifact_source'],
121 ['artifacts.id'], ),
122 PrimaryKeyConstraint('id'),
123 mysql_engine='InnoDB',
124 mysql_charset='utf8',
125 extend_existing=True)
126
127 op.create_index('ix_artifact_dependencies_dest_id',
128 'artifact_dependencies',
129 ['artifact_dest'],
130 unique=False)
131 op.create_index('ix_artifact_dependencies_direct_dependencies',
132 'artifact_dependencies',
133 ['artifact_source', 'is_direct'],
134 unique=False)
135 op.create_index('ix_artifact_dependencies_origin_id',
136 'artifact_dependencies',
137 ['artifact_origin'],
138 unique=False)
139 op.create_index('ix_artifact_dependencies_source_id',
140 'artifact_dependencies',
141 ['artifact_source'],
142 unique=False)
143
144
145def _add_artifact_properties_table():
146 op.create_table('artifact_properties',
147 Column('id', String(length=36), nullable=False),
148 Column('artifact_id', String(length=36), nullable=False),
149 Column('name', String(length=255), nullable=False),
150 Column('string_value', String(length=255), nullable=True),
151 Column('int_value', Integer(), nullable=True),
152 Column('numeric_value', Numeric(), nullable=True),
153 Column('bool_value', Boolean(), nullable=True),
154 Column('text_value', Text(), nullable=True),
155 Column('created_at', DateTime(), nullable=False),
156 Column('updated_at', DateTime(), nullable=False),
157 Column('position', Integer(), nullable=True),
158 ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
159 PrimaryKeyConstraint('id'),
160 mysql_engine='InnoDB',
161 mysql_charset='utf8',
162 extend_existing=True)
163
164 op.create_index('ix_artifact_properties_artifact_id',
165 'artifact_properties',
166 ['artifact_id'],
167 unique=False)
168 op.create_index('ix_artifact_properties_name',
169 'artifact_properties',
170 ['name'],
171 unique=False)
172
173
174def _add_artifact_tags_table():
175 op.create_table('artifact_tags',
176 Column('id', String(length=36), nullable=False),
177 Column('artifact_id', String(length=36), nullable=False),
178 Column('value', String(length=255), nullable=False),
179 Column('created_at', DateTime(), nullable=False),
180 Column('updated_at', DateTime(), nullable=False),
181 ForeignKeyConstraint(['artifact_id'], ['artifacts.id'], ),
182 PrimaryKeyConstraint('id'),
183 mysql_engine='InnoDB',
184 mysql_charset='utf8',
185 extend_existing=True)
186
187 op.create_index('ix_artifact_tags_artifact_id',
188 'artifact_tags',
189 ['artifact_id'],
190 unique=False)
191 op.create_index('ix_artifact_tags_artifact_id_tag_value',
192 'artifact_tags',
193 ['artifact_id', 'value'],
194 unique=False)
195
196
197def _add_artifact_blob_locations_table():
198 op.create_table('artifact_blob_locations',
199 Column('id', String(length=36), nullable=False),
200 Column('blob_id', String(length=36), nullable=False),
201 Column('value', Text(), nullable=False),
202 Column('created_at', DateTime(), nullable=False),
203 Column('updated_at', DateTime(), nullable=False),
204 Column('position', Integer(), nullable=True),
205 Column('status', String(length=36), nullable=True),
206 ForeignKeyConstraint(['blob_id'], ['artifact_blobs.id'], ),
207 PrimaryKeyConstraint('id'),
208 mysql_engine='InnoDB',
209 mysql_charset='utf8',
210 extend_existing=True)
211
212 op.create_index('ix_artifact_blob_locations_blob_id',
213 'artifact_blob_locations',
214 ['blob_id'],
215 unique=False)
216
217
218def upgrade():
219 _add_artifacts_table()
220 _add_artifact_blobs_table()
221 _add_artifact_dependencies_table()
222 _add_artifact_properties_table()
223 _add_artifact_tags_table()
224 _add_artifact_blob_locations_table()
diff --git a/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py
new file mode 100644
index 0000000..399c77e
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/add_images_tables.py
@@ -0,0 +1,201 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16from alembic import op
17from sqlalchemy import sql
18from sqlalchemy.schema import (
19 Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint)
20
21from glance.db.sqlalchemy.migrate_repo.schema import (
22 Boolean, DateTime, Integer, BigInteger, String, Text) # noqa
23from glance.db.sqlalchemy.models import JSONEncodedDict
24
25
26def _add_images_table():
27 op.create_table('images',
28 Column('id', String(length=36), nullable=False),
29 Column('name', String(length=255), nullable=True),
30 Column('size', BigInteger(), nullable=True),
31 Column('status', String(length=30), nullable=False),
32 Column('is_public', Boolean(), nullable=False),
33 Column('created_at', DateTime(), nullable=False),
34 Column('updated_at', DateTime(), nullable=True),
35 Column('deleted_at', DateTime(), nullable=True),
36 Column('deleted', Boolean(), nullable=False),
37 Column('disk_format', String(length=20), nullable=True),
38 Column('container_format',
39 String(length=20),
40 nullable=True),
41 Column('checksum', String(length=32), nullable=True),
42 Column('owner', String(length=255), nullable=True),
43 Column('min_disk', Integer(), nullable=False),
44 Column('min_ram', Integer(), nullable=False),
45 Column('protected',
46 Boolean(),
47 server_default=sql.false(),
48 nullable=False),
49 Column('virtual_size', BigInteger(), nullable=True),
50 PrimaryKeyConstraint('id'),
51 mysql_engine='InnoDB',
52 mysql_charset='utf8',
53 extend_existing=True)
54
55 op.create_index('checksum_image_idx',
56 'images',
57 ['checksum'],
58 unique=False)
59 op.create_index('ix_images_deleted',
60 'images',
61 ['deleted'],
62 unique=False)
63 op.create_index('ix_images_is_public',
64 'images',
65 ['is_public'],
66 unique=False)
67 op.create_index('owner_image_idx',
68 'images',
69 ['owner'],
70 unique=False)
71
72
73def _add_image_properties_table():
74 op.create_table('image_properties',
75 Column('id', Integer(), nullable=False),
76 Column('image_id', String(length=36), nullable=False),
77 Column('name', String(length=255), nullable=False),
78 Column('value', Text(), nullable=True),
79 Column('created_at', DateTime(), nullable=False),
80 Column('updated_at', DateTime(), nullable=True),
81 Column('deleted_at', DateTime(), nullable=True),
82 Column('deleted', Boolean(), nullable=False),
83 PrimaryKeyConstraint('id'),
84 ForeignKeyConstraint(['image_id'], ['images.id'], ),
85 UniqueConstraint('image_id',
86 'name',
87 name='ix_image_properties_image_id_name'),
88 mysql_engine='InnoDB',
89 mysql_charset='utf8',
90 extend_existing=True)
91
92 op.create_index('ix_image_properties_deleted',
93 'image_properties',
94 ['deleted'],
95 unique=False)
96 op.create_index('ix_image_properties_image_id',
97 'image_properties',
98 ['image_id'],
99 unique=False)
100
101
102def _add_image_locations_table():
103 op.create_table('image_locations',
104 Column('id', Integer(), nullable=False),
105 Column('image_id', String(length=36), nullable=False),
106 Column('value', Text(), nullable=False),
107 Column('created_at', DateTime(), nullable=False),
108 Column('updated_at', DateTime(), nullable=True),
109 Column('deleted_at', DateTime(), nullable=True),
110 Column('deleted', Boolean(), nullable=False),
111 Column('meta_data', JSONEncodedDict(), nullable=True),
112 Column('status',
113 String(length=30),
114 server_default='active',
115 nullable=False),
116 PrimaryKeyConstraint('id'),
117 ForeignKeyConstraint(['image_id'], ['images.id'], ),
118 mysql_engine='InnoDB',
119 mysql_charset='utf8',
120 extend_existing=True)
121
122 op.create_index('ix_image_locations_deleted',
123 'image_locations',
124 ['deleted'],
125 unique=False)
126 op.create_index('ix_image_locations_image_id',
127 'image_locations',
128 ['image_id'],
129 unique=False)
130
131
132def _add_image_members_table():
133 deleted_member_constraint = 'image_members_image_id_member_deleted_at_key'
134 op.create_table('image_members',
135 Column('id', Integer(), nullable=False),
136 Column('image_id', String(length=36), nullable=False),
137 Column('member', String(length=255), nullable=False),
138 Column('can_share', Boolean(), nullable=False),
139 Column('created_at', DateTime(), nullable=False),
140 Column('updated_at', DateTime(), nullable=True),
141 Column('deleted_at', DateTime(), nullable=True),
142 Column('deleted', Boolean(), nullable=False),
143 Column('status',
144 String(length=20),
145 server_default='pending',
146 nullable=False),
147 ForeignKeyConstraint(['image_id'], ['images.id'], ),
148 PrimaryKeyConstraint('id'),
149 UniqueConstraint('image_id',
150 'member',
151 'deleted_at',
152 name=deleted_member_constraint),
153 mysql_engine='InnoDB',
154 mysql_charset='utf8',
155 extend_existing=True)
156
157 op.create_index('ix_image_members_deleted',
158 'image_members',
159 ['deleted'],
160 unique=False)
161 op.create_index('ix_image_members_image_id',
162 'image_members',
163 ['image_id'],
164 unique=False)
165 op.create_index('ix_image_members_image_id_member',
166 'image_members',
167 ['image_id', 'member'],
168 unique=False)
169
170
171def _add_images_tags_table():
172 op.create_table('image_tags',
173 Column('id', Integer(), nullable=False),
174 Column('image_id', String(length=36), nullable=False),
175 Column('value', String(length=255), nullable=False),
176 Column('created_at', DateTime(), nullable=False),
177 Column('updated_at', DateTime(), nullable=True),
178 Column('deleted_at', DateTime(), nullable=True),
179 Column('deleted', Boolean(), nullable=False),
180 ForeignKeyConstraint(['image_id'], ['images.id'], ),
181 PrimaryKeyConstraint('id'),
182 mysql_engine='InnoDB',
183 mysql_charset='utf8',
184 extend_existing=True)
185
186 op.create_index('ix_image_tags_image_id',
187 'image_tags',
188 ['image_id'],
189 unique=False)
190 op.create_index('ix_image_tags_image_id_tag_value',
191 'image_tags',
192 ['image_id', 'value'],
193 unique=False)
194
195
196def upgrade():
197 _add_images_table()
198 _add_image_properties_table()
199 _add_image_locations_table()
200 _add_image_members_table()
201 _add_images_tags_table()
diff --git a/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py
new file mode 100644
index 0000000..96fa733
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/add_metadefs_tables.py
@@ -0,0 +1,171 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16from alembic import op
17from sqlalchemy.schema import (
18 Column, PrimaryKeyConstraint, ForeignKeyConstraint, UniqueConstraint)
19
20from glance.db.sqlalchemy.migrate_repo.schema import (
21 Boolean, DateTime, Integer, String, Text) # noqa
22from glance.db.sqlalchemy.models import JSONEncodedDict
23
24
25def _add_metadef_namespaces_table():
26 op.create_table('metadef_namespaces',
27 Column('id', Integer(), nullable=False),
28 Column('namespace', String(length=80), nullable=False),
29 Column('display_name', String(length=80), nullable=True),
30 Column('description', Text(), nullable=True),
31 Column('visibility', String(length=32), nullable=True),
32 Column('protected', Boolean(), nullable=True),
33 Column('owner', String(length=255), nullable=False),
34 Column('created_at', DateTime(), nullable=False),
35 Column('updated_at', DateTime(), nullable=True),
36 PrimaryKeyConstraint('id'),
37 UniqueConstraint('namespace',
38 name='uq_metadef_namespaces_namespace'),
39 mysql_engine='InnoDB',
40 mysql_charset='utf8',
41 extend_existing=True)
42
43 op.create_index('ix_metadef_namespaces_owner',
44 'metadef_namespaces',
45 ['owner'],
46 unique=False)
47
48
49def _add_metadef_resource_types_table():
50 op.create_table('metadef_resource_types',
51 Column('id', Integer(), nullable=False),
52 Column('name', String(length=80), nullable=False),
53 Column('protected', Boolean(), nullable=False),
54 Column('created_at', DateTime(), nullable=False),
55 Column('updated_at', DateTime(), nullable=True),
56 PrimaryKeyConstraint('id'),
57 UniqueConstraint('name',
58 name='uq_metadef_resource_types_name'),
59 mysql_engine='InnoDB',
60 mysql_charset='utf8',
61 extend_existing=True)
62
63
64def _add_metadef_namespace_resource_types_table():
65 op.create_table('metadef_namespace_resource_types',
66 Column('resource_type_id', Integer(), nullable=False),
67 Column('namespace_id', Integer(), nullable=False),
68 Column('properties_target',
69 String(length=80),
70 nullable=True),
71 Column('prefix', String(length=80), nullable=True),
72 Column('created_at', DateTime(), nullable=False),
73 Column('updated_at', DateTime(), nullable=True),
74 ForeignKeyConstraint(['namespace_id'],
75 ['metadef_namespaces.id'], ),
76 ForeignKeyConstraint(['resource_type_id'],
77 ['metadef_resource_types.id'], ),
78 PrimaryKeyConstraint('resource_type_id', 'namespace_id'),
79 mysql_engine='InnoDB',
80 mysql_charset='utf8',
81 extend_existing=True)
82
83 op.create_index('ix_metadef_ns_res_types_namespace_id',
84 'metadef_namespace_resource_types',
85 ['namespace_id'],
86 unique=False)
87
88
89def _add_metadef_objects_table():
90 ns_id_name_constraint = 'uq_metadef_objects_namespace_id_name'
91
92 op.create_table('metadef_objects',
93 Column('id', Integer(), nullable=False),
94 Column('namespace_id', Integer(), nullable=False),
95 Column('name', String(length=80), nullable=False),
96 Column('description', Text(), nullable=True),
97 Column('required', Text(), nullable=True),
98 Column('json_schema', JSONEncodedDict(), nullable=False),
99 Column('created_at', DateTime(), nullable=False),
100 Column('updated_at', DateTime(), nullable=True),
101 ForeignKeyConstraint(['namespace_id'],
102 ['metadef_namespaces.id'], ),
103 PrimaryKeyConstraint('id'),
104 UniqueConstraint('namespace_id',
105 'name',
106 name=ns_id_name_constraint),
107 mysql_engine='InnoDB',
108 mysql_charset='utf8',
109 extend_existing=True)
110
111 op.create_index('ix_metadef_objects_name',
112 'metadef_objects',
113 ['name'],
114 unique=False)
115
116
117def _add_metadef_properties_table():
118 ns_id_name_constraint = 'uq_metadef_properties_namespace_id_name'
119 op.create_table('metadef_properties',
120 Column('id', Integer(), nullable=False),
121 Column('namespace_id', Integer(), nullable=False),
122 Column('name', String(length=80), nullable=False),
123 Column('json_schema', JSONEncodedDict(), nullable=False),
124 Column('created_at', DateTime(), nullable=False),
125 Column('updated_at', DateTime(), nullable=True),
126 ForeignKeyConstraint(['namespace_id'],
127 ['metadef_namespaces.id'], ),
128 PrimaryKeyConstraint('id'),
129 UniqueConstraint('namespace_id',
130 'name',
131 name=ns_id_name_constraint),
132 mysql_engine='InnoDB',
133 mysql_charset='utf8',
134 extend_existing=True)
135
136 op.create_index('ix_metadef_properties_name',
137 'metadef_properties',
138 ['name'],
139 unique=False)
140
141
142def _add_metadef_tags_table():
143 op.create_table('metadef_tags',
144 Column('id', Integer(), nullable=False),
145 Column('namespace_id', Integer(), nullable=False),
146 Column('name', String(length=80), nullable=False),
147 Column('created_at', DateTime(), nullable=False),
148 Column('updated_at', DateTime(), nullable=True),
149 ForeignKeyConstraint(['namespace_id'],
150 ['metadef_namespaces.id'], ),
151 PrimaryKeyConstraint('id'),
152 UniqueConstraint('namespace_id',
153 'name',
154 name='uq_metadef_tags_namespace_id_name'),
155 mysql_engine='InnoDB',
156 mysql_charset='utf8',
157 extend_existing=True)
158
159 op.create_index('ix_metadef_tags_name',
160 'metadef_tags',
161 ['name'],
162 unique=False)
163
164
165def upgrade():
166 _add_metadef_namespaces_table()
167 _add_metadef_resource_types_table()
168 _add_metadef_namespace_resource_types_table()
169 _add_metadef_objects_table()
170 _add_metadef_properties_table()
171 _add_metadef_tags_table()
diff --git a/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py b/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py
new file mode 100644
index 0000000..d199557
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/add_tasks_tables.py
@@ -0,0 +1,66 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16from alembic import op
17from sqlalchemy.schema import (
18 Column, PrimaryKeyConstraint, ForeignKeyConstraint)
19
20from glance.db.sqlalchemy.migrate_repo.schema import (
21 Boolean, DateTime, String, Text) # noqa
22from glance.db.sqlalchemy.models import JSONEncodedDict
23
24
25def _add_tasks_table():
26 op.create_table('tasks',
27 Column('id', String(length=36), nullable=False),
28 Column('type', String(length=30), nullable=False),
29 Column('status', String(length=30), nullable=False),
30 Column('owner', String(length=255), nullable=False),
31 Column('expires_at', DateTime(), nullable=True),
32 Column('created_at', DateTime(), nullable=False),
33 Column('updated_at', DateTime(), nullable=True),
34 Column('deleted_at', DateTime(), nullable=True),
35 Column('deleted', Boolean(), nullable=False),
36 PrimaryKeyConstraint('id'),
37 mysql_engine='InnoDB',
38 mysql_charset='utf8',
39 extend_existing=True)
40
41 op.create_index('ix_tasks_deleted', 'tasks', ['deleted'], unique=False)
42 op.create_index('ix_tasks_owner', 'tasks', ['owner'], unique=False)
43 op.create_index('ix_tasks_status', 'tasks', ['status'], unique=False)
44 op.create_index('ix_tasks_type', 'tasks', ['type'], unique=False)
45 op.create_index('ix_tasks_updated_at',
46 'tasks',
47 ['updated_at'],
48 unique=False)
49
50
51def _add_task_info_table():
52 op.create_table('task_info',
53 Column('task_id', String(length=36), nullable=False),
54 Column('input', JSONEncodedDict(), nullable=True),
55 Column('result', JSONEncodedDict(), nullable=True),
56 Column('message', Text(), nullable=True),
57 ForeignKeyConstraint(['task_id'], ['tasks.id'], ),
58 PrimaryKeyConstraint('task_id'),
59 mysql_engine='InnoDB',
60 mysql_charset='utf8',
61 extend_existing=True)
62
63
64def upgrade():
65 _add_tasks_table()
66 _add_task_info_table()
diff --git a/glance/db/sqlalchemy/alembic_migrations/alembic.ini b/glance/db/sqlalchemy/alembic_migrations/alembic.ini
new file mode 100644
index 0000000..640a9af
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/alembic.ini
@@ -0,0 +1,69 @@
1# A generic, single database configuration.
2
3[alembic]
4# path to migration scripts
5script_location = %(here)s
6
7# template used to generate migration files
8# file_template = %%(rev)s_%%(slug)s
9
10# max length of characters to apply to the
11# "slug" field
12#truncate_slug_length = 40
13
14# set to 'true' to run the environment during
15# the 'revision' command, regardless of autogenerate
16# revision_environment = false
17
18# set to 'true' to allow .pyc and .pyo files without
19# a source .py file to be detected as revisions in the
20# versions/ directory
21# sourceless = false
22
23# version location specification; this defaults
24# to alembic_migrations/versions. When using multiple version
25# directories, initial revisions must be specified with --version-path
26# version_locations = %(here)s/bar %(here)s/bat alembic_migrations/versions
27
28# the output encoding used when revision files
29# are written from script.py.mako
30# output_encoding = utf-8
31
32# Uncomment and update to your sql connection string if wishing to run
33# alembic directly from command line
34#sqlalchemy.url =
35
36# Logging configuration
37[loggers]
38keys = root,sqlalchemy,alembic
39
40[handlers]
41keys = console
42
43[formatters]
44keys = generic
45
46[logger_root]
47level = WARN
48handlers = console
49qualname =
50
51[logger_sqlalchemy]
52level = WARN
53handlers =
54qualname = sqlalchemy.engine
55
56[logger_alembic]
57level = INFO
58handlers =
59qualname = alembic
60
61[handler_console]
62class = StreamHandler
63args = (sys.stderr,)
64level = NOTSET
65formatter = generic
66
67[formatter_generic]
68format = %(levelname)-5.5s [%(name)s] %(message)s
69datefmt = %H:%M:%S
diff --git a/glance/db/sqlalchemy/alembic_migrations/env.py b/glance/db/sqlalchemy/alembic_migrations/env.py
new file mode 100644
index 0000000..0de0d82
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/env.py
@@ -0,0 +1,92 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16from __future__ import with_statement
17from logging import config as log_config
18
19from alembic import context
20from sqlalchemy import engine_from_config, pool
21
22from glance.db.sqlalchemy import models
23from glance.db.sqlalchemy import models_glare
24from glance.db.sqlalchemy import models_metadef
25
26# this is the Alembic Config object, which provides
27# access to the values within the .ini file in use.
28config = context.config
29
30# other values from the config, defined by the needs of env.py,
31# can be acquired:
32# my_important_option = config.get_main_option("my_important_option")
33# ... etc.
34
35# Interpret the config file for Python logging.
36# This line sets up loggers basically.
37log_config.fileConfig(config.config_file_name)
38
39# add your model's MetaData object here
40# for 'autogenerate' support
41target_metadata = models.BASE.metadata
42for table in models_glare.BASE.metadata.sorted_tables:
43 target_metadata._add_table(table.name, table.schema, table)
44for table in models_metadef.BASE_DICT.metadata.sorted_tables:
45 target_metadata._add_table(table.name, table.schema, table)
46
47
48def run_migrations_offline():
49 """Run migrations in 'offline' mode.
50
51 This configures the context with just a URL
52 and not an Engine, though an Engine is acceptable
53 here as well. By skipping the Engine creation
54 we don't even need a DBAPI to be available.
55
56 Calls to context.execute() here emit the given string to the
57 script output.
58
59 """
60 url = config.get_main_option("sqlalchemy.url")
61 context.configure(
62 url=url, target_metadata=target_metadata, literal_binds=True)
63
64 with context.begin_transaction():
65 context.run_migrations()
66
67
68def run_migrations_online():
69 """Run migrations in 'online' mode.
70
71 In this scenario we need to create an Engine
72 and associate a connection with the context.
73
74 """
75 connectable = engine_from_config(
76 config.get_section(config.config_ini_section),
77 prefix='sqlalchemy.',
78 poolclass=pool.NullPool)
79
80 with connectable.connect() as connection:
81 context.configure(
82 connection=connection,
83 target_metadata=target_metadata
84 )
85
86 with context.begin_transaction():
87 context.run_migrations()
88
89if context.is_offline_mode():
90 run_migrations_offline()
91else:
92 run_migrations_online()
diff --git a/glance/db/sqlalchemy/alembic_migrations/migrate.cfg b/glance/db/sqlalchemy/alembic_migrations/migrate.cfg
new file mode 100644
index 0000000..8ddf050
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/migrate.cfg
@@ -0,0 +1,20 @@
1[db_settings]
2# Used to identify which repository this database is versioned under.
3# You can use the name of your project.
4repository_id=Glance Migrations
5
6# The name of the database table used to track the schema version.
7# This name shouldn't already be used by your project.
8# If this is changed once a database is under version control, you'll need to
9# change the table name in each database too.
10version_table=alembic_version
11
12# When committing a change script, Migrate will attempt to generate the
13# sql for all supported databases; normally, if one of them fails - probably
14# because you don't have that database installed - it is ignored and the
15# commit continues, perhaps ending successfully.
16# Databases in this list MUST compile successfully during a commit, or the
17# entire commit will fail. List the databases your application will actually
18# be using to ensure your updates to that database work properly.
19# This must be a list; example: ['postgres','sqlite']
20required_dbs=[]
diff --git a/glance/db/sqlalchemy/alembic_migrations/script.py.mako b/glance/db/sqlalchemy/alembic_migrations/script.py.mako
new file mode 100644
index 0000000..8323caa
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/script.py.mako
@@ -0,0 +1,20 @@
1"""${message}
2
3Revision ID: ${up_revision}
4Revises: ${down_revision | comma,n}
5Create Date: ${create_date}
6
7"""
8
9# revision identifiers, used by Alembic.
10revision = ${repr(up_revision)}
11down_revision = ${repr(down_revision)}
12branch_labels = ${repr(branch_labels)}
13depends_on = ${repr(depends_on)}
14
15from alembic import op
16import sqlalchemy as sa
17${imports if imports else ""}
18
19def upgrade():
20 ${upgrades if upgrades else "pass"}
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py b/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py
new file mode 100644
index 0000000..2d56680
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/liberty_initial.py
@@ -0,0 +1,40 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""liberty initial
17
18Revision ID: liberty
19Revises:
20Create Date: 2016-08-03 16:06:59.657433
21
22"""
23
24from glance.db.sqlalchemy.alembic_migrations import add_artifacts_tables
25from glance.db.sqlalchemy.alembic_migrations import add_images_tables
26from glance.db.sqlalchemy.alembic_migrations import add_metadefs_tables
27from glance.db.sqlalchemy.alembic_migrations import add_tasks_tables
28
29# revision identifiers, used by Alembic.
30revision = 'liberty'
31down_revision = None
32branch_labels = None
33depends_on = None
34
35
36def upgrade():
37 add_images_tables.upgrade()
38 add_tasks_tables.upgrade()
39 add_metadefs_tables.upgrade()
40 add_artifacts_tables.upgrade()
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py
new file mode 100644
index 0000000..5180c67
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka01_add_image_created_updated_idx.py
@@ -0,0 +1,47 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""add index on created_at and updated_at columns of 'images' table
17
18Revision ID: mitaka01
19Revises: liberty
20Create Date: 2016-08-03 17:19:35.306161
21
22"""
23
24from alembic import op
25from sqlalchemy import MetaData, Table, Index
26
27
28# revision identifiers, used by Alembic.
29revision = 'mitaka01'
30down_revision = 'liberty'
31branch_labels = None
32depends_on = None
33
34CREATED_AT_INDEX = 'created_at_image_idx'
35UPDATED_AT_INDEX = 'updated_at_image_idx'
36
37
38def upgrade():
39 migrate_engine = op.get_bind()
40 meta = MetaData(bind=migrate_engine)
41
42 images = Table('images', meta, autoload=True)
43
44 created_index = Index(CREATED_AT_INDEX, images.c.created_at)
45 created_index.create(migrate_engine)
46 updated_index = Index(UPDATED_AT_INDEX, images.c.updated_at)
47 updated_index.create(migrate_engine)
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py
new file mode 100644
index 0000000..9416c68
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/mitaka02_update_metadef_os_nova_server.py
@@ -0,0 +1,42 @@
1# Copyright 2016 Rackspace
2# Copyright 2013 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16"""update metadef os_nova_server
17
18Revision ID: mitaka02
19Revises: mitaka01
20Create Date: 2016-08-03 17:23:23.041663
21
22"""
23
24from alembic import op
25from sqlalchemy import MetaData, Table
26
27
28# revision identifiers, used by Alembic.
29revision = 'mitaka02'
30down_revision = 'mitaka01'
31branch_labels = None
32depends_on = None
33
34
35def upgrade():
36 migrate_engine = op.get_bind()
37 meta = MetaData(bind=migrate_engine)
38
39 resource_types_table = Table('metadef_resource_types', meta, autoload=True)
40
41 resource_types_table.update(values={'name': 'OS::Nova::Server'}).where(
42 resource_types_table.c.name == 'OS::Nova::Instance').execute()
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py
new file mode 100644
index 0000000..5d66513
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.py
@@ -0,0 +1,72 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13"""add visibility to and remove is_public from images
14
15Revision ID: ocata01
16Revises: mitaka02
17Create Date: 2017-01-20 12:58:16.647499
18
19"""
20
21import os
22
23from alembic import op
24from sqlalchemy import Column, Enum, MetaData, select, Table, not_, and_
25import sqlparse
26
27# revision identifiers, used by Alembic.
28revision = 'ocata01'
29down_revision = 'mitaka02'
30branch_labels = None
31depends_on = None
32
33
34def upgrade():
35 migrate_engine = op.get_bind()
36 meta = MetaData(bind=migrate_engine)
37
38 engine_name = migrate_engine.engine.name
39 if engine_name == 'sqlite':
40 sql_file = os.path.splitext(__file__)[0]
41 sql_file += '.sql'
42 with open(sql_file, 'r') as sqlite_script:
43 sql = sqlparse.format(sqlite_script.read(), strip_comments=True)
44 for statement in sqlparse.split(sql):
45 op.execute(statement)
46 return
47
48 enum = Enum('private', 'public', 'shared', 'community', metadata=meta,
49 name='image_visibility')
50 enum.create()
51 v_col = Column('visibility', enum, nullable=False, server_default='shared')
52 op.add_column('images', v_col)
53
54 op.create_index('visibility_image_idx', 'images', ['visibility'])
55
56 images = Table('images', meta, autoload=True)
57 images.update(values={'visibility': 'public'}).where(
58 images.c.is_public).execute()
59
60 image_members = Table('image_members', meta, autoload=True)
61
62 # NOTE(dharinic): Mark all the non-public images as 'private' first
63 images.update().values(visibility='private').where(
64 not_(images.c.is_public)).execute()
65 # NOTE(dharinic): Identify 'shared' images from the above
66 images.update().values(visibility='shared').where(and_(
67 images.c.visibility == 'private', images.c.id.in_(select(
68 [image_members.c.image_id]).distinct().where(
69 not_(image_members.c.deleted))))).execute()
70
71 op.drop_index('ix_images_is_public', 'images')
72 op.drop_column('images', 'is_public')
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql
new file mode 100644
index 0000000..0e848cc
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/ocata01_add_visibility_remove_is_public.sql
@@ -0,0 +1,162 @@
1CREATE TEMPORARY TABLE images_backup (
2 id VARCHAR(36) NOT NULL,
3 name VARCHAR(255),
4 size INTEGER,
5 status VARCHAR(30) NOT NULL,
6 is_public BOOLEAN NOT NULL,
7 created_at DATETIME NOT NULL,
8 updated_at DATETIME,
9 deleted_at DATETIME,
10 deleted BOOLEAN NOT NULL,
11 disk_format VARCHAR(20),
12 container_format VARCHAR(20),
13 checksum VARCHAR(32),
14 owner VARCHAR(255),
15 min_disk INTEGER NOT NULL,
16 min_ram INTEGER NOT NULL,
17 protected BOOLEAN DEFAULT 0 NOT NULL,
18 virtual_size INTEGER,
19 PRIMARY KEY (id),
20 CHECK (is_public IN (0, 1)),
21 CHECK (deleted IN (0, 1)),
22 CHECK (protected IN (0, 1))
23);
24
25INSERT INTO images_backup
26 SELECT id,
27 name,
28 size,
29 status,
30 is_public,
31 created_at,
32 updated_at,
33 deleted_at,
34 deleted,
35 disk_format,
36 container_format,
37 checksum,
38 owner,
39 min_disk,
40 min_ram,
41 protected,
42 virtual_size
43 FROM images;
44
45DROP TABLE images;
46
47CREATE TABLE images (
48 id VARCHAR(36) NOT NULL,
49 name VARCHAR(255),
50 size INTEGER,
51 status VARCHAR(30) NOT NULL,
52 created_at DATETIME NOT NULL,
53 updated_at DATETIME,
54 deleted_at DATETIME,
55 deleted BOOLEAN NOT NULL,
56 disk_format VARCHAR(20),
57 container_format VARCHAR(20),
58 checksum VARCHAR(32),
59 owner VARCHAR(255),
60 min_disk INTEGER NOT NULL,
61 min_ram INTEGER NOT NULL,
62 protected BOOLEAN DEFAULT 0 NOT NULL,
63 virtual_size INTEGER,
64 visibility VARCHAR(9) DEFAULT 'shared' NOT NULL,
65 PRIMARY KEY (id),
66 CHECK (deleted IN (0, 1)),
67 CHECK (protected IN (0, 1)),
68 CONSTRAINT image_visibility CHECK (visibility IN ('private', 'public', 'shared', 'community'))
69);
70
71CREATE INDEX checksum_image_idx ON images (checksum);
72CREATE INDEX visibility_image_idx ON images (visibility);
73CREATE INDEX ix_images_deleted ON images (deleted);
74CREATE INDEX owner_image_idx ON images (owner);
75CREATE INDEX created_at_image_idx ON images (created_at);
76CREATE INDEX updated_at_image_idx ON images (updated_at);
77
78-- Copy over all the 'public' rows
79
80INSERT INTO images (
81 id,
82 name,
83 size,
84 status,
85 created_at,
86 updated_at,
87 deleted_at,
88 deleted,
89 disk_format,
90 container_format,
91 checksum,
92 owner,
93 min_disk,
94 min_ram,
95 protected,
96 virtual_size
97 )
98 SELECT id,
99 name,
100 size,
101 status,
102 created_at,
103 updated_at,
104 deleted_at,
105 deleted,
106 disk_format,
107 container_format,
108 checksum,
109 owner,
110 min_disk,
111 min_ram,
112 protected,
113 virtual_size
114 FROM images_backup
115 WHERE is_public=1;
116
117
118UPDATE images SET visibility='public';
119
120-- Now copy over the 'private' rows
121
122INSERT INTO images (
123 id,
124 name,
125 size,
126 status,
127 created_at,
128 updated_at,
129 deleted_at,
130 deleted,
131 disk_format,
132 container_format,
133 checksum,
134 owner,
135 min_disk,
136 min_ram,
137 protected,
138 virtual_size
139 )
140 SELECT id,
141 name,
142 size,
143 status,
144 created_at,
145 updated_at,
146 deleted_at,
147 deleted,
148 disk_format,
149 container_format,
150 checksum,
151 owner,
152 min_disk,
153 min_ram,
154 protected,
155 virtual_size
156 FROM images_backup
157 WHERE is_public=0;
158
159UPDATE images SET visibility='private' WHERE visibility='shared';
160UPDATE images SET visibility='shared' WHERE visibility='private' AND id IN (SELECT DISTINCT image_id FROM image_members WHERE deleted != 1);
161
162DROP TABLE images_backup;
diff --git a/glance/tests/unit/test_manage.py b/glance/tests/unit/test_manage.py
index 16a76ab..0211e10 100644
--- a/glance/tests/unit/test_manage.py
+++ b/glance/tests/unit/test_manage.py
@@ -15,11 +15,8 @@
15 15
16import fixtures 16import fixtures
17import mock 17import mock
18from oslo_db.sqlalchemy import migration
19from six.moves import StringIO
20 18
21from glance.cmd import manage 19from glance.cmd import manage
22from glance.db import migration as db_migration
23from glance.db.sqlalchemy import api as db_api 20from glance.db.sqlalchemy import api as db_api
24from glance.db.sqlalchemy import metadata as db_metadata 21from glance.db.sqlalchemy import metadata as db_metadata
25from glance.tests import utils as test_utils 22from glance.tests import utils as test_utils
@@ -51,48 +48,35 @@ class TestManageBase(test_utils.BaseTestCase):
51 48
52class TestLegacyManage(TestManageBase): 49class TestLegacyManage(TestManageBase):
53 50
54 @mock.patch.object(migration, 'db_version') 51 @mock.patch.object(manage.DbCommands, 'version')
55 def test_legacy_db_version(self, db_version): 52 def test_legacy_db_version(self, db_upgrade):
56 with mock.patch('sys.stdout', new_callable=StringIO): 53 self._main_test_helper(['glance.cmd.manage', 'db_version'],
57 self._main_test_helper(['glance.cmd.manage', 'db_version'], 54 manage.DbCommands.version)
58 migration.db_version,
59 db_api.get_engine(),
60 db_migration.MIGRATE_REPO_PATH, 0)
61 55
62 @mock.patch.object(migration, 'db_sync') 56 @mock.patch.object(manage.DbCommands, 'sync')
63 def test_legacy_db_sync(self, db_sync): 57 def test_legacy_db_sync(self, db_sync):
64 self._main_test_helper(['glance.cmd.manage', 'db_sync'], 58 self._main_test_helper(['glance.cmd.manage', 'db_sync'],
65 migration.db_sync, 59 manage.DbCommands.sync, None)
66 db_api.get_engine(),
67 db_migration.MIGRATE_REPO_PATH, None)
68 60
69 @mock.patch.object(migration, 'db_sync') 61 @mock.patch.object(manage.DbCommands, 'upgrade')
70 def test_legacy_db_upgrade(self, db_sync): 62 def test_legacy_db_upgrade(self, db_upgrade):
71 self._main_test_helper(['glance.cmd.manage', 'db_upgrade'], 63 self._main_test_helper(['glance.cmd.manage', 'db_upgrade'],
72 migration.db_sync, 64 manage.DbCommands.upgrade, None)
73 db_api.get_engine(),
74 db_migration.MIGRATE_REPO_PATH, None)
75 65
76 @mock.patch.object(migration, 'db_version_control') 66 @mock.patch.object(manage.DbCommands, 'version_control')
77 def test_legacy_db_version_control(self, db_version_control): 67 def test_legacy_db_version_control(self, db_version_control):
78 self._main_test_helper(['glance.cmd.manage', 'db_version_control'], 68 self._main_test_helper(['glance.cmd.manage', 'db_version_control'],
79 migration.db_version_control, 69 manage.DbCommands.version_control, None)
80 db_api.get_engine(),
81 db_migration.MIGRATE_REPO_PATH, None)
82 70
83 @mock.patch.object(migration, 'db_sync') 71 @mock.patch.object(manage.DbCommands, 'sync')
84 def test_legacy_db_sync_version(self, db_sync): 72 def test_legacy_db_sync_version(self, db_sync):
85 self._main_test_helper(['glance.cmd.manage', 'db_sync', '20'], 73 self._main_test_helper(['glance.cmd.manage', 'db_sync', 'liberty'],
86 migration.db_sync, 74 manage.DbCommands.sync, 'liberty')
87 db_api.get_engine(),
88 db_migration.MIGRATE_REPO_PATH, '20')
89 75
90 @mock.patch.object(migration, 'db_sync') 76 @mock.patch.object(manage.DbCommands, 'upgrade')
91 def test_legacy_db_upgrade_version(self, db_sync): 77 def test_legacy_db_upgrade_version(self, db_upgrade):
92 self._main_test_helper(['glance.cmd.manage', 'db_upgrade', '20'], 78 self._main_test_helper(['glance.cmd.manage', 'db_upgrade', 'liberty'],
93 migration.db_sync, 79 manage.DbCommands.upgrade, 'liberty')
94 db_api.get_engine(),
95 db_migration.MIGRATE_REPO_PATH, '20')
96 80
97 def test_db_metadefs_unload(self): 81 def test_db_metadefs_unload(self):
98 db_metadata.db_unload_metadefs = mock.Mock() 82 db_metadata.db_unload_metadefs = mock.Mock()
@@ -157,48 +141,36 @@ class TestLegacyManage(TestManageBase):
157 141
158class TestManage(TestManageBase): 142class TestManage(TestManageBase):
159 143
160 @mock.patch.object(migration, 'db_version') 144 @mock.patch.object(manage.DbCommands, 'version')
161 def test_db_version(self, db_version): 145 def test_db_version(self, version):
162 with mock.patch('sys.stdout', new_callable=StringIO): 146 self._main_test_helper(['glance.cmd.manage', 'db', 'version'],
163 self._main_test_helper(['glance.cmd.manage', 'db', 'version'], 147 manage.DbCommands.version)
164 migration.db_version,
165 db_api.get_engine(),
166 db_migration.MIGRATE_REPO_PATH, 0)
167 148
168 @mock.patch.object(migration, 'db_sync') 149 @mock.patch.object(manage.DbCommands, 'sync')
169 def test_db_sync(self, db_sync): 150 def test_db_sync(self, sync):
170 self._main_test_helper(['glance.cmd.manage', 'db', 'sync'], 151 self._main_test_helper(['glance.cmd.manage', 'db', 'sync'],
171 migration.db_sync, 152 manage.DbCommands.sync)
172 db_api.get_engine(),
173 db_migration.MIGRATE_REPO_PATH, None)
174 153
175 @mock.patch.object(migration, 'db_sync') 154 @mock.patch.object(manage.DbCommands, 'upgrade')
176 def test_db_upgrade(self, db_sync): 155 def test_db_upgrade(self, upgrade):
177 self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'], 156 self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade'],
178 migration.db_sync, 157 manage.DbCommands.upgrade)
179 db_api.get_engine(),
180 db_migration.MIGRATE_REPO_PATH, None)
181 158
182 @mock.patch.object(migration, 'db_version_control') 159 @mock.patch.object(manage.DbCommands, 'version_control')
183 def test_db_version_control(self, db_version_control): 160 def test_db_version_control(self, version_control):
184 self._main_test_helper(['glance.cmd.manage', 'db', 'version_control'], 161 self._main_test_helper(['glance.cmd.manage', 'db', 'version_control'],
185 migration.db_version_control, 162 manage.DbCommands.version_control)
186 db_api.get_engine(), 163
187 db_migration.MIGRATE_REPO_PATH, None) 164 @mock.patch.object(manage.DbCommands, 'sync')
188 165 def test_db_sync_version(self, sync):
189 @mock.patch.object(migration, 'db_sync') 166 self._main_test_helper(['glance.cmd.manage', 'db', 'sync', 'liberty'],
190 def test_db_sync_version(self, db_sync): 167 manage.DbCommands.sync, 'liberty')
191 self._main_test_helper(['glance.cmd.manage', 'db', 'sync', '20'], 168
192 migration.db_sync, 169 @mock.patch.object(manage.DbCommands, 'upgrade')
193 db_api.get_engine(), 170 def test_db_upgrade_version(self, upgrade):
194 db_migration.MIGRATE_REPO_PATH, '20') 171 self._main_test_helper(['glance.cmd.manage', 'db',
195 172 'upgrade', 'liberty'],
196 @mock.patch.object(migration, 'db_sync') 173 manage.DbCommands.upgrade, 'liberty')
197 def test_db_upgrade_version(self, db_sync):
198 self._main_test_helper(['glance.cmd.manage', 'db', 'upgrade', '20'],
199 migration.db_sync,
200 db_api.get_engine(),
201 db_migration.MIGRATE_REPO_PATH, '20')
202 174
203 def test_db_metadefs_unload(self): 175 def test_db_metadefs_unload(self):
204 db_metadata.db_unload_metadefs = mock.Mock() 176 db_metadata.db_unload_metadefs = mock.Mock()
diff --git a/requirements.txt b/requirements.txt
index ff3ef37..16369c2 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -12,6 +12,8 @@ Routes!=2.0,!=2.1,!=2.3.0,>=1.12.3;python_version=='2.7' # MIT
12Routes!=2.0,!=2.3.0,>=1.12.3;python_version!='2.7' # MIT 12Routes!=2.0,!=2.3.0,>=1.12.3;python_version!='2.7' # MIT
13WebOb>=1.6.0 # MIT 13WebOb>=1.6.0 # MIT
14sqlalchemy-migrate>=0.9.6 # Apache-2.0 14sqlalchemy-migrate>=0.9.6 # Apache-2.0
15sqlparse>=0.2.2 # BSD
16alembic>=0.8.10 # MIT
15httplib2>=0.7.5 # MIT 17httplib2>=0.7.5 # MIT
16pycrypto>=2.6 # Public Domain 18pycrypto>=2.6 # Public Domain
17oslo.config!=3.18.0,>=3.14.0 # Apache-2.0 19oslo.config!=3.18.0,>=3.14.0 # Apache-2.0