summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorHemanth Makkapati <hemanth.makkapati@rackspace.com>2017-01-29 13:31:53 -0600
committerHemanth Makkapati <hemanth.makkapati@rackspace.com>2017-02-02 00:16:04 +0000
commit9859df2d1fc84a16dc08fad38bcb503f0a4e7b3f (patch)
tree406b6768e8713314ceb670830247d0f241f4a0e1
parent0f0354a8b890a456c90744a3a5fcf6927a1edc3b (diff)
Add expand/migrate/contract migrations for CI
This patch adds equivalent expand/migrate/contract migration scripts for Community Images. The expand migration 'ocata_expand01_add_visibility.py' creates a new migration branch 'expand' from the last known migration i.e., 'mitaka02'. Similarly, the contract migration 'ocata_contract01_drop_is_public.py' creates another new migration branch called 'contract' from the last known migration. The data migration 'ocata_migrate01_community_images.py' migrates all rows in the database at once. There is possibility of performance degradation while the data migrations are running. Change-Id: I34f5623d6804e9fe594e6b5b196ea4a162578196 Partially-Implements: blueprint database-strategy-for-rolling-upgrades Co-Authored-By: Hemanth Makkapati <hemanth.makkapati@rackspace.com> Depends-On: Ie839e0f240436dce7b151de5b464373516ff5a64
Notes
Notes (review): Code-Review+2: Dharini Chandrasekar <dharini.chandrasekar@intel.com> Code-Review+2: Steve Lewis (stevelle) <steve.lewis@rackspace.com> Code-Review+2: Nikhil Komawar <nik.komawar@gmail.com> Code-Review+2: Brian Rosmaita <brian.rosmaita@rackspace.com> Workflow+1: Brian Rosmaita <brian.rosmaita@rackspace.com> Verified+2: Jenkins Submitted-by: Jenkins Submitted-at: Thu, 02 Feb 2017 18:27:38 +0000 Reviewed-on: https://review.openstack.org/424774 Project: openstack/glance Branch: refs/heads/master
-rw-r--r--glance/cmd/manage.py15
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/data_migrations/ocata_migrate01_community_images.py103
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py67
-rw-r--r--glance/db/sqlalchemy/alembic_migrations/versions/ocata_expand01_add_visibility.py151
-rw-r--r--glance/tests/functional/db/migrations/test_ocata_contract01.py64
-rw-r--r--glance/tests/functional/db/migrations/test_ocata_expand01.py174
-rw-r--r--glance/tests/functional/db/migrations/test_ocata_migrate01.py147
-rw-r--r--glance/tests/functional/db/test_migrations.py7
8 files changed, 725 insertions, 3 deletions
diff --git a/glance/cmd/manage.py b/glance/cmd/manage.py
index 5a854bf..ffe652a 100644
--- a/glance/cmd/manage.py
+++ b/glance/cmd/manage.py
@@ -129,6 +129,11 @@ class DbCommands(object):
129 129
130 def expand(self): 130 def expand(self):
131 """Run the expansion phase of a rolling upgrade procedure.""" 131 """Run the expansion phase of a rolling upgrade procedure."""
132 engine = db_api.get_engine()
133 if engine.engine.name != 'mysql':
134 sys.exit(_('Rolling upgrades are currently supported only for '
135 'MySQL'))
136
132 expand_head = alembic_migrations.get_alembic_branch_head( 137 expand_head = alembic_migrations.get_alembic_branch_head(
133 db_migration.EXPAND_BRANCH) 138 db_migration.EXPAND_BRANCH)
134 if not expand_head: 139 if not expand_head:
@@ -146,6 +151,11 @@ class DbCommands(object):
146 151
147 def contract(self): 152 def contract(self):
148 """Run the contraction phase of a rolling upgrade procedure.""" 153 """Run the contraction phase of a rolling upgrade procedure."""
154 engine = db_api.get_engine()
155 if engine.engine.name != 'mysql':
156 sys.exit(_('Rolling upgrades are currently supported only for '
157 'MySQL'))
158
149 contract_head = alembic_migrations.get_alembic_branch_head( 159 contract_head = alembic_migrations.get_alembic_branch_head(
150 db_migration.CONTRACT_BRANCH) 160 db_migration.CONTRACT_BRANCH)
151 if not contract_head: 161 if not contract_head:
@@ -178,6 +188,11 @@ class DbCommands(object):
178 'curr_revs': curr_heads}) 188 'curr_revs': curr_heads})
179 189
180 def migrate(self): 190 def migrate(self):
191 engine = db_api.get_engine()
192 if engine.engine.name != 'mysql':
193 sys.exit(_('Rolling upgrades are currently supported only for '
194 'MySQL'))
195
181 curr_heads = alembic_migrations.get_current_alembic_heads() 196 curr_heads = alembic_migrations.get_current_alembic_heads()
182 expand_head = alembic_migrations.get_alembic_branch_head( 197 expand_head = alembic_migrations.get_alembic_branch_head(
183 db_migration.EXPAND_BRANCH) 198 db_migration.EXPAND_BRANCH)
diff --git a/glance/db/sqlalchemy/alembic_migrations/data_migrations/ocata_migrate01_community_images.py b/glance/db/sqlalchemy/alembic_migrations/data_migrations/ocata_migrate01_community_images.py
new file mode 100644
index 0000000..76c66c2
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/data_migrations/ocata_migrate01_community_images.py
@@ -0,0 +1,103 @@
1# Copyright 2016 Rackspace
2# Copyright 2016 Intel Corporation
3#
4# Licensed under the Apache License, Version 2.0 (the "License"); you may
5# not use this file except in compliance with the License. You may obtain
6# a copy of the License at
7#
8# http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
12# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
13# License for the specific language governing permissions and limitations
14# under the License.
15
16from sqlalchemy import MetaData, select, Table, and_, not_
17
18
19def has_migrations(engine):
20 """Returns true if at least one data row can be migrated.
21
22 There are rows left to migrate if:
23 #1 There exists a row with visibility not set yet.
24 Or
25 #2 There exists a private image with active members but its visibility
26 isn't set to 'shared' yet.
27
28 Note: This method can return a false positive if data migrations
29 are running in the background as it's being called.
30 """
31 meta = MetaData(engine)
32 images = Table('images', meta, autoload=True)
33
34 rows_with_null_visibility = (select([images.c.id])
35 .where(images.c.visibility == None)
36 .limit(1)
37 .execute())
38
39 if rows_with_null_visibility.rowcount == 1:
40 return True
41
42 image_members = Table('image_members', meta, autoload=True)
43 rows_with_pending_shared = (select[images.c.id]
44 .where(and_(
45 images.c.visibility == 'private',
46 images.c.id.in_(
47 select([image_members.c.image_id])
48 .distinct()
49 .where(not_(image_members.c.deleted))))
50 )
51 .limit(1)
52 .execute())
53 if rows_with_pending_shared.rowcount == 1:
54 return True
55
56 return False
57
58
59def _mark_all_public_images_with_public_visibility(images):
60 migrated_rows = (images
61 .update().values(visibility='public')
62 .where(images.c.is_public)
63 .execute())
64 return migrated_rows.rowcount
65
66
67def _mark_all_non_public_images_with_private_visibility(images):
68 migrated_rows = (images
69 .update().values(visibility='private')
70 .where(not_(images.c.is_public))
71 .execute())
72 return migrated_rows.rowcount
73
74
75def _mark_all_private_images_with_members_as_shared_visibility(images,
76 image_members):
77 migrated_rows = (images
78 .update().values(visibility='shared')
79 .where(and_(images.c.visibility == 'private',
80 images.c.id.in_(
81 select([image_members.c.image_id])
82 .distinct()
83 .where(not_(image_members.c.deleted)))))
84 .execute())
85 return migrated_rows.rowcount
86
87
88def _migrate_all(engine):
89 meta = MetaData(engine)
90 images = Table('images', meta, autoload=True)
91 image_members = Table('image_members', meta, autoload=True)
92
93 num_rows = _mark_all_public_images_with_public_visibility(images)
94 num_rows += _mark_all_non_public_images_with_private_visibility(images)
95 num_rows += _mark_all_private_images_with_members_as_shared_visibility(
96 images, image_members)
97
98 return num_rows
99
100
101def migrate(engine):
102 """Set visibility column based on is_public and image members."""
103 return _migrate_all(engine)
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py b/glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py
new file mode 100644
index 0000000..09fbed5
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/ocata_contract01_drop_is_public.py
@@ -0,0 +1,67 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13"""remove is_public from images
14
15Revision ID: ocata_contract01
16Revises: mitaka02
17Create Date: 2017-01-27 12:58:16.647499
18
19"""
20
21from alembic import op
22from sqlalchemy import MetaData, Table
23
24from glance.db import migration
25
26# revision identifiers, used by Alembic.
27revision = 'ocata_contract01'
28down_revision = 'mitaka02'
29branch_labels = migration.CONTRACT_BRANCH
30depends_on = 'expand'
31
32
33MYSQL_DROP_INSERT_TRIGGER = """
34DROP TRIGGER insert_visibility;
35"""
36
37MYSQL_DROP_UPDATE_TRIGGER = """
38DROP TRIGGER update_visibility;
39"""
40
41
42def _drop_column():
43 op.drop_index('ix_images_is_public', 'images')
44 op.drop_column('images', 'is_public')
45
46
47def _drop_triggers(engine):
48 engine_name = engine.engine.name
49 if engine_name == "mysql":
50 op.execute(MYSQL_DROP_INSERT_TRIGGER)
51 op.execute(MYSQL_DROP_UPDATE_TRIGGER)
52
53
54def _set_nullability_and_default_on_visibility(meta):
55 # NOTE(hemanthm): setting the default on 'visibility' column
56 # to 'shared'. Also, marking it as non-nullable.
57 images = Table('images', meta, autoload=True)
58 images.c.visibility.alter(nullable=False, server_default='shared')
59
60
61def upgrade():
62 migrate_engine = op.get_bind()
63 meta = MetaData(bind=migrate_engine)
64
65 _drop_column()
66 _drop_triggers(migrate_engine)
67 _set_nullability_and_default_on_visibility(meta)
diff --git a/glance/db/sqlalchemy/alembic_migrations/versions/ocata_expand01_add_visibility.py b/glance/db/sqlalchemy/alembic_migrations/versions/ocata_expand01_add_visibility.py
new file mode 100644
index 0000000..665f260
--- /dev/null
+++ b/glance/db/sqlalchemy/alembic_migrations/versions/ocata_expand01_add_visibility.py
@@ -0,0 +1,151 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13"""add visibility to images
14
15Revision ID: ocata_expand01
16Revises: mitaka02
17Create Date: 2017-01-27 12:58:16.647499
18
19"""
20
21from alembic import op
22from sqlalchemy import Column, Enum, MetaData, Table
23
24from glance.db import migration
25
26# revision identifiers, used by Alembic.
27revision = 'ocata_expand01'
28down_revision = 'mitaka02'
29branch_labels = migration.EXPAND_BRANCH
30depends_on = None
31
32ERROR_MESSAGE = 'Invalid visibility value'
33MYSQL_INSERT_TRIGGER = """
34CREATE TRIGGER insert_visibility BEFORE INSERT ON images
35FOR EACH ROW
36BEGIN
37 -- NOTE(abashmak):
38 -- The following IF/ELSE block implements a priority decision tree.
39 -- Strict order MUST be followed to correctly cover all the edge cases.
40
41 -- Edge case: neither is_public nor visibility specified
42 -- (or both specified as NULL):
43 IF NEW.is_public <=> NULL AND NEW.visibility <=> NULL THEN
44 SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
45 -- Edge case: both is_public and visibility specified:
46 ELSEIF NOT(NEW.is_public <=> NULL OR NEW.visibility <=> NULL) THEN
47 SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
48 -- Inserting with is_public, set visibility accordingly:
49 ELSEIF NOT NEW.is_public <=> NULL THEN
50 IF NEW.is_public = 1 THEN
51 SET NEW.visibility = 'public';
52 ELSE
53 SET NEW.visibility = 'shared';
54 END IF;
55 -- Inserting with visibility, set is_public accordingly:
56 ELSEIF NOT NEW.visibility <=> NULL THEN
57 IF NEW.visibility = 'public' THEN
58 SET NEW.is_public = 1;
59 ELSE
60 SET NEW.is_public = 0;
61 END IF;
62 -- Edge case: either one of: is_public or visibility,
63 -- is explicitly set to NULL:
64 ELSE
65 SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
66 END IF;
67END;
68"""
69
70MYSQL_UPDATE_TRIGGER = """
71CREATE TRIGGER update_visibility BEFORE UPDATE ON images
72FOR EACH ROW
73BEGIN
74 -- Case: new value specified for is_public:
75 IF NOT NEW.is_public <=> OLD.is_public THEN
76 -- Edge case: is_public explicitly set to NULL:
77 IF NEW.is_public <=> NULL THEN
78 SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
79 -- Edge case: new value also specified for visibility
80 ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
81 SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
82 -- Case: visibility not specified or specified as OLD value:
83 -- NOTE(abashmak): There is no way to reliably determine which
84 -- of the above two cases occurred, but allowing to proceed with
85 -- the update in either case does not break the model for both
86 -- N and N-1 services.
87 ELSE
88 -- Set visibility according to the value of is_public:
89 IF NEW.is_public <=> 1 THEN
90 SET NEW.visibility = 'public';
91 ELSE
92 SET NEW.visibility = 'shared';
93 END IF;
94 END IF;
95 -- Case: new value specified for visibility:
96 ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
97 -- Edge case: visibility explicitly set to NULL:
98 IF NEW.visibility <=> NULL THEN
99 SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
100 -- Edge case: new value also specified for is_public
101 ELSEIF NOT NEW.is_public <=> OLD.is_public THEN
102 SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
103 -- Case: is_public not specified or specified as OLD value:
104 -- NOTE(abashmak): There is no way to reliably determine which
105 -- of the above two cases occurred, but allowing to proceed with
106 -- the update in either case does not break the model for both
107 -- N and N-1 services.
108 ELSE
109 -- Set is_public according to the value of visibility:
110 IF NEW.visibility <=> 'public' THEN
111 SET NEW.is_public = 1;
112 ELSE
113 SET NEW.is_public = 0;
114 END IF;
115 END IF;
116 END IF;
117END;
118"""
119
120
121def _add_visibility_column(meta):
122 enum = Enum('private', 'public', 'shared', 'community', metadata=meta,
123 name='image_visibility')
124 enum.create()
125 v_col = Column('visibility', enum, nullable=True, server_default=None)
126 op.add_column('images', v_col)
127 op.create_index('visibility_image_idx', 'images', ['visibility'])
128
129
130def _add_triggers(engine):
131 if engine.engine.name == 'mysql':
132 op.execute(MYSQL_INSERT_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
133 ERROR_MESSAGE))
134 op.execute(MYSQL_UPDATE_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
135 ERROR_MESSAGE, ERROR_MESSAGE))
136
137
138def _change_nullability_and_default_on_is_public(meta):
139 # NOTE(hemanthm): we mark is_public as nullable so that when new versions
140 # add data only to be visibility column, is_public can be null.
141 images = Table('images', meta, autoload=True)
142 images.c.is_public.alter(nullable=True, server_default=None)
143
144
145def upgrade():
146 migrate_engine = op.get_bind()
147 meta = MetaData(bind=migrate_engine)
148
149 _add_visibility_column(meta)
150 _change_nullability_and_default_on_is_public(meta)
151 _add_triggers(migrate_engine)
diff --git a/glance/tests/functional/db/migrations/test_ocata_contract01.py b/glance/tests/functional/db/migrations/test_ocata_contract01.py
new file mode 100644
index 0000000..b2049f6
--- /dev/null
+++ b/glance/tests/functional/db/migrations/test_ocata_contract01.py
@@ -0,0 +1,64 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13import datetime
14
15from oslo_db.sqlalchemy import test_base
16from oslo_db.sqlalchemy import utils as db_utils
17
18from glance.tests.functional.db import test_migrations
19
20
21class TestOcataContract01Mixin(test_migrations.AlembicMigrationsMixin):
22
23 def _get_revisions(self, config):
24 return test_migrations.AlembicMigrationsMixin._get_revisions(
25 self, config, head='ocata_contract01')
26
27 def _pre_upgrade_ocata_contract01(self, engine):
28 images = db_utils.get_table(engine, 'images')
29 now = datetime.datetime.now()
30 self.assertIn('is_public', images.c)
31 self.assertIn('visibility', images.c)
32 self.assertTrue(images.c.is_public.nullable)
33 self.assertTrue(images.c.visibility.nullable)
34
35 # inserting a public image record
36 public_temp = dict(deleted=False,
37 created_at=now,
38 status='active',
39 is_public=True,
40 min_disk=0,
41 min_ram=0,
42 id='public_id_before_expand')
43 images.insert().values(public_temp).execute()
44
45 # inserting a private image record
46 shared_temp = dict(deleted=False,
47 created_at=now,
48 status='active',
49 is_public=False,
50 min_disk=0,
51 min_ram=0,
52 id='private_id_before_expand')
53 images.insert().values(shared_temp).execute()
54
55 def _check_ocata_contract01(self, engine, data):
56 # check that after contract 'is_public' column is dropped
57 images = db_utils.get_table(engine, 'images')
58 self.assertNotIn('is_public', images.c)
59 self.assertIn('visibility', images.c)
60
61
62class TestOcataContract01MySQL(TestOcataContract01Mixin,
63 test_base.MySQLOpportunisticTestCase):
64 pass
diff --git a/glance/tests/functional/db/migrations/test_ocata_expand01.py b/glance/tests/functional/db/migrations/test_ocata_expand01.py
new file mode 100644
index 0000000..ef68498
--- /dev/null
+++ b/glance/tests/functional/db/migrations/test_ocata_expand01.py
@@ -0,0 +1,174 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13import datetime
14
15from oslo_db.sqlalchemy import test_base
16from oslo_db.sqlalchemy import utils as db_utils
17
18from glance.tests.functional.db import test_migrations
19
20
21class TestOcataExpand01Mixin(test_migrations.AlembicMigrationsMixin):
22
23 def _get_revisions(self, config):
24 return test_migrations.AlembicMigrationsMixin._get_revisions(
25 self, config, head='ocata_expand01')
26
27 def _pre_upgrade_ocata_expand01(self, engine):
28 images = db_utils.get_table(engine, 'images')
29 now = datetime.datetime.now()
30 self.assertIn('is_public', images.c)
31 self.assertNotIn('visibility', images.c)
32 self.assertFalse(images.c.is_public.nullable)
33
34 # inserting a public image record
35 public_temp = dict(deleted=False,
36 created_at=now,
37 status='active',
38 is_public=True,
39 min_disk=0,
40 min_ram=0,
41 id='public_id_before_expand')
42 images.insert().values(public_temp).execute()
43
44 # inserting a private image record
45 shared_temp = dict(deleted=False,
46 created_at=now,
47 status='active',
48 is_public=False,
49 min_disk=0,
50 min_ram=0,
51 id='private_id_before_expand')
52 images.insert().values(shared_temp).execute()
53
54 def _check_ocata_expand01(self, engine, data):
55 # check that after migration, 'visibility' column is introduced
56 images = db_utils.get_table(engine, 'images')
57 self.assertIn('visibility', images.c)
58 self.assertIn('is_public', images.c)
59 self.assertTrue(images.c.is_public.nullable)
60 self.assertTrue(images.c.visibility.nullable)
61
62 # tests visibility set to None for existing images
63 rows = (images.select()
64 .where(images.c.id.like('%_before_expand'))
65 .order_by(images.c.id)
66 .execute()
67 .fetchall())
68
69 self.assertEqual(2, len(rows))
70 # private image first
71 self.assertEqual(0, rows[0]['is_public'])
72 self.assertEqual('private_id_before_expand', rows[0]['id'])
73 self.assertIsNone(rows[0]['visibility'])
74 # then public image
75 self.assertEqual(1, rows[1]['is_public'])
76 self.assertEqual('public_id_before_expand', rows[1]['id'])
77 self.assertIsNone(rows[1]['visibility'])
78
79 self._test_trigger_old_to_new(images)
80 self._test_trigger_new_to_old(images)
81
82 def _test_trigger_new_to_old(self, images):
83 now = datetime.datetime.now()
84 # inserting a public image record after expand
85 public_temp = dict(deleted=False,
86 created_at=now,
87 status='active',
88 visibility='public',
89 min_disk=0,
90 min_ram=0,
91 id='public_id_new_to_old')
92 images.insert().values(public_temp).execute()
93
94 # inserting a private image record after expand
95 shared_temp = dict(deleted=False,
96 created_at=now,
97 status='active',
98 visibility='private',
99 min_disk=0,
100 min_ram=0,
101 id='private_id_new_to_old')
102 images.insert().values(shared_temp).execute()
103
104 # inserting a shared image record after expand
105 shared_temp = dict(deleted=False,
106 created_at=now,
107 status='active',
108 visibility='shared',
109 min_disk=0,
110 min_ram=0,
111 id='shared_id_new_to_old')
112 images.insert().values(shared_temp).execute()
113
114 # test visibility is set appropriately by the trigger for new images
115 rows = (images.select()
116 .where(images.c.id.like('%_new_to_old'))
117 .order_by(images.c.id)
118 .execute()
119 .fetchall())
120
121 self.assertEqual(3, len(rows))
122 # private image first
123 self.assertEqual(0, rows[0]['is_public'])
124 self.assertEqual('private_id_new_to_old', rows[0]['id'])
125 self.assertEqual('private', rows[0]['visibility'])
126 # then public image
127 self.assertEqual(1, rows[1]['is_public'])
128 self.assertEqual('public_id_new_to_old', rows[1]['id'])
129 self.assertEqual('public', rows[1]['visibility'])
130 # then shared image
131 self.assertEqual(0, rows[2]['is_public'])
132 self.assertEqual('shared_id_new_to_old', rows[2]['id'])
133 self.assertEqual('shared', rows[2]['visibility'])
134
135 def _test_trigger_old_to_new(self, images):
136 now = datetime.datetime.now()
137 # inserting a public image record after expand
138 public_temp = dict(deleted=False,
139 created_at=now,
140 status='active',
141 is_public=True,
142 min_disk=0,
143 min_ram=0,
144 id='public_id_old_to_new')
145 images.insert().values(public_temp).execute()
146 # inserting a private image record after expand
147 shared_temp = dict(deleted=False,
148 created_at=now,
149 status='active',
150 is_public=False,
151 min_disk=0,
152 min_ram=0,
153 id='private_id_old_to_new')
154 images.insert().values(shared_temp).execute()
155 # tests visibility is set appropriately by the trigger for new images
156 rows = (images.select()
157 .where(images.c.id.like('%_old_to_new'))
158 .order_by(images.c.id)
159 .execute()
160 .fetchall())
161 self.assertEqual(2, len(rows))
162 # private image first
163 self.assertEqual(0, rows[0]['is_public'])
164 self.assertEqual('private_id_old_to_new', rows[0]['id'])
165 self.assertEqual('shared', rows[0]['visibility'])
166 # then public image
167 self.assertEqual(1, rows[1]['is_public'])
168 self.assertEqual('public_id_old_to_new', rows[1]['id'])
169 self.assertEqual('public', rows[1]['visibility'])
170
171
172class TestOcataExpand01MySQL(TestOcataExpand01Mixin,
173 test_base.MySQLOpportunisticTestCase):
174 pass
diff --git a/glance/tests/functional/db/migrations/test_ocata_migrate01.py b/glance/tests/functional/db/migrations/test_ocata_migrate01.py
new file mode 100644
index 0000000..93b7c97
--- /dev/null
+++ b/glance/tests/functional/db/migrations/test_ocata_migrate01.py
@@ -0,0 +1,147 @@
1# Licensed under the Apache License, Version 2.0 (the "License"); you may
2# not use this file except in compliance with the License. You may obtain
3# a copy of the License at
4#
5# http://www.apache.org/licenses/LICENSE-2.0
6#
7# Unless required by applicable law or agreed to in writing, software
8# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
9# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
10# License for the specific language governing permissions and limitations
11# under the License.
12
13import datetime
14
15from oslo_db.sqlalchemy import test_base
16from oslo_db.sqlalchemy import utils as db_utils
17
18from glance.db.sqlalchemy.alembic_migrations import data_migrations
19from glance.tests.functional.db import test_migrations
20
21
22class TestOcataMigrate01Mixin(test_migrations.AlembicMigrationsMixin):
23
24 def _get_revisions(self, config):
25 return test_migrations.AlembicMigrationsMixin._get_revisions(
26 self, config, head='ocata_expand01')
27
28 def _pre_upgrade_ocata_expand01(self, engine):
29 images = db_utils.get_table(engine, 'images')
30 image_members = db_utils.get_table(engine, 'image_members')
31 now = datetime.datetime.now()
32
33 # inserting a public image record
34 public_temp = dict(deleted=False,
35 created_at=now,
36 status='active',
37 is_public=True,
38 min_disk=0,
39 min_ram=0,
40 id='public_id')
41 images.insert().values(public_temp).execute()
42
43 # inserting a non-public image record for 'shared' visibility test
44 shared_temp = dict(deleted=False,
45 created_at=now,
46 status='active',
47 is_public=False,
48 min_disk=0,
49 min_ram=0,
50 id='shared_id')
51 images.insert().values(shared_temp).execute()
52
53 # inserting a non-public image records for 'private' visibility test
54 private_temp = dict(deleted=False,
55 created_at=now,
56 status='active',
57 is_public=False,
58 min_disk=0,
59 min_ram=0,
60 id='private_id_1')
61 images.insert().values(private_temp).execute()
62
63 private_temp = dict(deleted=False,
64 created_at=now,
65 status='active',
66 is_public=False,
67 min_disk=0,
68 min_ram=0,
69 id='private_id_2')
70 images.insert().values(private_temp).execute()
71
72 # adding an active as well as a deleted image member for checking
73 # 'shared' visibility
74 temp = dict(deleted=False,
75 created_at=now,
76 image_id='shared_id',
77 member='fake_member_452',
78 can_share=True,
79 id=45)
80 image_members.insert().values(temp).execute()
81
82 temp = dict(deleted=True,
83 created_at=now,
84 image_id='shared_id',
85 member='fake_member_453',
86 can_share=True,
87 id=453)
88 image_members.insert().values(temp).execute()
89
90 # adding an image member, but marking it deleted,
91 # for testing 'private' visibility
92 temp = dict(deleted=True,
93 created_at=now,
94 image_id='private_id_2',
95 member='fake_member_451',
96 can_share=True,
97 id=451)
98 image_members.insert().values(temp).execute()
99
100 # adding an active image member for the 'public' image,
101 # to test it remains public regardless.
102 temp = dict(deleted=False,
103 created_at=now,
104 image_id='public_id',
105 member='fake_member_450',
106 can_share=True,
107 id=450)
108 image_members.insert().values(temp).execute()
109
110 def _check_ocata_expand01(self, engine, data):
111 images = db_utils.get_table(engine, 'images')
112
113 # check that visibility is null for existing images
114 rows = (images.select()
115 .order_by(images.c.id)
116 .execute()
117 .fetchall())
118 self.assertEqual(4, len(rows))
119 for row in rows:
120 self.assertIsNone(row['visibility'])
121
122 # run data migrations
123 data_migrations.migrate(engine)
124
125 # check that visibility is set appropriately for all images
126 rows = (images.select()
127 .order_by(images.c.id)
128 .execute()
129 .fetchall())
130 self.assertEqual(4, len(rows))
131 # private_id_1 has private visibility
132 self.assertEqual('private_id_1', rows[0]['id'])
133 self.assertEqual('private', rows[0]['visibility'])
134 # private_id_2 has private visibility
135 self.assertEqual('private_id_2', rows[1]['id'])
136 self.assertEqual('private', rows[1]['visibility'])
137 # public_id has public visibility
138 self.assertEqual('public_id', rows[2]['id'])
139 self.assertEqual('public', rows[2]['visibility'])
140 # shared_id has shared visibility
141 self.assertEqual('shared_id', rows[3]['id'])
142 self.assertEqual('shared', rows[3]['visibility'])
143
144
145class TestOcataMigrate01MySQL(TestOcataMigrate01Mixin,
146 test_base.MySQLOpportunisticTestCase):
147 pass
diff --git a/glance/tests/functional/db/test_migrations.py b/glance/tests/functional/db/test_migrations.py
index 0d596ad..c364a11 100644
--- a/glance/tests/functional/db/test_migrations.py
+++ b/glance/tests/functional/db/test_migrations.py
@@ -23,7 +23,7 @@ from oslo_db.sqlalchemy import test_base
23from oslo_db.sqlalchemy import test_migrations 23from oslo_db.sqlalchemy import test_migrations
24import sqlalchemy.types as types 24import sqlalchemy.types as types
25 25
26from glance.db import migration as dm 26from glance.db import migration as db_migration
27from glance.db.sqlalchemy import alembic_migrations 27from glance.db.sqlalchemy import alembic_migrations
28from glance.db.sqlalchemy.alembic_migrations import versions 28from glance.db.sqlalchemy.alembic_migrations import versions
29from glance.db.sqlalchemy import models 29from glance.db.sqlalchemy import models
@@ -34,10 +34,11 @@ import glance.tests.utils as test_utils
34 34
35class AlembicMigrationsMixin(object): 35class AlembicMigrationsMixin(object):
36 36
37 def _get_revisions(self, config): 37 def _get_revisions(self, config, head=None):
38 head = head or db_migration.LATEST_REVISION
38 scripts_dir = alembic_script.ScriptDirectory.from_config(config) 39 scripts_dir = alembic_script.ScriptDirectory.from_config(config)
39 revisions = list(scripts_dir.walk_revisions(base='base', 40 revisions = list(scripts_dir.walk_revisions(base='base',
40 head=dm.LATEST_REVISION)) 41 head=head))
41 revisions = list(reversed(revisions)) 42 revisions = list(reversed(revisions))
42 revisions = [rev.revision for rev in revisions] 43 revisions = [rev.revision for rev in revisions]
43 return revisions 44 return revisions