Merge "Add expand/migrate/contract migrations for CI"

This commit is contained in:
Jenkins 2017-02-02 18:27:38 +00:00 committed by Gerrit Code Review
commit 3f98eb08fa
8 changed files with 725 additions and 3 deletions

View File

@ -129,6 +129,11 @@ class DbCommands(object):
def expand(self):
"""Run the expansion phase of a rolling upgrade procedure."""
engine = db_api.get_engine()
if engine.engine.name != 'mysql':
sys.exit(_('Rolling upgrades are currently supported only for '
'MySQL'))
expand_head = alembic_migrations.get_alembic_branch_head(
db_migration.EXPAND_BRANCH)
if not expand_head:
@ -146,6 +151,11 @@ class DbCommands(object):
def contract(self):
"""Run the contraction phase of a rolling upgrade procedure."""
engine = db_api.get_engine()
if engine.engine.name != 'mysql':
sys.exit(_('Rolling upgrades are currently supported only for '
'MySQL'))
contract_head = alembic_migrations.get_alembic_branch_head(
db_migration.CONTRACT_BRANCH)
if not contract_head:
@ -178,6 +188,11 @@ class DbCommands(object):
'curr_revs': curr_heads})
def migrate(self):
engine = db_api.get_engine()
if engine.engine.name != 'mysql':
sys.exit(_('Rolling upgrades are currently supported only for '
'MySQL'))
curr_heads = alembic_migrations.get_current_alembic_heads()
expand_head = alembic_migrations.get_alembic_branch_head(
db_migration.EXPAND_BRANCH)

View File

@ -0,0 +1,103 @@
# Copyright 2016 Rackspace
# Copyright 2016 Intel Corporation
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
from sqlalchemy import MetaData, select, Table, and_, not_
def has_migrations(engine):
"""Returns true if at least one data row can be migrated.
There are rows left to migrate if:
#1 There exists a row with visibility not set yet.
Or
#2 There exists a private image with active members but its visibility
isn't set to 'shared' yet.
Note: This method can return a false positive if data migrations
are running in the background as it's being called.
"""
meta = MetaData(engine)
images = Table('images', meta, autoload=True)
rows_with_null_visibility = (select([images.c.id])
.where(images.c.visibility == None)
.limit(1)
.execute())
if rows_with_null_visibility.rowcount == 1:
return True
image_members = Table('image_members', meta, autoload=True)
rows_with_pending_shared = (select[images.c.id]
.where(and_(
images.c.visibility == 'private',
images.c.id.in_(
select([image_members.c.image_id])
.distinct()
.where(not_(image_members.c.deleted))))
)
.limit(1)
.execute())
if rows_with_pending_shared.rowcount == 1:
return True
return False
def _mark_all_public_images_with_public_visibility(images):
migrated_rows = (images
.update().values(visibility='public')
.where(images.c.is_public)
.execute())
return migrated_rows.rowcount
def _mark_all_non_public_images_with_private_visibility(images):
migrated_rows = (images
.update().values(visibility='private')
.where(not_(images.c.is_public))
.execute())
return migrated_rows.rowcount
def _mark_all_private_images_with_members_as_shared_visibility(images,
image_members):
migrated_rows = (images
.update().values(visibility='shared')
.where(and_(images.c.visibility == 'private',
images.c.id.in_(
select([image_members.c.image_id])
.distinct()
.where(not_(image_members.c.deleted)))))
.execute())
return migrated_rows.rowcount
def _migrate_all(engine):
meta = MetaData(engine)
images = Table('images', meta, autoload=True)
image_members = Table('image_members', meta, autoload=True)
num_rows = _mark_all_public_images_with_public_visibility(images)
num_rows += _mark_all_non_public_images_with_private_visibility(images)
num_rows += _mark_all_private_images_with_members_as_shared_visibility(
images, image_members)
return num_rows
def migrate(engine):
"""Set visibility column based on is_public and image members."""
return _migrate_all(engine)

View File

@ -0,0 +1,67 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""remove is_public from images
Revision ID: ocata_contract01
Revises: mitaka02
Create Date: 2017-01-27 12:58:16.647499
"""
from alembic import op
from sqlalchemy import MetaData, Table
from glance.db import migration
# revision identifiers, used by Alembic.
revision = 'ocata_contract01'
down_revision = 'mitaka02'
branch_labels = migration.CONTRACT_BRANCH
depends_on = 'expand'
MYSQL_DROP_INSERT_TRIGGER = """
DROP TRIGGER insert_visibility;
"""
MYSQL_DROP_UPDATE_TRIGGER = """
DROP TRIGGER update_visibility;
"""
def _drop_column():
op.drop_index('ix_images_is_public', 'images')
op.drop_column('images', 'is_public')
def _drop_triggers(engine):
engine_name = engine.engine.name
if engine_name == "mysql":
op.execute(MYSQL_DROP_INSERT_TRIGGER)
op.execute(MYSQL_DROP_UPDATE_TRIGGER)
def _set_nullability_and_default_on_visibility(meta):
# NOTE(hemanthm): setting the default on 'visibility' column
# to 'shared'. Also, marking it as non-nullable.
images = Table('images', meta, autoload=True)
images.c.visibility.alter(nullable=False, server_default='shared')
def upgrade():
migrate_engine = op.get_bind()
meta = MetaData(bind=migrate_engine)
_drop_column()
_drop_triggers(migrate_engine)
_set_nullability_and_default_on_visibility(meta)

View File

@ -0,0 +1,151 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""add visibility to images
Revision ID: ocata_expand01
Revises: mitaka02
Create Date: 2017-01-27 12:58:16.647499
"""
from alembic import op
from sqlalchemy import Column, Enum, MetaData, Table
from glance.db import migration
# revision identifiers, used by Alembic.
revision = 'ocata_expand01'
down_revision = 'mitaka02'
branch_labels = migration.EXPAND_BRANCH
depends_on = None
ERROR_MESSAGE = 'Invalid visibility value'
MYSQL_INSERT_TRIGGER = """
CREATE TRIGGER insert_visibility BEFORE INSERT ON images
FOR EACH ROW
BEGIN
-- NOTE(abashmak):
-- The following IF/ELSE block implements a priority decision tree.
-- Strict order MUST be followed to correctly cover all the edge cases.
-- Edge case: neither is_public nor visibility specified
-- (or both specified as NULL):
IF NEW.is_public <=> NULL AND NEW.visibility <=> NULL THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
-- Edge case: both is_public and visibility specified:
ELSEIF NOT(NEW.is_public <=> NULL OR NEW.visibility <=> NULL) THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
-- Inserting with is_public, set visibility accordingly:
ELSEIF NOT NEW.is_public <=> NULL THEN
IF NEW.is_public = 1 THEN
SET NEW.visibility = 'public';
ELSE
SET NEW.visibility = 'shared';
END IF;
-- Inserting with visibility, set is_public accordingly:
ELSEIF NOT NEW.visibility <=> NULL THEN
IF NEW.visibility = 'public' THEN
SET NEW.is_public = 1;
ELSE
SET NEW.is_public = 0;
END IF;
-- Edge case: either one of: is_public or visibility,
-- is explicitly set to NULL:
ELSE
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
END IF;
END;
"""
MYSQL_UPDATE_TRIGGER = """
CREATE TRIGGER update_visibility BEFORE UPDATE ON images
FOR EACH ROW
BEGIN
-- Case: new value specified for is_public:
IF NOT NEW.is_public <=> OLD.is_public THEN
-- Edge case: is_public explicitly set to NULL:
IF NEW.is_public <=> NULL THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
-- Edge case: new value also specified for visibility
ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
-- Case: visibility not specified or specified as OLD value:
-- NOTE(abashmak): There is no way to reliably determine which
-- of the above two cases occurred, but allowing to proceed with
-- the update in either case does not break the model for both
-- N and N-1 services.
ELSE
-- Set visibility according to the value of is_public:
IF NEW.is_public <=> 1 THEN
SET NEW.visibility = 'public';
ELSE
SET NEW.visibility = 'shared';
END IF;
END IF;
-- Case: new value specified for visibility:
ELSEIF NOT NEW.visibility <=> OLD.visibility THEN
-- Edge case: visibility explicitly set to NULL:
IF NEW.visibility <=> NULL THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
-- Edge case: new value also specified for is_public
ELSEIF NOT NEW.is_public <=> OLD.is_public THEN
SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT = '%s';
-- Case: is_public not specified or specified as OLD value:
-- NOTE(abashmak): There is no way to reliably determine which
-- of the above two cases occurred, but allowing to proceed with
-- the update in either case does not break the model for both
-- N and N-1 services.
ELSE
-- Set is_public according to the value of visibility:
IF NEW.visibility <=> 'public' THEN
SET NEW.is_public = 1;
ELSE
SET NEW.is_public = 0;
END IF;
END IF;
END IF;
END;
"""
def _add_visibility_column(meta):
enum = Enum('private', 'public', 'shared', 'community', metadata=meta,
name='image_visibility')
enum.create()
v_col = Column('visibility', enum, nullable=True, server_default=None)
op.add_column('images', v_col)
op.create_index('visibility_image_idx', 'images', ['visibility'])
def _add_triggers(engine):
if engine.engine.name == 'mysql':
op.execute(MYSQL_INSERT_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
ERROR_MESSAGE))
op.execute(MYSQL_UPDATE_TRIGGER % (ERROR_MESSAGE, ERROR_MESSAGE,
ERROR_MESSAGE, ERROR_MESSAGE))
def _change_nullability_and_default_on_is_public(meta):
# NOTE(hemanthm): we mark is_public as nullable so that when new versions
# add data only to be visibility column, is_public can be null.
images = Table('images', meta, autoload=True)
images.c.is_public.alter(nullable=True, server_default=None)
def upgrade():
migrate_engine = op.get_bind()
meta = MetaData(bind=migrate_engine)
_add_visibility_column(meta)
_change_nullability_and_default_on_is_public(meta)
_add_triggers(migrate_engine)

View File

@ -0,0 +1,64 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
from glance.tests.functional.db import test_migrations
class TestOcataContract01Mixin(test_migrations.AlembicMigrationsMixin):
def _get_revisions(self, config):
return test_migrations.AlembicMigrationsMixin._get_revisions(
self, config, head='ocata_contract01')
def _pre_upgrade_ocata_contract01(self, engine):
images = db_utils.get_table(engine, 'images')
now = datetime.datetime.now()
self.assertIn('is_public', images.c)
self.assertIn('visibility', images.c)
self.assertTrue(images.c.is_public.nullable)
self.assertTrue(images.c.visibility.nullable)
# inserting a public image record
public_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0,
id='public_id_before_expand')
images.insert().values(public_temp).execute()
# inserting a private image record
shared_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='private_id_before_expand')
images.insert().values(shared_temp).execute()
def _check_ocata_contract01(self, engine, data):
# check that after contract 'is_public' column is dropped
images = db_utils.get_table(engine, 'images')
self.assertNotIn('is_public', images.c)
self.assertIn('visibility', images.c)
class TestOcataContract01MySQL(TestOcataContract01Mixin,
test_base.MySQLOpportunisticTestCase):
pass

View File

@ -0,0 +1,174 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
from glance.tests.functional.db import test_migrations
class TestOcataExpand01Mixin(test_migrations.AlembicMigrationsMixin):
def _get_revisions(self, config):
return test_migrations.AlembicMigrationsMixin._get_revisions(
self, config, head='ocata_expand01')
def _pre_upgrade_ocata_expand01(self, engine):
images = db_utils.get_table(engine, 'images')
now = datetime.datetime.now()
self.assertIn('is_public', images.c)
self.assertNotIn('visibility', images.c)
self.assertFalse(images.c.is_public.nullable)
# inserting a public image record
public_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0,
id='public_id_before_expand')
images.insert().values(public_temp).execute()
# inserting a private image record
shared_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='private_id_before_expand')
images.insert().values(shared_temp).execute()
def _check_ocata_expand01(self, engine, data):
# check that after migration, 'visibility' column is introduced
images = db_utils.get_table(engine, 'images')
self.assertIn('visibility', images.c)
self.assertIn('is_public', images.c)
self.assertTrue(images.c.is_public.nullable)
self.assertTrue(images.c.visibility.nullable)
# tests visibility set to None for existing images
rows = (images.select()
.where(images.c.id.like('%_before_expand'))
.order_by(images.c.id)
.execute()
.fetchall())
self.assertEqual(2, len(rows))
# private image first
self.assertEqual(0, rows[0]['is_public'])
self.assertEqual('private_id_before_expand', rows[0]['id'])
self.assertIsNone(rows[0]['visibility'])
# then public image
self.assertEqual(1, rows[1]['is_public'])
self.assertEqual('public_id_before_expand', rows[1]['id'])
self.assertIsNone(rows[1]['visibility'])
self._test_trigger_old_to_new(images)
self._test_trigger_new_to_old(images)
def _test_trigger_new_to_old(self, images):
now = datetime.datetime.now()
# inserting a public image record after expand
public_temp = dict(deleted=False,
created_at=now,
status='active',
visibility='public',
min_disk=0,
min_ram=0,
id='public_id_new_to_old')
images.insert().values(public_temp).execute()
# inserting a private image record after expand
shared_temp = dict(deleted=False,
created_at=now,
status='active',
visibility='private',
min_disk=0,
min_ram=0,
id='private_id_new_to_old')
images.insert().values(shared_temp).execute()
# inserting a shared image record after expand
shared_temp = dict(deleted=False,
created_at=now,
status='active',
visibility='shared',
min_disk=0,
min_ram=0,
id='shared_id_new_to_old')
images.insert().values(shared_temp).execute()
# test visibility is set appropriately by the trigger for new images
rows = (images.select()
.where(images.c.id.like('%_new_to_old'))
.order_by(images.c.id)
.execute()
.fetchall())
self.assertEqual(3, len(rows))
# private image first
self.assertEqual(0, rows[0]['is_public'])
self.assertEqual('private_id_new_to_old', rows[0]['id'])
self.assertEqual('private', rows[0]['visibility'])
# then public image
self.assertEqual(1, rows[1]['is_public'])
self.assertEqual('public_id_new_to_old', rows[1]['id'])
self.assertEqual('public', rows[1]['visibility'])
# then shared image
self.assertEqual(0, rows[2]['is_public'])
self.assertEqual('shared_id_new_to_old', rows[2]['id'])
self.assertEqual('shared', rows[2]['visibility'])
def _test_trigger_old_to_new(self, images):
now = datetime.datetime.now()
# inserting a public image record after expand
public_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0,
id='public_id_old_to_new')
images.insert().values(public_temp).execute()
# inserting a private image record after expand
shared_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='private_id_old_to_new')
images.insert().values(shared_temp).execute()
# tests visibility is set appropriately by the trigger for new images
rows = (images.select()
.where(images.c.id.like('%_old_to_new'))
.order_by(images.c.id)
.execute()
.fetchall())
self.assertEqual(2, len(rows))
# private image first
self.assertEqual(0, rows[0]['is_public'])
self.assertEqual('private_id_old_to_new', rows[0]['id'])
self.assertEqual('shared', rows[0]['visibility'])
# then public image
self.assertEqual(1, rows[1]['is_public'])
self.assertEqual('public_id_old_to_new', rows[1]['id'])
self.assertEqual('public', rows[1]['visibility'])
class TestOcataExpand01MySQL(TestOcataExpand01Mixin,
test_base.MySQLOpportunisticTestCase):
pass

View File

@ -0,0 +1,147 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
import datetime
from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import utils as db_utils
from glance.db.sqlalchemy.alembic_migrations import data_migrations
from glance.tests.functional.db import test_migrations
class TestOcataMigrate01Mixin(test_migrations.AlembicMigrationsMixin):
def _get_revisions(self, config):
return test_migrations.AlembicMigrationsMixin._get_revisions(
self, config, head='ocata_expand01')
def _pre_upgrade_ocata_expand01(self, engine):
images = db_utils.get_table(engine, 'images')
image_members = db_utils.get_table(engine, 'image_members')
now = datetime.datetime.now()
# inserting a public image record
public_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=True,
min_disk=0,
min_ram=0,
id='public_id')
images.insert().values(public_temp).execute()
# inserting a non-public image record for 'shared' visibility test
shared_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='shared_id')
images.insert().values(shared_temp).execute()
# inserting a non-public image records for 'private' visibility test
private_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='private_id_1')
images.insert().values(private_temp).execute()
private_temp = dict(deleted=False,
created_at=now,
status='active',
is_public=False,
min_disk=0,
min_ram=0,
id='private_id_2')
images.insert().values(private_temp).execute()
# adding an active as well as a deleted image member for checking
# 'shared' visibility
temp = dict(deleted=False,
created_at=now,
image_id='shared_id',
member='fake_member_452',
can_share=True,
id=45)
image_members.insert().values(temp).execute()
temp = dict(deleted=True,
created_at=now,
image_id='shared_id',
member='fake_member_453',
can_share=True,
id=453)
image_members.insert().values(temp).execute()
# adding an image member, but marking it deleted,
# for testing 'private' visibility
temp = dict(deleted=True,
created_at=now,
image_id='private_id_2',
member='fake_member_451',
can_share=True,
id=451)
image_members.insert().values(temp).execute()
# adding an active image member for the 'public' image,
# to test it remains public regardless.
temp = dict(deleted=False,
created_at=now,
image_id='public_id',
member='fake_member_450',
can_share=True,
id=450)
image_members.insert().values(temp).execute()
def _check_ocata_expand01(self, engine, data):
images = db_utils.get_table(engine, 'images')
# check that visibility is null for existing images
rows = (images.select()
.order_by(images.c.id)
.execute()
.fetchall())
self.assertEqual(4, len(rows))
for row in rows:
self.assertIsNone(row['visibility'])
# run data migrations
data_migrations.migrate(engine)
# check that visibility is set appropriately for all images
rows = (images.select()
.order_by(images.c.id)
.execute()
.fetchall())
self.assertEqual(4, len(rows))
# private_id_1 has private visibility
self.assertEqual('private_id_1', rows[0]['id'])
self.assertEqual('private', rows[0]['visibility'])
# private_id_2 has private visibility
self.assertEqual('private_id_2', rows[1]['id'])
self.assertEqual('private', rows[1]['visibility'])
# public_id has public visibility
self.assertEqual('public_id', rows[2]['id'])
self.assertEqual('public', rows[2]['visibility'])
# shared_id has shared visibility
self.assertEqual('shared_id', rows[3]['id'])
self.assertEqual('shared', rows[3]['visibility'])
class TestOcataMigrate01MySQL(TestOcataMigrate01Mixin,
test_base.MySQLOpportunisticTestCase):
pass

View File

@ -23,7 +23,7 @@ from oslo_db.sqlalchemy import test_base
from oslo_db.sqlalchemy import test_migrations
import sqlalchemy.types as types
from glance.db import migration as dm
from glance.db import migration as db_migration
from glance.db.sqlalchemy import alembic_migrations
from glance.db.sqlalchemy.alembic_migrations import versions
from glance.db.sqlalchemy import models
@ -34,10 +34,11 @@ import glance.tests.utils as test_utils
class AlembicMigrationsMixin(object):
def _get_revisions(self, config):
def _get_revisions(self, config, head=None):
head = head or db_migration.LATEST_REVISION
scripts_dir = alembic_script.ScriptDirectory.from_config(config)
revisions = list(scripts_dir.walk_revisions(base='base',
head=dm.LATEST_REVISION))
head=head))
revisions = list(reversed(revisions))
revisions = [rev.revision for rev in revisions]
return revisions