diff --git a/glance/db/sqlalchemy/migrate_repo/versions/029_location_meta_data_pickle_to_string.py b/glance/db/sqlalchemy/migrate_repo/versions/029_location_meta_data_pickle_to_string.py new file mode 100644 index 0000000000..fa524d35c6 --- /dev/null +++ b/glance/db/sqlalchemy/migrate_repo/versions/029_location_meta_data_pickle_to_string.py @@ -0,0 +1,75 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Rackspace Hosting +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import pickle + +import sqlalchemy +from sqlalchemy import MetaData, Table, Column +from glance.db.sqlalchemy import models + + +def upgrade(migrate_engine): + meta = sqlalchemy.schema.MetaData(migrate_engine) + image_locations = Table('image_locations', meta, autoload=True) + new_meta_data = Column('storage_meta_data', models.JSONEncodedDict, + default={}) + new_meta_data.create(image_locations) + + noe = pickle.dumps({}) + s = sqlalchemy.sql.select([image_locations]).\ + where(image_locations.c.meta_data != noe) + conn = migrate_engine.connect() + res = conn.execute(s) + + for row in res: + meta_data = row['meta_data'] + x = pickle.loads(meta_data) + if x != {}: + stmt = image_locations.update().\ + where(image_locations.c.id == row['id']).\ + values(storage_meta_data=x) + conn.execute(stmt) + conn.close() + image_locations.columns['meta_data'].drop() + image_locations.columns['storage_meta_data'].alter(name='meta_data') + + +def downgrade(migrate_engine): + meta = sqlalchemy.schema.MetaData(migrate_engine) + image_locations = Table('image_locations', meta, autoload=True) + old_meta_data = Column('old_meta_data', sqlalchemy.PickleType(), + default={}) + old_meta_data.create(image_locations) + + noj = json.dumps({}) + s = sqlalchemy.sql.select([image_locations]).\ + where(image_locations.c.meta_data != noj) + conn = migrate_engine.connect() + res = conn.execute(s) + + for row in res: + x = row['meta_data'] + meta_data = json.loads(x) + if meta_data != {}: + stmt = image_locations.update().\ + where(image_locations.c.id == row['id']).\ + values(old_meta_data=meta_data) + conn.execute(stmt) + conn.close() + image_locations.columns['meta_data'].drop() + image_locations.columns['old_meta_data'].alter(name='meta_data') diff --git a/glance/db/sqlalchemy/models.py b/glance/db/sqlalchemy/models.py index 958055c645..1df697ec26 100644 --- a/glance/db/sqlalchemy/models.py +++ b/glance/db/sqlalchemy/models.py @@ -19,12 +19,14 @@ """ SQLAlchemy models for glance data """ +import json from sqlalchemy import Column, Integer, String, BigInteger from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy import ForeignKey, DateTime, Boolean, Text, PickleType +from sqlalchemy import ForeignKey, DateTime, Boolean, Text from sqlalchemy.orm import relationship, backref, object_mapper +from sqlalchemy.types import TypeDecorator from sqlalchemy import Index, UniqueConstraint from glance.openstack.common import timeutils @@ -38,6 +40,22 @@ def compile_big_int_sqlite(type_, compiler, **kw): return 'INTEGER' +class JSONEncodedDict(TypeDecorator): + """Represents an immutable structure as a json-encoded string""" + + impl = Text + + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + + class ModelBase(object): """Base class for Nova and Glance Models""" __table_args__ = {'mysql_engine': 'InnoDB'} @@ -169,7 +187,7 @@ class ImageLocation(BASE, ModelBase): image_id = Column(String(36), ForeignKey('images.id'), nullable=False) image = relationship(Image, backref=backref('locations')) value = Column(Text(), nullable=False) - meta_data = Column(PickleType(), default={}) + meta_data = Column(JSONEncodedDict(), default={}) class ImageMember(BASE, ModelBase): diff --git a/glance/tests/unit/test_migrations.py b/glance/tests/unit/test_migrations.py index 97546ff15d..f4769203b0 100644 --- a/glance/tests/unit/test_migrations.py +++ b/glance/tests/unit/test_migrations.py @@ -28,6 +28,7 @@ if possible. import commands import ConfigParser import datetime +import json import os import pickle import urlparse @@ -935,3 +936,60 @@ class TestMigrations(utils.BaseTestCase): if idx.name == owner_index] self.assertNotIn((owner_index, columns), index_data) + + def _pre_upgrade_029(self, engine): + image_locations = get_table(engine, 'image_locations') + + meta_data = {'somelist': ['a', 'b', 'c'], 'avalue': 'hello', + 'adict': {}} + + now = datetime.datetime.now() + image_id = 'fake_029_id' + url = 'file:///some/place/onthe/fs029' + + images = get_table(engine, 'images') + temp = dict(deleted=False, + created_at=now, + updated_at=now, + status='active', + is_public=True, + min_disk=0, + min_ram=0, + id=image_id) + images.insert().values(temp).execute() + + pickle_md = pickle.dumps(meta_data) + temp = dict(deleted=False, + created_at=now, + updated_at=now, + image_id=image_id, + value=url, + meta_data=pickle_md) + image_locations.insert().values(temp).execute() + + return meta_data, image_id + + def _check_029(self, engine, data): + meta_data = data[0] + image_id = data[1] + image_locations = get_table(engine, 'image_locations') + + records = image_locations.select().\ + where(image_locations.c.image_id == image_id).execute().fetchall() + + for r in records: + d = json.loads(r['meta_data']) + self.assertEqual(d, meta_data) + + def _post_downgrade_029(self, engine): + image_id = 'fake_029_id' + + image_locations = get_table(engine, 'image_locations') + + records = image_locations.select().\ + where(image_locations.c.image_id == image_id).execute().fetchall() + + for r in records: + md = r['meta_data'] + d = pickle.loads(md) + self.assertEqual(type(d), dict)