132 lines
5.4 KiB
Python
132 lines
5.4 KiB
Python
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
|
# not use this file except in compliance with the License. You may obtain
|
|
# a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
|
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
|
# License for the specific language governing permissions and limitations
|
|
# under the License.
|
|
import hashlib
|
|
|
|
import migrate
|
|
from oslo_serialization import jsonutils
|
|
import sqlalchemy as sa
|
|
|
|
|
|
m_tables = [('metadata_text', sa.Text, True),
|
|
('metadata_bool', sa.Boolean, False),
|
|
('metadata_int', sa.BigInteger, False),
|
|
('metadata_float', sa.Float(53), False)]
|
|
|
|
|
|
def _migrate_meta_tables(meta, col, new_col, new_fk):
|
|
for t_name, t_type, t_nullable in m_tables:
|
|
m_table = sa.Table(t_name, meta, autoload=True)
|
|
m_table_new = sa.Table(
|
|
'%s_new' % t_name, meta,
|
|
sa.Column('id', sa.Integer, sa.ForeignKey(new_fk),
|
|
primary_key=True),
|
|
sa.Column('meta_key', sa.String(255),
|
|
primary_key=True),
|
|
sa.Column('value', t_type, nullable=t_nullable),
|
|
mysql_engine='InnoDB',
|
|
mysql_charset='utf8',
|
|
)
|
|
m_table_new.create()
|
|
|
|
if m_table.select().scalar() is not None:
|
|
m_table_new.insert().from_select(
|
|
['id', 'meta_key', 'value'],
|
|
sa.select([new_col, m_table.c.meta_key,
|
|
m_table.c.value]).where(
|
|
col == m_table.c.id).group_by(
|
|
new_col, m_table.c.meta_key, m_table.c.value)).execute()
|
|
|
|
m_table.drop()
|
|
if meta.bind.engine.name != 'sqlite':
|
|
sa.Index('ix_%s_meta_key' % t_name,
|
|
m_table_new.c.meta_key).create()
|
|
m_table_new.rename(t_name)
|
|
|
|
|
|
def upgrade(migrate_engine):
|
|
meta = sa.MetaData(bind=migrate_engine)
|
|
resource = sa.Table(
|
|
'resource', meta,
|
|
sa.Column('internal_id', sa.Integer, primary_key=True),
|
|
sa.Column('resource_id', sa.String(255)),
|
|
sa.Column('user_id', sa.String(255)),
|
|
sa.Column('project_id', sa.String(255)),
|
|
sa.Column('source_id', sa.String(255)),
|
|
sa.Column('resource_metadata', sa.Text),
|
|
sa.Column('metadata_hash', sa.String(32)),
|
|
mysql_engine='InnoDB',
|
|
mysql_charset='utf8')
|
|
resource.create()
|
|
|
|
# copy resource data in to resource table
|
|
sample = sa.Table('sample', meta, autoload=True)
|
|
sa.Column('metadata_hash', sa.String(32)).create(sample)
|
|
for row in sa.select([sample.c.id, sample.c.resource_metadata]).execute():
|
|
sample.update().where(sample.c.id == row['id']).values(
|
|
{sample.c.metadata_hash:
|
|
hashlib.md5(jsonutils.dumps(
|
|
row['resource_metadata'],
|
|
sort_keys=True)).hexdigest()}).execute()
|
|
query = sa.select([sample.c.resource_id, sample.c.user_id,
|
|
sample.c.project_id, sample.c.source_id,
|
|
sample.c.resource_metadata,
|
|
sample.c.metadata_hash]).distinct()
|
|
for row in query.execute():
|
|
resource.insert().values(
|
|
resource_id=row['resource_id'],
|
|
user_id=row['user_id'],
|
|
project_id=row['project_id'],
|
|
source_id=row['source_id'],
|
|
resource_metadata=row['resource_metadata'],
|
|
metadata_hash=row['metadata_hash']).execute()
|
|
# link sample records to new resource records
|
|
sa.Column('resource_id_new', sa.Integer).create(sample)
|
|
for row in sa.select([resource]).execute():
|
|
(sample.update().
|
|
where(sa.and_(
|
|
sample.c.resource_id == row['resource_id'],
|
|
sample.c.user_id == row['user_id'],
|
|
sample.c.project_id == row['project_id'],
|
|
sample.c.source_id == row['source_id'],
|
|
sample.c.metadata_hash == row['metadata_hash'])).
|
|
values({sample.c.resource_id_new: row['internal_id']}).execute())
|
|
|
|
sample.c.resource_id.drop()
|
|
sample.c.metadata_hash.drop()
|
|
sample.c.resource_id_new.alter(name='resource_id')
|
|
# re-bind metadata to pick up alter name change
|
|
meta = sa.MetaData(bind=migrate_engine)
|
|
sample = sa.Table('sample', meta, autoload=True)
|
|
resource = sa.Table('resource', meta, autoload=True)
|
|
if migrate_engine.name != 'sqlite':
|
|
sa.Index('ix_resource_resource_id', resource.c.resource_id).create()
|
|
sa.Index('ix_sample_user_id', sample.c.user_id).drop()
|
|
sa.Index('ix_sample_project_id', sample.c.project_id).drop()
|
|
sa.Index('ix_sample_resource_id', sample.c.resource_id).create()
|
|
sa.Index('ix_sample_meter_id_resource_id',
|
|
sample.c.meter_id, sample.c.resource_id).create()
|
|
|
|
params = {'columns': [sample.c.resource_id],
|
|
'refcolumns': [resource.c.internal_id]}
|
|
if migrate_engine.name == 'mysql':
|
|
params['name'] = 'fk_sample_resource_internal_id'
|
|
migrate.ForeignKeyConstraint(**params).create()
|
|
|
|
sample.c.user_id.drop()
|
|
sample.c.project_id.drop()
|
|
sample.c.source_id.drop()
|
|
sample.c.resource_metadata.drop()
|
|
|
|
_migrate_meta_tables(meta, sample.c.id, sample.c.resource_id,
|
|
'resource.internal_id')
|