Database migration

Creation of the database using the migration tools.

Change-Id: I339fb7dc17de6ec940cc6cc13a1e48dc088432e9
This commit is contained in:
Fabio Verboso 2018-08-07 15:53:36 +02:00
parent d27ee53304
commit 8d90b36e66
11 changed files with 607 additions and 13 deletions

330
iotronic/cmd/dbsync.py Normal file
View File

@ -0,0 +1,330 @@
# Copyright 2013 Hewlett-Packard Development Company, L.P.
# All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
"""
Run storage database migration.
"""
from __future__ import print_function
import sys
from iotronic.common import context
from iotronic.common.i18n import _
from iotronic.common import service
from iotronic.db import api as db_api
from iotronic.db import migration
from oslo_config import cfg
CONF = cfg.CONF
dbapi = db_api.get_instance()
# NOTE(rloo): This is a list of functions to perform online data migrations
# (from previous releases) for this release, in batches. It may be empty.
# The migration functions should be ordered by execution order; from earlier
# to later releases.
#
# Each migration function takes two arguments -- the context and maximum
# number of objects to migrate, and returns a 2-tuple -- the total number of
# objects that need to be migrated at the beginning of the function, and the
# number migrated. If the function determines that no migrations are needed,
# it returns (0, 0).
#
# Example of a function docstring:
#
# def sample_data_migration(context, max_count):
# """Sample method to migrate data to new format.
#
# :param context: an admin context
# :param max_count: The maximum number of objects to migrate. Must be
# >= 0. If zero, all the objects will be migrated.
# :returns: A 2-tuple -- the total number of objects that need to be
# migrated (at the beginning of this call) and the number
# of migrated objects.
# """
# NOTE(vdrok): Do not access objects' attributes, instead only provide object
# and attribute name tuples, so that not to trigger the load of the whole
# object, in case it is lazy loaded. The attribute will be accessed when needed
# by doing getattr on the object
ONLINE_MIGRATIONS = (
)
class DBCommand(object):
def _check_versions(self):
"""Check the versions of objects.
Check that the object versions are compatible with this release
of iotronic. It does this by comparing the objects' .version field
in the database, with the expected versions of these objects.
If it isn't compatible, we exit the program, returning 2.
"""
if migration.version() is None:
# no tables, nothing to check
return
"""
try:
if not dbapi.check_versions():
sys.stderr.write(
_('The database is not compatible with this '
'release of iotronic (%s). Please run '
'"iotronic-dbsync online_data_migrations" using '
'the previous release.\n')
% version.version_info.release_string())
# NOTE(rloo): We return 1 in online_data_migrations() to
# indicate that there are more objects to migrate,
# so don't use 1 here.
sys.exit(2)
except exception.DatabaseVersionTooOld:
sys.stderr.write(
_('The database version is not compatible with this '
'release of iotronic (%s). This can happen if you are '
'attempting to upgrade from a version older than '
'the previous release (skip versions upgrade). '
'This is an unsupported upgrade method. '
'Please run "iotronic-dbsync upgrade" using the previous '
'releases for a fast-forward upgrade.\n')
% version.version_info.release_string())
sys.exit(2)
"""
def upgrade(self):
self._check_versions()
migration.upgrade(CONF.command.revision)
def revision(self):
migration.revision(CONF.command.message, CONF.command.autogenerate)
def stamp(self):
migration.stamp(CONF.command.revision)
def version(self):
print(migration.version())
def create_schema(self):
migration.create_schema()
def online_data_migrations(self):
self._check_versions()
self._run_online_data_migrations(max_count=CONF.command.max_count,
options=CONF.command.options)
def _run_migration_functions(self, context, max_count, options):
"""Runs the migration functions.
Runs the data migration functions in the ONLINE_MIGRATIONS list.
It makes sure the total number of object migrations doesn't exceed the
specified max_count. A migration of an object will typically migrate
one row of data inside the database.
:param: context: an admin context
:param: max_count: the maximum number of objects (rows) to migrate;
a value >= 1.
:param: options: migration options - dict mapping migration name
to a dictionary of options for this migration.
:raises: Exception from the migration function
:returns: Boolean value indicating whether migrations are done. Returns
False if max_count objects have been migrated (since at that
point, it is unknown whether all migrations are done). Returns
True if migrations are all done (i.e. fewer than max_count objects
were migrated when the migrations are done).
"""
total_migrated = 0
for migration_func_obj, migration_func_name in ONLINE_MIGRATIONS:
migration_func = getattr(migration_func_obj, migration_func_name)
migration_opts = options.get(migration_func_name, {})
num_to_migrate = max_count - total_migrated
try:
total_to_do, num_migrated = migration_func(context,
num_to_migrate,
**migration_opts)
except Exception as e:
print(_("Error while running %(migration)s: %(err)s.")
% {'migration': migration_func.__name__, 'err': e},
file=sys.stderr)
raise
print(_('%(migration)s() migrated %(done)i of %(total)i objects.')
% {'migration': migration_func.__name__,
'total': total_to_do,
'done': num_migrated})
total_migrated += num_migrated
if total_migrated >= max_count:
# NOTE(rloo). max_count objects have been migrated so we have
# to stop. We return False because there is no look-ahead so
# we don't know if the migrations have been all done. All we
# know is that we've migrated max_count. It is possible that
# the migrations are done and that there aren't any more to
# migrate after this, but that would involve checking:
# 1. num_migrated == total_to_do (easy enough), AND
# 2. whether there are other migration functions and whether
# they need to do any object migrations (not so easy to
# check)
return False
return True
def _run_online_data_migrations(self, max_count=None, options=None):
"""Perform online data migrations for the release.
Online data migrations are done by running all the data migration
functions in the ONLINE_MIGRATIONS list. If max_count is None, all
the functions will be run in batches of 50 objects, until the
migrations are done. Otherwise, this will run (some of) the functions
until max_count objects have been migrated.
:param: max_count: the maximum number of individual object migrations
or modified rows, a value >= 1. If None, migrations are run in a
loop in batches of 50, until completion.
:param: options: options to pass to migrations. List of values in the
form of <migration name>.<option>=<value>
:raises: SystemExit. With exit code of:
0: when all migrations are complete.
1: when objects were migrated and the command needs to be
re-run (because there might be more objects to be migrated)
127: if max_count is < 1 or any option is invalid
:raises: Exception from a migration function
"""
parsed_options = {}
if options:
for option in options:
try:
migration, key_value = option.split('.', 1)
key, value = key_value.split('=', 1)
except ValueError:
print(_("Malformed option %s") % option)
sys.exit(127)
else:
parsed_options.setdefault(migration, {})[key] = value
admin_context = context.get_admin_context()
finished_migrating = False
if max_count is None:
max_count = 50
print(_('Running batches of %i until migrations have been '
'completed.') % max_count)
while not finished_migrating:
finished_migrating = self._run_migration_functions(
admin_context, max_count, parsed_options)
print(_('Data migrations have completed.'))
sys.exit(0)
if max_count < 1:
print(_('"max-count" must be a positive value.'), file=sys.stderr)
sys.exit(127)
finished_migrating = self._run_migration_functions(admin_context,
max_count,
parsed_options)
if finished_migrating:
print(_('Data migrations have completed.'))
sys.exit(0)
else:
print(_('Data migrations have not completed. Please re-run.'))
sys.exit(1)
def add_command_parsers(subparsers):
command_object = DBCommand()
parser = subparsers.add_parser(
'upgrade',
help=_("Upgrade the database schema to the latest version. "
"Optionally, use --revision to specify an alembic revision "
"string to upgrade to. It returns 2 (error) if the database is "
"not compatible with this version. If this happens, the "
"'iotronic-dbsync online_data_migrations' command should be"
" run using the previous version of iotronic, before upgrading"
" and running this command."))
parser.set_defaults(func=command_object.upgrade)
parser.add_argument('--revision', nargs='?')
parser = subparsers.add_parser('stamp')
parser.add_argument('--revision', nargs='?')
parser.set_defaults(func=command_object.stamp)
parser = subparsers.add_parser(
'revision',
help=_("Create a new alembic revision. "
"Use --message to set the message string."))
parser.add_argument('-m', '--message')
parser.add_argument('--autogenerate', action='store_true')
parser.set_defaults(func=command_object.revision)
parser = subparsers.add_parser(
'version',
help=_("Print the current version information and exit."))
parser.set_defaults(func=command_object.version)
parser = subparsers.add_parser(
'create_schema',
help=_("Create the database schema."))
parser.set_defaults(func=command_object.create_schema)
parser = subparsers.add_parser(
'online_data_migrations',
help=_("Perform online data migrations for the release. If "
"--max-count is specified, at most max-count objects will be "
"migrated. If not specified, all objects will be migrated "
"(in batches to avoid locking the database for long periods of "
"time). "
"The command returns code 0 (success) after migrations are "
"finished or there are no data to migrate. It returns code "
"1 (error) if there are still pending objects to be migrated. "
"Before upgrading to a newer release, this command must be run "
"until code 0 is returned. "
"It returns 127 (error) if max-count is < 1. "
"It returns 2 (error) if the database is not compatible with "
"this release. If this happens, this command should be run "
"using the previous release of iotronic, before upgrading and "
"running this command."))
parser.add_argument(
'--max-count', metavar='<number>', dest='max_count', type=int,
help=_("Maximum number of objects to migrate. If unspecified, all "
"objects are migrated."))
parser.add_argument(
'--option', metavar='<migration.opt=val>', action='append',
dest='options', default=[],
help=_("Options to pass to the migrations in the form of "
"<migration name>.<option>=<value>"))
parser.set_defaults(func=command_object.online_data_migrations)
def main():
command_opt = cfg.SubCommandOpt('command',
title='Command',
help=_('Available commands'),
handler=add_command_parsers)
CONF.register_cli_opt(command_opt)
# this is hack to work with previous usage of iotronic-dbsync
# pls change it to iotronic-dbsync upgrade
valid_commands = set([
'upgrade', 'revision',
'version', 'stamp', 'create_schema',
'online_data_migrations',
])
if not set(sys.argv) & valid_commands:
sys.argv.append('upgrade')
service.prepare_service(sys.argv)
CONF.command.func()

View File

@ -625,3 +625,7 @@ class NoPortsManaged(NotFound):
class NetworkError(IotronicException):
message = _("Network operation failure.")
class DatabaseVersionTooOld(IotronicException):
_msg_fmt = _("Database version is too old")

View File

@ -10,7 +10,3 @@ Upgrade can be performed by:
$ iotronic-dbsync - for backward compatibility
$ iotronic-dbsync upgrade
# iotronic-dbsync upgrade --revision head
Downgrading db:
$ iotronic-dbsync downgrade
$ iotronic-dbsync downgrade --revision base

View File

@ -13,6 +13,7 @@
from logging import config as log_config
from alembic import context
from oslo_db.sqlalchemy import enginefacade
try:
# NOTE(whaom): This is to register the DB2 alembic code which
@ -21,7 +22,6 @@ try:
except ImportError:
pass
from iotronic.db.sqlalchemy import api as sqla_api
from iotronic.db.sqlalchemy import models
# this is the Alembic Config object, which provides
@ -50,7 +50,7 @@ def run_migrations_online():
and associate a connection with the context.
"""
engine = sqla_api.get_engine()
engine = enginefacade.writer.get_engine()
with engine.connect() as connection:
context.configure(connection=connection,
target_metadata=target_metadata)

View File

@ -16,7 +16,3 @@ ${imports if imports else ""}
def upgrade():
${upgrades if upgrades else "pass"}
def downgrade():
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,179 @@
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
revision = 'df35e9cbeaff'
down_revision = None
from alembic import op
import iotronic.db.sqlalchemy.models
import sqlalchemy as sa
def upgrade():
op.create_table('boards',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=36), nullable=True),
sa.Column('code', sa.String(length=25), nullable=True),
sa.Column('status', sa.String(length=15), nullable=True),
sa.Column('name', sa.String(length=255), nullable=True),
sa.Column('type', sa.String(length=255), nullable=True),
sa.Column('agent', sa.String(length=255), nullable=True),
sa.Column('owner', sa.String(length=36), nullable=True),
sa.Column('project', sa.String(length=36), nullable=True),
sa.Column('mobile', sa.Boolean(), nullable=True),
sa.Column('config',
iotronic.db.sqlalchemy.models.JSONEncodedDict(),
nullable=True),
sa.Column('extra',
iotronic.db.sqlalchemy.models.JSONEncodedDict(),
nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('code', name='uniq_boards0code'),
sa.UniqueConstraint('uuid', name='uniq_boards0uuid')
)
op.create_table('conductors',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('hostname', sa.String(length=255),
nullable=False),
sa.Column('online', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('hostname',
name='uniq_conductors0hostname')
)
op.create_table('plugins',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=36), nullable=True),
sa.Column('name', sa.String(length=36), nullable=True),
sa.Column('owner', sa.String(length=36), nullable=True),
sa.Column('public', sa.Boolean(), nullable=True),
sa.Column('code', sa.TEXT(), nullable=True),
sa.Column('callable', sa.Boolean(), nullable=True),
sa.Column('parameters',
iotronic.db.sqlalchemy.models.JSONEncodedDict(),
nullable=True),
sa.Column('extra',
iotronic.db.sqlalchemy.models.JSONEncodedDict(),
nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('uuid', name='uniq_plugins0uuid')
)
op.create_table('services',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('uuid', sa.String(length=36), nullable=True),
sa.Column('name', sa.String(length=36), nullable=True),
sa.Column('project', sa.String(length=36), nullable=True),
sa.Column('port', sa.Integer(), nullable=True),
sa.Column('protocol', sa.String(length=3), nullable=True),
sa.Column('extra',
iotronic.db.sqlalchemy.models.JSONEncodedDict(),
nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('uuid', name='uniq_services0uuid')
)
op.create_table('wampagents',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('hostname', sa.String(length=255),
nullable=False),
sa.Column('wsurl', sa.String(length=255), nullable=False),
sa.Column('online', sa.Boolean(), nullable=True),
sa.Column('ragent', sa.Boolean(), nullable=True),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('hostname',
name='uniq_wampagentss0hostname')
)
op.create_table('exposed_services',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('board_uuid', sa.String(length=36),
nullable=True),
sa.Column('service_uuid', sa.String(length=36),
nullable=True),
sa.Column('public_port', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['board_uuid'], ['boards.uuid'], ),
sa.ForeignKeyConstraint(['service_uuid'],
['services.uuid'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('injection_plugins',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('board_uuid', sa.String(length=36),
nullable=True),
sa.Column('plugin_uuid', sa.String(length=36),
nullable=True),
sa.Column('onboot', sa.Boolean(), nullable=True),
sa.Column('status', sa.String(length=15), nullable=True),
sa.ForeignKeyConstraint(['board_uuid'], ['boards.uuid'], ),
sa.ForeignKeyConstraint(['plugin_uuid'],
['plugins.uuid'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('locations',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('longitude', sa.String(length=18),
nullable=True),
sa.Column('latitude', sa.String(length=18), nullable=True),
sa.Column('altitude', sa.String(length=18), nullable=True),
sa.Column('board_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['board_id'], ['boards.id'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('ports_on_boards',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('board_uuid', sa.String(length=40),
nullable=True),
sa.Column('uuid', sa.String(length=40), nullable=True),
sa.Column('VIF_name', sa.String(length=30), nullable=True),
sa.Column('MAC_add', sa.String(length=32), nullable=True),
sa.Column('ip', sa.String(length=36), nullable=True),
sa.Column('network', sa.String(length=36), nullable=True),
sa.ForeignKeyConstraint(['board_uuid'], ['boards.uuid'], ),
sa.PrimaryKeyConstraint('id')
)
op.create_table('sessions',
sa.Column('created_at', sa.DateTime(), nullable=True),
sa.Column('updated_at', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('valid', sa.Boolean(), nullable=True),
sa.Column('session_id', sa.String(length=15),
nullable=True),
sa.Column('board_uuid', sa.String(length=36),
nullable=True),
sa.Column('board_id', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['board_id'], ['boards.id'], ),
sa.PrimaryKeyConstraint('id'),
sa.UniqueConstraint('board_uuid',
name='uniq_board_uuid0board_uuid'),
sa.UniqueConstraint('session_id',
name='uniq_session_id0session_id')
)
def downgrade():
pass

View File

@ -20,8 +20,8 @@ import alembic
from alembic import config as alembic_config
import alembic.migration as alembic_migration
from oslo_db import exception as db_exc
from oslo_db.sqlalchemy import enginefacade
from iotronic.db.sqlalchemy import api as sqla_api
from iotronic.db.sqlalchemy import models
@ -38,7 +38,7 @@ def version(config=None, engine=None):
:rtype: string
"""
if engine is None:
engine = sqla_api.get_engine()
engine = enginefacade.writer.get_engine()
with engine.connect() as conn:
context = alembic_migration.MigrationContext.configure(conn)
return context.get_current_revision()
@ -62,7 +62,7 @@ def create_schema(config=None, engine=None):
Can be used for initial installation instead of upgrade('head').
"""
if engine is None:
engine = sqla_api.get_engine()
engine = enginefacade.writer.get_engine()
# NOTE(viktors): If we will use metadata.create_all() for non empty db
# schema, it will only add the new tables, but leave

View File

@ -21,6 +21,11 @@ classifier =
console_scripts =
iotronic-conductor = iotronic.cmd.conductor:main
iotronic-wamp-agent = iotronic.cmd.wamp_agent:main
iotronic-dbsync = iotronic.cmd.dbsync:main
iotronic.database.migration_backend =
sqlalchemy = iotronic.db.sqlalchemy.migration
[options]
build_scripts =

84
utils/all.sql Normal file

File diff suppressed because one or more lines are too long