Add db sync support for masakari

Added migration scripts for failover segment and host.
User will be able to sync the database with masakari-manage
command.

Change-Id: Ic37e681c6cb1ca6a341a77b738abb0a43a2910c4

README.rst

@@ -55,6 +55,17 @@ $ sudo python setup.py install
 9. To run masakari-api simply use following binary:
 $ masakari-api
 
+
+Configure masakari database
+---------------------------
+
+1. Create 'masakari' database
+
+2. After running setup.py for masakari '$ sudo python setup.py install'
+    run 'masakari-manage' command to sync the database
+    $ masakari-manage db sync
+
+
 Features
 --------
 

masakari/cmd/manage.py

@@ -0,0 +1,186 @@
+#!/usr/bin/env python
+# Copyright 2016 NTT DATA
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+  CLI interface for masakari management.
+"""
+
+
+import logging as python_logging
+import os
+import sys
+
+from oslo_config import cfg
+from oslo_db.sqlalchemy import migration
+from oslo_log import log as logging
+
+import masakari.conf
+from masakari.db import api as db_api
+from masakari.db.sqlalchemy import migration as db_migration
+from masakari import exception
+from masakari.i18n import _
+from masakari import version
+
+
+CONF = masakari.conf.CONF
+logging.register_options(CONF)
+
+
+# Decorators for actions
+def args(*args, **kwargs):
+    def _decorator(func):
+        func.__dict__.setdefault('args', []).insert(0, (args, kwargs))
+        return func
+    return _decorator
+
+
+def _db_error(caught_exception):
+    print('%s' % caught_exception)
+    print(_("The above error may show that the database has not "
+            "been created.\nPlease create a database using "
+            "'masakari-manage db sync' before running this command."))
+    sys.exit(1)
+
+
+class DbCommands(object):
+    """Class for managing the database."""
+
+    def __init__(self):
+        pass
+
+    @args('version', nargs='?', default=None, type=int,
+          help='Database version')
+    def sync(self, version=None):
+        """Sync the database up to the most recent version."""
+        try:
+            return db_migration.db_sync(version)
+        except exception.InvalidInput as ex:
+            print(ex)
+            sys.exit(1)
+
+    def version(self):
+        """Print the current database version."""
+        print(migration.db_version(db_api.get_engine(),
+                                   db_migration.MIGRATE_REPO_PATH,
+                                   db_migration.INIT_VERSION))
+
+
+CATEGORIES = {
+    'db': DbCommands,
+}
+
+
+def methods_of(obj):
+    """Return non-private methods from an object.
+
+    Get all callable methods of an object that don't start with underscore
+    :return: a list of tuples of the form (method_name, method)
+    """
+    result = []
+    for i in dir(obj):
+        if callable(getattr(obj, i)) and not i.startswith('_'):
+            result.append((i, getattr(obj, i)))
+    return result
+
+
+def add_command_parsers(subparsers):
+    for category in CATEGORIES:
+        command_object = CATEGORIES[category]()
+
+        parser = subparsers.add_parser(category)
+        parser.set_defaults(command_object=command_object)
+
+        category_subparsers = parser.add_subparsers(dest='action')
+
+        for (action, action_fn) in methods_of(command_object):
+            parser = category_subparsers.add_parser(action)
+
+            action_kwargs = []
+            for args, kwargs in getattr(action_fn, 'args', []):
+                parser.add_argument(*args, **kwargs)
+
+            parser.set_defaults(action_fn=action_fn)
+            parser.set_defaults(action_kwargs=action_kwargs)
+
+
+CONF.register_cli_opt(cfg.SubCommandOpt('category',
+                                        title='Command categories',
+                                        help='Available categories',
+                                        handler=add_command_parsers))
+
+
+def get_arg_string(args):
+    arg = None
+    if args[0] == '-':
+        # NOTE(Dinesh_Bhor): args starts with FLAGS.oparser.prefix_chars
+        # is optional args. Notice that cfg module takes care of
+        # actual ArgParser so prefix_chars is always '-'.
+        if args[1] == '-':
+            # This is long optional arg
+            arg = args[2:]
+        else:
+            arg = args[1:]
+    else:
+        arg = args
+
+    return arg
+
+
+def fetch_func_args(func):
+    fn_args = []
+    for args, kwargs in getattr(func, 'args', []):
+        arg = get_arg_string(args[0])
+        fn_args.append(getattr(CONF.category, arg))
+
+    return fn_args
+
+
+def main():
+    """Parse options and call the appropriate class/method."""
+    script_name = sys.argv[0]
+    if len(sys.argv) < 2:
+        print(_("\nOpenStack masakari version: %(version)s\n") %
+              {'version': version.version_string()})
+        print(script_name + " category action [<args>]")
+        print(_("Available categories:"))
+        for category in CATEGORIES:
+            print(_("\t%s") % category)
+        sys.exit(2)
+
+    try:
+        CONF(sys.argv[1:], project='masakari',
+             version=version.version_string())
+        logging.setup(CONF, "masakari")
+        python_logging.captureWarnings(True)
+    except cfg.ConfigDirNotFoundError as details:
+        print(_("Invalid directory: %s") % details)
+        sys.exit(2)
+    except cfg.ConfigFilesNotFoundError:
+        cfgfile = CONF.config_file[-1] if CONF.config_file else None
+        if cfgfile and not os.access(cfgfile, os.R_OK):
+            st = os.stat(cfgfile)
+            print(_("Could not read %s. Re-running with sudo") % cfgfile)
+            try:
+                os.execvp('sudo', ['sudo', '-u', '#%s' % st.st_uid] + sys.argv)
+            except Exception:
+                print(_('sudo failed, continuing as if nothing happened'))
+
+        print(_('Please re-run masakari-manage as root.'))
+        sys.exit(2)
+
+    fn = CONF.category.action_fn
+    fn_args = fetch_func_args(fn)
+    fn(*fn_args)

masakari/config.py

@@ -15,6 +15,7 @@ from oslo_log import log
 
 from masakari.common import config
 import masakari.conf
+from masakari.db.sqlalchemy import api as sqlalchemy_api
 from masakari import version
 
 
@@ -33,3 +34,6 @@ def parse_args(argv, default_config_files=None, configure_db=True,
          project='masakari',
          version=version.version_string(),
          default_config_files=default_config_files)
+
+    if configure_db:
+        sqlalchemy_api.configure(CONF)

masakari/db/__init__.py

@@ -0,0 +1,19 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""
+DB abstraction for Masakari
+"""
+
+from masakari.db.api import *  # noqa

masakari/db/api.py

@@ -0,0 +1,38 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Defines interface for DB access.
+
+Functions in this module are imported into the masakari.db namespace.
+Call these functions from masakari.db namespace, not the masakari.db.api
+namespace.
+"""
+
+from oslo_db import concurrency
+
+import masakari.conf
+
+CONF = masakari.conf.CONF
+
+_BACKEND_MAPPING = {'sqlalchemy': 'masakari.db.sqlalchemy.api'}
+
+IMPL = concurrency.TpoolDbapiWrapper(CONF, backend_mapping=_BACKEND_MAPPING)
+
+# The maximum value a signed INT type may have
+MAX_INT = 0x7FFFFFFF
+
+
+def get_engine():
+    """Returns database engine"""
+    return IMPL.get_engine()

masakari/db/migration.py

@@ -0,0 +1,25 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""Database setup and migration commands."""
+
+from masakari.db.sqlalchemy import migration
+
+IMPL = migration
+
+
+def db_sync(version=None):
+    """Migrate the database to `version` or the most recent version."""
+    return IMPL.db_sync(version=version)

masakari/db/sqlalchemy/api.py

@@ -0,0 +1,72 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+"""Implementation of SQLAlchemy backend."""
+
+import sys
+
+from oslo_db.sqlalchemy import enginefacade
+
+import masakari.conf
+
+
+CONF = masakari.conf.CONF
+
+main_context_manager = enginefacade.transaction_context()
+
+
+def _get_db_conf(conf_group, connection=None):
+
+    return {'connection': connection or conf_group.connection,
+            'slave_connection': conf_group.slave_connection,
+            'sqlite_fk': False,
+            '__autocommit': True,
+            'expire_on_commit': False,
+            'mysql_sql_mode': conf_group.mysql_sql_mode,
+            'idle_timeout': conf_group.idle_timeout,
+            'connection_debug': conf_group.connection_debug,
+            'max_pool_size': conf_group.max_pool_size,
+            'max_overflow': conf_group.max_overflow,
+            'pool_timeout': conf_group.pool_timeout,
+            'sqlite_synchronous': conf_group.sqlite_synchronous,
+            'connection_trace': conf_group.connection_trace,
+            'max_retries': conf_group.max_retries,
+            'retry_interval': conf_group.retry_interval}
+
+
+def _context_manager_from_context(context):
+    if context:
+        try:
+            return context.db_connection
+        except AttributeError:
+            pass
+
+
+def get_backend():
+    """The backend is this module itself."""
+    return sys.modules[__name__]
+
+
+def configure(conf):
+    main_context_manager.configure(**_get_db_conf(conf.database))
+
+
+def get_engine(use_slave=False, context=None):
+    """Get a database engine object.
+
+    :param use_slave: Whether to use the slave connection
+    :param context: The request context that can contain a context manager
+    """
+    ctxt_mgr = _context_manager_from_context(context) or main_context_manager
+    return ctxt_mgr.get_legacy_facade().get_engine(use_slave=use_slave)

masakari/db/sqlalchemy/migrate_repo/README.txt

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

masakari/db/sqlalchemy/migrate_repo/manage.py

@@ -0,0 +1,25 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+
+from migrate.versioning.shell import main
+
+from masakari.db.sqlalchemy import migrate_repo
+
+
+if __name__ == '__main__':
+    main(debug='False',
+         repository=os.path.abspath(os.path.dirname(migrate_repo.__file__)))

masakari/db/sqlalchemy/migrate_repo/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=masakari
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfuly during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

masakari/db/sqlalchemy/migrate_repo/versions/001_add_failover_segments_table.py

@@ -0,0 +1,60 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from migrate.changeset import UniqueConstraint
+from sqlalchemy import Column, MetaData, Table, Index
+from sqlalchemy import Integer, DateTime, String, Enum, Text
+
+
+def define_failover_segments_table(meta):
+
+    failover_segments = Table('failover_segments',
+                              meta,
+                              Column('created_at', DateTime, nullable=False),
+                              Column('updated_at', DateTime),
+                              Column('deleted_at', DateTime),
+                              Column('deleted', Integer),
+                              Column('id', Integer, primary_key=True,
+                                     nullable=False),
+                              Column('uuid', String(36), nullable=False),
+                              Column('name', String(255), nullable=False),
+                              Column('service_type', String(255),
+                                     nullable=False),
+                              Column('description', Text),
+                              Column('recovery_method',
+                                     Enum('auto', 'reserved_host',
+                                          'auto_priority',
+                                          'rh_priority',
+                                          name='recovery_methods'),
+                                     nullable=False),
+                              UniqueConstraint('name', 'deleted',
+                                               name='uniq_segment0name0deleted'
+                                               ),
+                              UniqueConstraint('uuid',
+                                               name='uniq_segments0uuid'),
+                              Index('segments_service_type_idx',
+                                    'service_type'),
+                              mysql_engine='InnoDB',
+                              mysql_charset='utf8',
+                              extend_existing=True)
+
+    return failover_segments
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+    table = define_failover_segments_table(meta)
+    table.create()

masakari/db/sqlalchemy/migrate_repo/versions/002_add_hosts_table.py

@@ -0,0 +1,56 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from migrate import ForeignKeyConstraint, UniqueConstraint
+from sqlalchemy import Column, MetaData, Table, Index
+from sqlalchemy import Integer, DateTime, String, Boolean, Text
+
+
+def define_hosts_table(meta):
+    failover_segments = Table('failover_segments', meta, autoload=True)
+    hosts = Table('hosts',
+                  meta,
+                  Column('created_at', DateTime, nullable=False),
+                  Column('updated_at', DateTime),
+                  Column('deleted_at', DateTime),
+                  Column('deleted', Integer),
+                  Column('id', Integer, primary_key=True,
+                         nullable=False),
+                  Column('uuid', String(36), nullable=False),
+                  Column('name', String(255), nullable=False),
+                  Column('reserved', Boolean, default=False),
+                  Column('type', String(255), nullable=False),
+                  Column('control_attributes', Text, nullable=False),
+                  Column('failover_segment_id', String(36), nullable=False),
+                  Column('on_maintenance', Boolean, default=False),
+                  UniqueConstraint('failover_segment_id', 'name', 'deleted',
+                                   name='uniq_host0name0deleted'),
+                  UniqueConstraint('uuid', name='uniq_host0uuid'),
+                  ForeignKeyConstraint(columns=['failover_segment_id'],
+                                       refcolumns=[failover_segments.c.uuid],
+                                       name='fk_failover_segments_uuid'),
+                  Index('hosts_type_idx', 'type'),
+                  mysql_engine='InnoDB',
+                  mysql_charset='utf8',
+                  extend_existing=True)
+
+    return hosts
+
+
+def upgrade(migrate_engine):
+    meta = MetaData()
+    meta.bind = migrate_engine
+    table = define_hosts_table(meta)
+    table.create()

masakari/db/sqlalchemy/migration.py

@@ -0,0 +1,76 @@
+# Copyright 2016 NTT Data.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import os
+import threading
+
+from oslo_config import cfg
+from oslo_db import exception as oslo_exception
+from oslo_db import options
+from stevedore import driver
+
+from masakari import db
+from masakari.db import api as db_api
+from masakari import exception
+from masakari.i18n import _
+
+INIT_VERSION = 0
+
+_IMPL = None
+_LOCK = threading.Lock()
+
+options.set_defaults(cfg.CONF)
+
+MIGRATE_REPO_PATH = os.path.join(
+    os.path.abspath(os.path.dirname(__file__)),
+    'migrate_repo',
+)
+
+
+def get_backend():
+    global _IMPL
+    if _IMPL is None:
+        with _LOCK:
+            if _IMPL is None:
+                _IMPL = driver.DriverManager(
+                    "masakari.database.migration_backend",
+                    cfg.CONF.database.backend).driver
+    return _IMPL
+
+
+def db_sync(version=None, init_version=INIT_VERSION, engine=None):
+    if engine is None:
+        engine = db_api.get_engine()
+
+    current_db_version = get_backend().db_version(engine,
+                                                  MIGRATE_REPO_PATH,
+                                                  init_version)
+
+    if version and int(version) < current_db_version:
+        msg = _('Database schema downgrade is not allowed.')
+        raise exception.InvalidInput(reason=msg)
+
+    if version and int(version) > db.MAX_INT:
+        message = _('Version should be less than or equal to %(max_version)d.'
+                    ) % {'max_version': db.MAX_INT}
+        raise exception.InvalidInput(reason=message)
+
+    try:
+        return get_backend().db_sync(engine=engine,
+                                     abs_path=MIGRATE_REPO_PATH,
+                                     version=version,
+                                     init_version=init_version)
+    except oslo_exception.DbMigrationError as exc:
+        raise exception.InvalidInput(reason=exc)

masakari/exception.py

@@ -161,6 +161,10 @@ class Invalid(MasakariException):
     code = 400
 
 
+class InvalidInput(Invalid):
+    msg_fmt = _("Invalid input received: %(reason)s")
+
+
 class InvalidAPIVersionString(Invalid):
     msg_fmt = _("API Version String %(version)s is of invalid format. Must "
                 "be of format MajorNum.MinorNum.")

setup.cfg

@@ -26,6 +26,10 @@ packages =
 [entry_points]
 console_scripts =
     masakari-api = masakari.cmd.api:main
+    masakari-manage = masakari.cmd.manage:main
+
+masakari.database.migration_backend =
+    sqlalchemy = oslo_db.sqlalchemy.migration
 
 masakari.api.v1.extensions =
     versions = masakari.api.openstack.ha.versionsV1:Versions