diff --git a/nimble/cmd/dbsync.py b/nimble/cmd/dbsync.py new file mode 100644 index 00000000..7cd20645 --- /dev/null +++ b/nimble/cmd/dbsync.py @@ -0,0 +1,91 @@ +# Copyright 2016 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +Run storage database migration. +""" + +import sys + +from oslo_config import cfg + +from nimble.common.i18n import _ +from nimble.common import service +from nimble.conf import CONF +from nimble.db import migration + + +class DBCommand(object): + + def upgrade(self): + migration.upgrade(CONF.command.revision) + + def revision(self): + migration.revision(CONF.command.message, CONF.command.autogenerate) + + def stamp(self): + migration.stamp(CONF.command.revision) + + def version(self): + print(migration.version()) + + def create_schema(self): + migration.create_schema() + + +def add_command_parsers(subparsers): + command_object = DBCommand() + + parser = subparsers.add_parser( + 'upgrade', + help=_("Upgrade the database schema to the latest version. " + "Optionally, use --revision to specify an alembic revision " + "string to upgrade to.")) + parser.set_defaults(func=command_object.upgrade) + parser.add_argument('--revision', nargs='?') + + parser = subparsers.add_parser('stamp') + parser.add_argument('--revision', nargs='?') + parser.set_defaults(func=command_object.stamp) + + parser = subparsers.add_parser( + 'revision', + help=_("Create a new alembic revision. " + "Use --message to set the message string.")) + parser.add_argument('-m', '--message') + parser.add_argument('--autogenerate', action='store_true') + parser.set_defaults(func=command_object.revision) + + parser = subparsers.add_parser( + 'version', + help=_("Print the current version information and exit.")) + parser.set_defaults(func=command_object.version) + + parser = subparsers.add_parser( + 'create_schema', + help=_("Create the database schema.")) + parser.set_defaults(func=command_object.create_schema) + + +def main(): + command_opt = cfg.SubCommandOpt('command', + title='Command', + help=_('Available commands'), + handler=add_command_parsers) + + CONF.register_cli_opt(command_opt) + + service.prepare_service(sys.argv) + CONF.command.func() diff --git a/nimble/common/paths.py b/nimble/common/paths.py new file mode 100644 index 00000000..556fc00b --- /dev/null +++ b/nimble/common/paths.py @@ -0,0 +1,50 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# Copyright 2012 Red Hat, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +from nimble.conf import CONF + + +def basedir_def(*args): + """Return an uninterpolated path relative to $pybasedir.""" + return os.path.join('$pybasedir', *args) + + +def bindir_def(*args): + """Return an uninterpolated path relative to $bindir.""" + return os.path.join('$bindir', *args) + + +def state_path_def(*args): + """Return an uninterpolated path relative to $state_path.""" + return os.path.join('$state_path', *args) + + +def basedir_rel(*args): + """Return a path relative to $pybasedir.""" + return os.path.join(CONF.pybasedir, *args) + + +def bindir_rel(*args): + """Return a path relative to $bindir.""" + return os.path.join(CONF.bindir, *args) + + +def state_path_rel(*args): + """Return a path relative to $state_path.""" + return os.path.join(CONF.state_path, *args) diff --git a/nimble/conf/__init__.py b/nimble/conf/__init__.py index 0ea01ea2..4516235d 100644 --- a/nimble/conf/__init__.py +++ b/nimble/conf/__init__.py @@ -16,7 +16,9 @@ from oslo_config import cfg from nimble.conf import api +from nimble.conf import database CONF = cfg.CONF api.register_opts(CONF) +database.register_opts(CONF) diff --git a/nimble/conf/database.py b/nimble/conf/database.py new file mode 100644 index 00000000..f73914b3 --- /dev/null +++ b/nimble/conf/database.py @@ -0,0 +1,28 @@ +# Copyright 2016 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_config import cfg + +from nimble.common.i18n import _ + +opts = [ + cfg.StrOpt('mysql_engine', + default='InnoDB', + help=_('MySQL engine to use.')) +] + + +def register_opts(conf): + conf.register_opts(opts, group='database') diff --git a/nimble/db/__init__.py b/nimble/db/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nimble/db/api.py b/nimble/db/api.py new file mode 100644 index 00000000..4ed39454 --- /dev/null +++ b/nimble/db/api.py @@ -0,0 +1,42 @@ +# Copyright 2016 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +Base classes for storage engines +""" + +import abc + +from oslo_config import cfg +from oslo_db import api as db_api +import six + + +_BACKEND_MAPPING = {'sqlalchemy': 'nimble.db.sqlalchemy.api'} +IMPL = db_api.DBAPI.from_config(cfg.CONF, backend_mapping=_BACKEND_MAPPING, + lazy=True) + + +def get_instance(): + """Return a DB API instance.""" + return IMPL + + +@six.add_metaclass(abc.ABCMeta) +class Connection(object): + """Base class for storage system connections.""" + + @abc.abstractmethod + def __init__(self): + """Constructor.""" diff --git a/nimble/db/migration.py b/nimble/db/migration.py new file mode 100644 index 00000000..aa780fb6 --- /dev/null +++ b/nimble/db/migration.py @@ -0,0 +1,52 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Database setup and migration commands.""" + +from oslo_config import cfg +from stevedore import driver + +_IMPL = None + + +def get_backend(): + global _IMPL + if not _IMPL: + cfg.CONF.import_opt('backend', 'oslo_db.options', group='database') + _IMPL = driver.DriverManager("nimble.database.migration_backend", + cfg.CONF.database.backend).driver + return _IMPL + + +def upgrade(version=None): + """Migrate the database to `version` or the most recent version.""" + return get_backend().upgrade(version) + + +def version(): + return get_backend().version() + + +def stamp(version): + return get_backend().stamp(version) + + +def revision(message, autogenerate): + return get_backend().revision(message, autogenerate) + + +def create_schema(): + return get_backend().create_schema() diff --git a/nimble/db/sqlalchemy/__init__.py b/nimble/db/sqlalchemy/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/nimble/db/sqlalchemy/alembic.ini b/nimble/db/sqlalchemy/alembic.ini new file mode 100644 index 00000000..a7689803 --- /dev/null +++ b/nimble/db/sqlalchemy/alembic.ini @@ -0,0 +1,54 @@ +# A generic, single database configuration. + +[alembic] +# path to migration scripts +script_location = %(here)s/alembic + +# template used to generate migration files +# file_template = %%(rev)s_%%(slug)s + +# max length of characters to apply to the +# "slug" field +#truncate_slug_length = 40 + +# set to 'true' to run the environment during +# the 'revision' command, regardless of autogenerate +# revision_environment = false + +#sqlalchemy.url = driver://user:pass@localhost/dbname + + +# Logging configuration +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/nimble/db/sqlalchemy/alembic/README b/nimble/db/sqlalchemy/alembic/README new file mode 100644 index 00000000..6cdb5854 --- /dev/null +++ b/nimble/db/sqlalchemy/alembic/README @@ -0,0 +1,12 @@ +Please see https://alembic.readthedocs.org/en/latest/index.html for general documentation + +To create alembic migrations use: +$ nimble-dbsync revision --message --autogenerate + +Stamp db with most recent migration version, without actually running migrations +$ nimble-dbsync stamp --revision head + +Upgrade can be performed by: +$ nimble-dbsync - for backward compatibility +$ nimble-dbsync upgrade +# nimble-dbsync upgrade --revision head diff --git a/nimble/db/sqlalchemy/alembic/env.py b/nimble/db/sqlalchemy/alembic/env.py new file mode 100644 index 00000000..6169f63c --- /dev/null +++ b/nimble/db/sqlalchemy/alembic/env.py @@ -0,0 +1,61 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from logging import config as log_config + +from alembic import context +from oslo_db.sqlalchemy import enginefacade + +try: + # NOTE(whaom): This is to register the DB2 alembic code which + # is an optional runtime dependency. + from ibm_db_alembic.ibm_db import IbmDbImpl # noqa +except ImportError: + pass + +from nimble.db.sqlalchemy import models + +# this is the Alembic Config object, which provides +# access to the values within the .ini file in use. +config = context.config + +# Interpret the config file for Python logging. +# This line sets up loggers basically. +log_config.fileConfig(config.config_file_name) + +# add your model's MetaData object here +# for 'autogenerate' support +# from myapp import mymodel +target_metadata = models.Base.metadata + +# other values from the config, defined by the needs of env.py, +# can be acquired: +# my_important_option = config.get_main_option("my_important_option") +# ... etc. + + +def run_migrations_online(): + """Run migrations in 'online' mode. + + In this scenario we need to create an Engine + and associate a connection with the context. + + """ + engine = enginefacade.get_legacy_facade().get_engine() + with engine.connect() as connection: + context.configure(connection=connection, + target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +run_migrations_online() diff --git a/nimble/db/sqlalchemy/alembic/script.py.mako b/nimble/db/sqlalchemy/alembic/script.py.mako new file mode 100644 index 00000000..3b1c960c --- /dev/null +++ b/nimble/db/sqlalchemy/alembic/script.py.mako @@ -0,0 +1,18 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision} +Create Date: ${create_date} + +""" + +# revision identifiers, used by Alembic. +revision = ${repr(up_revision)} +down_revision = ${repr(down_revision)} + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +def upgrade(): + ${upgrades if upgrades else "pass"} diff --git a/nimble/db/sqlalchemy/api.py b/nimble/db/sqlalchemy/api.py new file mode 100644 index 00000000..f7650d9d --- /dev/null +++ b/nimble/db/sqlalchemy/api.py @@ -0,0 +1,59 @@ +# Copyright 2016 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""SQLAlchemy storage backend.""" + +import threading + +from oslo_db.sqlalchemy import enginefacade +from oslo_log import log + +from nimble.db import api + +LOG = log.getLogger(__name__) + + +_CONTEXT = threading.local() + + +def get_backend(): + """The backend is this module itself.""" + return Connection() + + +def _session_for_read(): + return enginefacade.reader.using(_CONTEXT) + + +def _session_for_write(): + return enginefacade.writer.using(_CONTEXT) + + +def model_query(model, *args, **kwargs): + """Query helper for simpler session usage. + + :param session: if present, the session to use + """ + + with _session_for_read() as session: + query = session.query(model, *args) + return query + + +class Connection(api.Connection): + """SqlAlchemy connection.""" + + def __init__(self): + pass diff --git a/nimble/db/sqlalchemy/migration.py b/nimble/db/sqlalchemy/migration.py new file mode 100644 index 00000000..bd77b5ee --- /dev/null +++ b/nimble/db/sqlalchemy/migration.py @@ -0,0 +1,113 @@ +# Copyright 2010 United States Government as represented by the +# Administrator of the National Aeronautics and Space Administration. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os + +import alembic +from alembic import config as alembic_config +import alembic.migration as alembic_migration +from oslo_db import exception as db_exc +from oslo_db.sqlalchemy import enginefacade + +from nimble.db.sqlalchemy import models + + +def _alembic_config(): + path = os.path.join(os.path.dirname(__file__), 'alembic.ini') + config = alembic_config.Config(path) + return config + + +def version(config=None, engine=None): + """Current database version. + + :returns: Database version + :rtype: string + """ + if engine is None: + engine = enginefacade.get_legacy_facade().get_engine() + with engine.connect() as conn: + context = alembic_migration.MigrationContext.configure(conn) + return context.get_current_revision() + + +def upgrade(revision, config=None): + """Used for upgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'head' + config = config or _alembic_config() + + alembic.command.upgrade(config, revision or 'head') + + +def create_schema(config=None, engine=None): + """Create database schema from models description. + + Can be used for initial installation instead of upgrade('head'). + """ + if engine is None: + engine = enginefacade.get_legacy_facade().get_engine() + + # NOTE(viktors): If we will use metadata.create_all() for non empty db + # schema, it will only add the new tables, but leave + # existing as is. So we should avoid of this situation. + if version(engine=engine) is not None: + raise db_exc.DbMigrationError("DB schema is already under version" + " control. Use upgrade() instead") + + models.Base.metadata.create_all(engine) + stamp('head', config=config) + + +def downgrade(revision, config=None): + """Used for downgrading database. + + :param version: Desired database version + :type version: string + """ + revision = revision or 'base' + config = config or _alembic_config() + return alembic.command.downgrade(config, revision) + + +def stamp(revision, config=None): + """Stamps database with provided revision. + + Don't run any migrations. + + :param revision: Should match one from repository or head - to stamp + database with most recent revision + :type revision: string + """ + config = config or _alembic_config() + return alembic.command.stamp(config, revision=revision) + + +def revision(message=None, autogenerate=False, config=None): + """Creates template for migration. + + :param message: Text that will be used for migration title + :type message: string + :param autogenerate: If True - generates diff based on current database + state + :type autogenerate: bool + """ + config = config or _alembic_config() + return alembic.command.revision(config, message=message, + autogenerate=autogenerate) diff --git a/nimble/db/sqlalchemy/models.py b/nimble/db/sqlalchemy/models.py new file mode 100644 index 00000000..f73d6629 --- /dev/null +++ b/nimble/db/sqlalchemy/models.py @@ -0,0 +1,54 @@ +# Copyright 2016 Huawei Technologies Co.,LTD. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +SQLAlchemy models for baremetal compute service. +""" + +from oslo_db import options as db_options +from oslo_db.sqlalchemy import models +import six.moves.urllib.parse as urlparse +from sqlalchemy.ext.declarative import declarative_base + +from nimble.common import paths +from nimble.conf import CONF + +_DEFAULT_SQL_CONNECTION = 'sqlite:///' + paths.state_path_def('nimble.sqlite') + + +db_options.set_defaults(CONF, _DEFAULT_SQL_CONNECTION, 'nimble.sqlite') + + +def table_args(): + engine_name = urlparse.urlparse(CONF.database.connection).scheme + if engine_name == 'mysql': + return {'mysql_engine': CONF.database.mysql_engine, + 'mysql_charset': "utf8"} + return None + + +class NimbleBase(models.TimestampMixin, + models.ModelBase): + + metadata = None + + def as_dict(self): + d = {} + for c in self.__table__.columns: + d[c.name] = self[c.name] + return d + + +Base = declarative_base(cls=NimbleBase) diff --git a/requirements.txt b/requirements.txt index bb49ba09..fa2fdd9e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,8 @@ # process, which may cause wedges in the gate later. pbr>=1.6 # Apache-2.0 +SQLAlchemy<1.1.0,>=1.0.10 # MIT +alembic>=0.8.4 # MIT eventlet!=0.18.3,>=0.18.2 # MIT WebOb>=1.2.3 # MIT oslo.concurrency>=3.8.0 # Apache-2.0 diff --git a/setup.cfg b/setup.cfg index 57f3e134..2881f6b5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,6 +26,7 @@ packages = [entry_points] console_scripts = nimble-api = nimble.cmd.api:main + nimble-dbsync = nimble.cmd.dbsync:main [build_sphinx] source-dir = doc/source