Add second migrate_repo for cells v2 database migrations

This sets up the framework for adding database migrations to run on the
new api database.  It splits migration commands so that they can run on
the current 'main' database or the new 'api' database.

bp: cells-v2-mapping
Change-Id: I73dee6fd51b74bd670a6dffa07e875bf86891e97

nova/db/migration.py

@@ -21,19 +21,19 @@ from nova.db.sqlalchemy import migration
 IMPL = migration
 
 
-def db_sync(version=None):
+def db_sync(version=None, database='main'):
     """Migrate the database to `version` or the most recent version."""
-    return IMPL.db_sync(version=version)
+    return IMPL.db_sync(version=version, database=database)
 
 
-def db_version():
+def db_version(database='main'):
     """Display the current database version."""
-    return IMPL.db_version()
+    return IMPL.db_version(database=database)
 
 
-def db_initial_version():
+def db_initial_version(database='main'):
     """The starting version for the database."""
-    return IMPL.db_initial_version()
+    return IMPL.db_initial_version(database=database)
 
 
 def db_null_instance_uuid_scan(delete=False):

nova/db/sqlalchemy/api_migrations/migrate_repo/README

@@ -0,0 +1,4 @@
+This is a database migration repository.
+
+More information at
+http://code.google.com/p/sqlalchemy-migrate/

nova/db/sqlalchemy/api_migrations/migrate_repo/migrate.cfg

@@ -0,0 +1,20 @@
+[db_settings]
+# Used to identify which repository this database is versioned under.
+# You can use the name of your project.
+repository_id=nova_api
+
+# The name of the database table used to track the schema version.
+# This name shouldn't already be used by your project.
+# If this is changed once a database is under version control, you'll need to
+# change the table name in each database too.
+version_table=migrate_version
+
+# When committing a change script, Migrate will attempt to generate the
+# sql for all supported databases; normally, if one of them fails - probably
+# because you don't have that database installed - it is ignored and the
+# commit continues, perhaps ending successfully.
+# Databases in this list MUST compile successfully during a commit, or the
+# entire commit will fail. List the databases your application will actually
+# be using to ensure your updates to that database work properly.
+# This must be a list; example: ['postgres','sqlite']
+required_dbs=[]

nova/db/sqlalchemy/migration.py

@@ -28,23 +28,25 @@ from nova.db.sqlalchemy import api as db_session
 from nova import exception
 from nova.i18n import _
 
-INIT_VERSION = 215
-_REPOSITORY = None
+INIT_VERSION = {}
+INIT_VERSION['main'] = 215
+INIT_VERSION['api'] = 0
+_REPOSITORY = {}
 
 LOG = logging.getLogger(__name__)
 
 get_engine = db_session.get_engine
 
 
-def db_sync(version=None):
+def db_sync(version=None, database='main'):
     if version is not None:
         try:
             version = int(version)
         except ValueError:
             raise exception.NovaException(_("version should be an integer"))
 
-    current_version = db_version()
-    repository = _find_migrate_repo()
+    current_version = db_version(database)
+    repository = _find_migrate_repo(database)
     if version is None or version > current_version:
         return versioning_api.upgrade(get_engine(), repository, version)
     else:
@@ -52,8 +54,8 @@ def db_sync(version=None):
                                         version)
 
 
-def db_version():
-    repository = _find_migrate_repo()
+def db_version(database='main'):
+    repository = _find_migrate_repo(database)
     try:
         return versioning_api.db_version(get_engine(), repository)
     except versioning_exceptions.DatabaseNotControlledError as exc:
@@ -62,7 +64,7 @@ def db_version():
         meta.reflect(bind=engine)
         tables = meta.tables
         if len(tables) == 0:
-            db_version_control(INIT_VERSION)
+            db_version_control(INIT_VERSION[database])
             return versioning_api.db_version(get_engine(), repository)
         else:
             LOG.exception(exc)
@@ -72,8 +74,8 @@ def db_version():
                 _("Upgrade DB using Essex release first."))
 
 
-def db_initial_version():
-    return INIT_VERSION
+def db_initial_version(database='main'):
+    return INIT_VERSION[database]
 
 
 def _process_null_records(table, col_name, check_fkeys, delete=False):
@@ -151,12 +153,15 @@ def db_version_control(version=None):
     return version
 
 
-def _find_migrate_repo():
+def _find_migrate_repo(database='main'):
     """Get the path for the migrate repository."""
     global _REPOSITORY
+    rel_path = 'migrate_repo'
+    if database == 'api':
+        rel_path = os.path.join('api_migrations', 'migrate_repo')
     path = os.path.join(os.path.abspath(os.path.dirname(__file__)),
-                        'migrate_repo')
+                        rel_path)
     assert os.path.exists(path)
-    if _REPOSITORY is None:
-        _REPOSITORY = Repository(path)
-    return _REPOSITORY
+    if _REPOSITORY.get(database) is None:
+        _REPOSITORY[database] = Repository(path)
+    return _REPOSITORY[database]

nova/tests/functional/db/api/test_migrations.py

@@ -0,0 +1,149 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+"""
+Tests for database migrations.
+There are "opportunistic" tests which allows testing against all 3 databases
+(sqlite in memory, mysql, pg) in a properly configured unit test environment.
+
+For the opportunistic testing you need to set up db's named 'openstack_citest'
+with user 'openstack_citest' and password 'openstack_citest' on localhost. The
+test will then use that db and u/p combo to run the tests.
+
+For postgres on Ubuntu this can be done with the following commands::
+
+| sudo -u postgres psql
+| postgres=# create user openstack_citest with createdb login password
+|       'openstack_citest';
+| postgres=# create database openstack_citest with owner openstack_citest;
+
+"""
+
+import logging
+import os
+
+from migrate.versioning import repository
+import mock
+from oslo_config import cfg
+from oslo_db.sqlalchemy import test_base
+from oslo_db.sqlalchemy import test_migrations
+
+from nova.db import migration
+from nova.db.sqlalchemy.api_migrations import migrate_repo
+from nova.db.sqlalchemy import migration as sa_migration
+from nova import test
+
+
+CONF = cfg.CONF
+LOG = logging.getLogger(__name__)
+
+
+class NovaAPIModelsSync(test_migrations.ModelsMigrationsSync):
+    """Test that the models match the database after migrations are run."""
+
+    def db_sync(self, engine):
+        with mock.patch.object(sa_migration, 'get_engine',
+                               return_value=engine):
+            sa_migration.db_sync(database='api')
+
+    @property
+    def migrate_engine(self):
+        return self.engine
+
+    def get_engine(self):
+        return self.migrate_engine
+
+    def test_models_sync(self):
+        # TODO(alaski): Remove this override to run the test when there are
+        # models
+        pass
+
+    def get_metadata(self):
+        # TODO(alaski): Add model metadata once the first model is defined
+        pass
+
+    def include_object(self, object_, name, type_, reflected, compare_to):
+        if type_ == 'table':
+            # migrate_version is a sqlalchemy-migrate control table and
+            # isn't included in the model.
+            if name == 'migrate_version':
+                return False
+
+        return True
+
+
+class TestNovaAPIMigrationsSQLite(NovaAPIModelsSync,
+                                  test_base.DbTestCase,
+                                  test.NoDBTestCase):
+    pass
+
+
+class TestNovaAPIMigrationsMySQL(NovaAPIModelsSync,
+                                 test_base.MySQLOpportunisticTestCase,
+                                 test.NoDBTestCase):
+    pass
+
+
+class TestNovaAPIMigrationsPostgreSQL(NovaAPIModelsSync,
+        test_base.PostgreSQLOpportunisticTestCase, test.NoDBTestCase):
+    pass
+
+
+class NovaAPIMigrationsWalk(test_migrations.WalkVersionsMixin):
+    snake_walk = True
+    downgrade = True
+
+    def setUp(self):
+        super(NovaAPIMigrationsWalk, self).setUp()
+        # NOTE(viktors): We should reduce log output because it causes issues,
+        #                when we run tests with testr
+        migrate_log = logging.getLogger('migrate')
+        old_level = migrate_log.level
+        migrate_log.setLevel(logging.WARN)
+        self.addCleanup(migrate_log.setLevel, old_level)
+
+    @property
+    def INIT_VERSION(self):
+        return migration.db_initial_version('api')
+
+    @property
+    def REPOSITORY(self):
+        return repository.Repository(
+            os.path.abspath(os.path.dirname(migrate_repo.__file__)))
+
+    @property
+    def migration_api(self):
+        return sa_migration.versioning_api
+
+    @property
+    def migrate_engine(self):
+        return self.engine
+
+    def test_walk_versions(self):
+        self.walk_versions(self.snake_walk, self.downgrade)
+
+
+class TestNovaAPIMigrationsWalkSQLite(NovaAPIMigrationsWalk,
+                                      test_base.DbTestCase,
+                                      test.NoDBTestCase):
+    pass
+
+
+class TestNovaAPIMigrationsWalkMySQL(NovaAPIMigrationsWalk,
+                                     test_base.MySQLOpportunisticTestCase,
+                                     test.NoDBTestCase):
+    pass
+
+
+class TestNovaAPIMigrationsWalkPostgreSQL(NovaAPIMigrationsWalk,
+        test_base.PostgreSQLOpportunisticTestCase, test.NoDBTestCase):
+    pass