neutron-db-manage: add has_offline_migrations command

This command should be used by operators and deployment tools to
determine whether full neutron-server shutdown is needed for database
upgrade.

The change also makes neutron-db-manage tool to return the cumulative
result of commands being issued (in most cases it will still be 0 only,
since our command handlers implicitly return None).

DocImpact: Update doc to add new command 'has_offline_migrations' to
'neutron-db-manage' tool. The command determines whether full
neutron-server shutdown is needed for database upgrade.

Closes-Bug: #1519118
Change-Id: I7c5a4882ad4f80459ebe69c9a9c43cc60ce50200
Co-Authored-By: Martin Hickey <martin.hickey@ie.ibm.com>

doc/source/devref/alembic_migrations.rst

@@ -445,6 +445,14 @@ non-expansive migration rules, if any::
 
 and finally, start your neutron-server again.
 
+If you have multiple neutron-server instances in your cloud, and there are
+pending contract scripts not applied to the database, full shutdown of all
+those services is required before 'upgrade --contract' is executed. You can
+determine whether there are any pending contract scripts by checking return
+code for the following command::
+
+ neutron-db-manage has_offline_migrations
+
 If you are not interested in applying safe migration rules while the service is
 running, you can still upgrade database the old way, by stopping the service,
 and then applying all available rules::

doc/source/devref/upgrade.rst

@@ -90,12 +90,19 @@ Database upgrade is split into two parts:
 
 Each part represents a separate alembic branch.
 
-:ref:`More info on alembic scripts <alembic_migrations>`.
-
 The former step can be executed while old neutron-server code is running. The
 latter step requires *all* neutron-server instances to be shut down. Once it's
 complete, neutron-servers can be started again.
 
+.. note::
+    Full shutdown of neutron-server instances can be skipped depending on
+    whether there are pending contract scripts not applied to the database::
+
+     $ neutron-db-manage has_offline_migrations
+     Command will return a message if there are pending contract scripts.
+
+:ref:`More info on alembic scripts <alembic_migrations>`.
+
 Agents upgrade
 ~~~~~~~~~~~~~~
 

neutron/db/migration/alembic_migrations/env.py

@@ -16,12 +16,12 @@ from logging import config as logging_config
 
 from alembic import context
 from oslo_config import cfg
-from oslo_db.sqlalchemy import session
 import sqlalchemy as sa
 from sqlalchemy import event
 
 from neutron.db.migration.alembic_migrations import external
 from neutron.db.migration import autogen
+from neutron.db.migration.connection import DBConnection
 from neutron.db.migration.models import head  # noqa
 from neutron.db import model_base
 
@@ -109,24 +109,15 @@ def run_migrations_online():
     """
     set_mysql_engine()
     connection = config.attributes.get('connection')
-    new_engine = connection is None
-    if new_engine:
-        engine = session.create_engine(neutron_config.database.connection)
-        connection = engine.connect()
-    context.configure(
-        connection=connection,
-        target_metadata=target_metadata,
-        include_object=include_object,
-        process_revision_directives=autogen.process_revision_directives
-    )
-
-    try:
+    with DBConnection(neutron_config.database.connection, connection) as conn:
+        context.configure(
+            connection=conn,
+            target_metadata=target_metadata,
+            include_object=include_object,
+            process_revision_directives=autogen.process_revision_directives
+        )
         with context.begin_transaction():
             context.run_migrations()
-    finally:
-        if new_engine:
-            connection.close()
-            engine.dispose()
 
 
 if context.is_offline_mode():

neutron/db/migration/cli.py

@@ -17,6 +17,7 @@ import os
 from alembic import command as alembic_command
 from alembic import config as alembic_config
 from alembic import environment
+from alembic import migration as alembic_migration
 from alembic import script as alembic_script
 from alembic import util as alembic_util
 import debtcollector
@@ -29,6 +30,7 @@ import six
 from neutron._i18n import _
 from neutron.common import utils
 from neutron.db import migration
+from neutron.db.migration.connection import DBConnection
 
 
 HEAD_FILENAME = 'HEAD'
@@ -435,6 +437,32 @@ def update_head_file(config):
         f.write('\n'.join(head))
 
 
+def _get_current_database_heads(config):
+    with DBConnection(config.neutron_config.database.connection) as conn:
+        opts = {
+            'version_table': get_alembic_version_table(config)
+        }
+        context = alembic_migration.MigrationContext.configure(
+            conn, opts=opts)
+        return context.get_current_heads()
+
+
+def has_offline_migrations(config, cmd):
+    heads_map = _get_heads_map(config)
+    if heads_map[CONTRACT_BRANCH] not in _get_current_database_heads(config):
+        # If there is at least one contract revision not applied to database,
+        # it means we should shut down all neutron-server instances before
+        # proceeding with upgrade.
+        project = config.get_main_option('neutron_project')
+        alembic_util.msg(_('Need to apply migrations from %(project)s '
+                           'contract branch. This will require all Neutron '
+                           'server instances to be shutdown before '
+                           'proceeding with the upgrade.') %
+            {"project": project})
+        return True
+    return False
+
+
 def add_command_parsers(subparsers):
     for name in ['current', 'history', 'branches', 'heads']:
         parser = add_alembic_subparser(subparsers, name)
@@ -477,6 +505,13 @@ def add_command_parsers(subparsers):
     add_branch_options(parser)
     parser.set_defaults(func=do_revision)
 
+    parser = subparsers.add_parser(
+        'has_offline_migrations',
+        help='Determine whether there are pending migration scripts that '
+             'require full shutdown for all services that directly access '
+             'database.')
+    parser.set_defaults(func=has_offline_migrations)
+
 
 command_opt = cfg.SubCommandOpt('command',
                                 title='Command',
@@ -610,6 +645,21 @@ def _get_subproject_base(subproject):
     return entrypoint.module_name.split('.')[0]
 
 
+def get_alembic_version_table(config):
+    script_dir = alembic_script.ScriptDirectory.from_config(config)
+    alembic_version_table = [None]
+
+    def alembic_version_table_from_env(rev, context):
+        alembic_version_table[0] = context.version_table
+        return []
+
+    with environment.EnvironmentContext(config, script_dir,
+                                       fn=alembic_version_table_from_env):
+        script_dir.run_env()
+
+    return alembic_version_table[0]
+
+
 def get_alembic_configs():
     '''Return a list of alembic configs, one per project.
     '''
@@ -688,6 +738,12 @@ def get_engine_config():
 def main():
     CONF(project='neutron')
     validate_cli_options()
+    return_val = False
     for config in get_alembic_configs():
         #TODO(gongysh) enable logging
-        CONF.command.func(config, CONF.command.name)
+        return_val |= bool(CONF.command.func(config, CONF.command.name))
+
+    if CONF.command.name == 'has_offline_migrations' and not return_val:
+        alembic_util.msg(_('No offline migrations pending.'))
+
+    return return_val

neutron/db/migration/connection.py

@@ -0,0 +1,41 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_db.sqlalchemy import session
+
+
+class DBConnection(object):
+    """Context manager class which handles a DB connection.
+
+       An existing connection can be passed as a parameter. When
+       nested block is complete the new connection will be closed.
+       This class is not thread safe.
+    """
+
+    def __init__(self, connection_url, connection=None):
+        self.connection = connection
+        self.connection_url = connection_url
+        self.new_engine = False
+
+    def __enter__(self):
+        self.new_engine = self.connection is None
+        if self.new_engine:
+            self.engine = session.create_engine(self.connection_url)
+            self.connection = self.engine.connect()
+        return self.connection
+
+    def __exit__(self, type, value, traceback):
+        if self.new_engine:
+            try:
+                self.connection.close()
+            finally:
+                self.engine.dispose()

neutron/tests/functional/db/test_migrations.py

@@ -284,6 +284,20 @@ class TestModelsMigrationsMysql(_TestModelsMigrations,
                    and table != 'alembic_version']
             self.assertEqual(0, len(res), "%s non InnoDB tables created" % res)
 
+    def _test_has_offline_migrations(self, revision, expected):
+        engine = self.get_engine()
+        cfg.CONF.set_override('connection', engine.url, group='database')
+        migration.do_alembic_command(self.alembic_config, 'upgrade', revision)
+        self.assertEqual(expected,
+                         migration.has_offline_migrations(self.alembic_config,
+                                                          'unused'))
+
+    def test_has_offline_migrations_pending_contract_scripts(self):
+        self._test_has_offline_migrations('kilo', True)
+
+    def test_has_offline_migrations_all_heads_upgraded(self):
+        self._test_has_offline_migrations('heads', False)
+
 
 class TestModelsMigrationsPsql(_TestModelsMigrations,
                                base.PostgreSQLTestCase):