Retry online_data_migrations until exit code 0 or 2.

The command 'cinder-manage db online_data_migrations' has three possible
exit codes:

- 0: The migrations have been completed successfully.
- 1: Some migrations have been completed, but more objects need to be
  migrated.
- 2: Some migrations are generating errors and manual intervention is
  needed.

This change will run in a loop the command online_data_migrations in
batches of 5000 objects, when the exit code is 1 runs again, when the
exit code is 0 the loop is stopped, when the exit code is 2 a
CalledProcessError exception is raised.

Change-Id: Iee99e9ab9777392f05c1a0db55951d97773bb0ba

hooks/cinder_utils.py

@@ -38,6 +38,7 @@ from charmhelpers.core.hookenv import (
     log,
     DEBUG,
     INFO,
+    ERROR,
     hook_name,
 )
 
@@ -177,6 +178,10 @@ MEMCACHED_CONF = '/etc/memcached.conf'
 WSGI_CINDER_API_CONF = '/etc/apache2/sites-enabled/wsgi-openstack-api.conf'
 PACKAGE_CINDER_API_CONF = '/etc/apache2/conf-enabled/cinder-wsgi.conf'
 
+# max number of objects to include in a batch/query when running
+# online_data_migrations.
+MAX_OBJECTS_COUNT = '5000'
+
 VERSION_PACKAGE = 'cinder-common'
 
 TEMPLATES = 'templates/'
@@ -801,9 +806,36 @@ def migrate_database(upgrade=False):
         if need_online_migration_msg not in e.output:
             raise
 
-        log("Running online_data_migrations", level=INFO)
-        subprocess.check_call(['cinder-manage', 'db',
-                               'online_data_migrations'])
+        # Re-run online_data_migrations until the exit code is 0 or 2.
+        # See https://docs.openstack.org/cinder/latest/admin/upgrades.html
+        num_runs = 1
+        cmd_online_data_migrations = ['cinder-manage', 'db',
+                                      'online_data_migrations',
+                                      '--max_count', MAX_OBJECTS_COUNT]
+        exit_code = 1
+        while exit_code == 1:
+            log("Running online_data_migrations (%d iteration(s))" % num_runs,
+                level=INFO)
+            result = subprocess.run(cmd_online_data_migrations, check=False)
+            exit_code = result.returncode
+            # log as ERROR when the exit code 2 which is a fatal failure.
+            level = ERROR if exit_code == 2 else DEBUG
+            log("%s: exit code %s" % (" ".join(cmd_online_data_migrations),
+                                      result.returncode),
+                level=level)
+            log("stdout: %s" % result.stdout, level=level)
+            log("stderr: %s" % result.stderr, level=level)
+
+            if exit_code == 2:
+                # If no further migrations are possible, the exit status will
+                # be 2 if some migrations are still generating errors, which
+                # requires intervention to resolve.
+                raise subprocess.CalledProcessError(result.returncode,
+                                                    result.args,
+                                                    result.stdout,
+                                                    result.stderr)
+            num_runs += 1
+
         log("Re-running database migration", level=INFO)
         subprocess.check_call(cmd)
 

unit_tests/test_cinder_utils.py

@@ -770,18 +770,31 @@ class TestCinderUtils(CharmTestCase):
         rid = 'cluster:0'
         self.relation_ids.return_value = [rid]
         with patch('subprocess.check_output') as check_output, \
-             patch('subprocess.check_call') as check_call:
+             patch('subprocess.check_call') as check_call, \
+             patch('subprocess.run') as run:
             check_output.side_effect = subprocess.CalledProcessError(
                 1, 'cinder db-manage sync',
                 ("Please run `cinder-manage db online_data_migrations`. "
                  "There are still untyped volumes unmigrated."))
+            run_result = MagicMock()
+            run_result.returncode = 0
+            run.return_value = run_result
             cinder_utils.migrate_database(upgrade=True)
             check_output.assert_called_with(['cinder-manage', 'db', 'sync'],
                                             universal_newlines=True)
             check_call.assert_has_calls([
-                call(['cinder-manage', 'db', 'online_data_migrations']),
                 call(['cinder-manage', 'db', 'sync'])
             ])
+            run.assert_has_calls([
+                call(['cinder-manage', 'db', 'online_data_migrations',
+                      '--max_count', cinder_utils.MAX_OBJECTS_COUNT],
+                     check=False),
+            ])
+
+            run_result.returncode = 2
+            self.assertRaises(subprocess.CalledProcessError,
+                              cinder_utils.migrate_database,
+                              upgrade=True)
 
     @patch.object(cinder_utils, 'resource_map')
     def test_register_configs(self, resource_map):