From c19b58a9f312bbe2ef0183f08e6773431eba6fe6 Mon Sep 17 00:00:00 2001
From: Martin Schuppert <mschuppert@redhat.com>
Date: Wed, 10 Oct 2018 15:16:02 +0200
Subject: [PATCH] Verify nova api migration finished before start placement

There is a deployment race where nova-placement fails to start if
the nova api db migration have not finished before starting it.
We start nova placement early to make sure it is up before the
nova-compute services get started. Since in HA scenario there is
no sync in between the nodes on the current worked deployment step
we might have the situation that the placement service gets started
on C1/2 when the nova api db sync is not yet finished on C0.

We have two possibilities:
1) start placement later and verify that nova-computes recover correct
2) verify that db migration on nova_api db finished before start nova-
placement on the controllers

This is a way to address 2) and should be the less invasive change as
we make sure placement gets started after the nova api db sync finished
and does not move the start to a complete different step which might
introduce any unforeseen behavior.

Change-Id: Ifc4346658cea5355bf89f6d7023ed846d9162b3e
---
 docker/services/nova-placement.yaml           | 20 ++++-
 .../nova_wait_for_db_sync.py                  | 73 +++++++++++++++++++
 2 files changed, 91 insertions(+), 2 deletions(-)
 create mode 100755 docker_config_scripts/nova_wait_for_db_sync.py

diff --git a/docker/services/nova-placement.yaml b/docker/services/nova-placement.yaml
index 1782154253..f77c183bc5 100644
--- a/docker/services/nova-placement.yaml
+++ b/docker/services/nova-placement.yaml
@@ -115,14 +115,30 @@ outputs:
             - path: /var/log/nova
               owner: nova:nova
               recurse: true
+      docker_config_scripts:
+        nova_wait_for_db_sync.py:
+          mode: "0700"
+          content: { get_file: ../../docker_config_scripts/nova_wait_for_db_sync.py }
       docker_config:
         step_2:
           get_attr: [NovaPlacementLogging, docker_config, step_2]
         # start this early so it is up before computes start reporting
         step_3:
-          nova_placement:
+          nova_wait_for_db_sync:
             start_order: 1
-            image: {get_param: DockerNovaPlacementImage}
+            image: &nova_placement_image {get_param: DockerNovaPlacementImage}
+            net: host
+            user: root
+            privileged: false
+            detach: false
+            volumes:
+              - /var/lib/nova:/var/lib/nova:shared
+              - /var/lib/docker-config-scripts/:/docker-config-scripts/
+              - /var/lib/config-data/puppet-generated/nova_placement/etc/nova:/etc/nova:ro
+            command: "/docker-config-scripts/nova_wait_for_db_sync.py"
+          nova_placement:
+            start_order: 2
+            image: *nova_placement_image
             net: host
             user: root
             restart: always
diff --git a/docker_config_scripts/nova_wait_for_db_sync.py b/docker_config_scripts/nova_wait_for_db_sync.py
new file mode 100755
index 0000000000..10e7f7d99d
--- /dev/null
+++ b/docker_config_scripts/nova_wait_for_db_sync.py
@@ -0,0 +1,73 @@
+#!/usr/bin/env python
+#
+# Copyright 2018 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+# shell script to check if nova API DB migrations finished after X attempts.
+# Default max is 60 iterations with 10s (default) timeout in between.
+
+from __future__ import print_function
+
+import logging
+import os
+import sys
+import time
+
+from migrate.versioning import api as versioning_api
+
+from nova import config
+from nova.db.sqlalchemy import migration
+
+
+logging.basicConfig(stream=sys.stdout, level=logging.DEBUG)
+LOG = logging.getLogger('nova_wait_for_db_sync')
+
+iterations = 60
+timeout = 10
+nova_cfg = '/etc/nova/nova.conf'
+
+if __name__ == '__main__':
+    if os.path.isfile(nova_cfg):
+        config.parse_args(sys.argv)
+    else:
+        LOG.error('Nova configuration file %s does not exist', nova_cfg)
+        sys.exit(1)
+
+    repo = migration._find_migrate_repo('api')
+    max_migration_number = versioning_api.version(repo.path)
+    LOG.info("Max migration number from files: %i", max_migration_number)
+
+    # wait for db miration to be finished, or fail
+    while iterations > 1:
+        iterations -= 1
+        try:
+            db_migration_number = migration.db_version('api')
+            if db_migration_number == max_migration_number:
+                LOG.info('Nova API DB sync finished. Migration number %i',
+                         db_migration_number)
+                sys.exit(0)
+                break
+            else:
+                LOG.info('Nova API DB sync not yet finished. Migration' +
+                         'number DB/files (%i/%i)',
+                         db_migration_number,
+                         max_migration_number)
+                time.sleep(timeout)
+        except Exception as e:
+            LOG.error('uuups something went wrong: %s ' + e.message)
+            break
+
+    sys.exit(1)
+
+# vim: set et ts=4 sw=4 :