From b6b2b562f03bbaddfd8b6612fb3afeff20315397 Mon Sep 17 00:00:00 2001
From: Dougal Matthews <dougal@redhat.com>
Date: Tue, 13 Sep 2016 09:49:07 +0100
Subject: [PATCH] Verify that the Deployment Plan creation was successful

The creation of the default deployment plan was recently broken but the error
went unnoticed as CI never failed. This then led to another obscure bug as
something worked due only to the default plan not existing.

This change waits for the plan creation to finish and verifies that it
completed without an error. A timeout is added, should, for some reason the
creation ever stall.

NOTE: This will not raise an error if plan creation fails, since it is
      currently broken. This will be fixed in this following patch:
      https://review.openstack.org/373446

Partial-Bug: #1622683
Change-Id: Id8001fee677fe321b42892264e3c55afb6790e1c
---
 instack_undercloud/tests/test_undercloud.py | 23 +++++++++++++++++++++
 instack_undercloud/undercloud.py            | 23 +++++++++++++++++++--
 2 files changed, 44 insertions(+), 2 deletions(-)

diff --git a/instack_undercloud/tests/test_undercloud.py b/instack_undercloud/tests/test_undercloud.py
index 0c62ae337..23390871b 100644
--- a/instack_undercloud/tests/test_undercloud.py
+++ b/instack_undercloud/tests/test_undercloud.py
@@ -530,6 +530,7 @@ class TestPostConfig(base.BaseTestCase):
     def test_create_default_plan(self):
         mock_mistral = mock.Mock()
         mock_mistral.environments.list.return_value = []
+        mock_mistral.executions.get.return_value = mock.Mock(state="SUCCESS")
 
         undercloud._create_default_plan(mock_mistral)
         mock_mistral.executions.create.assert_called_once_with(
@@ -556,6 +557,28 @@ class TestPostConfig(base.BaseTestCase):
         mock_mistral.executions.create.assert_called_once_with(
             'tripleo.validations.v1.copy_ssh_key')
 
+    @mock.patch('time.sleep')
+    def test_create_default_plan_timeout(self, mock_sleep):
+        mock_mistral = mock.Mock()
+        mock_mistral.environments.list.return_value = []
+        mock_mistral.executions.get.return_value = mock.Mock(state="RUNNING")
+
+        self.assertRaises(
+            RuntimeError,
+            undercloud._create_default_plan, mock_mistral, timeout=0)
+
+    def test_create_default_plan_failed(self):
+        mock_mistral = mock.Mock()
+        mock_mistral.environments.list.return_value = []
+        mock_mistral.executions.get.return_value = mock.Mock(state="ERROR")
+
+        # TODO(dmatthews): Switch to this check after this lands:
+        #   https://review.openstack.org/#/c/371347/
+        # self.assertRaises(
+        #     RuntimeError,
+        #     undercloud._create_default_plan, mock_mistral)
+        undercloud._create_default_plan(mock_mistral)
+
     @mock.patch('instack_undercloud.undercloud._run_command')
     def test_copy_stackrc(self, mock_run):
         undercloud._copy_stackrc()
diff --git a/instack_undercloud/undercloud.py b/instack_undercloud/undercloud.py
index 4f318cc5c..544899c21 100644
--- a/instack_undercloud/undercloud.py
+++ b/instack_undercloud/undercloud.py
@@ -26,6 +26,7 @@ import socket
 import subprocess
 import sys
 import tempfile
+import time
 import uuid
 
 from keystoneclient import exceptions as ks_exceptions
@@ -1099,7 +1100,7 @@ def _clean_os_refresh_config():
     _run_command(args, name='Clean os-refresh-config')
 
 
-def _create_default_plan(mistral):
+def _create_default_plan(mistral, timeout=60):
     plan_name = 'overcloud'
     queue_name = str(uuid.uuid4())
 
@@ -1108,11 +1109,29 @@ def _create_default_plan(mistral):
                  plan_name)
         return
 
-    mistral.executions.create(
+    execution = mistral.executions.create(
         'tripleo.plan_management.v1.create_default_deployment_plan',
         workflow_input={'container': plan_name, 'queue_name': queue_name}
     )
 
+    timeout_at = time.time() + timeout
+
+    while time.time() < timeout_at:
+        exe = mistral.executions.get(execution.id)
+        if exe.state == "RUNNING":
+            time.sleep(5)
+            continue
+        if exe.state == "SUCCESS":
+            return
+        else:
+            LOG.warning("Failed to create the default Deployment Plan.")
+            return
+    else:
+        exe = mistral.executions.get(execution.id)
+        LOG.error("Timed out waiting for execution %s to finish. State: %s",
+                  exe.id, exe.state)
+        raise RuntimeError("Timed out creating the default Deployment Plan.")
+
 
 def _prepare_ssh_environment(mistral):
     mistral.executions.create('tripleo.validations.v1.copy_ssh_key')