Verify that the Deployment Plan creation was successful

The creation of the default deployment plan was recently broken but the error
went unnoticed as CI never failed. This then led to another obscure bug as
something worked due only to the default plan not existing.

This change waits for the plan creation to finish and verifies that it
completed without an error. A timeout is added, should, for some reason the
creation ever stall.

NOTE: This will not raise an error if plan creation fails, since it is
      currently broken. This will be fixed in this following patch:
      https://review.openstack.org/373446

Partial-Bug: #1622683
Change-Id: Id8001fee677fe321b42892264e3c55afb6790e1c

instack_undercloud/tests/test_undercloud.py

@@ -530,6 +530,7 @@ class TestPostConfig(base.BaseTestCase):
     def test_create_default_plan(self):
         mock_mistral = mock.Mock()
         mock_mistral.environments.list.return_value = []
+        mock_mistral.executions.get.return_value = mock.Mock(state="SUCCESS")
 
         undercloud._create_default_plan(mock_mistral)
         mock_mistral.executions.create.assert_called_once_with(
@@ -556,6 +557,28 @@ class TestPostConfig(base.BaseTestCase):
         mock_mistral.executions.create.assert_called_once_with(
             'tripleo.validations.v1.copy_ssh_key')
 
+    @mock.patch('time.sleep')
+    def test_create_default_plan_timeout(self, mock_sleep):
+        mock_mistral = mock.Mock()
+        mock_mistral.environments.list.return_value = []
+        mock_mistral.executions.get.return_value = mock.Mock(state="RUNNING")
+
+        self.assertRaises(
+            RuntimeError,
+            undercloud._create_default_plan, mock_mistral, timeout=0)
+
+    def test_create_default_plan_failed(self):
+        mock_mistral = mock.Mock()
+        mock_mistral.environments.list.return_value = []
+        mock_mistral.executions.get.return_value = mock.Mock(state="ERROR")
+
+        # TODO(dmatthews): Switch to this check after this lands:
+        #   https://review.openstack.org/#/c/371347/
+        # self.assertRaises(
+        #     RuntimeError,
+        #     undercloud._create_default_plan, mock_mistral)
+        undercloud._create_default_plan(mock_mistral)
+
     @mock.patch('instack_undercloud.undercloud._run_command')
     def test_copy_stackrc(self, mock_run):
         undercloud._copy_stackrc()

instack_undercloud/undercloud.py

@@ -26,6 +26,7 @@ import socket
 import subprocess
 import sys
 import tempfile
+import time
 import uuid
 
 from keystoneclient import exceptions as ks_exceptions
@@ -1099,7 +1100,7 @@ def _clean_os_refresh_config():
     _run_command(args, name='Clean os-refresh-config')
 
 
-def _create_default_plan(mistral):
+def _create_default_plan(mistral, timeout=60):
     plan_name = 'overcloud'
     queue_name = str(uuid.uuid4())
 
@@ -1108,11 +1109,29 @@ def _create_default_plan(mistral):
                  plan_name)
         return
 
-    mistral.executions.create(
+    execution = mistral.executions.create(
         'tripleo.plan_management.v1.create_default_deployment_plan',
         workflow_input={'container': plan_name, 'queue_name': queue_name}
     )
 
+    timeout_at = time.time() + timeout
+
+    while time.time() < timeout_at:
+        exe = mistral.executions.get(execution.id)
+        if exe.state == "RUNNING":
+            time.sleep(5)
+            continue
+        if exe.state == "SUCCESS":
+            return
+        else:
+            LOG.warning("Failed to create the default Deployment Plan.")
+            return
+    else:
+        exe = mistral.executions.get(execution.id)
+        LOG.error("Timed out waiting for execution %s to finish. State: %s",
+                  exe.id, exe.state)
+        raise RuntimeError("Timed out creating the default Deployment Plan.")
+
 
 def _prepare_ssh_environment(mistral):
     mistral.executions.create('tripleo.validations.v1.copy_ssh_key')