Fail Fuel Master deployment if puppet or check fails

Puppet apply for Fuel Master deployment runs with
--detailed-exitcodes in order to capture any failures
and fails if any resource fails to apply, including
docker containers.
Cleaned up some obsolete lines about hiera and astute.yaml.

Fixed automated environment deployment scripts to detect
failed deployments and fail as well.

Change-Id: I29238a27f1335d63a6c82274a6dab9b25e1ecf0a
Related-Bug: #1390129

actions/master-node-create-and-install.sh

@@ -52,3 +52,5 @@ enable_outbound_network_for_product_vm $vm_master_ip $vm_master_username $vm_mas
 echo
 echo "Master node has been installed."
 
+#Sleep 10s to wait for Cobbler to settle
+sleep 10

functions/product.sh

@@ -1,4 +1,4 @@
-#!/bin/bash 
+#!/bin/bash
 
 #    Copyright 2013 Mirantis, Inc.
 #
@@ -27,7 +27,7 @@ is_product_vm_operational() {
     prompt=$4
 
     # Log in into the VM, see if Puppet has completed its run
-    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line 
+    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line
     result=$(
         expect << ENDOFEXPECT
         spawn ssh $ssh_options $username@$ip
@@ -35,7 +35,7 @@ is_product_vm_operational() {
         expect "*?assword:*"
         send "$password\r"
         expect "$prompt"
-        send "grep -o 'Fuel node deployment complete' /var/log/puppet/bootstrap_admin_node.log\r"
+        send "grep 'Fuel node deployment' /var/log/puppet/bootstrap_admin_node.log\r"
         expect "$prompt"
 ENDOFEXPECT
     )
@@ -51,10 +51,14 @@ ENDOFEXPECT
 
     for line in $result; do
         IFS="${OIFS}"
-        if [[ $line == Fuel* ]]; then
-	    IFS="${NIFS}"
+        if [[ "$line" == Fuel*complete* ]]; then
+            IFS="${NIFS}"
             return 0;
-        fi    
+        elif [[ "$line" == Fuel*FAILED* ]]; then
+            IFS="${NIFS}"
+            echo "$line" 1>&2
+            exit 1
+        fi
         IFS="${NIFS}"
     done
 
@@ -70,8 +74,14 @@ wait_for_product_vm_to_install() {
     echo "Waiting for product VM to install. Please do NOT abort the script..."
 
     # Loop until master node gets successfully installed
+    maxdelay=3000
     while ! is_product_vm_operational $ip $username $password "$prompt"; do
         sleep 5
+        ((waited += 5))
+        if (( waited >= maxdelay )); then
+          echo "Installation timed out! ($maxdelay seconds)" 1>&2
+          exit 1
+        fi
     done
 }
 
@@ -110,12 +120,12 @@ enable_outbound_network_for_product_vm() {
     else
       echo "OK"
     fi
-    
+
     # Enable internet access on inside the VMs
     echo -n "Enabling outbound network/internet access for the product VM... "
 
     # Log in into the VM, configure and bring up the NAT interface, set default gateway, check internet connectivity
-    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line 
+    # Looks a bit ugly, but 'end of expect' has to be in the very beginning of the line
     result=$(
         expect << ENDOFEXPECT
         spawn ssh $ssh_options $username@$ip