Wait for hypervisor-stats to reflect ironic nodes

When enrolling nodes into Ironic, poll nova's hypervisor-stats until
the hypervisor count reflects the number of enrolled nodes.  This
eliminates a race where devstack completes and an instance is spawned
before the first post-enrollment periodic task ticks on the Nova side,
which has recently started popping up in the gate.

Conflicts:
	lib/ironic

Change-Id: Ib3d8005e0094ee8af2d5fcb65aca6cd92736da90
Closes-bug: #1398128
(cherry picked from commit c78e467109)

lib/ironic

@@ -462,9 +462,23 @@ function create_bridge_and_vms {
     create_ovs_taps
 }
 
+function wait_for_nova_resources {
+    # After nodes have been enrolled, we need to wait for n-cpu's periodic
+    # task populate the resource tracker with available nodes.  Wait for 2
+    # minutes before timing out.
+    local expected_count=$1
+    echo_summary "Waiting 2 minutes for Nova resource tracker to pick up $expected_count Ironic nodes"
+    for i in $(seq 1 120); do
+        if [ $(nova hypervisor-stats | grep " count " | get_field 2) -ge $expected_count ]; then
+            return 0
+        fi
+        sleep 1
+    done
+    die $LINENO "Nova hypervisor-stats did not register at least $expected_count nodes"
+}
+
 function enroll_vms {
     local chassis_id=$(ironic chassis-create -d "ironic test chassis" | grep " uuid " | get_field 2)
-    local idx=0
 
     if [[ "$IRONIC_DEPLOY_DRIVER" == "pxe_ssh" ]] ; then
         local _IRONIC_DEPLOY_KERNEL_KEY=pxe_deploy_kernel
@@ -474,6 +488,7 @@ function enroll_vms {
         local _IRONIC_DEPLOY_RAMDISK_KEY=deploy_ramdisk
     fi
 
+    local total_nodes=0
     while read MAC; do
 
         local node_id=$(ironic node-create --chassis_uuid $chassis_id \
@@ -493,7 +508,7 @@ function enroll_vms {
 
         ironic port-create --address $MAC --node_uuid $node_id
 
-        idx=$((idx+1))
+        total_nodes=$((total_nodes+1))
     done < $IRONIC_VM_MACS_CSV_FILE
 
     # create the nova flavor
@@ -508,6 +523,10 @@ function enroll_vms {
     # from the flavor after the completion of
     # https://blueprints.launchpad.net/ironic/+spec/add-node-instance-info
     nova flavor-key baremetal set "cpu_arch"="x86_64" "baremetal:deploy_kernel_id"="$IRONIC_DEPLOY_KERNEL_ID" "baremetal:deploy_ramdisk_id"="$IRONIC_DEPLOY_RAMDISK_ID"
+
+    if [ "$VIRT_DRIVER" == "ironic" ]; then
+        wait_for_nova_resources $total_nodes
+    fi
 }
 
 function configure_iptables {