sanity checkpoints

After we supposedly shut down Folsom, actually check that it's all
the way down.

After we start grizzly, before we do anything, make sure that the
services look up, and the final vm that should have survived, did.

Change-Id: I13159c84444c3b3e9744acac7110b5cbd5ff9233

check-sanity

@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+
+# ``upgrade-sanity``
+
+# this is a set of sanity checks that should be run after upgrade to make sure
+# the environment actually looks like we expect, if not, die horribly
+
+echo "*********************************************************************"
+echo "Begin $0"
+echo "*********************************************************************"
+
+# Keep track of the devstack directory
+GRENADE_DIR=$(cd $(dirname "$0") && pwd)
+
+# Import common functions
+source $GRENADE_DIR/functions
+
+# Determine what system we are running on.  This provides ``os_VENDOR``,
+# ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
+# and ``DISTRO``
+GetDistro
+
+# Source params
+source $GRENADE_DIR/grenaderc
+
+# We need the OS_ credentials
+source $TARGET_DEVSTACK_DIR/openrc
+
+# all the services should actually be running that we expect
+NOT_RUNNING=""
+for name in ${GRIZZLY_SERVICES}; do
+    if ! is_running ${name}; then
+        NOT_RUNNING="$NOT_RUNNING $name"
+    fi
+done
+
+if [[ -n "$NOT_RUNNING" ]]; then
+    echo "The following services are not running after upgrade: $NOT_RUNNING"
+    # do a process dump at the end, just to see how bad things really are
+    ps auxw
+    exit 1
+fi
+
+# ensure that the peltast vm survived the upgrade
+OS_USERNAME=admin nova list --all-tenants | grep peltast > /dev/null
+RC=$?
+if [[ $RC -ne 0 ]]; then
+    echo "peltast vm did not survice the upgrade, something is wrong"
+    OS_USERNAME=admin nova list --all-tenants
+    exit 1
+fi

functions

@@ -1305,6 +1305,15 @@ function get_pip_command() {
     fi
 }
 
+# Find out if a process exists by partial name.
+function is_running() {
+    local name=$1
+    ps auxw | grep -v grep | grep ${name} > /dev/null
+    RC=$?
+    # some times I really hate bash reverse binary logic
+    return $RC
+}
+
 # Restore xtrace
 $XTRACE
 

grenade.sh

@@ -272,6 +272,10 @@ if [[ "$RUN_TARGET" == "True" ]]; then
         stop $STOP upgrade-tempest 290
     fi
 
+    # Upgrade Checks
+    echo_summary "Running upgrade sanity check"
+    $GRENADE_DIR/check-sanity || die $LINENO "Failure in check-sanity"
+    stop $STOP check-sanity 310
 
     # Upgrade Tests
     # =============
@@ -281,7 +285,7 @@ if [[ "$RUN_TARGET" == "True" ]]; then
     if [[ "$TARGET_RUN_EXERCISES" == "True" ]]; then
         $TARGET_DEVSTACK_DIR/exercise.sh
     fi
-    stop $STOP target-exercise 310
+    stop $STOP target-exercise 320
 
 
     # Save databases

grenaderc

@@ -72,3 +72,14 @@ RECLONE=${RECLONE:-no}
 # Set these after localrc so user can pick-n-choose
 BASE_RUN_EXERCISES=${RUN_EXERCISES:=True}
 TARGET_RUN_EXERCISES=${RUN_EXERCISES:=True}
+
+# TODO(sdague): make these complete afterwards
+# The services we'd expect at each release
+FOLSOM_SERVICES="nova-api nova-compute keystone glance-api cinder-api"
+
+# The services we'd expect at each release
+GRIZZLY_SERVICES="nova-api nova-conductor nova-compute keystone glance-api cinder-api"
+
+# Local variables:
+# mode: shell-script
+# End:

stop-base

@@ -20,8 +20,6 @@ GetDistro
 # Source params
 source $GRENADE_DIR/grenaderc
 
-# For debugging
-set -o xtrace
 
 # Duplicate some setup bits from base DevStack
 source $BASE_DEVSTACK_DIR/stackrc
@@ -29,6 +27,10 @@ DATA_DIR=${STACK_ROOT}/data
 
 source $BASE_DEVSTACK_DIR/lib/cinder
 
+# For debugging
+set -o xtrace
+
+echo "Shutting down devstack in screen"
 
 # Shut down running processes in screen - most of them
 SCREEN=$(which screen)
@@ -63,3 +65,17 @@ fi
     # Unplumb the Swift data
 sudo umount ${DATA_DIR}/swift/drives/images/swift.img || /bin/true
 
+# ensure everything is shut down
+STILL_RUNNING=""
+for name in ${FOLSOM_SERVICES}; do
+    if is_running ${name}; then
+        STILL_RUNNING="$STILL_RUNNING $name"
+    fi
+done
+
+if [[ -n "$STILL_RUNNING" ]]; then
+    echo "The following services are still running: $STILL_RUNNING"
+    # do a process dump at the end, just to see how bad things really are
+    ps auxw
+    exit 1
+fi