322 lines
9.1 KiB
Bash
Executable File
322 lines
9.1 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# ``grenade.sh`` is an OpenStack upgrade test harness to exercise the
|
|
# OpenStack upgrade process. It uses DevStack to perform the initial
|
|
# OpenStack install.
|
|
|
|
# Grenade assumes it is running on the system that will be hosting the
|
|
# upgrade processes
|
|
|
|
# ``grenade.sh [-b] [-t] [-s stop-label] [-q]``
|
|
#
|
|
# ``-b`` Run only the base part
|
|
# ``-t`` Run only the target part (assumes a base run is in place)
|
|
# ``-q`` Quiet mode
|
|
# ``-s stop-label`` is the name of the step after which the script will stop.
|
|
# This is useful for debugging upgrades.
|
|
|
|
# Keep track of the Grenade directory
|
|
GRENADE_DIR=$(cd $(dirname "$0") && pwd)
|
|
|
|
# Import common functions
|
|
source $GRENADE_DIR/functions
|
|
|
|
# Determine what system we are running on. This provides ``os_VENDOR``,
|
|
# ``os_RELEASE``, ``os_UPDATE``, ``os_PACKAGE``, ``os_CODENAME``
|
|
# and ``DISTRO``
|
|
GetDistro
|
|
|
|
# Source params
|
|
source $GRENADE_DIR/grenaderc
|
|
|
|
RUN_BASE=$(trueorfalse True $RUN_BASE)
|
|
RUN_TARGET=$(trueorfalse True $RUN_TARGET)
|
|
VERBOSE=$(trueorfalse True $VERBOSE)
|
|
|
|
while getopts bqs:t c; do
|
|
case $c in
|
|
b)
|
|
RUN_TARGET=False
|
|
;;
|
|
q)
|
|
VERBOSE=False
|
|
;;
|
|
s)
|
|
STOP=$2
|
|
;;
|
|
t)
|
|
RUN_BASE=False
|
|
;;
|
|
esac
|
|
done
|
|
shift `expr $OPTIND - 1`
|
|
|
|
function echo_summary() {
|
|
echo $@ >&6
|
|
}
|
|
|
|
function echo_nolog() {
|
|
echo $@ >&3
|
|
}
|
|
|
|
function stop() {
|
|
stop=$1
|
|
shift
|
|
if [[ "$@" =~ "$stop" ]]; then
|
|
echo "STOP called for $1"
|
|
exit 1
|
|
fi
|
|
}
|
|
|
|
# Logging
|
|
# =======
|
|
|
|
# Set up logging
|
|
# Set ``LOGFILE`` to turn on logging
|
|
# Append '.xxxxxxxx' to the given name to maintain history
|
|
# where 'xxxxxxxx' is a representation of the date the file was created
|
|
if [[ -n "$LOGFILE" ]]; then
|
|
LOGDAYS=${LOGDAYS:-7}
|
|
TIMESTAMP_FORMAT=${TIMESTAMP_FORMAT:-"%F-%H%M%S"}
|
|
CURRENT_LOG_TIME=$(date "+$TIMESTAMP_FORMAT")
|
|
|
|
# First clean up old log files. Use the user-specified ``LOGFILE``
|
|
# as the template to search for, appending '.*' to match the date
|
|
# we added on earlier runs.
|
|
LOGDIR=$(dirname "$LOGFILE")
|
|
LOGNAME=$(basename "$LOGFILE")
|
|
echo "Creating $LOGDIR...."
|
|
|
|
mkdir -p $LOGDIR
|
|
find $LOGDIR -maxdepth 1 -name $LOGNAME.\* -mtime +$LOGDAYS -exec rm {} \;
|
|
LOGFILE=$LOGFILE.${CURRENT_LOG_TIME}
|
|
SUMFILE=$LOGFILE.${CURRENT_LOG_TIME}.summary
|
|
|
|
# Redirect output according to config
|
|
# Copy stdout to fd 3
|
|
exec 3>&1
|
|
if [[ "$VERBOSE" == "True" ]]; then
|
|
# Redirect stdout/stderr to tee to write the log file
|
|
exec 1> >( tee "${LOGFILE}" ) 2>&1
|
|
# Set up a second fd for output
|
|
exec 6> >( tee "${SUMFILE}" )
|
|
else
|
|
# Set fd 1 and 2 to primary logfile
|
|
exec 1> "${LOGFILE}" 2>&1
|
|
# Set fd 6 to summary logfile and stdout
|
|
exec 6> >( tee "${SUMFILE}" /dev/fd/3 )
|
|
fi
|
|
|
|
echo_summary "grenade.sh log $LOGFILE"
|
|
# Specified logfile name always links to the most recent log
|
|
ln -sf $LOGFILE $LOGDIR/$LOGNAME
|
|
ln -sf $SUMFILE $LOGDIR/$LOGNAME.summary
|
|
else
|
|
# Set up output redirection without log files
|
|
# Copy stdout to fd 3
|
|
exec 3>&1
|
|
if [[ "$VERBOSE" != "yes" ]]; then
|
|
# Throw away stdout and stderr
|
|
exec 1>/dev/null 2>&1
|
|
fi
|
|
# Always send summary fd to original stdout
|
|
exec 6>&3
|
|
fi
|
|
|
|
# Set up logging of screen windows
|
|
# Set ``SCREEN_LOGDIR`` to turn on logging of screen windows to the
|
|
# directory specified in ``SCREEN_LOGDIR``, we will log to the the file
|
|
# ``screen-$SERVICE_NAME-$TIMESTAMP.log`` in that dir and have a link
|
|
# ``screen-$SERVICE_NAME.log`` to the latest log file.
|
|
# Logs are kept for as long specified in ``LOGDAYS``.
|
|
if [[ -n "$SCREEN_LOGDIR" ]]; then
|
|
|
|
# We make sure the directory is created.
|
|
if [[ -d "$SCREEN_LOGDIR" ]]; then
|
|
# We cleanup the old logs
|
|
find $SCREEN_LOGDIR -maxdepth 1 -name screen-\*.log -mtime +$LOGDAYS -exec rm {} \;
|
|
else
|
|
mkdir -p $SCREEN_LOGDIR
|
|
fi
|
|
fi
|
|
|
|
# This script exits on an error so that errors don't compound and you see
|
|
# only the first error that occurred.
|
|
set -o errexit
|
|
|
|
# Print the commands being run so that we can see the command that triggers
|
|
# an error. It is also useful for following allowing as the install occurs.
|
|
set -o xtrace
|
|
|
|
|
|
# More Setup
|
|
# ==========
|
|
|
|
# Set up for smoke tests (default to True)
|
|
RUN_SMOKE=${RUN_SMOKE:=True}
|
|
BASE_RUN_SMOKE=${BASE_RUN_SMOKE:-$RUN_SMOKE}
|
|
TARGET_RUN_SMOKE=${TARGET_RUN_SMOKE:-$RUN_SMOKE}
|
|
|
|
# Install 'Base' Build of OpenStack
|
|
# =================================
|
|
|
|
if [[ "$RUN_BASE" == "True" ]]; then
|
|
#echo_summary "Sourcing base DevStack config"
|
|
#source $BASE_DEVSTACK_DIR/stackrc
|
|
|
|
echo_summary "Running prep-base"
|
|
$GRENADE_DIR/prep-base
|
|
stop $STOP prep-base 01
|
|
|
|
echo_summary "Running base stack.sh"
|
|
cd $BASE_DEVSTACK_DIR
|
|
GIT_BASE=$GIT_BASE ./stack.sh
|
|
stop $STOP stack.sh 10
|
|
|
|
# Cache downloaded instances
|
|
# --------------------------
|
|
|
|
echo_summary "Caching downloaded images"
|
|
mkdir -p $BASE_RELEASE_DIR/images
|
|
echo "Images: $IMAGE_URLS"
|
|
for image_url in ${IMAGE_URLS//,/ }; do
|
|
IMAGE_FNAME=`basename "$image_url"`
|
|
if [[ -r $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME ]]; then
|
|
rsync -av $BASE_DEVSTACK_DIR/files/$IMAGE_FNAME $BASE_RELEASE_DIR/images
|
|
fi
|
|
done
|
|
rsync -av $BASE_DEVSTACK_DIR/files/images/ $BASE_RELEASE_DIR/images
|
|
stop $STOP image-cache 20
|
|
|
|
# Operation
|
|
# ---------
|
|
|
|
# Validate the install
|
|
echo_summary "Running base smoke test"
|
|
if [[ "$BASE_RUN_SMOKE" == "True" ]]; then
|
|
cd $BASE_RELEASE_DIR/tempest
|
|
tox -esmoke
|
|
fi
|
|
stop $STOP base-smoke 110
|
|
|
|
# Create a project, users and instances
|
|
echo_summary "Creating Javelin project"
|
|
$GRENADE_DIR/setup-javelin
|
|
stop $STOP setup-javelin 120
|
|
|
|
# Save some stuff before we shut that whole thing down
|
|
echo_summary "Saving current state information"
|
|
$GRENADE_DIR/save-state
|
|
stop $STOP save-state 130
|
|
|
|
# Shut down running code
|
|
echo_summary "Shutting down base"
|
|
# unstack.sh is too aggressive in cleaning up by default
|
|
# so we'll do it ourselves...
|
|
$GRENADE_DIR/stop-base
|
|
stop $STOP stop-base 140
|
|
fi
|
|
|
|
|
|
# Upgrades
|
|
# ========
|
|
|
|
if [[ "$RUN_TARGET" == "True" ]]; then
|
|
# Get target bits ready
|
|
echo_summary "Running prep-target"
|
|
$GRENADE_DIR/prep-target
|
|
stop $STOP prep-target 210
|
|
|
|
# Upgrade OS packages and known Python updates
|
|
echo_summary "Running upgrade-packages"
|
|
#$GRENADE_DIR/upgrade-packages
|
|
stop $STOP upgrade-packages 220
|
|
|
|
# Upgrade DevStack
|
|
echo_summary "Running upgrade-devstack"
|
|
#$GRENADE_DIR/upgrade-devstack
|
|
stop $STOP upgrade-devstack 230
|
|
|
|
# Upgrade Infra
|
|
echo_summary "Running upgrade-infra"
|
|
$GRENADE_DIR/upgrade-infra || die $LINENO "Failure in upgrade-infra"
|
|
stop $STOP upgrade-infra 232
|
|
|
|
# Upgrade Oslo
|
|
echo_summary "Running upgrade-oslo"
|
|
$GRENADE_DIR/upgrade-oslo || die $LINENO "Failure in upgrade-oslo"
|
|
stop $STOP upgrade-oslo 235
|
|
|
|
# Upgrade Keystone
|
|
echo_summary "Running upgrade-keystone"
|
|
$GRENADE_DIR/upgrade-keystone || die $LINENO "Failure in upgrade-keystone"
|
|
stop $STOP upgrade-keystone 240
|
|
|
|
# Upgrade Swift
|
|
echo_summary "Running upgrade-swift"
|
|
$GRENADE_DIR/upgrade-swift || die $LINENO "Failure in upgrade-swift"
|
|
stop $STOP upgrade-swift 245
|
|
|
|
# Upgrade Glance
|
|
echo_summary "Running upgrade-glance"
|
|
$GRENADE_DIR/upgrade-glance || die $LINENO "Failure in upgrade-glance"
|
|
stop $STOP upgrade-glance 250
|
|
|
|
# Upgrade Nova
|
|
echo_summary "Running upgrade-nova"
|
|
$GRENADE_DIR/upgrade-nova || die $LINENO "Failure in upgrade-nova"
|
|
stop $STOP upgrade-nova 260
|
|
|
|
# Upgrade Cinder
|
|
echo_summary "Running upgrade-cinder"
|
|
$GRENADE_DIR/upgrade-cinder || die $LINENO "Failure in upgrade-cinder"
|
|
stop $STOP upgrade-cinder 270
|
|
|
|
# Upgrade Tempest
|
|
if [[ "$ENABLE_TEMPEST" == "True" ]]; then
|
|
echo_summary "Running upgrade-tempest"
|
|
$GRENADE_DIR/upgrade-tempest || die $LINENO "Failure in upgrade-tempest"
|
|
stop $STOP upgrade-tempest 290
|
|
fi
|
|
|
|
# Upgrade Horizon
|
|
echo_summary "Running upgrade-horizon"
|
|
$GRENADE_DIR/upgrade-horizon || die $LINENO "Failure in upgrade-horizon"
|
|
stop $STOP upgrade-horizon 240
|
|
|
|
# Upgrade Checks
|
|
echo_summary "Running upgrade sanity check"
|
|
$GRENADE_DIR/check-sanity || die $LINENO "Failure in check-sanity"
|
|
stop $STOP check-sanity 310
|
|
|
|
# Upgrade Tests
|
|
# =============
|
|
|
|
# Validate the upgrade
|
|
if [[ "$TARGET_RUN_SMOKE" == "True" ]]; then
|
|
echo_summary "Running tempest scenario and smoke tests"
|
|
cd $TARGET_RELEASE_DIR/tempest
|
|
tox -esmoke
|
|
stop $STOP run-smoke 330
|
|
fi
|
|
|
|
# Save databases
|
|
# --------------
|
|
|
|
echo_summary "Sourcing target DevStack config"
|
|
source $TARGET_DEVSTACK_DIR/stackrc
|
|
echo_summary "Dumping target databases"
|
|
mkdir -p $SAVE_DIR
|
|
for db in keystone glance nova cinder; do
|
|
mysqldump -uroot -p$MYSQL_PASSWORD $db >$SAVE_DIR/$db.sql.$TARGET_RELEASE
|
|
done
|
|
fi
|
|
|
|
|
|
# Fin
|
|
# ===
|
|
|
|
echo_summary "Grenade has completed the pre-programmed upgrade scripts."
|
|
# Indicate how long this took to run (bash maintained variable ``SECONDS``)
|
|
echo_summary "grenade.sh completed in $SECONDS seconds."
|