Implement a data plane down time test for upgrades
This patch implements a VM using the demo project, then consistently executes a connectivity test through L3 and a disk access test on the test instance while the upgrade script executes. The intent is to determine whether any data plane down time is experienced while executing the upgrade. Once the upgrade is complete, the testing is stopped and if there are any failures, the job will exit as failed. Change-Id: Ie3f1738436c72eb82d7f6ca45d84024d11f3d923
This commit is contained in:
Jesse Pretorius
parent
7e4d83e9b2
commit
5640b278da
|
|
@ -189,6 +189,9 @@ if [[ "${ACTION}" == "upgrade" ]]; then
|
|||
unset GROUP_VARS_PATH
|
||||
unset HOST_VARS_PATH
|
||||
|
||||
# Kick off the data plane tester
|
||||
$(dirname "${0}")/../tests/data-plane-test.sh &
|
||||
|
||||
# Fetch script to execute API availability tests, then
|
||||
# background them while the upgrade runs.
|
||||
get_bowling_ball_tests
|
||||
|
|
@ -199,10 +202,30 @@ if [[ "${ACTION}" == "upgrade" ]]; then
|
|||
# upgrade is irreversable.
|
||||
echo 'YES' | bash "$(dirname "${0}")/run-upgrade.sh"
|
||||
|
||||
# Terminate the API availability tests
|
||||
kill_bowling_ball_tests
|
||||
# Wait to let all the processes finish before looking for output.
|
||||
|
||||
# Terminate the data plane tester
|
||||
rm -f /var/run/data-plane-test.socket
|
||||
|
||||
# Wait 10s for the tests to complete
|
||||
sleep 10
|
||||
|
||||
# Output the API availability test results
|
||||
print_bowling_ball_results
|
||||
|
||||
# Check for any data plane failures, and fail if there are
|
||||
if ! egrep -q "^FAIL: 0$" /var/log/data-plane-test.log; then
|
||||
echo -e "\n\nFAIL: The L3 data plane check failed!\n\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for any disk access failures, and fail if there are
|
||||
if ! egrep -q "^FAIL: 0$" /var/log/disk-access-test.log; then
|
||||
echo -e "\n\nFAIL: The disk access data plane check failed!\n\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
fi
|
||||
|
||||
exit_success
|
||||
|
|
|
|||
|
|
@ -0,0 +1,180 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Shell Opts ----------------------------------------------------------------
|
||||
set -e
|
||||
|
||||
## Vars ----------------------------------------------------------------------
|
||||
|
||||
# Log file location
|
||||
TEST_LOG_FILE="/var/log/data-plane-test.log"
|
||||
|
||||
# The test instance name
|
||||
INSTANCE_NAME="test1"
|
||||
|
||||
# Test script socket file location
|
||||
TEST_SOCKET_FILE="/var/run/data-plane-test.socket"
|
||||
|
||||
# Disk access test script
|
||||
DISK_ACCESS_SCRIPT="/opt/openstack-ansible/tests/disk-access-test.sh"
|
||||
|
||||
# Disk access log file
|
||||
DISK_ACCESS_LOG="~/disk-access-test.log"
|
||||
|
||||
# Setup counters
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
# SSH/SCP prefixes
|
||||
CMD_SCP_PREFIX="sshpass -p cubswin:) scp -o StrictHostKeyChecking=no"
|
||||
CMD_SSH_PREFIX="sshpass -p cubswin:) ssh -o StrictHostKeyChecking=no"
|
||||
|
||||
## Functions -----------------------------------------------------------------
|
||||
|
||||
# Create a demorc file with auth credentials
|
||||
# and other state tracking information.
|
||||
setup_demorc() {
|
||||
cp /root/openrc /root/demorc
|
||||
sed -i 's/OS_PROJECT_NAME=.*/OS_PROJECT_NAME=demo/' /root/demorc
|
||||
sed -i 's/OS_TENANT_NAME=.*/OS_TENANT_NAME=demo/' /root/demorc
|
||||
sed -i 's/OS_USERNAME=.*/OS_USERNAME=demo/' /root/demorc
|
||||
sed -i 's/OS_PASSWORD=.*/OS_PASSWORD=demo/' /root/demorc
|
||||
echo "INSTANCE_NAME=${INSTANCE_NAME}" >> /root/demorc
|
||||
echo "TEST_LOG_FILE=${TEST_LOG_FILE}" >> /root/demorc
|
||||
echo "TEST_SOCKET_FILE=${TEST_SOCKET_FILE}" >> /root/demorc
|
||||
}
|
||||
|
||||
# Log results
|
||||
result_log() {
|
||||
# We want the output format to be:
|
||||
# YYYY-MM-DD HH:MM:SS <result>
|
||||
STAMP=$(date -u "+%Y-%m-%d %H:%M:%S")
|
||||
echo "${STAMP} ${1}" >> ${TEST_LOG_FILE}
|
||||
}
|
||||
|
||||
# Tests to execute
|
||||
tests() {
|
||||
# A simple end-to-end test to verify that we can login via the floating
|
||||
# IP address and can read data from the disk.
|
||||
CMD_CONNECT="timeout 1s ${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS}"
|
||||
if ${CMD_CONNECT} cat /etc/issue > /dev/null; then
|
||||
result_log PASS
|
||||
PASS=$((PASS+1))
|
||||
else
|
||||
result_log FAIL
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Steps to execute when finishing
|
||||
finish() {
|
||||
finish_disk_test
|
||||
rm -f ${TEST_SOCKET_FILE} > /dev/null
|
||||
echo "PASS: ${PASS}" >> ${TEST_LOG_FILE}
|
||||
echo "FAIL: ${FAIL}" >> ${TEST_LOG_FILE}
|
||||
}
|
||||
|
||||
# Setup the disk access test
|
||||
setup_disk_test() {
|
||||
# Copy the disk access test script to the instance
|
||||
${CMD_SCP_PREFIX} ${DISK_ACCESS_SCRIPT} cirros@${INSTANCE_PUBLIC_ADDRESS}:~
|
||||
|
||||
# Initiate the disk access test in the instance
|
||||
CMD_TO_START="sudo /bin/sh disk-access-test.sh > ${DISK_ACCESS_LOG} &"
|
||||
${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS} "${CMD_TO_START}"
|
||||
}
|
||||
|
||||
# Finish the disk access test
|
||||
finish_disk_test() {
|
||||
# Remove the socket file to stop the test
|
||||
${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS} sudo rm -f /var/run/disk-access-test.socket
|
||||
|
||||
# Wait 2s for test to finalise
|
||||
sleep 2
|
||||
|
||||
# Fetch the log file with the results
|
||||
${CMD_SCP_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS}:${DISK_ACCESS_LOG} /var/log/disk-access-log.log
|
||||
}
|
||||
|
||||
# Setup the trap for the interrupt
|
||||
trap finish SIGHUP SIGINT SIGTERM
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
# Create the demorc file if it doesn't exist
|
||||
if [[ ! -f /root/demorc ]]; then
|
||||
setup_demorc
|
||||
fi
|
||||
|
||||
# Fetch the environment variables to be used
|
||||
source /root/demorc
|
||||
|
||||
# Create a volume for the test instance to use for the disk access test
|
||||
if [ -z ${INSTANCE_VOLUME_UUID+x} ]; then
|
||||
INSTANCE_VOLUME_UUID=$(openstack volume create --size 1 ${INSTANCE_NAME} --column id --format value)
|
||||
echo "INSTANCE_VOLUME_UUID=${INSTANCE_VOLUME_UUID}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# Register the private network UUID
|
||||
if [ -z ${INSTANCE_NETWORK_UUID+x} ]; then
|
||||
INSTANCE_NETWORK_UUID=$(openstack network show private --column id --format value)
|
||||
echo "INSTANCE_NETWORK_UUID=${INSTANCE_NETWORK_UUID}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# If a test instance does not exist, create it
|
||||
if [ -z ${INSTANCE_UUID+x} ]; then
|
||||
INSTANCE_UUID=$(openstack server create --flavor tempest1 --image cirros ${INSTANCE_NAME} --nic net-id=${INSTANCE_NETWORK_UUID} --column id --format value)
|
||||
echo "INSTANCE_UUID=${INSTANCE_UUID}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# If a floating IP address has not been allocated, do so
|
||||
if [ -z ${INSTANCE_PUBLIC_ADDRESS+x} ]; then
|
||||
INSTANCE_PUBLIC_ADDRESS=$(openstack floating ip create public --column floating_ip_address --format value)
|
||||
echo "INSTANCE_PUBLIC_ADDRESS=${INSTANCE_PUBLIC_ADDRESS}" >> /root/demorc
|
||||
fi
|
||||
|
||||
# If the floating IP is not associated with the test instance, associate it
|
||||
if ! openstack server show test1 --column addresses --format value | grep -q ${INSTANCE_PUBLIC_ADDRESS}; then
|
||||
openstack server add floating ip ${INSTANCE_NAME} ${INSTANCE_PUBLIC_ADDRESS}
|
||||
fi
|
||||
|
||||
# Wait for the volume to be ready
|
||||
while [[ "$(openstack volume show ${INSTANCE_VOLUME_UUID} --column status --format value)" != "available" ]]; do
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Attach the volume to the test instance
|
||||
openstack server add volume ${INSTANCE_UUID} ${INSTANCE_VOLUME_UUID}
|
||||
|
||||
# Wait for the volume to show as in-use
|
||||
while [[ "$(openstack volume show ${INSTANCE_VOLUME_UUID} --column status --format value)" != "in-use" ]]; do
|
||||
sleep 4
|
||||
done
|
||||
|
||||
# Start the disk access test in the instance
|
||||
setup_disk_test
|
||||
|
||||
# Setup the socket file to allow termination later
|
||||
echo $$ > ${TEST_SOCKET_FILE}
|
||||
|
||||
# Execute the test loop
|
||||
while [ -f "${TEST_SOCKET_FILE}" ]; do
|
||||
tests
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# This point will only be reached if the
|
||||
# socket file is removed
|
||||
finish
|
||||
|
|
@ -0,0 +1,82 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Shell Opts ----------------------------------------------------------------
|
||||
set -e
|
||||
|
||||
## Vars ----------------------------------------------------------------------
|
||||
|
||||
# Test script socket file location
|
||||
TEST_SOCKET_FILE="/var/run/disk-access-test.socket"
|
||||
|
||||
# The location to write to
|
||||
TEST_DATA_FILE="/mnt/test"
|
||||
|
||||
# Setup counters
|
||||
PASS=0
|
||||
FAIL=0
|
||||
|
||||
## Functions -----------------------------------------------------------------
|
||||
|
||||
# Tests to execute
|
||||
tests() {
|
||||
# We want the output format to be:
|
||||
# YYYY-MM-DD HH:MM:SS <result>
|
||||
echo -n "$(date -u '+%Y-%m-%d %H:%M:%S') "
|
||||
# A simple disk write test to validate whether
|
||||
# we are able to write to disk.
|
||||
CMD_WRITE="timeout 1s dd bs=1M count=50 if=/dev/zero of=${TEST_DATA_FILE} conv=fdatasync"
|
||||
if ${CMD_WRITE}; then
|
||||
echo "PASS"
|
||||
PASS=$((PASS+1))
|
||||
else
|
||||
echo "FAIL"
|
||||
FAIL=$((FAIL+1))
|
||||
fi
|
||||
}
|
||||
|
||||
# Steps to execute when finishing
|
||||
finish() {
|
||||
rm -f ${TEST_SOCKET_FILE} > /dev/null
|
||||
echo "PASS: ${PASS}"
|
||||
echo "FAIL: ${FAIL}"
|
||||
}
|
||||
|
||||
# Setup the trap for the interrupt
|
||||
trap finish SIGHUP SIGINT SIGTERM
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
# Partition the volume
|
||||
echo ';' | sfdisk --quiet /dev/vdb > /dev/null
|
||||
|
||||
# Format the volume
|
||||
mkfs /dev/vdb1 > /dev/null
|
||||
|
||||
# Mount the volume
|
||||
mount /dev/vdb1 /mnt
|
||||
|
||||
# Setup the socket file to allow termination later
|
||||
echo $$ > ${TEST_SOCKET_FILE}
|
||||
|
||||
# Execute the test loop
|
||||
while [ -f "${TEST_SOCKET_FILE}" ]; do
|
||||
tests
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# This point will only be reached if the
|
||||
# socket file is removed
|
||||
finish
|
||||
|
|
@ -25,6 +25,7 @@ packages_install:
|
|||
- lvm2
|
||||
- python
|
||||
- python-devel
|
||||
- sshpass
|
||||
- tmux
|
||||
- vim
|
||||
- xfsprogs
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ packages_install:
|
|||
- lvm2
|
||||
- python2.7
|
||||
- python-dev
|
||||
- sshpass
|
||||
- tmux
|
||||
- vim
|
||||
- vlan
|
||||
|
|
|
|||
Loading…
Reference in New Issue