Remove broken uptime tests during upgrade

The uptime tests during upgrades have not been
working for some time, and we really need a better
way to implement them anyway.

To at least allow upgrade tests to work without
their interference, we remove the scripts and
implementation.

Change-Id: I33f7994a50ba5fcea7fca171dfca6b4d32011900
(cherry picked from commit 889e059909)
This commit is contained in:
Jesse Pretorius 2018-09-09 11:58:31 +01:00
parent 181f570fe9
commit ff907e177b
4 changed files with 0 additions and 329 deletions

View File

@ -200,43 +200,11 @@ if [[ "${ACTION}" == "upgrade" ]]; then
# Source the current scripts-library.sh functions
source "${OSA_CLONE_DIR}/scripts/scripts-library.sh"
# Kick off the data plane tester
bash ${OSA_CLONE_DIR}/tests/data-plane-test.sh &> /var/log/data-plane-error.log &
# Fetch script to execute API availability tests, then
# background them while the upgrade runs.
get_bowling_ball_tests
start_bowling_ball_tests
# To execute the upgrade script we need to provide
# an affirmative response to the warning that the
# upgrade is irreversable.
echo 'YES' | bash "${OSA_CLONE_DIR}/scripts/run-upgrade.sh"
# Terminate the API availability tests
kill_bowling_ball_tests
# Terminate the data plane tester
rm -f /var/run/data-plane-test.socket
# Wait 10s for the tests to complete
sleep 10
# Output the API availability test results
print_bowling_ball_results
# Check for any data plane failures, and fail if there are
if ! egrep -q "^FAIL: 0$" /var/log/data-plane-test.log; then
echo -e "\n\nFAIL: The L3 data plane check failed!\n\n"
exit 1
fi
# Check for any disk access failures, and fail if there are
if ! egrep -q "^FAIL: 0$" /var/log/disk-access-test.log; then
echo -e "\n\nFAIL: The disk access data plane check failed!\n\n"
exit 1
fi
fi
exit_success

View File

@ -365,36 +365,6 @@ function get_pip {
--isolated
}
function get_bowling_ball_tests {
# Retrieve the latest bowling ball test script in case we don't already have it.
if [ -f scripts/rolling_tests.py ]; then
return
fi
curl --silent https://raw.githubusercontent.com/openstack/openstack-ansible-ops/master/bowling_ball/rolling_tests.py > scripts/rolling_tests.py
}
function start_bowling_ball_tests {
# The tests will pull Keystone information from the env vars defined in openrc
source ~/openrc
# Get the list of services to test for from the script, so we're not hard coding.
for SERVICE in $(python ./scripts/rolling_tests.py list | cut -d - -f 1); do
# Start the scripts in the background and wait between each invocation.
# Without the wait, they do not all launch.
python ./scripts/rolling_tests.py $SERVICE &
sleep 1
echo "Started $SERVICE test in background"
done
}
function kill_bowling_ball_tests {
pkill -f rolling_tests
}
function print_bowling_ball_results {
grep "failure rate" /var/log/*_rolling.log
}
## Signal traps --------------------------------------------------------------
# Trap all Death Signals and Errors
trap "exit_fail ${LINENO} $? 'Received STOP Signal'" SIGHUP SIGINT SIGTERM

View File

@ -1,185 +0,0 @@
#!/bin/bash
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## Shell Opts ----------------------------------------------------------------
set -e -x
## Vars ----------------------------------------------------------------------
# Log file location
TEST_LOG_FILE="/var/log/data-plane-test.log"
# The test instance name
INSTANCE_NAME="test1"
# Test script socket file location
TEST_SOCKET_FILE="/var/run/data-plane-test.socket"
# Disk access test script
DISK_ACCESS_SCRIPT="/opt/openstack-ansible/tests/disk-access-test.sh"
# Disk access log file
DISK_ACCESS_LOG="~/disk-access-test.log"
# Setup counters
PASS=0
FAIL=0
# SSH/SCP prefixes
CMD_SCP_PREFIX="sshpass -p cubswin:) scp -o StrictHostKeyChecking=no"
CMD_SSH_PREFIX="sshpass -p cubswin:) ssh -o StrictHostKeyChecking=no"
## Functions -----------------------------------------------------------------
# Create a demorc file with auth credentials
# and other state tracking information.
setup_demorc() {
cp /root/openrc /root/demorc
sed -i 's/OS_PROJECT_NAME=.*/OS_PROJECT_NAME=demo/' /root/demorc
sed -i 's/OS_TENANT_NAME=.*/OS_TENANT_NAME=demo/' /root/demorc
sed -i 's/OS_USERNAME=.*/OS_USERNAME=demo/' /root/demorc
sed -i 's/OS_PASSWORD=.*/OS_PASSWORD=demo/' /root/demorc
echo "INSTANCE_NAME=${INSTANCE_NAME}" >> /root/demorc
echo "TEST_LOG_FILE=${TEST_LOG_FILE}" >> /root/demorc
echo "TEST_SOCKET_FILE=${TEST_SOCKET_FILE}" >> /root/demorc
}
# Log results
result_log() {
# We want the output format to be:
# YYYY-MM-DD HH:MM:SS <result>
STAMP=$(date -u "+%Y-%m-%d %H:%M:%S")
echo "${STAMP} ${1}" >> ${TEST_LOG_FILE}
}
# Tests to execute
tests() {
# A simple end-to-end test to verify that we can login via the floating
# IP address and can read data from the disk.
CMD_CONNECT="timeout 1s ${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS}"
if ${CMD_CONNECT} cat /etc/issue > /dev/null; then
result_log PASS
PASS=$((PASS+1))
else
result_log FAIL
FAIL=$((FAIL+1))
fi
}
# Steps to execute when finishing
finish() {
finish_disk_test
rm -f ${TEST_SOCKET_FILE} > /dev/null
echo "PASS: ${PASS}" >> ${TEST_LOG_FILE}
echo "FAIL: ${FAIL}" >> ${TEST_LOG_FILE}
}
# Setup the disk access test
setup_disk_test() {
# Copy the disk access test script to the instance
${CMD_SCP_PREFIX} ${DISK_ACCESS_SCRIPT} cirros@${INSTANCE_PUBLIC_ADDRESS}:~
# Initiate the disk access test in the instance
CMD_TO_START="sudo /bin/sh disk-access-test.sh > ${DISK_ACCESS_LOG} &"
${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS} "${CMD_TO_START}"
}
# Finish the disk access test
finish_disk_test() {
# Remove the socket file to stop the test
${CMD_SSH_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS} sudo rm -f /var/run/disk-access-test.socket
# Wait 2s for test to finalise
sleep 2
# Fetch the log file with the results
${CMD_SCP_PREFIX} cirros@${INSTANCE_PUBLIC_ADDRESS}:${DISK_ACCESS_LOG} /var/log/disk-access-log.log
}
# Setup the trap for the interrupt
trap finish SIGHUP SIGINT SIGTERM
## Main ----------------------------------------------------------------------
# Create the demorc file if it doesn't exist
if [[ ! -f /root/demorc ]]; then
setup_demorc
fi
# Fetch the environment variables to be used
source /root/demorc
# Create a volume for the test instance to use for the disk access test
if [ -z ${INSTANCE_VOLUME_UUID+x} ]; then
INSTANCE_VOLUME_UUID=$(openstack volume create --size 1 ${INSTANCE_NAME} --column id --format value)
echo "INSTANCE_VOLUME_UUID=${INSTANCE_VOLUME_UUID}" >> /root/demorc
fi
# Register the private network UUID
if [ -z ${INSTANCE_NETWORK_UUID+x} ]; then
INSTANCE_NETWORK_UUID=$(openstack network show private --column id --format value)
echo "INSTANCE_NETWORK_UUID=${INSTANCE_NETWORK_UUID}" >> /root/demorc
fi
# If a test instance does not exist, create it
if [ -z ${INSTANCE_UUID+x} ]; then
INSTANCE_UUID=$(openstack server create --flavor tempest1 --image cirros ${INSTANCE_NAME} --nic net-id=${INSTANCE_NETWORK_UUID} --column id --format value)
echo "INSTANCE_UUID=${INSTANCE_UUID}" >> /root/demorc
fi
# If a floating IP address has not been allocated, do so
if [ -z ${INSTANCE_PUBLIC_ADDRESS+x} ]; then
INSTANCE_PUBLIC_ADDRESS=$(openstack floating ip create public --column floating_ip_address --format value)
echo "INSTANCE_PUBLIC_ADDRESS=${INSTANCE_PUBLIC_ADDRESS}" >> /root/demorc
fi
# Wait for the server to be ready
while [[ "$(openstack server show ${INSTANCE_UUID} --column status --format value)" != "ACTIVE" ]]; do
sleep 4
done
# If the floating IP is not associated with the test instance, associate it
if ! openstack server show ${INSTANCE_UUID} --column addresses --format value | grep -q ${INSTANCE_PUBLIC_ADDRESS}; then
openstack server add floating ip ${INSTANCE_UUID} ${INSTANCE_PUBLIC_ADDRESS}
fi
# Wait for the volume to be ready
while [[ "$(openstack volume show ${INSTANCE_VOLUME_UUID} --column status --format value)" != "available" ]]; do
sleep 4
done
# Attach the volume to the test instance
openstack server add volume ${INSTANCE_UUID} ${INSTANCE_VOLUME_UUID}
# Wait for the volume to show as in-use
while [[ "$(openstack volume show ${INSTANCE_VOLUME_UUID} --column status --format value)" != "in-use" ]]; do
sleep 4
done
# Start the disk access test in the instance
setup_disk_test
# Setup the socket file to allow termination later
echo $$ > ${TEST_SOCKET_FILE}
# Execute the test loop
while [ -f "${TEST_SOCKET_FILE}" ]; do
tests
sleep 1
done
# This point will only be reached if the
# socket file is removed
finish

View File

@ -1,82 +0,0 @@
#!/bin/bash
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## Shell Opts ----------------------------------------------------------------
set -e
## Vars ----------------------------------------------------------------------
# Test script socket file location
TEST_SOCKET_FILE="/var/run/disk-access-test.socket"
# The location to write to
TEST_DATA_FILE="/mnt/test"
# Setup counters
PASS=0
FAIL=0
## Functions -----------------------------------------------------------------
# Tests to execute
tests() {
# We want the output format to be:
# YYYY-MM-DD HH:MM:SS <result>
echo -n "$(date -u '+%Y-%m-%d %H:%M:%S') "
# A simple disk write test to validate whether
# we are able to write to disk.
CMD_WRITE="timeout 1s dd bs=1M count=50 if=/dev/zero of=${TEST_DATA_FILE} conv=fdatasync"
if ${CMD_WRITE}; then
echo "PASS"
PASS=$((PASS+1))
else
echo "FAIL"
FAIL=$((FAIL+1))
fi
}
# Steps to execute when finishing
finish() {
rm -f ${TEST_SOCKET_FILE} > /dev/null
echo "PASS: ${PASS}"
echo "FAIL: ${FAIL}"
}
# Setup the trap for the interrupt
trap finish SIGHUP SIGINT SIGTERM
## Main ----------------------------------------------------------------------
# Partition the volume
echo ';' | sfdisk --quiet /dev/vdb > /dev/null
# Format the volume
mkfs /dev/vdb1 > /dev/null
# Mount the volume
mount /dev/vdb1 /mnt
# Setup the socket file to allow termination later
echo $$ > ${TEST_SOCKET_FILE}
# Execute the test loop
while [ -f "${TEST_SOCKET_FILE}" ]; do
tests
sleep 1
done
# This point will only be reached if the
# socket file is removed
finish