From a6c7d4b4dbd98eeba4d3c33f51d893b7cac021d8 Mon Sep 17 00:00:00 2001 From: "Mauro S. M. Rodrigues" Date: Tue, 4 Feb 2014 15:26:19 +0000 Subject: [PATCH] Remove exercises and run tempest smoke tests instead Smoke and Scenarios tests can offer a better coverage and also it's good to centralize all tests in tempest. Change-Id: Iacb60ab011d2c7d90885b339b3c8454bd3de8219 Partial-bug: #1023131 --- HACKING.rst | 71 ----------------------------------------- README.rst | 2 +- TODO.rst | 5 --- devstack.localrc.base | 1 - devstack.localrc.target | 1 - grenade.sh | 30 +++++++---------- grenaderc | 11 ++----- 7 files changed, 16 insertions(+), 105 deletions(-) diff --git a/HACKING.rst b/HACKING.rst index fa3d1162..42dc5444 100644 --- a/HACKING.rst +++ b/HACKING.rst @@ -55,74 +55,3 @@ Markdown formatting in the comments; use it sparingly. Specifically, ``grenade. uses Markdown headers to divide the script into logical sections. .. _shocco: http://rtomayko.github.com/shocco/ - - -Exercises ---------- - -The scripts in the exercises directory are meant to 1) perform additional -operational checks on certain aspects of OpenStack; and b) set up some instances -and data that can be used to verify the upgrade process is non-destructive -for the end-user. - -* Begin and end with a banner that stands out in a sea of script logs to aid - in debugging failures, particularly in automated testing situations. If the - end banner is not displayed, the script ended prematurely and can be assumed - to have failed. - - :: - - echo "**************************************************" - echo "Begin Grenade Exercise: $0" - echo "**************************************************" - ... - set +o xtrace - echo "**************************************************" - echo "End Grenade Exercise: $0" - echo "**************************************************" - -* The scripts will generally have the shell ``xtrace`` attribute set to display - the actual commands being executed, and the ``errexit`` attribute set to exit - the script on non-zero exit codes:: - - # This script exits on an error so that errors don't compound and you see - # only the first error that occurred. - set -o errexit - - # Print the commands being run so that we can see the command that triggers - # an error. It is also useful for following allowing as the install occurs. - set -o xtrace - -* Settings and configuration are stored in ``exerciserc``, which must be - sourced after ``grenaderc``:: - - # Import exercise configuration - source $TOP_DIR/exerciserc - -* There are a couple of helper functions in the common ``functions`` sub-script - that will check for non-zero exit codes and unset environment variables and - print a message and exit the script. These should be called after most client - commands that are not otherwise checked to short-circuit long timeouts - (instance boot failure, for example):: - - swift post $CONTAINER - if [[ $? != 0 ]]; then - die $LINENO "Failure creating container $CONTAINER" - fi - - FLOATING_IP=`euca-allocate-address | cut -f2` - die_if_not_set $LINENO FLOATING_IP "Failure allocating floating IP" - -* If you want an exercise to be skipped when for example a service wasn't - enabled for the exercise to be run, you can exit your exercise with the - special exitcode 55 and it will be detected as skipped. - -* The exercise scripts should only use the various OpenStack client binaries to - interact with OpenStack. This specifically excludes any ``*-manage`` tools - as those assume direct access to configuration and databases, as well as direct - database access from the exercise itself. - -* The exercise MUST clean up after itself even if it is not successful. This is - different from current DevStack practice. The exercise SHOULD also clean up - or graciously handle possible artifacts left over from previous runs if executed - again. diff --git a/README.rst b/README.rst index 4a91f570..435fef31 100644 --- a/README.rst +++ b/README.rst @@ -37,7 +37,7 @@ Process ------- * Install base OpenStack using current stable/ DevStack -* Perform basic testing (exercise.sh) +* Perform basic testing (tempest's smoke and scenarios tests) * Create some artifacts in a new project ('javelin') for comparison after the upgrade process. * Install current target DevStack to support the upgrades diff --git a/TODO.rst b/TODO.rst index c39f8576..315900cf 100644 --- a/TODO.rst +++ b/TODO.rst @@ -11,8 +11,3 @@ Upgrade Scripts Needed Known Upgrade Failures ====================== - -Known Exercise Failures -======================= - -* Instance deletion not working (both OSAPI and EC2) diff --git a/devstack.localrc.base b/devstack.localrc.base index a3d98ccd..826aedfa 100644 --- a/devstack.localrc.base +++ b/devstack.localrc.base @@ -25,7 +25,6 @@ SCREEN_LOGDIR=$DEST/logs/screen SCREEN_NAME=${SCREEN_NAME:-stack} SERVICE_PASSWORD=$ADMIN_PASSWORD SERVICE_TOKEN=cd0d1a03-b701-4fcb-801a-8b4d0bc3d06e -SKIP_EXERCISES=boot_from_volume,bundle,euca STACK_LOG=stack.sh.log VOLUME_BACKING_FILE_SIZE=10000M diff --git a/devstack.localrc.target b/devstack.localrc.target index cba4ddf5..6f4cfa46 100644 --- a/devstack.localrc.target +++ b/devstack.localrc.target @@ -25,7 +25,6 @@ SCREEN_LOGDIR=$DEST/logs/screen SCREEN_NAME=${SCREEN_NAME:-stack} SERVICE_PASSWORD=$ADMIN_PASSWORD SERVICE_TOKEN=cd0d1a03-b701-4fcb-801a-8b4d0bc3d06e -SKIP_EXERCISES=boot_from_volume,bundle,euca STACK_LOG=stack.sh.log VOLUME_BACKING_FILE_SIZE=10000M diff --git a/grenade.sh b/grenade.sh index 77954e11..dd772155 100755 --- a/grenade.sh +++ b/grenade.sh @@ -152,12 +152,10 @@ set -o xtrace # More Setup # ========== -# Set up for exercises -BASE_RUN_EXERCISES=${BASE_RUN_EXERCISES:-RUN_EXERCISES} -TARGET_RUN_EXERCISES=${TARGET_RUN_EXERCISES:-RUN_EXERCISES} - -# Set up for smoke tests (default to False) -TARGET_RUN_SMOKE=${TARGET_RUN_SMOKE:=False} +# Set up for smoke tests (default to True) +RUN_SMOKE=${RUN_SMOKE:=True} +BASE_RUN_SMOKE=${BASE_RUN_SMOKE:-$RUN_SMOKE} +TARGET_RUN_SMOKE=${TARGET_RUN_SMOKE:-$RUN_SMOKE} # Install 'Base' Build of OpenStack # ================================= @@ -194,11 +192,12 @@ if [[ "$RUN_BASE" == "True" ]]; then # --------- # Validate the install - echo_summary "Running base exercises" - if [[ "$BASE_RUN_EXERCISES" == "True" ]]; then - $BASE_DEVSTACK_DIR/exercise.sh + echo_summary "Running base smoke test" + if [[ "$BASE_RUN_SMOKE" == "True" ]]; then + cd $BASE_RELEASE_DIR/tempest + tox -esmoke fi - stop $STOP base-exercise 110 + stop $STOP base-smoke 110 # Create a project, users and instances echo_summary "Creating Javelin project" @@ -294,15 +293,10 @@ if [[ "$RUN_TARGET" == "True" ]]; then # ============= # Validate the upgrade - echo_summary "Running target exercises" - if [[ "$TARGET_RUN_EXERCISES" == "True" ]]; then - $TARGET_DEVSTACK_DIR/exercise.sh - fi - stop $STOP target-exercise 320 - if [[ "$TARGET_RUN_SMOKE" == "True" ]]; then - echo_summary "Running tempest smoke tests" - $TARGET_RELEASE_DIR/tempest/run_tests.sh -N -s + echo_summary "Running tempest scenario and smoke tests" + cd $TARGET_RELEASE_DIR/tempest + tox -esmoke stop $STOP run-smoke 330 fi diff --git a/grenaderc b/grenaderc index 53eaaaf2..a87bc7fd 100644 --- a/grenaderc +++ b/grenaderc @@ -29,11 +29,8 @@ TARGET_DEVSTACK_REPO=$BASE_DEVSTACK_REPO TARGET_DEVSTACK_BRANCH=stable/$TARGET_RELEASE TARGET_DEVSTACK_DIR=$TARGET_RELEASE_DIR/devstack -# Allow skipping the exercises -RUN_EXERCISES=True - # Allow skipping smoke tests -TARGET_RUN_SMOKE=False +RUN_SMOKE=True # Saved stuff SAVE_DIR=${STACK_ROOT}/save @@ -74,10 +71,8 @@ ENABLE_TEMPEST=${ENABLE_TEMPEST:=True} # We need RECLONE to get the updated branches; not everyone wants it though RECLONE=${RECLONE:-no} -# Set these after localrc so user can pick-n-choose -# Allow setting target exercises independently -BASE_RUN_EXERCISES=${RUN_EXERCISES:=True} -TARGET_RUN_EXERCISES=${TARGET_RUN_EXERCISES:-$BASE_RUN_EXERCISES} +BASE_RUN_SMOKE=${RUN_SMOKE:=True} +TARGET_RUN_SMOKE=${TARGET_RUN_SMOKE:-$BASE_RUN_SMOKE} # TODO(sdague): make these complete afterwards # The services we'd expect at each release