From 0876f880298565df456d2b5909766f26bc08f831 Mon Sep 17 00:00:00 2001 From: Ronelle Landy Date: Fri, 25 May 2018 14:50:22 +0000 Subject: [PATCH] Revert "Add dry run option to toci_quickstart" This reverts commit 418f5a639f2d5b2f68a8dca29fff7e04b8989185. Closes-Bug: 1773381 Change-Id: If3086bfc96d792edcd5ea8139fd21702ad65d279 --- docs/tripleo-quickstart-logs.html | 31 +------- scripts/oooq_common_functions.sh | 68 ------------------ toci_gate_test-oooq.sh | 5 -- toci_quickstart.sh | 113 ++++++++++++++++++++---------- 4 files changed, 78 insertions(+), 139 deletions(-) diff --git a/docs/tripleo-quickstart-logs.html b/docs/tripleo-quickstart-logs.html index 349c7162f..e1873d4c5 100644 --- a/docs/tripleo-quickstart-logs.html +++ b/docs/tripleo-quickstart-logs.html @@ -49,41 +49,12 @@ Quickstart itself.

- -

Variables used in the job run

-

The logs contain files showing variables used in the job run.

- - -

Dry Run option

-

As a debugging step, a job can be run manually with '-dryrun' -appended to the job name. When the "playbook dry run" option is invoked, -the playbooks will not execute and collect logs will not run but -certain log files, including 'toci_env_args_output.log', which -contains the environment variables used in the job, and -playbook_executions.log will still be produced in the logs -directory for inspection. This option serves to assist with -debugging and to test the testing scripts themselves.

- - diff --git a/scripts/oooq_common_functions.sh b/scripts/oooq_common_functions.sh index 4d85ee757..75931a428 100644 --- a/scripts/oooq_common_functions.sh +++ b/scripts/oooq_common_functions.sh @@ -88,71 +88,3 @@ function run_with_timeout { /usr/bin/timeout --preserve-status ${TIME_FOR_COMMAND}m ${COMMAND} } -function collect_logs { - cat <<-EOF > $LOGS_DIR/collect_logs.sh - #!/bin/bash - set -x - - export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''} - export STATS_TESTENV=${STATS_TESTENV:-''} - export STATS_OOOQ=${STATS_OOOQ:-''} - export START_JOB_TIME=${START_JOB_TIME:-''} - export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''} - export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''} - export REMAINING_TIME=${REMAINING_TIME:-''} - export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" - export OPT_WORKDIR=$LOCAL_WORKING_DIR - export WORKING_DIR="$HOME" - export LOGS_DIR=$WORKSPACE/logs - export VIRTUAL_ENV_DISABLE_PROMPT=1 - export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg - export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite - export ZUUL_CHANGES=${ZUUL_CHANGES:-''} - export NODES_FILE=${NODES_FILE:-''} - export TOCI_JOBTYPE=$TOCI_JOBTYPE - export STABLE_RELEASE=${STABLE_RELEASE:-''} - export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''} - - set +u - source $LOCAL_WORKING_DIR/bin/activate - set -u - source $OOOQ_DIR/ansible_ssh_env.sh - - /usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \ - echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details" - - cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true - - if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz - elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz - elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then - cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz - fi - - # Copy tempest.html to root dir - cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true - - # Copy tempest and .testrepository directory to /opt/stack/new/tempest and - # unzip - sudo mkdir -p /opt/stack/new - sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true - sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true - - # record the size of the logs directory - # -L, --dereference dereference all symbolic links - # Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe' - - du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true -EOF - - if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then - if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then - bash $LOGS_DIR/collect_logs.sh - # rename script to not to run it in multinode jobs - mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh - fi - fi - -} diff --git a/toci_gate_test-oooq.sh b/toci_gate_test-oooq.sh index 8a6d27770..ea1f370e5 100755 --- a/toci_gate_test-oooq.sh +++ b/toci_gate_test-oooq.sh @@ -117,8 +117,6 @@ TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60)) export EXTRA_VARS=${EXTRA_VARS:-""} export NODES_ARGS="" export EXTRANODE="" -# Set playbook execution status -export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0} export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml" LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" LWD=$LOCAL_WORKING_DIR @@ -211,9 +209,6 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do ;; gate) ;; - dryrun) - PLAYBOOK_DRY_RUN=1 - ;; *) # the rest should be node configuration NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/$JOB_TYPE_PART.yml" diff --git a/toci_quickstart.sh b/toci_quickstart.sh index 2ba6ce226..755309e4d 100755 --- a/toci_quickstart.sh +++ b/toci_quickstart.sh @@ -122,53 +122,94 @@ set -u source $OOOQ_DIR/ansible_ssh_env.sh [[ -n ${STATS_OOOQ:-''} ]] && export STATS_OOOQ=$(( $(date +%s) - STATS_OOOQ )) -# Debug step capture env variables -if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then - echo "-- Capture Environment Variables Used ---------" - echo "$(env)" | tee -a $LOGS_DIR/toci_env_args_output.log - declare -p | tee -a $LOGS_DIR/toci_env_args_output.log -fi -# Define repeated arg set -PLAYBOOK_REPEATED_ARGS="$QUICKSTART_INSTALL_CMD \ + +for playbook in $PLAYBOOKS; do + echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" + run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \ + "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \ $NODES_ARGS \ $FEATURESET_CONF \ $ENV_VARS \ $EXTRA_VARS \ - $DEFAULT_ARGS" + $DEFAULT_ARGS \ + --extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \ + $LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \ + 2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$? -echo "-- Playbooks Output --------------------------" -for playbook in $PLAYBOOKS; do - echo "$PLAYBOOK_REPEATED_ARGS \ - ${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG} \ - $LOCAL_WORKING_DIR/playbooks/$playbook ${PLAYBOOKS_ARGS[$playbook]:-}" \ - | sed 's/--/\n--/g' \ - | tee -a $LOGS_DIR/playbook_executions.log - echo "# --------------------------------------- " \ - | tee -a $LOGS_DIR/playbook_executions.log + # Print status of playbook run + [[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully" + [[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break done -if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then - exit_value=0 -else - for playbook in $PLAYBOOKS; do - echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" - run_with_timeout $START_JOB_TIME \ - $PLAYBOOK_REPEATED_ARGS \ - "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \ - $LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \ - --extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \ - 2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$? +[[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed" - # Print status of playbook run - [[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully" - [[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break - done +## LOGS COLLECTION - [[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed" +cat < $LOGS_DIR/collect_logs.sh +#!/bin/bash +set -x - ## LOGS COLLECTION - collect_logs +export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''} +export STATS_TESTENV=${STATS_TESTENV:-''} +export STATS_OOOQ=${STATS_OOOQ:-''} +export START_JOB_TIME=${START_JOB_TIME:-''} +export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''} +export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''} +export REMAINING_TIME=${REMAINING_TIME:-''} +export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart" +export OPT_WORKDIR=$LOCAL_WORKING_DIR +export WORKING_DIR="$HOME" +export LOGS_DIR=$WORKSPACE/logs +export VIRTUAL_ENV_DISABLE_PROMPT=1 +export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg +export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite +export ZUUL_CHANGES=${ZUUL_CHANGES:-''} +export NODES_FILE=${NODES_FILE:-''} +export TOCI_JOBTYPE=$TOCI_JOBTYPE +export STABLE_RELEASE=${STABLE_RELEASE:-''} +export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''} + +set +u +source $LOCAL_WORKING_DIR/bin/activate +set -u +source $OOOQ_DIR/ansible_ssh_env.sh + +/usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \ + echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details" + +cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true + +if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz +elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz +elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then + cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz +fi + +# Copy tempest.html to root dir +cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true + +# Copy tempest and .testrepository directory to /opt/stack/new/tempest and +# unzip +sudo mkdir -p /opt/stack/new +sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true +sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true + +# record the size of the logs directory +# -L, --dereference dereference all symbolic links +# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe' + +du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true +EOF + +if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then + if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then + bash $LOGS_DIR/collect_logs.sh + # rename script to not to run it in multinode jobs + mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh + fi fi popd