Revert "Add dry run option to toci_quickstart"
This reverts commit 418f5a639f.
Closes-Bug: 1773381
Change-Id: If3086bfc96d792edcd5ea8139fd21702ad65d279
This commit is contained in:
Ronelle Landy
parent
418f5a639f
commit
0876f88029
|
|
@ -49,41 +49,12 @@ Quickstart itself.</p>
|
|||
<ul>
|
||||
<li><a href='undercloud/home/zuul/'>undercloud/home/zuul/</a>
|
||||
- the source and log output of all templated shell scripts</li>
|
||||
<li><a href='undercloud/var/log/'>undercloud/var/log/</a> -
|
||||
directories and files copied from /var/log on the undercloud.
|
||||
If other overcloud/subnodes exist, similar $node/var/log
|
||||
directories will also exist in these logs.</li>
|
||||
<li><a href='undercloud/var/log/extra/'>undercloud/var/log/extra/</a> -
|
||||
extra system details like package list, and cpu info gathered from the
|
||||
undercloud</li>
|
||||
<li><a href='undercloud/var/lib/mistral'>undercloud/var/lib/mistral</a> -
|
||||
output of all ansible used by config-download to drive the overcloud deployment</li>
|
||||
<li><a href='undercloud/var/lib/mistral'>undercloud/var/lib/mistral</a> - output of all ansible used by config-download to drive the overcloud deployment</li>
|
||||
<li><a href='stackwiz'>stackwiz</a> - tempest test results</li>
|
||||
<li><a href='docs/build/'>docs/build/</a> - autogenerated documentation</li>
|
||||
</ul>
|
||||
|
||||
<h1>Variables used in the job run</h1>
|
||||
<p>The logs contain files showing variables used in the job run.</p>
|
||||
<ul>
|
||||
<li><a href='undercloud/var/log/extra/dump_variables_vars.json.txt.gz'>undercloud/var/log/extra/dump_variables_vars.json.txt.gz</a>
|
||||
- contains the variables used in the running the actual test</li>
|
||||
<li><a href='releases.sh'>releases.sh</a>
|
||||
- the output of the script setting release-related variables</li>
|
||||
<li><a href='playbook_executions.log'>playbook_executions.log</a>
|
||||
- prints out the complete commands, with all expanded arguments,
|
||||
to run the playbooks</li>
|
||||
</ul>
|
||||
|
||||
<h1>Dry Run option</h1>
|
||||
<p>As a debugging step, a job can be run manually with '-dryrun'
|
||||
appended to the job name. When the "playbook dry run" option is invoked,
|
||||
the playbooks will not execute and collect logs will not run but
|
||||
certain log files, including 'toci_env_args_output.log', which
|
||||
contains the environment variables used in the job, and
|
||||
playbook_executions.log will still be produced in the logs
|
||||
directory for inspection. This option serves to assist with
|
||||
debugging and to test the testing scripts themselves.</p>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -88,71 +88,3 @@ function run_with_timeout {
|
|||
/usr/bin/timeout --preserve-status ${TIME_FOR_COMMAND}m ${COMMAND}
|
||||
}
|
||||
|
||||
function collect_logs {
|
||||
cat <<-EOF > $LOGS_DIR/collect_logs.sh
|
||||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''}
|
||||
export STATS_TESTENV=${STATS_TESTENV:-''}
|
||||
export STATS_OOOQ=${STATS_OOOQ:-''}
|
||||
export START_JOB_TIME=${START_JOB_TIME:-''}
|
||||
export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''}
|
||||
export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''}
|
||||
export REMAINING_TIME=${REMAINING_TIME:-''}
|
||||
export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
|
||||
export OPT_WORKDIR=$LOCAL_WORKING_DIR
|
||||
export WORKING_DIR="$HOME"
|
||||
export LOGS_DIR=$WORKSPACE/logs
|
||||
export VIRTUAL_ENV_DISABLE_PROMPT=1
|
||||
export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg
|
||||
export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite
|
||||
export ZUUL_CHANGES=${ZUUL_CHANGES:-''}
|
||||
export NODES_FILE=${NODES_FILE:-''}
|
||||
export TOCI_JOBTYPE=$TOCI_JOBTYPE
|
||||
export STABLE_RELEASE=${STABLE_RELEASE:-''}
|
||||
export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''}
|
||||
|
||||
set +u
|
||||
source $LOCAL_WORKING_DIR/bin/activate
|
||||
set -u
|
||||
source $OOOQ_DIR/ansible_ssh_env.sh
|
||||
|
||||
/usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \
|
||||
echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details"
|
||||
|
||||
cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true
|
||||
|
||||
if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
fi
|
||||
|
||||
# Copy tempest.html to root dir
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true
|
||||
|
||||
# Copy tempest and .testrepository directory to /opt/stack/new/tempest and
|
||||
# unzip
|
||||
sudo mkdir -p /opt/stack/new
|
||||
sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true
|
||||
sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true
|
||||
|
||||
# record the size of the logs directory
|
||||
# -L, --dereference dereference all symbolic links
|
||||
# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe'
|
||||
|
||||
du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true
|
||||
EOF
|
||||
|
||||
if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then
|
||||
if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then
|
||||
bash $LOGS_DIR/collect_logs.sh
|
||||
# rename script to not to run it in multinode jobs
|
||||
mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh
|
||||
fi
|
||||
fi
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,8 +117,6 @@ TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60))
|
|||
export EXTRA_VARS=${EXTRA_VARS:-""}
|
||||
export NODES_ARGS=""
|
||||
export EXTRANODE=""
|
||||
# Set playbook execution status
|
||||
export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0}
|
||||
export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml"
|
||||
LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
|
||||
LWD=$LOCAL_WORKING_DIR
|
||||
|
|
@ -211,9 +209,6 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
|
|||
;;
|
||||
gate)
|
||||
;;
|
||||
dryrun)
|
||||
PLAYBOOK_DRY_RUN=1
|
||||
;;
|
||||
*)
|
||||
# the rest should be node configuration
|
||||
NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/$JOB_TYPE_PART.yml"
|
||||
|
|
|
|||
|
|
@ -122,53 +122,94 @@ set -u
|
|||
source $OOOQ_DIR/ansible_ssh_env.sh
|
||||
[[ -n ${STATS_OOOQ:-''} ]] && export STATS_OOOQ=$(( $(date +%s) - STATS_OOOQ ))
|
||||
|
||||
# Debug step capture env variables
|
||||
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
|
||||
echo "-- Capture Environment Variables Used ---------"
|
||||
echo "$(env)" | tee -a $LOGS_DIR/toci_env_args_output.log
|
||||
declare -p | tee -a $LOGS_DIR/toci_env_args_output.log
|
||||
fi
|
||||
|
||||
# Define repeated arg set
|
||||
PLAYBOOK_REPEATED_ARGS="$QUICKSTART_INSTALL_CMD \
|
||||
|
||||
for playbook in $PLAYBOOKS; do
|
||||
echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}"
|
||||
run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \
|
||||
"${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \
|
||||
$NODES_ARGS \
|
||||
$FEATURESET_CONF \
|
||||
$ENV_VARS \
|
||||
$EXTRA_VARS \
|
||||
$DEFAULT_ARGS"
|
||||
$DEFAULT_ARGS \
|
||||
--extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \
|
||||
$LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \
|
||||
2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$?
|
||||
|
||||
echo "-- Playbooks Output --------------------------"
|
||||
for playbook in $PLAYBOOKS; do
|
||||
echo "$PLAYBOOK_REPEATED_ARGS \
|
||||
${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG} \
|
||||
$LOCAL_WORKING_DIR/playbooks/$playbook ${PLAYBOOKS_ARGS[$playbook]:-}" \
|
||||
| sed 's/--/\n--/g' \
|
||||
| tee -a $LOGS_DIR/playbook_executions.log
|
||||
echo "# --------------------------------------- " \
|
||||
| tee -a $LOGS_DIR/playbook_executions.log
|
||||
# Print status of playbook run
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully"
|
||||
[[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break
|
||||
done
|
||||
|
||||
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
|
||||
exit_value=0
|
||||
else
|
||||
for playbook in $PLAYBOOKS; do
|
||||
echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}"
|
||||
run_with_timeout $START_JOB_TIME \
|
||||
$PLAYBOOK_REPEATED_ARGS \
|
||||
"${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \
|
||||
$LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \
|
||||
--extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \
|
||||
2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$?
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed"
|
||||
|
||||
# Print status of playbook run
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully"
|
||||
[[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break
|
||||
done
|
||||
## LOGS COLLECTION
|
||||
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed"
|
||||
cat <<EOF > $LOGS_DIR/collect_logs.sh
|
||||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
## LOGS COLLECTION
|
||||
collect_logs
|
||||
export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''}
|
||||
export STATS_TESTENV=${STATS_TESTENV:-''}
|
||||
export STATS_OOOQ=${STATS_OOOQ:-''}
|
||||
export START_JOB_TIME=${START_JOB_TIME:-''}
|
||||
export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''}
|
||||
export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''}
|
||||
export REMAINING_TIME=${REMAINING_TIME:-''}
|
||||
export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
|
||||
export OPT_WORKDIR=$LOCAL_WORKING_DIR
|
||||
export WORKING_DIR="$HOME"
|
||||
export LOGS_DIR=$WORKSPACE/logs
|
||||
export VIRTUAL_ENV_DISABLE_PROMPT=1
|
||||
export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg
|
||||
export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite
|
||||
export ZUUL_CHANGES=${ZUUL_CHANGES:-''}
|
||||
export NODES_FILE=${NODES_FILE:-''}
|
||||
export TOCI_JOBTYPE=$TOCI_JOBTYPE
|
||||
export STABLE_RELEASE=${STABLE_RELEASE:-''}
|
||||
export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''}
|
||||
|
||||
set +u
|
||||
source $LOCAL_WORKING_DIR/bin/activate
|
||||
set -u
|
||||
source $OOOQ_DIR/ansible_ssh_env.sh
|
||||
|
||||
/usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \
|
||||
echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details"
|
||||
|
||||
cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true
|
||||
|
||||
if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
fi
|
||||
|
||||
# Copy tempest.html to root dir
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true
|
||||
|
||||
# Copy tempest and .testrepository directory to /opt/stack/new/tempest and
|
||||
# unzip
|
||||
sudo mkdir -p /opt/stack/new
|
||||
sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true
|
||||
sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true
|
||||
|
||||
# record the size of the logs directory
|
||||
# -L, --dereference dereference all symbolic links
|
||||
# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe'
|
||||
|
||||
du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true
|
||||
EOF
|
||||
|
||||
if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then
|
||||
if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then
|
||||
bash $LOGS_DIR/collect_logs.sh
|
||||
# rename script to not to run it in multinode jobs
|
||||
mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh
|
||||
fi
|
||||
fi
|
||||
|
||||
popd
|
||||
|
|
|
|||
Loading…
Reference in New Issue