Add dry run option to toci_quickstart
With dry run, playbooks are not executed. The env variables, releases script output and playbooks commands are written out to files for debug. The review also moves the collect logs lines to a function for easier organization. Documentation for the dry run option and variable files is added. Depends-On: I4c8f389978717848e755f12856dd454d605c9137 Change-Id: Ib6ae184a07ba291d719cabb48b51d890b0485a49
This commit is contained in:
Ronelle Landy
Emilien Macchi
parent
b8612d0709
commit
418f5a639f
|
|
@ -49,12 +49,41 @@ Quickstart itself.</p>
|
|||
<ul>
|
||||
<li><a href='undercloud/home/zuul/'>undercloud/home/zuul/</a>
|
||||
- the source and log output of all templated shell scripts</li>
|
||||
<li><a href='undercloud/var/log/'>undercloud/var/log/</a> -
|
||||
directories and files copied from /var/log on the undercloud.
|
||||
If other overcloud/subnodes exist, similar $node/var/log
|
||||
directories will also exist in these logs.</li>
|
||||
<li><a href='undercloud/var/log/extra/'>undercloud/var/log/extra/</a> -
|
||||
extra system details like package list, and cpu info gathered from the
|
||||
undercloud</li>
|
||||
<li><a href='undercloud/var/lib/mistral'>undercloud/var/lib/mistral</a> - output of all ansible used by config-download to drive the overcloud deployment</li>
|
||||
<li><a href='undercloud/var/lib/mistral'>undercloud/var/lib/mistral</a> -
|
||||
output of all ansible used by config-download to drive the overcloud deployment</li>
|
||||
<li><a href='stackwiz'>stackwiz</a> - tempest test results</li>
|
||||
<li><a href='docs/build/'>docs/build/</a> - autogenerated documentation</li>
|
||||
</ul>
|
||||
|
||||
<h1>Variables used in the job run</h1>
|
||||
<p>The logs contain files showing variables used in the job run.</p>
|
||||
<ul>
|
||||
<li><a href='undercloud/var/log/extra/dump_variables_vars.json.txt.gz'>undercloud/var/log/extra/dump_variables_vars.json.txt.gz</a>
|
||||
- contains the variables used in the running the actual test</li>
|
||||
<li><a href='releases.sh'>releases.sh</a>
|
||||
- the output of the script setting release-related variables</li>
|
||||
<li><a href='playbook_executions.log'>playbook_executions.log</a>
|
||||
- prints out the complete commands, with all expanded arguments,
|
||||
to run the playbooks</li>
|
||||
</ul>
|
||||
|
||||
<h1>Dry Run option</h1>
|
||||
<p>As a debugging step, a job can be run manually with '-dryrun'
|
||||
appended to the job name. When the "playbook dry run" option is invoked,
|
||||
the playbooks will not execute and collect logs will not run but
|
||||
certain log files, including 'toci_env_args_output.log', which
|
||||
contains the environment variables used in the job, and
|
||||
playbook_executions.log will still be produced in the logs
|
||||
directory for inspection. This option serves to assist with
|
||||
debugging and to test the testing scripts themselves.</p>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -88,3 +88,71 @@ function run_with_timeout {
|
|||
/usr/bin/timeout --preserve-status ${TIME_FOR_COMMAND}m ${COMMAND}
|
||||
}
|
||||
|
||||
function collect_logs {
|
||||
cat <<-EOF > $LOGS_DIR/collect_logs.sh
|
||||
#!/bin/bash
|
||||
set -x
|
||||
|
||||
export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''}
|
||||
export STATS_TESTENV=${STATS_TESTENV:-''}
|
||||
export STATS_OOOQ=${STATS_OOOQ:-''}
|
||||
export START_JOB_TIME=${START_JOB_TIME:-''}
|
||||
export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''}
|
||||
export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''}
|
||||
export REMAINING_TIME=${REMAINING_TIME:-''}
|
||||
export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
|
||||
export OPT_WORKDIR=$LOCAL_WORKING_DIR
|
||||
export WORKING_DIR="$HOME"
|
||||
export LOGS_DIR=$WORKSPACE/logs
|
||||
export VIRTUAL_ENV_DISABLE_PROMPT=1
|
||||
export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg
|
||||
export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite
|
||||
export ZUUL_CHANGES=${ZUUL_CHANGES:-''}
|
||||
export NODES_FILE=${NODES_FILE:-''}
|
||||
export TOCI_JOBTYPE=$TOCI_JOBTYPE
|
||||
export STABLE_RELEASE=${STABLE_RELEASE:-''}
|
||||
export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''}
|
||||
|
||||
set +u
|
||||
source $LOCAL_WORKING_DIR/bin/activate
|
||||
set -u
|
||||
source $OOOQ_DIR/ansible_ssh_env.sh
|
||||
|
||||
/usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \
|
||||
echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details"
|
||||
|
||||
cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true
|
||||
|
||||
if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
fi
|
||||
|
||||
# Copy tempest.html to root dir
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true
|
||||
|
||||
# Copy tempest and .testrepository directory to /opt/stack/new/tempest and
|
||||
# unzip
|
||||
sudo mkdir -p /opt/stack/new
|
||||
sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true
|
||||
sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true
|
||||
|
||||
# record the size of the logs directory
|
||||
# -L, --dereference dereference all symbolic links
|
||||
# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe'
|
||||
|
||||
du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true
|
||||
EOF
|
||||
|
||||
if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then
|
||||
if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then
|
||||
bash $LOGS_DIR/collect_logs.sh
|
||||
# rename script to not to run it in multinode jobs
|
||||
mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh
|
||||
fi
|
||||
fi
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -117,6 +117,8 @@ TIMEOUT_SECS=$((DEVSTACK_GATE_TIMEOUT*60))
|
|||
export EXTRA_VARS=${EXTRA_VARS:-""}
|
||||
export NODES_ARGS=""
|
||||
export EXTRANODE=""
|
||||
# Set playbook execution status
|
||||
export PLAYBOOK_DRY_RUN=${PLAYBOOK_DRY_RUN:=0}
|
||||
export COLLECT_CONF="$TRIPLEO_ROOT/tripleo-ci/toci-quickstart/config/collect-logs.yml"
|
||||
LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
|
||||
LWD=$LOCAL_WORKING_DIR
|
||||
|
|
@ -209,6 +211,9 @@ for JOB_TYPE_PART in $(sed 's/-/ /g' <<< "${TOCI_JOBTYPE:-}") ; do
|
|||
;;
|
||||
gate)
|
||||
;;
|
||||
dryrun)
|
||||
PLAYBOOK_DRY_RUN=1
|
||||
;;
|
||||
*)
|
||||
# the rest should be node configuration
|
||||
NODES_FILE="$TRIPLEO_ROOT/tripleo-quickstart/config/nodes/$JOB_TYPE_PART.yml"
|
||||
|
|
|
|||
|
|
@ -122,94 +122,53 @@ set -u
|
|||
source $OOOQ_DIR/ansible_ssh_env.sh
|
||||
[[ -n ${STATS_OOOQ:-''} ]] && export STATS_OOOQ=$(( $(date +%s) - STATS_OOOQ ))
|
||||
|
||||
# Debug step capture env variables
|
||||
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
|
||||
echo "-- Capture Environment Variables Used ---------"
|
||||
echo "$(env)" | tee -a $LOGS_DIR/toci_env_args_output.log
|
||||
declare -p | tee -a $LOGS_DIR/toci_env_args_output.log
|
||||
fi
|
||||
|
||||
|
||||
for playbook in $PLAYBOOKS; do
|
||||
echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}"
|
||||
run_with_timeout $START_JOB_TIME $QUICKSTART_INSTALL_CMD \
|
||||
"${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \
|
||||
# Define repeated arg set
|
||||
PLAYBOOK_REPEATED_ARGS="$QUICKSTART_INSTALL_CMD \
|
||||
$NODES_ARGS \
|
||||
$FEATURESET_CONF \
|
||||
$ENV_VARS \
|
||||
$EXTRA_VARS \
|
||||
$DEFAULT_ARGS \
|
||||
--extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \
|
||||
$LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \
|
||||
2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$?
|
||||
$DEFAULT_ARGS"
|
||||
|
||||
# Print status of playbook run
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully"
|
||||
[[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break
|
||||
echo "-- Playbooks Output --------------------------"
|
||||
for playbook in $PLAYBOOKS; do
|
||||
echo "$PLAYBOOK_REPEATED_ARGS \
|
||||
${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG} \
|
||||
$LOCAL_WORKING_DIR/playbooks/$playbook ${PLAYBOOKS_ARGS[$playbook]:-}" \
|
||||
| sed 's/--/\n--/g' \
|
||||
| tee -a $LOGS_DIR/playbook_executions.log
|
||||
echo "# --------------------------------------- " \
|
||||
| tee -a $LOGS_DIR/playbook_executions.log
|
||||
done
|
||||
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed"
|
||||
if [[ "$PLAYBOOK_DRY_RUN" == "1" ]]; then
|
||||
exit_value=0
|
||||
else
|
||||
for playbook in $PLAYBOOKS; do
|
||||
echo "${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}"
|
||||
run_with_timeout $START_JOB_TIME \
|
||||
$PLAYBOOK_REPEATED_ARGS \
|
||||
"${RELEASE_ARGS[$playbook]:=$QUICKSTART_DEFAULT_RELEASE_ARG}" \
|
||||
$LOCAL_WORKING_DIR/playbooks/$playbook "${PLAYBOOKS_ARGS[$playbook]:-}" \
|
||||
--extra-vars ci_job_end_time=$(( START_JOB_TIME + REMAINING_TIME*60 )) \
|
||||
2>&1 | tee -a $LOGS_DIR/quickstart_install.log && exit_value=0 || exit_value=$?
|
||||
|
||||
## LOGS COLLECTION
|
||||
# Print status of playbook run
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run of $playbook passed successfully"
|
||||
[[ "$exit_value" != 0 ]] && echo "Playbook run of $playbook failed" && break
|
||||
done
|
||||
|
||||
cat <<EOF > $LOGS_DIR/collect_logs.sh
|
||||
#!/bin/bash
|
||||
set -x
|
||||
[[ "$exit_value" == 0 ]] && echo "Playbook run passed successfully" || echo "Playbook run failed"
|
||||
|
||||
export NODEPOOL_PROVIDER=${NODEPOOL_PROVIDER:-''}
|
||||
export STATS_TESTENV=${STATS_TESTENV:-''}
|
||||
export STATS_OOOQ=${STATS_OOOQ:-''}
|
||||
export START_JOB_TIME=${START_JOB_TIME:-''}
|
||||
export ZUUL_PIPELINE=${ZUUL_PIPELINE:-''}
|
||||
export DEVSTACK_GATE_TIMEOUT=${DEVSTACK_GATE_TIMEOUT:-''}
|
||||
export REMAINING_TIME=${REMAINING_TIME:-''}
|
||||
export LOCAL_WORKING_DIR="$WORKSPACE/.quickstart"
|
||||
export OPT_WORKDIR=$LOCAL_WORKING_DIR
|
||||
export WORKING_DIR="$HOME"
|
||||
export LOGS_DIR=$WORKSPACE/logs
|
||||
export VIRTUAL_ENV_DISABLE_PROMPT=1
|
||||
export ANSIBLE_CONFIG=$OOOQ_DIR/ansible.cfg
|
||||
export ARA_DATABASE=sqlite:///${LOCAL_WORKING_DIR}/ara.sqlite
|
||||
export ZUUL_CHANGES=${ZUUL_CHANGES:-''}
|
||||
export NODES_FILE=${NODES_FILE:-''}
|
||||
export TOCI_JOBTYPE=$TOCI_JOBTYPE
|
||||
export STABLE_RELEASE=${STABLE_RELEASE:-''}
|
||||
export QUICKSTART_RELEASE=${QUICKSTART_RELEASE:-''}
|
||||
|
||||
set +u
|
||||
source $LOCAL_WORKING_DIR/bin/activate
|
||||
set -u
|
||||
source $OOOQ_DIR/ansible_ssh_env.sh
|
||||
|
||||
/usr/bin/timeout --preserve-status 40m $QUICKSTART_COLLECTLOGS_CMD > $LOGS_DIR/quickstart_collect_logs.log || \
|
||||
echo "WARNING: quickstart collect-logs failed, check quickstart_collectlogs.log for details"
|
||||
|
||||
cp $LOGS_DIR/undercloud/var/log/postci.txt.gz $LOGS_DIR/ || true
|
||||
|
||||
if [[ -e $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/testrepository.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/pingtest.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
elif [[ -e $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ]]; then
|
||||
cp $LOGS_DIR/undercloud/home/$USER/undercloud_sanity.subunit.gz ${LOGS_DIR}/testrepository.subunit.gz
|
||||
fi
|
||||
|
||||
# Copy tempest.html to root dir
|
||||
cp $LOGS_DIR/undercloud/home/$USER/tempest/tempest.html.gz ${LOGS_DIR} || true
|
||||
|
||||
# Copy tempest and .testrepository directory to /opt/stack/new/tempest and
|
||||
# unzip
|
||||
sudo mkdir -p /opt/stack/new
|
||||
sudo cp -Rf $LOGS_DIR/undercloud/home/$USER/tempest /opt/stack/new || true
|
||||
sudo gzip -d -r /opt/stack/new/tempest/.testrepository || true
|
||||
|
||||
# record the size of the logs directory
|
||||
# -L, --dereference dereference all symbolic links
|
||||
# Note: tail -n +1 is to prevent the error 'Broken Pipe' e.g. 'sort: write failed: standard output: Broken pipe'
|
||||
|
||||
du -L -ch $LOGS_DIR/* | tail -n +1 | sort -rh | head -n 200 &> $LOGS_DIR/log-size.txt || true
|
||||
EOF
|
||||
|
||||
if [[ "${NODEPOOL_PROVIDER:-''}" == "rdo-cloud-tripleo" ]] || [[ "${NODEPOOL_PROVIDER:-''}" == "tripleo-test-cloud-rh1" ]]; then
|
||||
if [[ "$TOCI_JOBTYPE" =~ "ovb" ]]; then
|
||||
bash $LOGS_DIR/collect_logs.sh
|
||||
# rename script to not to run it in multinode jobs
|
||||
mv $LOGS_DIR/collect_logs.sh $LOGS_DIR/ovb_collect_logs.sh
|
||||
fi
|
||||
## LOGS COLLECTION
|
||||
collect_logs
|
||||
fi
|
||||
|
||||
popd
|
||||
|
|
|
|||
Loading…
Reference in New Issue