Revert "Move log collection to 'post' stage"
This reverts commit fd4db8ea90.
The playbook used here needs to be in every repo calling that job
(which is all our branches and all our repos).
I suggest we revert it until alll the other zuul v3 changes are settled.
We can then take a more organized approach.
Change-Id: I8ba967af69311fc0684635462d27cdc03728ca81
This commit is contained in:
Jean-Philippe Evrard
Jean-Philippe Evrard
parent
fd4db8ea90
commit
af3a08a5b1
|
|
@ -96,6 +96,14 @@ function execute_ansible_playbook {
|
|||
|
||||
}
|
||||
|
||||
function gate_job_exit_tasks {
|
||||
# This environment variable captures the exit code
|
||||
# which was present when the trap was initiated.
|
||||
# This would be the success/failure of the test.
|
||||
export TEST_EXIT_CODE=$?
|
||||
source "${COMMON_TESTS_PATH}/test-log-collect.sh"
|
||||
}
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
# NOTE(mhayden): CentOS images in the gate have several slow mirrors enabled
|
||||
|
|
@ -112,6 +120,9 @@ fi
|
|||
source "${COMMON_TESTS_PATH}/test-ansible-env-prep.sh"
|
||||
setup_ara
|
||||
|
||||
# Set gate job exit traps, this is run regardless of exit state when the job finishes.
|
||||
trap gate_job_exit_tasks EXIT
|
||||
|
||||
# Prepare the extra CLI parameters used in each execution
|
||||
set_ansible_parameters
|
||||
|
||||
|
|
|
|||
|
|
@ -33,19 +33,21 @@ export TESTING_HOME=${TESTING_HOME:-$HOME}
|
|||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
echo "#### BEGIN LOG COLLECTION ###"
|
||||
if [[ -d "/etc/nodepool" ]]; then
|
||||
|
||||
mkdir -vp \
|
||||
echo "#### BEGIN LOG COLLECTION ###"
|
||||
|
||||
mkdir -vp \
|
||||
"${WORKING_DIR}/logs/host" \
|
||||
"${WORKING_DIR}/logs/openstack" \
|
||||
"${WORKING_DIR}/logs/etc/host" \
|
||||
"${WORKING_DIR}/logs/etc/openstack" \
|
||||
|
||||
# NOTE(mhayden): We use sudo here to ensure that all logs are copied.
|
||||
sudo ${RSYNC_CMD} /var/log/ "${WORKING_DIR}/logs/host" || true
|
||||
if [ -d "/openstack/log" ]; then
|
||||
# NOTE(mhayden): We use sudo here to ensure that all logs are copied.
|
||||
sudo ${RSYNC_CMD} /var/log/ "${WORKING_DIR}/logs/host" || true
|
||||
if [ -d "/openstack/log" ]; then
|
||||
sudo ${RSYNC_CMD} /openstack/log/ "${WORKING_DIR}/logs/openstack" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# NOTE(cloudnull): This is collection thousands of files resulting in infra upload
|
||||
# issues. To remove immediate pressure this is being stopped and
|
||||
|
|
@ -63,61 +65,63 @@ fi
|
|||
# done
|
||||
# fi
|
||||
|
||||
# NOTE(mhayden): All of the files must be world-readable so that the log
|
||||
# pickup jobs will work properly. Without this, you get a "File not found"
|
||||
# when trying to read the files in the job results.
|
||||
# NOTE(odyssey4me): Using '--chown $(whoami) --chmod=ugo+rX' in the rsync
|
||||
# CMD to achieve this would be optimal, but the CentOS version of rsync
|
||||
# (3.0.x) does not support that option.
|
||||
sudo chmod -R ugo+rX "${WORKING_DIR}/logs/"
|
||||
sudo chown -R $(whoami) "${WORKING_DIR}/logs/"
|
||||
# NOTE(mhayden): All of the files must be world-readable so that the log
|
||||
# pickup jobs will work properly. Without this, you get a "File not found"
|
||||
# when trying to read the files in the job results.
|
||||
# NOTE(odyssey4me): Using '--chown $(whoami) --chmod=ugo+rX' in the rsync
|
||||
# CMD to achieve this would be optimal, but the CentOS version of rsync
|
||||
# (3.0.x) does not support that option.
|
||||
sudo chmod -R ugo+rX "${WORKING_DIR}/logs/"
|
||||
sudo chown -R $(whoami) "${WORKING_DIR}/logs/"
|
||||
|
||||
if [ ! -z "${ANSIBLE_LOG_DIR}" ]; then
|
||||
if [ ! -z "${ANSIBLE_LOG_DIR}" ]; then
|
||||
mkdir -p "${WORKING_DIR}/logs/ansible"
|
||||
${RSYNC_CMD} "${ANSIBLE_LOG_DIR}/" "${WORKING_DIR}/logs/ansible" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Rename all files gathered to have a .txt suffix so that the compressed
|
||||
# files are viewable via a web browser in OpenStack-CI.
|
||||
find "${WORKING_DIR}/logs/" -type f ! -name '*.html' -exec mv {} {}.txt \;
|
||||
# Rename all files gathered to have a .txt suffix so that the compressed
|
||||
# files are viewable via a web browser in OpenStack-CI.
|
||||
find "${WORKING_DIR}/logs/" -type f ! -name '*.html' -exec mv {} {}.txt \;
|
||||
|
||||
# Get the ara sqlite database
|
||||
${RSYNC_CMD} "${TESTING_HOME}/.ara/ansible.sqlite" "${WORKING_DIR}/logs/" || true
|
||||
# Get the ara sqlite database
|
||||
${RSYNC_CMD} "${TESTING_HOME}/.ara/ansible.sqlite" "${WORKING_DIR}/logs/" || true
|
||||
|
||||
# Generate the ARA report
|
||||
# In order to reduce the quantity of unnecessary log content
|
||||
# being kept in OpenStack-Infra we only generate the ARA report
|
||||
# when the test result is a failure. The ARA sqlite database is
|
||||
# still available for self generation if desired for successful
|
||||
# tests.
|
||||
if [[ "${TEST_EXIT_CODE}" != "0" ]]; then
|
||||
# Generate the ARA report
|
||||
# In order to reduce the quantity of unnecessary log content
|
||||
# being kept in OpenStack-Infra we only generate the ARA report
|
||||
# when the test result is a failure. The ARA sqlite database is
|
||||
# still available for self generation if desired for successful
|
||||
# tests.
|
||||
if [[ "${TEST_EXIT_CODE}" != "0" ]]; then
|
||||
echo "Generating ARA report due to non-zero exit code (${TEST_EXIT_CODE})."
|
||||
${ARA_CMD} "${WORKING_DIR}/logs/ara" || true
|
||||
else
|
||||
else
|
||||
echo "Not generating ARA report due to test pass."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Get a dmesg output so we can look for kernel failures
|
||||
dmesg > "${WORKING_DIR}/logs/dmesg.log.txt" || true
|
||||
# Get a dmesg output so we can look for kernel failures
|
||||
dmesg > "${WORKING_DIR}/logs/dmesg.log.txt" || true
|
||||
|
||||
# output ram usage
|
||||
free -m > "${WORKING_DIR}/logs/memory-available.txt" || true
|
||||
# output ram usage
|
||||
free -m > "${WORKING_DIR}/logs/memory-available.txt" || true
|
||||
|
||||
# Redhat package debugging
|
||||
if which yum &>/dev/null || which dnf &>/dev/null; then
|
||||
# Redhat package debugging
|
||||
if which yum &>/dev/null || which dnf &>/dev/null; then
|
||||
# Prefer dnf over yum for CentOS.
|
||||
which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
|
||||
sudo $RHT_PKG_MGR repolist -v > "${WORKING_DIR}/logs/redhat-rpm-repolist.txt" || true
|
||||
sudo $RHT_PKG_MGR list installed > "${WORKING_DIR}/logs/redhat-rpm-list-installed.txt" || true
|
||||
|
||||
# SUSE package debugging
|
||||
elif which zypper &>/dev/null; then
|
||||
# SUSE package debugging
|
||||
elif which zypper &>/dev/null; then
|
||||
sudo zypper lr -d > "${WORKING_DIR}/logs/suse-zypper-repolist.txt" || true
|
||||
sudo zypper pa -i > "${WORKING_DIR}/logs/suse-zypper-list-installed.txt" || true
|
||||
fi
|
||||
|
||||
# Compress the files gathered so that they do not take up too much space.
|
||||
# We use 'command' to ensure that we're not executing with some sort of alias.
|
||||
command gzip --best --recursive "${WORKING_DIR}/logs/"
|
||||
|
||||
echo "#### END LOG COLLECTION ###"
|
||||
|
||||
fi
|
||||
|
||||
# Compress the files gathered so that they do not take up too much space.
|
||||
# We use 'command' to ensure that we're not executing with some sort of alias.
|
||||
command gzip --best --recursive "${WORKING_DIR}/logs/"
|
||||
|
||||
echo "#### END LOG COLLECTION ###"
|
||||
|
|
|
|||
|
|
@ -112,7 +112,6 @@
|
|||
- ^Vagrantfile
|
||||
timeout: 5400 # 90 mins
|
||||
run: zuul.d/playbooks/run
|
||||
post-run: zuul.d/playbooks/post
|
||||
vars:
|
||||
tox_env: functional
|
||||
|
||||
|
|
@ -166,5 +165,4 @@
|
|||
to execute the tests from must be specified in the
|
||||
``osa_test_repo`` variable.
|
||||
run: zuul.d/playbooks/cross-repo-run
|
||||
post-run: zuul.d/playbooks/cross-repo-post
|
||||
nodeset: ubuntu-xenial
|
||||
|
|
|
|||
|
|
@ -1,36 +0,0 @@
|
|||
---
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- hosts: all
|
||||
tasks:
|
||||
- name: Run log collection script
|
||||
shell: |
|
||||
echo "TEST_EXIT_CODE: ${TEST_EXIT_CODE}"
|
||||
if [[ -d "tests/common" ]]; then
|
||||
source tests/common/test-log-collect.sh
|
||||
else
|
||||
echo "The common tests repo has not been cloned yet. Logs cannot be collected without it."
|
||||
fi
|
||||
args:
|
||||
executable: "/bin/bash"
|
||||
chdir: "{{ ansible_user_dir }}/src/git.openstack.org/{{ osa_test_repo }}"
|
||||
environment:
|
||||
TEST_EXIT_CODE: "{{ zuul_success }}"
|
||||
|
||||
- name: Copy logs back to the executor
|
||||
synchronize:
|
||||
src: "{{ ansible_user_dir }}/src/git.openstack.org/{{ osa_test_repo }}/logs/"
|
||||
dest: "{{ zuul.executor.log_root }}"
|
||||
mode: pull
|
||||
|
|
@ -1,36 +0,0 @@
|
|||
---
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- hosts: all
|
||||
tasks:
|
||||
- name: Run log collection script
|
||||
shell: |
|
||||
echo "TEST_EXIT_CODE: ${TEST_EXIT_CODE}"
|
||||
if [[ -d "tests/common" ]]; then
|
||||
source tests/common/test-log-collect.sh
|
||||
else
|
||||
echo "The common tests repo has not been cloned yet. Logs cannot be collected without it."
|
||||
fi
|
||||
args:
|
||||
executable: "/bin/bash"
|
||||
chdir: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
|
||||
environment:
|
||||
TEST_EXIT_CODE: "{{ zuul_success }}"
|
||||
|
||||
- name: Copy logs back to the executor
|
||||
synchronize:
|
||||
src: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}/logs/"
|
||||
dest: "{{ zuul.executor.log_root }}"
|
||||
mode: pull
|
||||
Loading…
Reference in New Issue