Revert "Revert "Move log collection to 'post' stage""
This reverts commit af3a08a5b1
but adds the tests repository as a required repo to make sure that
it works properly across other repositories. This will be tested
through another review using depends-on so that it does not
break the role tests again.
Change-Id: If9bded9d60e2fe839c0ec8c49cc5102b5a541474
Signed-off-by: Kevin Carter <kevin.carter@rackspace.com>
This commit is contained in:
Jesse Pretorius (odyssey4me)
Kevin Carter
parent
246c3b9720
commit
a27ea8cfda
|
|
@ -96,14 +96,6 @@ function execute_ansible_playbook {
|
|||
|
||||
}
|
||||
|
||||
function gate_job_exit_tasks {
|
||||
# This environment variable captures the exit code
|
||||
# which was present when the trap was initiated.
|
||||
# This would be the success/failure of the test.
|
||||
export TEST_EXIT_CODE=$?
|
||||
source "${COMMON_TESTS_PATH}/test-log-collect.sh"
|
||||
}
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
# NOTE(mhayden): CentOS images in the gate have several slow mirrors enabled
|
||||
|
|
@ -120,9 +112,6 @@ fi
|
|||
source "${COMMON_TESTS_PATH}/test-ansible-env-prep.sh"
|
||||
setup_ara
|
||||
|
||||
# Set gate job exit traps, this is run regardless of exit state when the job finishes.
|
||||
trap gate_job_exit_tasks EXIT
|
||||
|
||||
# Prepare the extra CLI parameters used in each execution
|
||||
set_ansible_parameters
|
||||
|
||||
|
|
|
|||
|
|
@ -33,21 +33,19 @@ export TESTING_HOME=${TESTING_HOME:-$HOME}
|
|||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
if [[ -d "/etc/nodepool" ]]; then
|
||||
echo "#### BEGIN LOG COLLECTION ###"
|
||||
|
||||
echo "#### BEGIN LOG COLLECTION ###"
|
||||
|
||||
mkdir -vp \
|
||||
mkdir -vp \
|
||||
"${WORKING_DIR}/logs/host" \
|
||||
"${WORKING_DIR}/logs/openstack" \
|
||||
"${WORKING_DIR}/logs/etc/host" \
|
||||
"${WORKING_DIR}/logs/etc/openstack" \
|
||||
|
||||
# NOTE(mhayden): We use sudo here to ensure that all logs are copied.
|
||||
sudo ${RSYNC_CMD} /var/log/ "${WORKING_DIR}/logs/host" || true
|
||||
if [ -d "/openstack/log" ]; then
|
||||
# NOTE(mhayden): We use sudo here to ensure that all logs are copied.
|
||||
sudo ${RSYNC_CMD} /var/log/ "${WORKING_DIR}/logs/host" || true
|
||||
if [ -d "/openstack/log" ]; then
|
||||
sudo ${RSYNC_CMD} /openstack/log/ "${WORKING_DIR}/logs/openstack" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# NOTE(cloudnull): This is collection thousands of files resulting in infra upload
|
||||
# issues. To remove immediate pressure this is being stopped and
|
||||
|
|
@ -65,63 +63,61 @@ if [[ -d "/etc/nodepool" ]]; then
|
|||
# done
|
||||
# fi
|
||||
|
||||
# NOTE(mhayden): All of the files must be world-readable so that the log
|
||||
# pickup jobs will work properly. Without this, you get a "File not found"
|
||||
# when trying to read the files in the job results.
|
||||
# NOTE(odyssey4me): Using '--chown $(whoami) --chmod=ugo+rX' in the rsync
|
||||
# CMD to achieve this would be optimal, but the CentOS version of rsync
|
||||
# (3.0.x) does not support that option.
|
||||
sudo chmod -R ugo+rX "${WORKING_DIR}/logs/"
|
||||
sudo chown -R $(whoami) "${WORKING_DIR}/logs/"
|
||||
# NOTE(mhayden): All of the files must be world-readable so that the log
|
||||
# pickup jobs will work properly. Without this, you get a "File not found"
|
||||
# when trying to read the files in the job results.
|
||||
# NOTE(odyssey4me): Using '--chown $(whoami) --chmod=ugo+rX' in the rsync
|
||||
# CMD to achieve this would be optimal, but the CentOS version of rsync
|
||||
# (3.0.x) does not support that option.
|
||||
sudo chmod -R ugo+rX "${WORKING_DIR}/logs/"
|
||||
sudo chown -R $(whoami) "${WORKING_DIR}/logs/"
|
||||
|
||||
if [ ! -z "${ANSIBLE_LOG_DIR}" ]; then
|
||||
if [ ! -z "${ANSIBLE_LOG_DIR}" ]; then
|
||||
mkdir -p "${WORKING_DIR}/logs/ansible"
|
||||
${RSYNC_CMD} "${ANSIBLE_LOG_DIR}/" "${WORKING_DIR}/logs/ansible" || true
|
||||
fi
|
||||
fi
|
||||
|
||||
# Rename all files gathered to have a .txt suffix so that the compressed
|
||||
# files are viewable via a web browser in OpenStack-CI.
|
||||
find "${WORKING_DIR}/logs/" -type f ! -name '*.html' -exec mv {} {}.txt \;
|
||||
# Rename all files gathered to have a .txt suffix so that the compressed
|
||||
# files are viewable via a web browser in OpenStack-CI.
|
||||
find "${WORKING_DIR}/logs/" -type f ! -name '*.html' -exec mv {} {}.txt \;
|
||||
|
||||
# Get the ara sqlite database
|
||||
${RSYNC_CMD} "${TESTING_HOME}/.ara/ansible.sqlite" "${WORKING_DIR}/logs/" || true
|
||||
# Get the ara sqlite database
|
||||
${RSYNC_CMD} "${TESTING_HOME}/.ara/ansible.sqlite" "${WORKING_DIR}/logs/" || true
|
||||
|
||||
# Generate the ARA report
|
||||
# In order to reduce the quantity of unnecessary log content
|
||||
# being kept in OpenStack-Infra we only generate the ARA report
|
||||
# when the test result is a failure. The ARA sqlite database is
|
||||
# still available for self generation if desired for successful
|
||||
# tests.
|
||||
if [[ "${TEST_EXIT_CODE}" != "0" ]]; then
|
||||
# Generate the ARA report
|
||||
# In order to reduce the quantity of unnecessary log content
|
||||
# being kept in OpenStack-Infra we only generate the ARA report
|
||||
# when the test result is a failure. The ARA sqlite database is
|
||||
# still available for self generation if desired for successful
|
||||
# tests.
|
||||
if [[ "${TEST_EXIT_CODE}" != "0" ]]; then
|
||||
echo "Generating ARA report due to non-zero exit code (${TEST_EXIT_CODE})."
|
||||
${ARA_CMD} "${WORKING_DIR}/logs/ara" || true
|
||||
else
|
||||
else
|
||||
echo "Not generating ARA report due to test pass."
|
||||
fi
|
||||
fi
|
||||
|
||||
# Get a dmesg output so we can look for kernel failures
|
||||
dmesg > "${WORKING_DIR}/logs/dmesg.log.txt" || true
|
||||
# Get a dmesg output so we can look for kernel failures
|
||||
dmesg > "${WORKING_DIR}/logs/dmesg.log.txt" || true
|
||||
|
||||
# output ram usage
|
||||
free -m > "${WORKING_DIR}/logs/memory-available.txt" || true
|
||||
# output ram usage
|
||||
free -m > "${WORKING_DIR}/logs/memory-available.txt" || true
|
||||
|
||||
# Redhat package debugging
|
||||
if which yum &>/dev/null || which dnf &>/dev/null; then
|
||||
# Redhat package debugging
|
||||
if which yum &>/dev/null || which dnf &>/dev/null; then
|
||||
# Prefer dnf over yum for CentOS.
|
||||
which dnf &>/dev/null && RHT_PKG_MGR='dnf' || RHT_PKG_MGR='yum'
|
||||
sudo $RHT_PKG_MGR repolist -v > "${WORKING_DIR}/logs/redhat-rpm-repolist.txt" || true
|
||||
sudo $RHT_PKG_MGR list installed > "${WORKING_DIR}/logs/redhat-rpm-list-installed.txt" || true
|
||||
|
||||
# SUSE package debugging
|
||||
elif which zypper &>/dev/null; then
|
||||
# SUSE package debugging
|
||||
elif which zypper &>/dev/null; then
|
||||
sudo zypper lr -d > "${WORKING_DIR}/logs/suse-zypper-repolist.txt" || true
|
||||
sudo zypper pa -i > "${WORKING_DIR}/logs/suse-zypper-list-installed.txt" || true
|
||||
fi
|
||||
|
||||
# Compress the files gathered so that they do not take up too much space.
|
||||
# We use 'command' to ensure that we're not executing with some sort of alias.
|
||||
command gzip --best --recursive "${WORKING_DIR}/logs/"
|
||||
|
||||
echo "#### END LOG COLLECTION ###"
|
||||
|
||||
fi
|
||||
|
||||
# Compress the files gathered so that they do not take up too much space.
|
||||
# We use 'command' to ensure that we're not executing with some sort of alias.
|
||||
command gzip --best --recursive "${WORKING_DIR}/logs/"
|
||||
|
||||
echo "#### END LOG COLLECTION ###"
|
||||
|
|
|
|||
|
|
@ -54,6 +54,7 @@
|
|||
- name: openstack/openstack-ansible-repo_server
|
||||
- name: openstack/openstack-ansible-rsyslog_client
|
||||
- name: openstack/openstack-ansible-rsyslog_server
|
||||
- name: openstack/openstack-ansible-tests
|
||||
- name: openstack/requirements
|
||||
timeout: 600 # 5 mins
|
||||
|
||||
|
|
@ -100,6 +101,7 @@
|
|||
- name: openstack/openstack-ansible-repo_server
|
||||
- name: openstack/openstack-ansible-rsyslog_client
|
||||
- name: openstack/openstack-ansible-rsyslog_server
|
||||
- name: openstack/openstack-ansible-tests
|
||||
- name: openstack/requirements
|
||||
irrelevant-files:
|
||||
- ^\.git.*
|
||||
|
|
@ -112,6 +114,7 @@
|
|||
- ^Vagrantfile
|
||||
timeout: 5400 # 90 mins
|
||||
run: zuul.d/playbooks/run.yml
|
||||
post-run: zuul.d/playbooks/post.yml
|
||||
vars:
|
||||
tox_env: functional
|
||||
|
||||
|
|
@ -165,4 +168,5 @@
|
|||
to execute the tests from must be specified in the
|
||||
``osa_test_repo`` variable.
|
||||
run: zuul.d/playbooks/cross-repo-run.yml
|
||||
post-run: zuul.d/playbooks/cross-repo-post.yml
|
||||
nodeset: ubuntu-xenial
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
---
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- hosts: all
|
||||
tasks:
|
||||
- name: Run log collection script
|
||||
shell: |
|
||||
echo "TEST_EXIT_CODE: ${TEST_EXIT_CODE}"
|
||||
if [[ -d "tests/common" ]]; then
|
||||
source tests/common/test-log-collect.sh
|
||||
else
|
||||
echo "The common tests repo has not been cloned yet. Logs cannot be collected without it."
|
||||
fi
|
||||
args:
|
||||
executable: "/bin/bash"
|
||||
chdir: "{{ ansible_user_dir }}/src/git.openstack.org/{{ osa_test_repo }}"
|
||||
environment:
|
||||
TEST_EXIT_CODE: "{{ zuul_success }}"
|
||||
|
||||
- name: Copy logs back to the executor
|
||||
synchronize:
|
||||
src: "{{ ansible_user_dir }}/src/git.openstack.org/{{ osa_test_repo }}/logs/"
|
||||
dest: "{{ zuul.executor.log_root }}"
|
||||
mode: pull
|
||||
|
|
@ -0,0 +1,36 @@
|
|||
---
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- hosts: all
|
||||
tasks:
|
||||
- name: Run log collection script
|
||||
shell: |
|
||||
echo "TEST_EXIT_CODE: ${TEST_EXIT_CODE}"
|
||||
if [[ -d "tests/common" ]]; then
|
||||
source tests/common/test-log-collect.sh
|
||||
else
|
||||
echo "The common tests repo has not been cloned yet. Logs cannot be collected without it."
|
||||
fi
|
||||
args:
|
||||
executable: "/bin/bash"
|
||||
chdir: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
|
||||
environment:
|
||||
TEST_EXIT_CODE: "{{ zuul_success }}"
|
||||
|
||||
- name: Copy logs back to the executor
|
||||
synchronize:
|
||||
src: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}/logs/"
|
||||
dest: "{{ zuul.executor.log_root }}"
|
||||
mode: pull
|
||||
Loading…
Reference in New Issue