Add a zuul job to deploy and teardown a tenks cluster

This stops short of enrolling the nodes in Ironic, but provides
some coverage of the earlier stages.

Change-Id: I48a0bafaff1555caea0a7d216ac6f6a4f11c2b55
Depends-On: https://review.openstack.org/#/c/615544/
Depends-On: https://review.openstack.org/#/c/616510/
Depends-On: https://review.openstack.org/#/c/616501/
This commit is contained in:
Will Szumski 2018-11-07 15:24:12 +00:00
parent 583d3232ae
commit b4bc89dfff
19 changed files with 337 additions and 0 deletions

View File

@ -22,9 +22,11 @@
- name: Perform Ironic node configuration
import_playbook: node_enrolment.yml
tags: openstack
- name: Register flavors in Nova
import_playbook: flavor_registration.yml
tags: openstack
- name: Clean up Tenks state
import_playbook: cleanup_state.yml

View File

@ -10,9 +10,11 @@
- name: Deregister flavors in Nova
import_playbook: flavor_registration.yml
tags: openstack
- name: Perform Ironic node deconfiguration
import_playbook: node_enrolment.yml
tags: openstack
- name: Deconfigure virtual node BMCs
import_playbook: node_bmc.yml

View File

@ -0,0 +1,13 @@
---
- name: tenks-deploy-teardown post run
hosts: all
pre_tasks:
- name: Include common variables
include_vars: common.yml
roles:
- role: tenks-diagnostics
tenks_diagnostics_phase: "post"
tenks_diagnostics_log_dir: "{{ logs_dir }}"
tenks_diagnostics_config_dir: "{{ config_dir }}"
tenks_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

View File

@ -0,0 +1,40 @@
---
- hosts: all
pre_tasks:
- name: Include common variables
include_vars: common.yml
roles:
- role: tenks-diagnostics
tenks_diagnostics_phase: "pre"
tenks_diagnostics_log_dir: "{{ logs_dir }}"
- role: tenks-ci-prep
tasks:
- name: Ensure config directory exists
file:
path: "{{ config_dir }}"
state: "directory"
- name: Template tenks overrides
template:
src: tenks-overrides.yml.j2
dest: "{{ tenks_overrides_path }}"
- name: Create virtualenv for tenks
pip:
requirements: "{{ tenks_src_dir }}/requirements.txt"
virtualenv: "{{ tenks_venv }}"
- name: Install Ansible Galaxy dependencies
vars:
ansible_galaxy_bin: "{{ tenks_venv }}/bin/ansible-galaxy"
command: >-
{{ ansible_galaxy_bin }} install -r {{ tenks_src_dir }}/requirements.yml
-p {{ tenks_src_dir }}/ansible/roles
chdir: "{{ tenks_src_dir }}"
- name: Ensure network bridge for physnet1 exists
command: "{{ item }}"
become: true
with_items:
- "ip l add breth1 type bridge"
- "ip l set breth1 up"

View File

@ -0,0 +1,32 @@
---
- hosts: all
vars:
ansible_playbook_bin: "{{ tenks_venv }}/bin/ansible-playbook"
tasks:
- name: Include common variables
include_vars: common.yml
- name: Deploy tenks cluster
shell:
cmd: >-
{{ ansible_playbook_bin }} -vvv --skip-tags openstack
--inventory ansible/inventory
--extra-vars=@{{ tenks_overrides_path }}
ansible/deploy.yml > {{ logs_dir }}/ansible/tenks-deploy
chdir: "{{ tenks_src_dir }}"
- name: Get some diagnostic info for the tenks cluster
include_role:
name: tenks-diagnostics
vars:
tenks_diagnostics_phase: "created"
tenks_diagnostics_log_dir: "{{ logs_dir }}"
- name: Teardown tenks cluster
shell:
cmd: >-
{{ ansible_playbook_bin }} -vvv --skip-tags openstack
--inventory ansible/inventory
--extra-vars=@{{ tenks_overrides_path }}
ansible/teardown.yml > {{ logs_dir }}/ansible/tenks-teardown
chdir: "{{ tenks_src_dir }}"

View File

@ -0,0 +1,31 @@
---
# This file holds the config given to Tenks when running the zuul job,
# tenks-deploy-teardown. It assumes the existence of the bridge `breth1`.
node_types:
type0:
memory_mb: 1024
vcpus: 1
volumes:
# There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent:
# https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290
- capacity: 4GiB
physical_networks:
- physnet1
console_log_enabled: true
specs:
- type: type0
count: 2
ironic_config:
resource_class: test-rc
network_interface: flat
nova_flavors:
- resource_class: test-rc
node_type: type0
physnet_mappings:
physnet1: breth1
python_upper_constraints_url: file://{{ upper_constraints_path }}

View File

@ -0,0 +1,10 @@
---
# Variables shared between the playbooks
tenks_src_dir: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/tenks'].src_dir }}"
upper_constraints_path: "{{ ansible_env.HOME ~ '/' ~ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
tenks_venv: "{{ ansible_env.HOME ~ '/' ~ 'venv-tenks' }}"
config_dir: "{{ ansible_env.HOME ~ '/' ~ 'tenks-config' }}"
tenks_overrides_path: "{{ config_dir ~ '/' ~ 'tenks-overrides.yml' }}"
logs_dir: "/tmp/logs"

View File

@ -0,0 +1,14 @@
---
- block:
# NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
# but the configure-mirrors role used by Zuul disables epel. Since we
# install epel-release and expect epel to be enabled, enable it here.
- name: Ensure yum-utils is installed
yum:
name: yum-utils
state: installed
- name: Enable the EPEL yum repository
command: yum-config-manager --enable epel
when: ansible_os_family == 'RedHat'
become: true

View File

@ -0,0 +1,6 @@
==================
Tenks Diagnostics
==================
Ansible role to collect diagnostic information following a CI job performing
integration testing.

View File

@ -0,0 +1,13 @@
---
# Job phase - one of 'pre', 'created, or 'post'. The created phase is
# designed to called when the virtual machines have been created.
tenks_diagnostics_phase:
# Directory on the remote host in which to save diagnostics.
tenks_diagnostics_logs_dir:
# Directory on the remote host in which configuration is stored.
tenks_diagnostics_config_dir:
# Directory on the executor in which to save logs.
tenks_diagnostics_executor_log_dir:

View File

@ -0,0 +1,26 @@
#!/bin/bash
# This script will query libvirt to get some information useful for
# debugging
# Environment variables:
# $LOG_DIR is the directory to copy logs to.
# $CONFIG_DIR is the directory to copy configuration from.
set +o errexit
copy_logs() {
if ! command -v virsh > /dev/null 2>&1; then
return 0
fi
virsh list --all > ${LOG_DIR}/libvirt_logs/list.txt
virsh list --all --name | while read vm; do
if [ "$vm" != "" ]; then
virsh dumpxml "$vm" > ${LOG_DIR}/libvirt_logs/"$vm".txt
fi
done
}
copy_logs

View File

@ -0,0 +1,57 @@
#!/bin/bash
# NOTE(wszumski): This has been adapted from tests/get_logs.sh in Kayobe.
# Environment variables:
# $LOG_DIR is the directory to copy logs to.
# $CONFIG_DIR is the directory to copy configuration from.
set +o errexit
copy_logs() {
if [[ -d ${CONFIG_DIR} ]]; then
cp -rnL ${CONFIG_DIR}/* ${LOG_DIR}/config/
fi
cp -rvnL /var/log/* ${LOG_DIR}/system_logs/
if [[ -x "$(command -v journalctl)" ]]; then
journalctl --no-pager > ${LOG_DIR}/system_logs/syslog.txt
fi
cp -r /etc/sudoers.d ${LOG_DIR}/system_logs/
cp /etc/sudoers ${LOG_DIR}/system_logs/sudoers.txt
df -h > ${LOG_DIR}/system_logs/df.txt
free > ${LOG_DIR}/system_logs/free.txt
parted -l > ${LOG_DIR}/system_logs/parted-l.txt
mount > ${LOG_DIR}/system_logs/mount.txt
env > ${LOG_DIR}/system_logs/env.txt
ip address > ${LOG_DIR}/system_logs/ip-address.txt
ip route > ${LOG_DIR}/system_logs/ip-route.txt
iptables-save > ${LOG_DIR}/system_logs/iptables.txt
if [ `command -v dpkg` ]; then
dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt
fi
if [ `command -v rpm` ]; then
rpm -qa > ${LOG_DIR}/system_logs/rpm-qa.txt
fi
# final memory usage and process list
ps -eo user,pid,ppid,lwp,%cpu,%mem,size,rss,cmd > ${LOG_DIR}/system_logs/ps.txt
# Rename files to .txt; this is so that when displayed via
# logs.openstack.org clicking results in the browser shows the
# files, rather than trying to send it to another app or make you
# download it, etc.
for f in $(find ${LOG_DIR}/{system_logs,libvirt_logs} -name "*.log"); do
mv $f ${f/.log/.txt}
done
chmod -R 777 ${LOG_DIR}
find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.txt' -execdir gzip -f -9 {} \+
find ${LOG_DIR}/{system_logs,libvirt_logs} -iname '*.json' -execdir gzip -f -9 {} \+
}
copy_logs

View File

@ -0,0 +1,14 @@
---
- name: Run diagnostics script
script: dump_libvirt_logs.sh
register: dump_libvirt_logs_result
become: true
failed_when: false
environment:
LOG_DIR: "{{ tenks_diagnostics_log_dir }}"
CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}"
- name: Print dump_libvirt_logs.sh output
debug:
msg: "{{ dump_libvirt_logs_result.stdout }}"

View File

@ -0,0 +1,2 @@
---
- include: "{{ tenks_diagnostics_phase }}.yml"

View File

@ -0,0 +1,24 @@
---
- name: Write host variables to a file
copy:
content: "{{ hostvars[inventory_hostname] | to_nice_json }}"
dest: "{{ tenks_diagnostics_log_dir }}/facts.json"
- name: Run diagnostics script
script: get_logs.sh
register: get_logs_result
become: true
failed_when: false
environment:
LOG_DIR: "{{ tenks_diagnostics_log_dir }}"
CONFIG_DIR: "{{ tenks_diagnostics_config_dir }}"
- name: Print get_logs.sh output
debug:
msg: "{{ get_logs_result.stdout }}"
- name: Download logs to executor
synchronize:
src: "{{ tenks_diagnostics_log_dir }}/"
dest: "{{ tenks_diagnostics_executor_log_dir }}/"
mode: pull

View File

@ -0,0 +1,16 @@
---
- name: Ensure node log directory exists
file:
path: "{{ tenks_diagnostics_log_dir }}"
state: "directory"
- name: Ensure node log subdirectories exist
file:
path: "{{ tenks_diagnostics_log_dir }}/{{ item }}"
state: "directory"
mode: 0777
with_items:
- "libvirt_logs"
- "config"
- "system_logs"
- "ansible"

21
zuul.d/base.yaml Normal file
View File

@ -0,0 +1,21 @@
---
- job:
name: tenks-deploy-teardown-base
description: |
Creates the tenks virtual machines and sets up the virtual BMC daemon.
This stops short of enrolling the machines in Ironic. The cluster is
then destroyed again.
pre-run: playbooks/tenks-deploy-teardown/pre.yml
run: playbooks/tenks-deploy-teardown/run.yml
post-run: playbooks/tenks-deploy-teardown/post.yml
required-projects:
- name: openstack/requirements
attempts: 1
timeout: 1800
irrelevant-files:
- ^.*\.rst$
- ^doc/.*
- ^releasenotes/.*
- ^setup.cfg$
- ^tox.ini$

View File

@ -7,3 +7,13 @@
vars:
tox_envlist: alint
- job:
name: tenks-deploy-teardown-centos
parent: tenks-deploy-teardown-base
nodeset: centos-7
- job:
name: tenks-deploy-teardown-ubuntu
parent: tenks-deploy-teardown-base
nodeset: ubuntu-bionic

View File

@ -9,8 +9,12 @@
jobs:
- build-openstack-releasenotes
- tenks-tox-ansible-lint
- tenks-deploy-teardown-centos
- tenks-deploy-teardown-ubuntu
gate:
queue: tenks
jobs:
- build-openstack-releasenotes
- tenks-tox-ansible-lint
- tenks-deploy-teardown-centos
- tenks-deploy-teardown-ubuntu