From d0e9c50fd282f685301f6750554786a794489a3c Mon Sep 17 00:00:00 2001 From: Will Miller Date: Mon, 10 Sep 2018 09:36:50 +0000 Subject: [PATCH] Add tenks-deploy.sh dev script tenks-deploy.sh deploys a minimal virtualized baremetal test cluster locally. It also adds it to the overcloud-base CI job. To make the new CI job work, we need to configure the firewall on the test machine to allow the baremetal machines to communicate with the openstack services. Change-Id: I7487a2606cf0bac71c5c63db41b2b518a6f6398b Depends-On: https://review.openstack.org/#/c/615939 Depends-On: https://review.openstack.org/#/c/618003 Story: 2004297 Task: 27850 --- dev/configure-firewall.sh | 19 ++ dev/functions | 263 ++++++++++++++++-- dev/install.sh | 2 +- dev/overcloud-test-baremetal.sh | 16 ++ ...overcloud-test.sh => overcloud-test-vm.sh} | 2 +- dev/ovs-vsctl | 4 + dev/tenks-deploy-config.yml | 36 +++ dev/tenks-deploy.sh | 26 ++ .../kayobe-overcloud-base/overrides.yml.j2 | 1 + playbooks/kayobe-overcloud-base/pre.yml | 5 + playbooks/kayobe-overcloud-base/run.yml | 19 +- .../overrides.yml.j2 | 1 + .../kayobe-overcloud-upgrade-base/run.yml | 37 ++- playbooks/kayobe-seed-base/overrides.yml.j2 | 1 + roles/kayobe-diagnostics/files/get_logs.sh | 2 + zuul.d/jobs.yaml | 2 + 16 files changed, 407 insertions(+), 29 deletions(-) create mode 100755 dev/configure-firewall.sh create mode 100755 dev/overcloud-test-baremetal.sh rename dev/{overcloud-test.sh => overcloud-test-vm.sh} (80%) create mode 100755 dev/ovs-vsctl create mode 100644 dev/tenks-deploy-config.yml create mode 100755 dev/tenks-deploy.sh diff --git a/dev/configure-firewall.sh b/dev/configure-firewall.sh new file mode 100755 index 000000000..b013ba07d --- /dev/null +++ b/dev/configure-firewall.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Simple script to configure the firewall. This should be +# executed from within the VM. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + config_init + configure_iptables +} + +main "$@" diff --git a/dev/functions b/dev/functions index 39e0415c2..beec19087 100644 --- a/dev/functions +++ b/dev/functions @@ -27,6 +27,9 @@ function config_defaults { # Path to the kayobe virtual environment. export KAYOBE_VENV_PATH="${KAYOBE_VENV_PATH:-${HOME}/kayobe-venv}" + # Path to the Tenks virtual environment. + export TENKS_VENV_PATH="${TENKS_VENV_PATH:-${HOME}/tenks-test-venv}" + # Whether to provision a VM for the seed host. export KAYOBE_SEED_VM_PROVISION=${KAYOBE_SEED_VM_PROVISION:-1} @@ -84,27 +87,37 @@ function install_dependencies { } function install_venv { - local venv_parent="$(dirname ${KAYOBE_VENV_PATH})" + # Install a virtualenv at $1. Install all the packages in proceeding + # arguments using pip. + venv_path="$1" + shift + pip_paths="$@" + local venv_parent="$(dirname ${venv_path})" if [[ ! -d "$venv_parent" ]]; then mkdir -p "$venv_parent" fi - if [[ ! -f "${KAYOBE_VENV_PATH}/bin/activate" ]]; then - echo "Creating kayobe virtual environment in ${KAYOBE_VENV_PATH}" - virtualenv "${KAYOBE_VENV_PATH}" + if [[ ! -f "${venv_path}/bin/activate" ]]; then + echo "Creating virtual environment in ${venv_path}" + virtualenv "${venv_path}" # NOTE: Virtualenv's activate and deactivate scripts reference an # unbound variable. set +u - source "${KAYOBE_VENV_PATH}/bin/activate" + source "${venv_path}/bin/activate" pip install -U pip - pip install "${KAYOBE_SOURCE_PATH}" + pip install $pip_paths deactivate set -u else - echo "Using existing kayobe virtual environment in ${KAYOBE_VENV_PATH}" + echo "Using existing virtual environment in ${venv_path}" fi } -function upgrade_venv { +function install_kayobe_venv { + # Install the Kayobe venv. + install_venv "${KAYOBE_VENV_PATH}" "${KAYOBE_SOURCE_PATH}" +} + +function upgrade_kayobe_venv { echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}" virtualenv "${KAYOBE_VENV_PATH}" # NOTE: Virtualenv's activate and deactivate scripts reference an @@ -252,7 +265,7 @@ function overcloud_upgrade { echo "single node OpenStack control plane." echo "Upgrading Kayobe" - upgrade_venv + upgrade_kayobe_venv environment_setup @@ -301,32 +314,240 @@ function overcloud_upgrade { echo "Control plane upgrade complete" } -function overcloud_test { - # Perform a simple smoke test against the cloud. - echo "Performing a simple smoke test" +function overcloud_test_init { + echo "Performing overcloud test init" environment_setup pip install python-openstackclient - source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" - if ! openstack image show cirros >/dev/null 2>&1; then + + # This guards init-runonce from running more than once + if mkdir /tmp/init-runonce > /dev/null 2>&1; then echo "Running kolla-ansible init-runonce" ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce else echo "Not running kolla-ansible init-runonce - resources exist" fi +} - echo "Creating a VM" - openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net demo1 - openstack server show demo1 - status=$(openstack server show demo1 -f value -c status) +function overcloud_test { + set -eu + + # function arguments + name="$1" + flavor="$2" + network="$3" + + node_config="{ 'name': '$name', 'flavor': '$flavor', 'network': '$network' }" + + overcloud_test_init + + # Perform a simple smoke test against the cloud. + echo "Performing a simple smoke test with node config: $node_config" + + echo "$name: Creating a server" + openstack server create --wait --image cirros --flavor "$flavor" --key-name mykey --network "$network" "$name" + echo "$name: Server created" + openstack server show "$name" + status=$(openstack server show "$name" -f value -c status) if [[ $status != ACTIVE ]]; then - echo "VM creation failed" + echo "$name: Node creation failed" return 1 fi # TODO(mgoddard): Test SSH connectivity to the VM. - echo "Deleting the VM" - openstack server delete --wait demo1 + echo "$name: Deleting the Node" + openstack server delete --wait "$name" +} + +function tenks_deploy { + set -eu + # Create a simple test Tenks deployment. Assumes that a bridge named + # 'breth1' exists. Arguments: + # $1: The path to the Tenks repo. + local tenks_path="$1" + + echo "Configuring Tenks" + + environment_setup + + # We don't want to use the Kayobe venv. + deactivate + + # Install the Tenks venv. + install_venv "${TENKS_VENV_PATH}" "$tenks_path" + source ${TENKS_VENV_PATH:-$HOME/tenks-test-venv}/bin/activate + + ansible-galaxy install \ + --role-file="$tenks_path/requirements.yml" \ + --roles-path="$tenks_path/ansible/roles/" + + local parent="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + + # Install a trivial script for ovs-vsctl that talks to containerised Open + # vSwitch. + sudo cp --no-clobber "$parent/ovs-vsctl" /usr/bin/ovs-vsctl + + source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" + + pip install python-openstackclient + + ansible-playbook \ + -vvv \ + --inventory "$tenks_path/ansible/inventory" \ + --extra-vars=@"$parent/tenks-deploy-config.yml" \ + "$tenks_path/ansible/deploy.yml" +} + +# General purpose + +# Prints backtrace info +# filename:lineno:function +# backtrace level +function backtrace { + local level=$1 + local deep + deep=$((${#BASH_SOURCE[@]} - 1)) + echo "[Call Trace]" + while [ $level -le $deep ]; do + echo "${BASH_SOURCE[$deep]}:${BASH_LINENO[$deep-1]}:${FUNCNAME[$deep-1]}" + deep=$((deep - 1)) + done +} + +# Prints line number and "message" then exits +# die $LINENO "message" +function die { + local exitcode=$? + set +o xtrace + local line=$1; shift + if [ $exitcode == 0 ]; then + exitcode=1 + fi + backtrace 2 + err $line "$*" + # Give buffers a second to flush + sleep 1 + exit $exitcode +} + + +# Prints line number and "message" in error format +# err $LINENO "message" +function err { + local exitcode=$? + local xtrace + xtrace=$(set +o | grep xtrace) + set +o xtrace + local msg="[ERROR] ${BASH_SOURCE[2]}:$1 $2" + echo "$msg" 1>&2; + if [[ -n ${LOGDIR} ]]; then + echo "$msg" >> "${LOGDIR}/error.log" + fi + $xtrace + return $exitcode +} + + +function die_if_module_not_loaded { + if ! grep -q $1 /proc/modules; then + die $LINENO "$1 kernel module is not loaded" + fi +} + +# running_in_container - Returns true otherwise false +function running_in_container { + [[ $(systemd-detect-virt --container) != 'none' ]] +} + +# enable_kernel_bridge_firewall - Enable kernel support for bridge firewalling +function enable_kernel_bridge_firewall { + # Load bridge module. This module provides access to firewall for bridged + # frames; and also on older kernels (pre-3.18) it provides sysctl knobs to + # enable/disable bridge firewalling + sudo modprobe bridge + # For newer kernels (3.18+), those sysctl settings are split into a separate + # kernel module (br_netfilter). Load it too, if present. + sudo modprobe br_netfilter 2>> /dev/null || : + # Enable bridge firewalling in case it's disabled in kernel (upstream + # default is enabled, but some distributions may decide to change it). + # This is at least needed for RHEL 7.2 and earlier releases. + for proto in ip ip6; do + sudo sysctl -w net.bridge.bridge-nf-call-${proto}tables=1 + done +} + +function to_bool { + if [[ "$1" =~ (y|Y|yes|Yes|YES|true|True|TRUE|on|On|ON) ]]; then + true + elif [[ "$1" =~ (n|N|no|No|NO|false|False|FALSE|off|Off|OFF) ]]; then + false + else + die $LINENO "$1 was not a valid yaml boolean" + fi +} + +function is_ipxe_enabled { + flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_ironic_ipxe)" + to_bool "$flag" +} + +function is_cinder_enabled { + flag="$(run_kayobe configuration dump --host controllers[0] --var-name kolla_enable_cinder)" + to_bool "$flag" +} + +function configure_iptables { + # NOTE(wszumski): adapted from the ironic devstack plugin, see: + # https://github.com/openstack/ironic/blob/36e87dc5b472d79470b783fbba9ce396e3cbb96e/devstack/lib/ironic#L2132 + + set -eu + environment_setup + + # FIXME(wszumski): set these variables with values from kayobe-config + HOST_IP='192.168.33.3' + INTERNAL_VIP='192.168.33.2' + IRONIC_TFTPSERVER_IP="$HOST_IP" + IRONIC_SERVICE_PORT=6385 + IRONIC_INSPECTOR_PORT=5050 + IRONIC_HTTP_SERVER="$INTERNAL_VIP" + GLANCE_SERVICE_PORT=9292 + IRONIC_HTTP_PORT=8089 + ISCSI_SERVICE_PORT=3260 + + # enable tftp natting for allowing connections to HOST_IP's tftp server + if ! running_in_container; then + sudo modprobe nf_conntrack_tftp + sudo modprobe nf_nat_tftp + + enable_kernel_bridge_firewall + else + die_if_module_not_loaded nf_conntrack_tftp + die_if_module_not_loaded nf_nat_tftp + fi + # explicitly allow DHCP - packets are occasionally being dropped here + sudo iptables -I INPUT -p udp --dport 67:68 --sport 67:68 -j ACCEPT || true + # nodes boot from TFTP and callback to the API server listening on $HOST_IP + sudo iptables -I INPUT -d $IRONIC_TFTPSERVER_IP -p udp --dport 69 -j ACCEPT || true + + sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true + # open ironic API on baremetal network + sudo iptables -I INPUT -d $IRONIC_HTTP_SERVER -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true + # allow IPA to connect to ironic API + sudo iptables -I FORWARD -p tcp --dport $IRONIC_SERVICE_PORT -j ACCEPT || true + # allow IPA to connect to ironic inspector + sudo iptables -I FORWARD -p tcp --dport $IRONIC_INSPECTOR_PORT -j ACCEPT || true + + # agent ramdisk gets instance image from swift + sudo iptables -I INPUT -d $INTERNAL_VIP -p tcp --dport ${SWIFT_DEFAULT_BIND_PORT:-8080} -j ACCEPT || true + sudo iptables -I INPUT -d $INTERNAL_VIP -p tcp --dport $GLANCE_SERVICE_PORT -j ACCEPT || true + + if is_ipxe_enabled; then + sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $IRONIC_HTTP_PORT -j ACCEPT || true + fi + + if is_cinder_enabled; then + sudo iptables -I INPUT -d $HOST_IP -p tcp --dport $ISCSI_SERVICE_PORT -j ACCEPT || true + fi } diff --git a/dev/install.sh b/dev/install.sh index 1637edf73..efb4fd61a 100755 --- a/dev/install.sh +++ b/dev/install.sh @@ -16,7 +16,7 @@ function main { KAYOBE_CONFIG_REQUIRED=0 config_init install_dependencies - install_venv + install_kayobe_venv } main diff --git a/dev/overcloud-test-baremetal.sh b/dev/overcloud-test-baremetal.sh new file mode 100755 index 000000000..3c99055e4 --- /dev/null +++ b/dev/overcloud-test-baremetal.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -eu +set -o pipefail + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + config_init + overcloud_test bm1 test-rc provision-net +} + +main diff --git a/dev/overcloud-test.sh b/dev/overcloud-test-vm.sh similarity index 80% rename from dev/overcloud-test.sh rename to dev/overcloud-test-vm.sh index b27e2fab8..44c37d8ca 100755 --- a/dev/overcloud-test.sh +++ b/dev/overcloud-test-vm.sh @@ -10,7 +10,7 @@ source "${PARENT}/functions" function main { config_init - overcloud_test + overcloud_test vm1 m1.tiny demo-net } main diff --git a/dev/ovs-vsctl b/dev/ovs-vsctl new file mode 100755 index 000000000..6136254d4 --- /dev/null +++ b/dev/ovs-vsctl @@ -0,0 +1,4 @@ +#!/usr/bin/env bash + + +sudo docker exec openvswitch_vswitchd ovs-vsctl "$@" diff --git a/dev/tenks-deploy-config.yml b/dev/tenks-deploy-config.yml new file mode 100644 index 000000000..4bda1ae28 --- /dev/null +++ b/dev/tenks-deploy-config.yml @@ -0,0 +1,36 @@ +--- +# This file holds the config given to Tenks when running `tenks-deploy.sh`. It +# assumes the existence of the bridge `breth1`. + +node_types: + type0: + memory_mb: 1024 + vcpus: 1 + volumes: + # There is a minimum disk space capacity requirement of 4GiB when using Ironic Python Agent: + # https://github.com/openstack/ironic-python-agent/blob/master/ironic_python_agent/utils.py#L290 + - capacity: 4GiB + physical_networks: + - physnet1 + console_log_enabled: true + +specs: + - type: type0 + count: 2 + ironic_config: + resource_class: test-rc + network_interface: flat + +nova_flavors: + - resource_class: test-rc + node_type: type0 + +physnet_mappings: + physnet1: breth1 + +deploy_kernel: ipa.vmlinuz +deploy_ramdisk: ipa.initramfs + +# Use a custom socket path to avoid a conflict with the nova_libvirt kolla +# container which bind mounts /var/run/libvirt. +libvirt_custom_socket_path: /var/run/libvirt-tenks diff --git a/dev/tenks-deploy.sh b/dev/tenks-deploy.sh new file mode 100755 index 000000000..506d4c6ed --- /dev/null +++ b/dev/tenks-deploy.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Simple script to configure and deploy a Tenks cluster. This should be +# executed from within the VM. Arguments: +# $1: The path to the Tenks repo. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + if [ -z ${1+x} ]; then + echo "Usage: $0 " + return 1 + fi + tenks_path="$1" + + config_init + tenks_deploy "$tenks_path" +} + +main "$@" diff --git a/playbooks/kayobe-overcloud-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-base/overrides.yml.j2 index 42edfffeb..0f80b73ad 100644 --- a/playbooks/kayobe-overcloud-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-base/overrides.yml.j2 @@ -8,6 +8,7 @@ kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].che kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}" kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}" kolla_upper_constraints_file: "/tmp/upper-constraints.txt" +kolla_openstack_logging_debug: True # Use the CI infra's PyPI mirror. pip_local_mirror: true diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml index 8e2811f62..33f8b4cd3 100644 --- a/playbooks/kayobe-overcloud-base/pre.yml +++ b/playbooks/kayobe-overcloud-base/pre.yml @@ -43,3 +43,8 @@ shell: cmd: dev/install.sh > {{ logs_dir }}/ansible/install chdir: "{{ kayobe_src_dir }}" + + - name: Configure the firewall + shell: + cmd: dev/configure-firewall.sh + chdir: "{{ kayobe_src_dir }}" diff --git a/playbooks/kayobe-overcloud-base/run.yml b/playbooks/kayobe-overcloud-base/run.yml index 8752409ea..bea0b13c6 100644 --- a/playbooks/kayobe-overcloud-base/run.yml +++ b/playbooks/kayobe-overcloud-base/run.yml @@ -4,6 +4,7 @@ kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}" kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" logs_dir: "/tmp/logs" + tenks_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/tenks'].src_dir }}" environment: KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" tasks: @@ -12,7 +13,21 @@ cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy chdir: "{{ kayobe_src_dir }}" - - name: Perform testing of the overcloud + - name: Ensure test Tenks cluster is deployed shell: - cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test + # Pass absolute source directory, since otherwise the `chdir` will + # cause this to fail. + cmd: dev/tenks-deploy.sh '{{ tenks_src_dir }}' > {{ logs_dir }}/ansible/tenks-deploy + chdir: "{{ kayobe_src_dir }}" + + - name: Perform testing of the virtualized machines + # We must do this before tenks-deploy as that will stop the nova_libvirt + # container + shell: + cmd: dev/overcloud-test-vm.sh > {{ logs_dir }}/ansible/overcloud-test-vm + chdir: "{{ kayobe_src_dir }}" + + - name: Perform testing of the baremetal machines + shell: + cmd: dev/overcloud-test-baremetal.sh > {{ logs_dir }}/ansible/overcloud-test-baremetal chdir: "{{ kayobe_src_dir }}" diff --git a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 index 4f91d4edd..383e97cd5 100644 --- a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 +++ b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 @@ -10,6 +10,7 @@ kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.opensta kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}" kolla_upper_constraints_file: "/tmp/upper-constraints.txt" {% endif %} +kolla_openstack_logging_debug: true # Use the CI infra's PyPI mirror. pip_local_mirror: true diff --git a/playbooks/kayobe-overcloud-upgrade-base/run.yml b/playbooks/kayobe-overcloud-upgrade-base/run.yml index 60011eb8b..766942cb5 100644 --- a/playbooks/kayobe-overcloud-upgrade-base/run.yml +++ b/playbooks/kayobe-overcloud-upgrade-base/run.yml @@ -6,6 +6,7 @@ previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}" previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" logs_dir: "/tmp/logs" + tenks_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/tenks'].src_dir }}" tasks: # Install the previous release of Kayobe, and use it to deploy a control @@ -17,10 +18,27 @@ cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade chdir: "{{ previous_kayobe_src_dir }}" + - name: Configure the firewall + # We run kayobe commands to determine variable values. So this must + # be run after install. + shell: + cmd: dev/configure-firewall.sh + # Does not exist in Rocky release, use script from later version. + chdir: "{{ kayobe_src_dir }}" + - name: Ensure overcloud is deployed shell: cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade chdir: "{{ previous_kayobe_src_dir }}" + + - name: Ensure test Tenks cluster is deployed + shell: + # Pass absolute source directory, since otherwise the `chdir` will + # cause this to fail. Don't use previous_kayobe_source_dir as tenks-deploy.sh + # does not exist there. + cmd: dev/tenks-deploy.sh '{{ tenks_src_dir }}' > {{ logs_dir }}/ansible/tenks-deploy + chdir: "{{ kayobe_src_dir }}" + environment: KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}" @@ -51,11 +69,16 @@ # Perform a smoke test against the previous release. - block: - - name: Perform testing of the overcloud prior to upgrade + - name: Perform testing of the VMs in the overcloud prior to upgrade shell: - cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-pre-upgrade + cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-vm-pre-upgrade chdir: "{{ previous_kayobe_src_dir }}" + - name: Perform testing of the baremetal machines in the overcloud prior to upgrade + shell: + cmd: dev/overcloud-test-baremetal.sh > {{ logs_dir }}/ansible/overcloud-test-bm-pre-upgrade + chdir: "{{ kayobe_src_dir }}" + # Upgrade Kayobe, and use it to perform an upgrade of the control plane. - name: Ensure overcloud is upgraded @@ -73,9 +96,15 @@ # Perform a smoke test against the upgraded current release. - - name: Perform testing of the upgraded overcloud + - name: Perform testing of VMs in the upgraded overcloud shell: - cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-post-upgrade + cmd: dev/overcloud-test-vm.sh > {{ logs_dir }}/ansible/overcloud-test-vm-post-upgrade chdir: "{{ kayobe_src_dir }}" + + - name: Perform testing of the baremetal machines in the upgraded overcloud + shell: + cmd: dev/overcloud-test-baremetal.sh > {{ logs_dir }}/ansible/overcloud-test-bm-post-upgrade + chdir: "{{ kayobe_src_dir }}" + environment: KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" diff --git a/playbooks/kayobe-seed-base/overrides.yml.j2 b/playbooks/kayobe-seed-base/overrides.yml.j2 index c9db6012a..34ad2462f 100644 --- a/playbooks/kayobe-seed-base/overrides.yml.j2 +++ b/playbooks/kayobe-seed-base/overrides.yml.j2 @@ -8,6 +8,7 @@ kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].che kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}" kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}" kolla_upper_constraints_file: "/tmp/upper-constraints.txt" +kolla_openstack_logging_debug: True # NOTE(mgoddard): We're using a cirros image, which doesn't require the # resolv.conf work around used for CentOS. diff --git a/roles/kayobe-diagnostics/files/get_logs.sh b/roles/kayobe-diagnostics/files/get_logs.sh index f3999ab17..a5916c37d 100644 --- a/roles/kayobe-diagnostics/files/get_logs.sh +++ b/roles/kayobe-diagnostics/files/get_logs.sh @@ -41,6 +41,8 @@ copy_logs() { ip address > ${LOG_DIR}/system_logs/ip-address.txt ip route > ${LOG_DIR}/system_logs/ip-route.txt + iptables-save > ${LOG_DIR}/system_logs/iptables.txt + if [ `command -v dpkg` ]; then dpkg -l > ${LOG_DIR}/system_logs/dpkg-l.txt fi diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 1a4d694f9..53361dbc7 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -64,6 +64,7 @@ override-checkout: stable/rocky - name: openstack/requirements override-checkout: stable/rocky + - name: openstack/tenks irrelevant-files: - ^.*\.rst$ - ^doc/.* @@ -99,6 +100,7 @@ override-checkout: stable/rocky - name: openstack/requirements override-checkout: stable/rocky + - name: openstack/tenks vars: # Name of the release to upgrade from. previous_release: queens