From 6266312fa1e45c04114343664d415d6da2b3da7b Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Fri, 17 Aug 2018 09:20:43 +0100 Subject: [PATCH] Test upgrades in CI There is currently no coverage of upgrades in CI, which leaves us open to regression in this infrequently tested but crucial area. This change adds the required scripts and Zuul configuration. A control plane is first deployed using the Kayobe stable/pike branch and associated default configuration. The control plane is tested by deploying then deleting a server instance. An upgrade to Queens is performed, using the Kayobe master branch, or code in review if applicable. The upgraded control plane is tested by deploying then deleting a second server instance. A workaround was required to restart the nova_compute service after the upgrade, since the SIGHUP sent to it by Kolla Ansible during upgrade appeared to be putting it into a degraded state. A future improvement to this test could be to leave a server instance running during the upgrade. Change-Id: I0e595524e39d1131fe3ec6aaf2aeec3ff3d6a536 Story: 2003472 Task: 24732 --- dev/functions | 89 ++++++++++++++++++- dev/overcloud-upgrade.sh | 20 +++++ doc/source/development/automated.rst | 12 +++ playbooks/kayobe-overcloud-base/pre.yml | 23 +---- .../globals.yml.j2 | 10 +++ .../overrides.yml.j2 | 16 ++++ .../kayobe-overcloud-upgrade-base/post.yml | 8 ++ .../kayobe-overcloud-upgrade-base/pre.yml | 69 ++++++++++++++ .../kayobe-overcloud-upgrade-base/run.yml | 83 +++++++++++++++++ playbooks/kayobe-seed-base/pre.yml | 23 +---- roles/kayobe-ci-prep/tasks/main.yml | 21 +++++ roles/kayobe-diagnostics/files/get_logs.sh | 1 + zuul.d/jobs.yaml | 38 ++++++++ zuul.d/project.yaml | 2 + 14 files changed, 370 insertions(+), 45 deletions(-) create mode 100755 dev/overcloud-upgrade.sh create mode 100644 playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2 create mode 100644 playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 create mode 100644 playbooks/kayobe-overcloud-upgrade-base/post.yml create mode 100644 playbooks/kayobe-overcloud-upgrade-base/pre.yml create mode 100644 playbooks/kayobe-overcloud-upgrade-base/run.yml create mode 100644 roles/kayobe-ci-prep/tasks/main.yml diff --git a/dev/functions b/dev/functions index 18ec53a11..dee3a375a 100644 --- a/dev/functions +++ b/dev/functions @@ -104,6 +104,19 @@ function install_venv { fi } +function upgrade_venv { + echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}" + virtualenv "${KAYOBE_VENV_PATH}" + # NOTE: Virtualenv's activate and deactivate scripts reference an + # unbound variable. + set +u + source "${KAYOBE_VENV_PATH}/bin/activate" + pip install -U pip + pip install -U "${KAYOBE_SOURCE_PATH}" + deactivate + set -u +} + # Deployment function is_deploy_image_built_locally { @@ -183,7 +196,18 @@ function overcloud_deploy { environment_setup echo "Bootstrapping the Ansible control host" - run_kayobe control host bootstrap + for i in $(seq 1 3); do + if run_kayobe control host bootstrap; then + chb_success=1 + break + fi + echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying" + done + if [[ -z ${chb_success+x} ]]; then + echo "Failed to bootstrap control host" + exit 1 + fi + echo "Bootstrapped control host after $i attempts" echo "Configuring the controller host" run_kayobe overcloud host configure @@ -216,6 +240,61 @@ function overcloud_deploy { echo "Control plane deployment complete" } +function overcloud_upgrade { + # Upgrade a kayobe control plane. + echo "Upgrading a kayobe development environment. This consists of a " + echo "single node OpenStack control plane." + + echo "Upgrading Kayobe" + upgrade_venv + + environment_setup + + echo "Upgrading the Ansible control host" + for i in $(seq 1 3); do + if run_kayobe control host upgrade; then + chu_success=1 + break + fi + echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying" + done + if [[ -z ${chu_success+x} ]]; then + echo "Failed to upgrade control host" + exit 1 + fi + echo "Upgraded control host after $i attempts" + + echo "Upgrading the controller host" + run_kayobe overcloud host upgrade + + if is_deploy_image_built_locally; then + echo "Building overcloud deployment images" + run_kayobe overcloud deployment image build --force-rebuild + else + echo "Not building overcloud deployment images" + fi + + if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then + echo "Building overcloud container images" + run_kayobe overcloud container image build + else + echo "Pulling overcloud container images" + run_kayobe overcloud container image pull + fi + + echo "Saving overcloud service configuration" + if ! run_kayobe overcloud service configuration save; then + # NOTE(mgoddard): This fails in CI due to a memory error while copying + # the IPA deployment images. + echo "FIXME: Saving service configuration failed. Ignoring for now" + fi + + echo "Deploying containerised overcloud services" + run_kayobe overcloud service upgrade + + echo "Control plane upgrade complete" +} + function overcloud_test { # Perform a simple smoke test against the cloud. echo "Performing a simple smoke test" @@ -224,9 +303,13 @@ function overcloud_test { pip install python-openstackclient - echo "Running kolla-ansible init-runonce" source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh" - ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce + if ! openstack image show cirros >/dev/null 2>&1; then + echo "Running kolla-ansible init-runonce" + ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce + else + echo "Not running kolla-ansible init-runonce - resources exist" + fi echo "Creating a VM" openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net demo1 diff --git a/dev/overcloud-upgrade.sh b/dev/overcloud-upgrade.sh new file mode 100755 index 000000000..cbb15855b --- /dev/null +++ b/dev/overcloud-upgrade.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -eu +set -o pipefail + +# Simple script to upgrade a development environment for an OpenStack +# controller in a Vagrant VM using kayobe. This should be executed from within +# the VM. + +PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +source "${PARENT}/functions" + + +function main { + config_init + overcloud_upgrade +} + +main diff --git a/doc/source/development/automated.rst b/doc/source/development/automated.rst index b3bbf6243..59e6bf9dd 100644 --- a/doc/source/development/automated.rst +++ b/doc/source/development/automated.rst @@ -93,6 +93,18 @@ plane:: Upon successful completion of this script, the control plane will be active. +The control plane can be tested by running the ``dev/overcloud-test.sh`` +script. This will run the ``init-runonce`` setup script provided by Kolla +Ansible that registers images, networks, flavors etc. It will then deploy a +virtual server instance, and delete it once it becomes active:: + + ./dev/overcloud-test.sh + +It is possible to test an upgrade by running the ``dev/overcloud-upgrade.sh`` +script:: + + ./dev/overcloud-upgrade.sh + Seed Hypervisor =============== diff --git a/playbooks/kayobe-overcloud-base/pre.yml b/playbooks/kayobe-overcloud-base/pre.yml index bed29c078..8e2811f62 100644 --- a/playbooks/kayobe-overcloud-base/pre.yml +++ b/playbooks/kayobe-overcloud-base/pre.yml @@ -10,28 +10,9 @@ - role: kayobe-diagnostics kayobe_diagnostics_phase: "pre" kayobe_diagnostics_log_dir: "{{ logs_dir }}" + + - role: kayobe-ci-prep tasks: - - name: Install dbus for debian system - apt: - name: dbus - when: - - ansible_os_family == 'Debian' - become: true - - - block: - # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, - # but the configure-mirrors role used by Zuul disables epel. Since we - # install epel-release and expect epel to be enabled, enable it here. - - name: Ensure yum-utils is installed - yum: - name: yum-utils - state: installed - - - name: Enable the EPEL yum repository - command: yum-config-manager --enable epel - when: ansible_os_family == 'RedHat' - become: true - # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both # the zuul and stack users. - name: Ensure upper-contraints.txt exists diff --git a/playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2 new file mode 100644 index 000000000..f7201a37a --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2 @@ -0,0 +1,10 @@ +--- +# The Queens kayobe configuration uses the fernet token provider. To avoid +# issues during upgrade, set the provider to fernet in the Pike deployment +# also. +keystone_token_provider: 'fernet' + +# Most development environments will use nested virtualisation, and we can't +# guarantee that nested KVM support is available. Use QEMU as a lowest common +# denominator. +nova_compute_virt_type: qemu \ No newline at end of file diff --git a/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 new file mode 100644 index 000000000..4f91d4edd --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2 @@ -0,0 +1,16 @@ +--- +# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as +# Ansible is run directly on the controller. +disable_selinux_do_reboot: false + +{% if not previous_release | default(false) %} +kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}" +kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}" +kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}" +kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}" +kolla_upper_constraints_file: "/tmp/upper-constraints.txt" +{% endif %} + +# Use the CI infra's PyPI mirror. +pip_local_mirror: true +pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple" diff --git a/playbooks/kayobe-overcloud-upgrade-base/post.yml b/playbooks/kayobe-overcloud-upgrade-base/post.yml new file mode 100644 index 000000000..31dccb283 --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/post.yml @@ -0,0 +1,8 @@ +--- +- hosts: all + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "post" + kayobe_diagnostics_log_dir: "/tmp/logs" + kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" diff --git a/playbooks/kayobe-overcloud-upgrade-base/pre.yml b/playbooks/kayobe-overcloud-upgrade-base/pre.yml new file mode 100644 index 000000000..7e78ea0f7 --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/pre.yml @@ -0,0 +1,69 @@ +--- +- hosts: primary + vars: + logs_dir: "/tmp/logs" + kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}" + kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}" + previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" + roles: + - role: kayobe-diagnostics + kayobe_diagnostics_phase: "pre" + kayobe_diagnostics_log_dir: "{{ logs_dir }}" + + - role: kayobe-ci-prep + tasks: + # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both + # the zuul and stack users. + - name: Ensure upper-contraints.txt exists + copy: + src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}" + dest: "/tmp" + mode: 0644 + remote_src: true + + - name: Ensure previous kayobe directory exists + file: + path: "{{ previous_kayobe_src_dir }}" + state: directory + + - name: Ensure previous kayobe repository is cloned + git: + repo: https://git.openstack.org/openstack/kayobe + dest: "{{ previous_kayobe_src_dir }}" + version: "stable/{{ previous_release | lower }}" + + - name: Ensure previous kayobe-config directory exists + file: + path: "{{ previous_kayobe_config_src_dir }}" + state: directory + + - name: Ensure kayobe-config repository is cloned + git: + repo: https://git.openstack.org/openstack/kayobe-config-dev + dest: "{{ previous_kayobe_config_src_dir }}" + version: "stable/{{ previous_release | lower }}" + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + vars: + previous_release: true + + - name: Ensure kayobe-config globals.yml config file exists + template: + src: globals.yml.j2 + dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml" + + # NOTE(mgoddard): The kayobe dev config by default expects a bridge - + # breth1 - to exist with an IP address of 192.168.33.3. + - name: Ensure all-in-one network bridge interface exists + command: "{{ item }}" + become: true + with_items: + - "ip l add breth1 type bridge" + - "ip l set breth1 up" + - "ip a add 192.168.33.3/24 dev breth1" diff --git a/playbooks/kayobe-overcloud-upgrade-base/run.yml b/playbooks/kayobe-overcloud-upgrade-base/run.yml new file mode 100644 index 000000000..8a3413f55 --- /dev/null +++ b/playbooks/kayobe-overcloud-upgrade-base/run.yml @@ -0,0 +1,83 @@ +--- +- hosts: primary + vars: + kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}" + kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}" + previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}" + previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}" + logs_dir: "/tmp/logs" + tasks: + + # Install the previous release of Kayobe, and use it to deploy a control + # plane. + + - block: + - name: Ensure kayobe is installed + shell: + cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade + chdir: "{{ previous_kayobe_src_dir }}" + + - name: Ensure overcloud is deployed + shell: + cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade + chdir: "{{ previous_kayobe_src_dir }}" + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}" + + # Update the Kayobe configuration to the current release. + + - name: Ensure kolla config directory exists + file: + path: "{{ kayobe_config_src_dir }}/etc/kolla" + state: directory + + - name: Copy across relevant kayobe-config files + copy: + src: "{{ previous_kayobe_config_src_dir }}/{{ item }}" + dest: "{{ kayobe_config_src_dir }}/{{ item }}" + remote_src: true + with_items: + - etc/kayobe/kolla/passwords.yml + - etc/kolla/admin-openrc.sh + - etc/kolla/public-openrc.sh + + # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes + # precedence over the standard config files. + - name: Ensure kayobe-config override config file exists + template: + src: overrides.yml.j2 + dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml" + + # Perform a smoke test against the previous release. + + - block: + - name: Perform testing of the overcloud prior to upgrade + shell: + cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-pre-upgrade + # NOTE(mgoddard): Currently need to use the new kayobe repo for + # testing, since the overcloud-test.sh script is not available in Pike. + chdir: "{{ kayobe_src_dir }}" + + # Upgrade Kayobe, and use it to perform an upgrade of the control plane. + + - name: Ensure overcloud is upgraded + shell: + cmd: dev/overcloud-upgrade.sh > {{ logs_dir }}/ansible/overcloud-upgrade + chdir: "{{ kayobe_src_dir }}" + + # FIXME(mgoddard): The nova-compute service does not seem to be correctly + # handling the SIGHUP after being upgraded, leading to "In shutdown, no new + # events can be scheduled" errors when booting an instance. + - name: Workaround for SIGHUP issue - restart nova-compute service + shell: + cmd: docker restart nova_compute + become: true + + # Perform a smoke test against the upgraded current release. + + - name: Perform testing of the upgraded overcloud + shell: + cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-post-upgrade + chdir: "{{ kayobe_src_dir }}" + environment: + KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}" diff --git a/playbooks/kayobe-seed-base/pre.yml b/playbooks/kayobe-seed-base/pre.yml index e7a6d3dd8..3fd9bbc1f 100644 --- a/playbooks/kayobe-seed-base/pre.yml +++ b/playbooks/kayobe-seed-base/pre.yml @@ -10,28 +10,9 @@ - role: kayobe-diagnostics kayobe_diagnostics_phase: "pre" kayobe_diagnostics_log_dir: "{{ logs_dir }}" + + - role: kayobe-ci-prep tasks: - - name: Install dbus for debian system - apt: - name: dbus - when: - - ansible_os_family == 'Debian' - become: true - - - block: - # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, - # but the configure-mirrors role used by Zuul disables epel. Since we - # install epel-release and expect epel to be enabled, enable it here. - - name: Ensure yum-utils is installed - yum: - name: yum-utils - state: installed - - - name: Enable the EPEL yum repository - command: yum-config-manager --enable epel - when: ansible_os_family == 'RedHat' - become: true - # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both # the zuul and stack users. - name: Ensure upper-contraints.txt exists diff --git a/roles/kayobe-ci-prep/tasks/main.yml b/roles/kayobe-ci-prep/tasks/main.yml new file mode 100644 index 000000000..cfe5c4c2a --- /dev/null +++ b/roles/kayobe-ci-prep/tasks/main.yml @@ -0,0 +1,21 @@ +--- +- name: Install dbus for debian system + apt: + name: dbus + when: + - ansible_os_family == 'Debian' + become: true + +- block: + # NOTE(mgoddard): The CentOS image used in CI has epel-release installed, + # but the configure-mirrors role used by Zuul disables epel. Since we + # install epel-release and expect epel to be enabled, enable it here. + - name: Ensure yum-utils is installed + yum: + name: yum-utils + state: installed + + - name: Enable the EPEL yum repository + command: yum-config-manager --enable epel + when: ansible_os_family == 'RedHat' + become: true diff --git a/roles/kayobe-diagnostics/files/get_logs.sh b/roles/kayobe-diagnostics/files/get_logs.sh index a3f797f37..f3999ab17 100644 --- a/roles/kayobe-diagnostics/files/get_logs.sh +++ b/roles/kayobe-diagnostics/files/get_logs.sh @@ -34,6 +34,7 @@ copy_logs() { df -h > ${LOG_DIR}/system_logs/df.txt free > ${LOG_DIR}/system_logs/free.txt + cat /etc/hosts > ${LOG_DIR}/system_logs/hosts.txt parted -l > ${LOG_DIR}/system_logs/parted-l.txt mount > ${LOG_DIR}/system_logs/mount.txt env > ${LOG_DIR}/system_logs/env.txt diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 081a6a8de..30670518c 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -77,6 +77,44 @@ parent: kayobe-overcloud-base nodeset: kayobe-centos +- job: + name: kayobe-overcloud-upgrade-base + description: | + Base job for testing overcloud upgrades. + + Configures the primary VM as an overcloud controller using the previous + OpenStack release, and upgrades it to the current release. + pre-run: playbooks/kayobe-overcloud-upgrade-base/pre.yml + run: playbooks/kayobe-overcloud-upgrade-base/run.yml + post-run: playbooks/kayobe-overcloud-upgrade-base/post.yml + attempts: 1 + timeout: 7200 + required-projects: + # Include kayobe to ensure other projects can use this job. + - name: openstack/kayobe + - name: openstack/kayobe-config-dev + - name: openstack/kolla + override-checkout: stable/queens + - name: openstack/kolla-ansible + override-checkout: stable/queens + - name: openstack/requirements + override-checkout: stable/queens + vars: + # Name of the release to upgrade from. + previous_release: pike + irrelevant-files: + - ^.*\.rst$ + - ^doc/.* + - ^releasenotes/.* + - ^setup.cfg$ + - ^tools/.*$ + - ^tox.ini$ + +- job: + name: kayobe-overcloud-upgrade-centos + parent: kayobe-overcloud-upgrade-base + nodeset: kayobe-centos + - job: name: kayobe-seed-base description: | diff --git a/zuul.d/project.yaml b/zuul.d/project.yaml index 59a06e9b6..b9a69e8cd 100644 --- a/zuul.d/project.yaml +++ b/zuul.d/project.yaml @@ -15,6 +15,7 @@ - kayobe-tox-ansible - kayobe-tox-molecule - kayobe-overcloud-centos + - kayobe-overcloud-upgrade-centos - kayobe-seed-centos gate: @@ -25,4 +26,5 @@ - kayobe-tox-ansible - kayobe-tox-molecule - kayobe-overcloud-centos + - kayobe-overcloud-upgrade-centos - kayobe-seed-centos