Test upgrades in CI

There is currently no coverage of upgrades in CI, which leaves us open
to regression in this infrequently tested but crucial area. This change
adds the required scripts and Zuul configuration.

A control plane is first deployed using the Kayobe stable/pike branch
and associated default configuration. The control plane is tested by
deploying then deleting a server instance. An upgrade to Queens is
performed, using the Kayobe master branch, or code in review if
applicable. The upgraded control plane is tested by deploying then
deleting a second server instance.

A workaround was required to restart the nova_compute service after the
upgrade, since the SIGHUP sent to it by Kolla Ansible during upgrade
appeared to be putting it into a degraded state.

A future improvement to this test could be to leave a server instance
running during the upgrade.

Change-Id: I0e595524e39d1131fe3ec6aaf2aeec3ff3d6a536
Story: 2003472
Task: 24732

dev/functions

@@ -104,6 +104,19 @@ function install_venv {
     fi
 }
 
+function upgrade_venv {
+    echo "Upgrading kayobe virtual environment in ${KAYOBE_VENV_PATH}"
+    virtualenv "${KAYOBE_VENV_PATH}"
+    # NOTE: Virtualenv's activate and deactivate scripts reference an
+    # unbound variable.
+    set +u
+    source "${KAYOBE_VENV_PATH}/bin/activate"
+    pip install -U pip
+    pip install -U "${KAYOBE_SOURCE_PATH}"
+    deactivate
+    set -u
+}
+
 # Deployment
 
 function is_deploy_image_built_locally {
@@ -183,7 +196,18 @@ function overcloud_deploy {
     environment_setup
 
     echo "Bootstrapping the Ansible control host"
-    run_kayobe control host bootstrap
+    for i in $(seq 1 3); do
+        if run_kayobe control host bootstrap; then
+            chb_success=1
+            break
+        fi
+        echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
+    done
+    if [[ -z ${chb_success+x} ]]; then
+        echo "Failed to bootstrap control host"
+        exit 1
+    fi
+    echo "Bootstrapped control host after $i attempts"
 
     echo "Configuring the controller host"
     run_kayobe overcloud host configure
@@ -216,6 +240,61 @@ function overcloud_deploy {
     echo "Control plane deployment complete"
 }
 
+function overcloud_upgrade {
+    # Upgrade a kayobe control plane.
+    echo "Upgrading a kayobe development environment. This consists of a "
+    echo "single node OpenStack control plane."
+
+    echo "Upgrading Kayobe"
+    upgrade_venv
+
+    environment_setup
+
+    echo "Upgrading the Ansible control host"
+    for i in $(seq 1 3); do
+        if run_kayobe control host upgrade; then
+            chu_success=1
+            break
+        fi
+        echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying"
+    done
+    if [[ -z ${chu_success+x} ]]; then
+        echo "Failed to upgrade control host"
+        exit 1
+    fi
+    echo "Upgraded control host after $i attempts"
+
+    echo "Upgrading the controller host"
+    run_kayobe overcloud host upgrade
+
+    if is_deploy_image_built_locally; then
+        echo "Building overcloud deployment images"
+        run_kayobe overcloud deployment image build --force-rebuild
+    else
+        echo "Not building overcloud deployment images"
+    fi
+
+    if [[ ${KAYOBE_OVERCLOUD_CONTAINER_IMAGE_BUILD} = 1 ]]; then
+        echo "Building overcloud container images"
+        run_kayobe overcloud container image build
+    else
+        echo "Pulling overcloud container images"
+        run_kayobe overcloud container image pull
+    fi
+
+    echo "Saving overcloud service configuration"
+    if ! run_kayobe overcloud service configuration save; then
+        # NOTE(mgoddard): This fails in CI due to a memory error while copying
+        # the IPA deployment images.
+        echo "FIXME: Saving service configuration failed. Ignoring for now"
+    fi
+
+    echo "Deploying containerised overcloud services"
+    run_kayobe overcloud service upgrade
+
+    echo "Control plane upgrade complete"
+}
+
 function overcloud_test {
     # Perform a simple smoke test against the cloud.
     echo "Performing a simple smoke test"
@@ -224,9 +303,13 @@ function overcloud_test {
 
     pip install python-openstackclient
 
-    echo "Running kolla-ansible init-runonce"
     source "${KOLLA_CONFIG_PATH:-/etc/kolla}/admin-openrc.sh"
-    ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce
+    if ! openstack image show cirros >/dev/null 2>&1; then
+        echo "Running kolla-ansible init-runonce"
+        ${KOLLA_VENV_PATH:-$HOME/kolla-venv}/share/kolla-ansible/init-runonce
+    else
+        echo "Not running kolla-ansible init-runonce - resources exist"
+    fi
 
     echo "Creating a VM"
     openstack server create --wait --image cirros --flavor m1.tiny --key-name mykey --network demo-net demo1

dev/overcloud-upgrade.sh

@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to upgrade a development environment for an OpenStack
+# controller in a Vagrant VM using kayobe.  This should be executed from within
+# the VM.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    overcloud_upgrade
+}
+
+main

doc/source/development/automated.rst

@@ -93,6 +93,18 @@ plane::
 
 Upon successful completion of this script, the control plane will be active.
 
+The control plane can be tested by running the ``dev/overcloud-test.sh``
+script. This will run the ``init-runonce`` setup script provided by Kolla
+Ansible that registers images, networks, flavors etc. It will then deploy a
+virtual server instance, and delete it once it becomes active::
+
+    ./dev/overcloud-test.sh
+
+It is possible to test an upgrade by running the ``dev/overcloud-upgrade.sh``
+script::
+
+    ./dev/overcloud-upgrade.sh
+
 Seed Hypervisor
 ===============
 

playbooks/kayobe-overcloud-base/pre.yml

@@ -10,28 +10,9 @@
     - role: kayobe-diagnostics
       kayobe_diagnostics_phase: "pre"
       kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - role: kayobe-ci-prep
   tasks:
-    - name: Install dbus for debian system
-      apt:
-        name: dbus
-      when:
-        - ansible_os_family == 'Debian'
-      become: true
-
-    - block:
-        # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
-        # but the configure-mirrors role used by Zuul disables epel. Since we
-        # install epel-release and expect epel to be enabled, enable it here.
-        - name: Ensure yum-utils is installed
-          yum:
-            name: yum-utils
-            state: installed
-
-        - name: Enable the EPEL yum repository
-          command: yum-config-manager --enable epel
-      when: ansible_os_family == 'RedHat'
-      become: true
-
     # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
     # the zuul and stack users.
     - name: Ensure upper-contraints.txt exists

playbooks/kayobe-overcloud-upgrade-base/globals.yml.j2

@@ -0,0 +1,10 @@
+---
+# The Queens kayobe configuration uses the fernet token provider. To avoid
+# issues during upgrade, set the provider to fernet in the Pike deployment
+# also.
+keystone_token_provider: 'fernet'
+
+# Most development environments will use nested virtualisation, and we can't
+# guarantee that nested KVM support is available. Use QEMU as a lowest common
+# denominator.
+nova_compute_virt_type: qemu

playbooks/kayobe-overcloud-upgrade-base/overrides.yml.j2

@@ -0,0 +1,16 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+{% if not previous_release | default(false) %}
+kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}"
+kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}"
+kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}"
+kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}"
+kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
+{% endif %}
+
+# Use the CI infra's PyPI mirror.
+pip_local_mirror: true
+pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"

playbooks/kayobe-overcloud-upgrade-base/post.yml

@@ -0,0 +1,8 @@
+---
+- hosts: all
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "post"
+      kayobe_diagnostics_log_dir: "/tmp/logs"
+      kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+      kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

playbooks/kayobe-overcloud-upgrade-base/pre.yml

@@ -0,0 +1,69 @@
+---
+- hosts: primary
+  vars:
+    logs_dir: "/tmp/logs"
+    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
+    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "pre"
+      kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - role: kayobe-ci-prep
+  tasks:
+    # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
+    # the zuul and stack users.
+    - name: Ensure upper-contraints.txt exists
+      copy:
+        src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
+        dest: "/tmp"
+        mode: 0644
+        remote_src: true
+
+    - name: Ensure previous kayobe directory exists
+      file:
+        path: "{{ previous_kayobe_src_dir }}"
+        state: directory
+
+    - name: Ensure previous kayobe repository is cloned
+      git:
+        repo: https://git.openstack.org/openstack/kayobe
+        dest: "{{ previous_kayobe_src_dir }}"
+        version: "stable/{{ previous_release | lower }}"
+
+    - name: Ensure previous kayobe-config directory exists
+      file:
+        path: "{{ previous_kayobe_config_src_dir }}"
+        state: directory
+
+    - name: Ensure kayobe-config repository is cloned
+      git:
+        repo: https://git.openstack.org/openstack/kayobe-config-dev
+        dest: "{{ previous_kayobe_config_src_dir }}"
+        version: "stable/{{ previous_release | lower }}"
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+      vars:
+        previous_release: true
+
+    - name: Ensure kayobe-config globals.yml config file exists
+      template:
+        src: globals.yml.j2
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/globals.yml"
+
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist with an IP address of 192.168.33.3.
+    - name: Ensure all-in-one network bridge interface exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add breth1 type bridge"
+        - "ip l set breth1 up"
+        - "ip a add 192.168.33.3/24 dev breth1"

playbooks/kayobe-overcloud-upgrade-base/run.yml

@@ -0,0 +1,83 @@
+---
+- hosts: primary
+  vars:
+    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
+    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+    logs_dir: "/tmp/logs"
+  tasks:
+
+    # Install the previous release of Kayobe, and use it to deploy a control
+    # plane.
+
+    - block:
+        - name: Ensure kayobe is installed
+          shell:
+            cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade
+            chdir: "{{ previous_kayobe_src_dir }}"
+
+        - name: Ensure overcloud is deployed
+          shell:
+            cmd: dev/overcloud-deploy.sh > {{ logs_dir }}/ansible/overcloud-deploy-pre-upgrade
+            chdir: "{{ previous_kayobe_src_dir }}"
+      environment:
+        KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
+
+    # Update the Kayobe configuration to the current release.
+
+    - name: Ensure kolla config directory exists
+      file:
+        path: "{{ kayobe_config_src_dir }}/etc/kolla"
+        state: directory
+
+    - name: Copy across relevant kayobe-config files
+      copy:
+        src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
+        dest: "{{ kayobe_config_src_dir }}/{{ item }}"
+        remote_src: true
+      with_items:
+        - etc/kayobe/kolla/passwords.yml
+        - etc/kolla/admin-openrc.sh
+        - etc/kolla/public-openrc.sh
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    # Perform a smoke test against the previous release.
+
+    - block:
+        - name: Perform testing of the overcloud prior to upgrade
+          shell:
+            cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-pre-upgrade
+            # NOTE(mgoddard): Currently need to use the new kayobe repo for
+            # testing, since the overcloud-test.sh script is not available in Pike.
+            chdir: "{{ kayobe_src_dir }}"
+
+        # Upgrade Kayobe, and use it to perform an upgrade of the control plane.
+
+        - name: Ensure overcloud is upgraded
+          shell:
+            cmd: dev/overcloud-upgrade.sh > {{ logs_dir }}/ansible/overcloud-upgrade
+            chdir: "{{ kayobe_src_dir }}"
+
+        # FIXME(mgoddard): The nova-compute service does not seem to be correctly
+        # handling the SIGHUP after being upgraded, leading to "In shutdown, no new
+        # events can be scheduled" errors when booting an instance.
+        - name: Workaround for SIGHUP issue - restart nova-compute service
+          shell:
+            cmd: docker restart nova_compute
+          become: true
+
+        # Perform a smoke test against the upgraded current release.
+
+        - name: Perform testing of the upgraded overcloud
+          shell:
+            cmd: dev/overcloud-test.sh > {{ logs_dir }}/ansible/overcloud-test-post-upgrade
+            chdir: "{{ kayobe_src_dir }}"
+      environment:
+        KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"

playbooks/kayobe-seed-base/pre.yml

@@ -10,28 +10,9 @@
     - role: kayobe-diagnostics
       kayobe_diagnostics_phase: "pre"
       kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - role: kayobe-ci-prep
   tasks:
-    - name: Install dbus for debian system
-      apt:
-        name: dbus
-      when:
-        - ansible_os_family == 'Debian'
-      become: true
-
-    - block:
-        # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
-        # but the configure-mirrors role used by Zuul disables epel. Since we
-        # install epel-release and expect epel to be enabled, enable it here.
-        - name: Ensure yum-utils is installed
-          yum:
-            name: yum-utils
-            state: installed
-
-        - name: Enable the EPEL yum repository
-          command: yum-config-manager --enable epel
-      when: ansible_os_family == 'RedHat'
-      become: true
-
     # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
     # the zuul and stack users.
     - name: Ensure upper-contraints.txt exists

roles/kayobe-ci-prep/tasks/main.yml

@@ -0,0 +1,21 @@
+---
+- name: Install dbus for debian system
+  apt:
+    name: dbus
+  when:
+    - ansible_os_family == 'Debian'
+  become: true
+
+- block:
+    # NOTE(mgoddard): The CentOS image used in CI has epel-release installed,
+    # but the configure-mirrors role used by Zuul disables epel. Since we
+    # install epel-release and expect epel to be enabled, enable it here.
+    - name: Ensure yum-utils is installed
+      yum:
+        name: yum-utils
+        state: installed
+
+    - name: Enable the EPEL yum repository
+      command: yum-config-manager --enable epel
+  when: ansible_os_family == 'RedHat'
+  become: true

roles/kayobe-diagnostics/files/get_logs.sh

@@ -34,6 +34,7 @@ copy_logs() {
 
     df -h > ${LOG_DIR}/system_logs/df.txt
     free  > ${LOG_DIR}/system_logs/free.txt
+    cat /etc/hosts  > ${LOG_DIR}/system_logs/hosts.txt
     parted -l > ${LOG_DIR}/system_logs/parted-l.txt
     mount > ${LOG_DIR}/system_logs/mount.txt
     env > ${LOG_DIR}/system_logs/env.txt

zuul.d/jobs.yaml

@@ -77,6 +77,44 @@
     parent: kayobe-overcloud-base
     nodeset: kayobe-centos
 
+- job:
+    name: kayobe-overcloud-upgrade-base
+    description: |
+      Base job for testing overcloud upgrades.
+
+      Configures the primary VM as an overcloud controller using the previous
+      OpenStack release, and upgrades it to the current release.
+    pre-run: playbooks/kayobe-overcloud-upgrade-base/pre.yml
+    run: playbooks/kayobe-overcloud-upgrade-base/run.yml
+    post-run: playbooks/kayobe-overcloud-upgrade-base/post.yml
+    attempts: 1
+    timeout: 7200
+    required-projects:
+      # Include kayobe to ensure other projects can use this job.
+      - name: openstack/kayobe
+      - name: openstack/kayobe-config-dev
+      - name: openstack/kolla
+        override-checkout: stable/queens
+      - name: openstack/kolla-ansible
+        override-checkout: stable/queens
+      - name: openstack/requirements
+        override-checkout: stable/queens
+    vars:
+      # Name of the release to upgrade from.
+      previous_release: pike
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+      - ^releasenotes/.*
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^tox.ini$
+
+- job:
+    name: kayobe-overcloud-upgrade-centos
+    parent: kayobe-overcloud-upgrade-base
+    nodeset: kayobe-centos
+
 - job:
     name: kayobe-seed-base
     description: |

zuul.d/project.yaml

@@ -15,6 +15,7 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos
+        - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos
 
     gate:
@@ -25,4 +26,5 @@
         - kayobe-tox-ansible
         - kayobe-tox-molecule
         - kayobe-overcloud-centos
+        - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos