Test upgrading seed services in CI

Adds the kayobe-seed-upgrade-centos job, which performs an upgrade of
the seed services from the previous release to the current release.

Change-Id: Ia3eb39cf81cb3618fd94c4456bd576b52098c946
Story: 2004308
Task: 27873

dev/functions

@@ -153,6 +153,38 @@ function run_kayobe {
     kayobe ${KAYOBE_EXTRA_ARGS} $*
 }
 
+function control_host_bootstrap {
+    echo "Bootstrapping the Ansible control host"
+    for i in $(seq 1 3); do
+        if run_kayobe control host bootstrap; then
+            chb_success=1
+            break
+        fi
+        echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
+    done
+    if [[ -z ${chb_success+x} ]]; then
+        die $LINENO "Failed to bootstrap control host"
+        exit 1
+    fi
+    echo "Bootstrapped control host after $i attempts"
+}
+
+function control_host_upgrade {
+    echo "Upgrading the Ansible control host"
+    for i in $(seq 1 3); do
+        if run_kayobe control host upgrade; then
+            chu_success=1
+            break
+        fi
+        echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying"
+    done
+    if [[ -z ${chu_success+x} ]]; then
+        die $LINENO "Failed to upgrade control host"
+        exit 1
+    fi
+    echo "Upgraded control host after $i attempts"
+}
+
 function seed_hypervisor_deploy {
     # Deploy a seed hypervisor.
     environment_setup
@@ -168,8 +200,7 @@ function seed_deploy {
     # Deploy a kayobe seed in a VM.
     environment_setup
 
-    echo "Bootstrapping the Ansible control host"
-    run_kayobe control host bootstrap
+    control_host_bootstrap
 
     if [[ ${KAYOBE_SEED_VM_PROVISION} = 1 ]]; then
         echo "Provisioning the seed VM"
@@ -179,7 +210,7 @@ function seed_deploy {
     echo "Configuring the seed host"
     run_kayobe seed host configure
 
-    # Note: This must currently be before host configure, because host
+    # Note: This must currently be done before host configure, because host
     # configure runs kolla-ansible.yml, which validates the presence of the
     # built deploy images.
     if is_deploy_image_built_locally; then
@@ -201,6 +232,37 @@ function seed_deploy {
     run_kayobe seed service deploy
 }
 
+function seed_upgrade {
+    # Upgrade a kayobe seed in a VM.
+    echo "Upgrading Kayobe"
+    upgrade_kayobe_venv
+
+    environment_setup
+
+    control_host_upgrade
+
+    echo "Upgrading the seed host"
+    run_kayobe seed host upgrade
+
+    if is_deploy_image_built_locally; then
+        echo "Building seed deployment images"
+        run_kayobe seed deployment image build --force-rebuild
+    else
+        echo "Not building seed deployment images"
+    fi
+
+    if [[ ${KAYOBE_SEED_CONTAINER_IMAGE_BUILD} = 1 ]]; then
+        echo "Building seed container images"
+        run_kayobe seed container image build
+    else
+        echo "Not pulling seed container images - no such command yet"
+        #run_kayobe seed container image pull
+    fi
+
+    echo "Upgrading containerised seed services"
+    run_kayobe seed service upgrade
+}
+
 function overcloud_deploy {
     # Deploy a kayobe control plane.
     echo "Deploying a kayobe development environment. This consists of a "
@@ -208,19 +270,7 @@ function overcloud_deploy {
 
     environment_setup
 
-    echo "Bootstrapping the Ansible control host"
-    for i in $(seq 1 3); do
-        if run_kayobe control host bootstrap; then
-            chb_success=1
-            break
-        fi
-        echo "Control host bootstrap failed - likely Ansible Galaxy flakiness. Retrying"
-    done
-    if [[ -z ${chb_success+x} ]]; then
-        echo "Failed to bootstrap control host"
-        exit 1
-    fi
-    echo "Bootstrapped control host after $i attempts"
+    control_host_bootstrap
 
     echo "Configuring the controller host"
     run_kayobe overcloud host configure
@@ -269,19 +319,7 @@ function overcloud_upgrade {
 
     environment_setup
 
-    echo "Upgrading the Ansible control host"
-    for i in $(seq 1 3); do
-        if run_kayobe control host upgrade; then
-            chu_success=1
-            break
-        fi
-        echo "Control host upgrade failed - likely Ansible Galaxy flakiness. Retrying"
-    done
-    if [[ -z ${chu_success+x} ]]; then
-        echo "Failed to upgrade control host"
-        exit 1
-    fi
-    echo "Upgraded control host after $i attempts"
+    control_host_upgrade
 
     echo "Upgrading the controller host"
     run_kayobe overcloud host upgrade

dev/seed-upgrade.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -eu
+set -o pipefail
+
+# Simple script to upgrade a development environment for a seed VM using
+# kayobe.  This should be executed from the hypervisor.
+
+PARENT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+source "${PARENT}/functions"
+
+
+function main {
+    config_init
+    seed_upgrade
+}
+
+main

doc/source/development/automated.rst

@@ -193,3 +193,8 @@ Upon successful completion of this script, the seed VM will be active.  The
 seed VM may be accessed via SSH as the ``stack`` user::
 
     ssh stack@192.168.33.5
+
+It is possible to test an upgrade by running the ``dev/seed-upgrade.sh``
+script::
+
+    ./dev/seed-upgrade.sh

playbooks/kayobe-seed-upgrade-base/bifrost-overrides.yml.j2

@@ -0,0 +1,7 @@
+---
+# Don't build an IPA deployment image, instead download upstream images.
+create_ipa_image: false
+download_ipa: true
+
+# Don't build a disk image. It takes time and can be unreliable.
+use_cirros: true

playbooks/kayobe-seed-upgrade-base/overrides.yml.j2

@@ -0,0 +1,33 @@
+---
+# NOTE(mgoddard): Don't reboot after disabling SELinux during CI testing, as
+# Ansible is run directly on the controller.
+disable_selinux_do_reboot: false
+
+{% if not is_previous_release | default(false) %}
+kolla_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla'].src_dir }}"
+kolla_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla'].checkout }}"
+kolla_ansible_source_url: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kolla-ansible'].src_dir }}"
+kolla_ansible_source_version: "{{ zuul.projects['git.openstack.org/openstack/kolla-ansible'].checkout }}"
+kolla_upper_constraints_file: "/tmp/upper-constraints.txt"
+{% endif %}
+
+# NOTE(mgoddard): We're using a cirros image, which doesn't require the
+# resolv.conf work around used for CentOS.
+overcloud_host_image_workaround_resolv_enabled: false
+
+# Use the CI infra's PyPI mirror.
+pip_local_mirror: true
+pip_index_url: "http://{{ zuul_site_mirror_fqdn }}/pypi/simple"
+
+{% if previous_release == "pike" %}
+# kayobe-config-dev in queens changed to use overlay by default. Specify
+# devicemapper explicitly to avoid changing.
+docker_storage_driver: devicemapper
+docker_storage_volume_thinpool_size: 45%VG
+
+# NOTE(mgoddard): Use a loopback-mounted LVM volume for docker storage since
+# the overlay driver doesn't work with the ansible template module until
+# ansible 2.4.0, and this is required by bifrost.
+seed_lvm_group_data_disks:
+  - /dev/loop0
+{% endif %}

playbooks/kayobe-seed-upgrade-base/post.yml

@@ -0,0 +1,9 @@
+---
+- hosts: all
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "post"
+      kayobe_diagnostics_log_dir: "/tmp/logs"
+      kayobe_diagnostics_config_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+      kayobe_diagnostics_previous_config_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+      kayobe_diagnostics_executor_log_dir: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}"

playbooks/kayobe-seed-upgrade-base/pre.yml

@@ -0,0 +1,85 @@
+---
+- hosts: primary
+  vars:
+    logs_dir: "/tmp/logs"
+    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+  roles:
+    - role: kayobe-diagnostics
+      kayobe_diagnostics_phase: "pre"
+      kayobe_diagnostics_log_dir: "{{ logs_dir }}"
+
+    - role: kayobe-ci-prep
+  tasks:
+    # NOTE(mgoddard): Copying upper constraints to somewhere accessible by both
+    # the zuul and stack users.
+    - name: Ensure upper-contraints.txt exists
+      copy:
+        src: "{{ zuul.projects['git.openstack.org/openstack/requirements'].src_dir ~ '/upper-constraints.txt' }}"
+        dest: "/tmp"
+        mode: 0644
+        remote_src: true
+
+    - name: Ensure previous kayobe directory exists
+      file:
+        path: "{{ previous_kayobe_src_dir }}"
+        state: directory
+
+    - name: Ensure previous kayobe repository is cloned
+      git:
+        repo: https://git.openstack.org/openstack/kayobe
+        dest: "{{ previous_kayobe_src_dir }}"
+        version: "stable/{{ previous_release | lower }}"
+
+    - name: Ensure previous kayobe-config directory exists
+      file:
+        path: "{{ previous_kayobe_config_src_dir }}"
+        state: directory
+
+    - name: Ensure kayobe-config repository is cloned
+      git:
+        repo: https://git.openstack.org/openstack/kayobe-config-dev
+        dest: "{{ previous_kayobe_config_src_dir }}"
+        version: "stable/{{ previous_release | lower }}"
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+      vars:
+        is_previous_release: true
+
+    - name: Ensure bifrost overrides directory exists
+      file:
+        path: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
+        state: "directory"
+
+    - name: Ensure bifrost overrides file exists
+      template:
+        src: bifrost-overrides.yml.j2
+        dest: "{{ previous_kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost/bifrost.yml"
+
+    - block:
+        # NOTE(mgoddard): Create a loopback device backed by a file for docker
+        # storage.  We do this since the overlay driver doesn't work with the
+        # ansible template module until ansible 2.4.0, and this is required by
+        # bifrost.
+        - name: Ensure a docker storage backing file exists
+          command: truncate -s 20G /tmp/docker-storage
+
+        - name: Ensure the docker storage loopback device is created
+          command: losetup /dev/loop0 /tmp/docker-storage
+          become: true
+      when: previous_release == "pike"
+
+    # NOTE(mgoddard): The kayobe dev config by default expects a bridge -
+    # breth1 - to exist on the seed with an IP address of 192.168.33.5.
+    - name: Ensure all-in-one network bridge interface exists
+      command: "{{ item }}"
+      become: true
+      with_items:
+        - "ip l add breth1 type bridge"
+        - "ip l set breth1 up"
+        - "ip a add 192.168.33.5/24 dev breth1"

playbooks/kayobe-seed-upgrade-base/run.yml

@@ -0,0 +1,64 @@
+---
+- hosts: primary
+  vars:
+    kayobe_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe'].src_dir }}"
+    kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/' ~ zuul.projects['git.openstack.org/openstack/kayobe-config-dev'].src_dir }}"
+    previous_kayobe_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe' }}"
+    previous_kayobe_config_src_dir: "{{ ansible_env.PWD ~ '/previous/kayobe-config' }}"
+    logs_dir: "/tmp/logs"
+  tasks:
+
+    # Install the previous release of Kayobe, and use it to deploy a control
+    # plane.
+
+    - block:
+        - name: Ensure kayobe is installed
+          shell:
+            cmd: dev/install.sh > {{ logs_dir }}/ansible/install-pre-upgrade
+            chdir: "{{ previous_kayobe_src_dir }}"
+
+        - name: Ensure seed is deployed
+          shell:
+            cmd: dev/seed-deploy.sh > {{ logs_dir }}/ansible/seed-deploy-pre-upgrade
+            chdir: "{{ previous_kayobe_src_dir }}"
+      environment:
+        KAYOBE_CONFIG_SOURCE_PATH: "{{ previous_kayobe_config_src_dir }}"
+        # Don't provision a seed VM - use the Zuul VM as the seed host.
+        KAYOBE_SEED_VM_PROVISION: 0
+
+    # Update the Kayobe configuration to the current release.
+
+    - name: Ensure bifrost overrides directory exists
+      file:
+        path: "{{ kayobe_config_src_dir }}/etc/kayobe/kolla/config/bifrost"
+        state: "directory"
+
+    - name: Copy across relevant kayobe-config files
+      copy:
+        src: "{{ previous_kayobe_config_src_dir }}/{{ item }}"
+        dest: "{{ kayobe_config_src_dir }}/{{ item }}"
+        remote_src: true
+      with_items:
+        - etc/kayobe/kolla/passwords.yml
+        - etc/kayobe/kolla/config/bifrost/bifrost.yml
+
+    # NOTE(mgoddard): Use the name zz-overrides.yml to ensure this takes
+    # precedence over the standard config files.
+    - name: Ensure kayobe-config override config file exists
+      template:
+        src: overrides.yml.j2
+        dest: "{{ kayobe_config_src_dir }}/etc/kayobe/zz-overrides.yml"
+
+    # TODO(mgoddard): Perform a smoke test against the previous release.
+
+    - block:
+        # Upgrade Kayobe, and use it to perform an upgrade of the seed.
+
+        - name: Ensure seed is upgraded
+          shell:
+            cmd: dev/seed-upgrade.sh > {{ logs_dir }}/ansible/seed-upgrade
+            chdir: "{{ kayobe_src_dir }}"
+
+        # TODO(mgoddard): Perform a smoke test against the upgraded current release.
+      environment:
+        KAYOBE_CONFIG_SOURCE_PATH: "{{ kayobe_config_src_dir }}"

zuul.d/jobs.yaml

@@ -150,3 +150,41 @@
     name: kayobe-seed-centos
     parent: kayobe-seed-base
     nodeset: kayobe-centos
+
+- job:
+    name: kayobe-seed-upgrade-base
+    description: |
+      Base job for testing seed upgrades.
+
+      Configures the primary VM as a seed using the previous OpenStack release,
+      and upgrades it to the current release.
+    pre-run: playbooks/kayobe-seed-upgrade-base/pre.yml
+    run: playbooks/kayobe-seed-upgrade-base/run.yml
+    post-run: playbooks/kayobe-seed-upgrade-base/post.yml
+    attempts: 1
+    timeout: 5400
+    required-projects:
+      # Include kayobe to ensure other projects can use this job.
+      - name: openstack/kayobe
+      - name: openstack/kayobe-config-dev
+      - name: openstack/kolla
+        override-checkout: stable/queens
+      - name: openstack/kolla-ansible
+        override-checkout: stable/queens
+      - name: openstack/requirements
+        override-checkout: stable/queens
+    vars:
+      # Name of the release to upgrade from.
+      previous_release: pike
+    irrelevant-files:
+      - ^.*\.rst$
+      - ^doc/.*
+      - ^releasenotes/.*
+      - ^setup.cfg$
+      - ^tools/.*$
+      - ^tox.ini$
+
+- job:
+    name: kayobe-seed-upgrade-centos
+    parent: kayobe-seed-upgrade-base
+    nodeset: kayobe-centos

zuul.d/project.yaml

@@ -17,6 +17,7 @@
         - kayobe-overcloud-centos
         - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos
+        - kayobe-seed-upgrade-centos
 
     gate:
       queue: kayobe
@@ -28,3 +29,4 @@
         - kayobe-overcloud-centos
         - kayobe-overcloud-upgrade-centos
         - kayobe-seed-centos
+        - kayobe-seed-upgrade-centos