Implement separate deploy and upgrade tests

The current functional test does a full upgrade process, but uses the same current role for both the previous version and the new version. Ideally we want to test the initial deploy using the previous series' role, so that we're actually testing a series upgrade, not just a rabbitmq upgrade. This patch separates the upgrade test out and makes the default test just be a deployment. This allows us to more clearly see where something breaks. Depends-On: https://review.openstack.org/543245 Change-Id: I43e1e224cd4237ac37b40fde5ce7c279331a012d
2018-02-10 17:28:54 +00:00 · 2018-02-10 17:28:54 +00:00 · 1d399625d6
parent 52f3b38b63
commit 1d399625d6
12 changed files with 217 additions and 92 deletions
--- a/tests/ansible-role-requirements.yml
+++ b/tests/ansible-role-requirements.yml
@ -18,3 +18,7 @@
  src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
  scm: git
  version: master
+- name: previous_rabbitmq_server
+  src: https://git.openstack.org/openstack/openstack-ansible-rabbitmq_server
+  scm: git
+  version: stable/pike
--- a/tests/inventory
+++ b/tests/inventory
@ -4,10 +4,10 @@ container1
 container2
 container3

-[rabbitmq_all_other]
+[rabbitmq_all]
 container1
 container2
 container3

 [all_containers:children]
-rabbitmq_all_other
+rabbitmq_all
--- a/tests/rabbitmq_server-overrides.yml
+++ b/tests/rabbitmq_server-overrides.yml
@ -0,0 +1,5 @@
+rabbitmq_cookie_token: secrete
+rabbitmq_ssl_cert: /etc/rabbitmq/rabbitmq.pem
+rabbitmq_ssl_key: /etc/rabbitmq/rabbitmq.key
+
+rabbitmq_hipe_compile: True
--- a/tests/test-install-previous-rabbitmq.yml
+++ b/tests/test-install-previous-rabbitmq.yml
@ -1,5 +1,5 @@
 ---
-# Copyright 2016, Rackspace US, Inc.
+# Copyright 2018, SUSE LINUX GmbH.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@ -13,33 +13,27 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.

+# TODO(odyssey4me):
+# If/when https://review.openstack.org/543245 is available to
+# this script in the previous stable branch, remove this play
+# and use it instead.
+
+# The rabbitmq needs access to hostvars for all containers in order
+# to set the values in /etc/hosts. We therefore collect all host/container
+# facts before continuing.
 - name: Gather facts
  hosts: "all"
  user: root
  gather_facts: true

-# The cluster must be stopped when doing major/minor upgrades
-# http://www.rabbitmq.com/clustering.html#upgrading
- name: Stop RabbitMQ nodes that are not the upgrader
-  hosts: "{{ rabbitmq_host_group }}[1:]"
-  serial: 1
-  max_fail_percentage: 0
+# Install previous version
+- name: Deploy previous rabbitmq version
+  hosts: rabbitmq_all
  user: root
-  tasks:
-    - name: "Stop RabbitMQ"
-      service:
-        name: "rabbitmq-server"
-        state: "stopped"
-      when: rabbitmq_upgrade | default(false) | bool
-  vars_files:
-    - test-vars.yml
-
- name: Install RabbitMQ server
-  hosts: "{{ rabbitmq_host_group }}"
  serial: 1
-  user: root
-  gather_facts: True
+  any_errors_fatal: true
+  gather_facts: true
  roles:
-    - role: rabbitmq_server
+    - role: "previous_rabbitmq_server"
  vars_files:
-    - test-vars.yml
+    - common/previous/test-vars.yml
--- a/tests/test-rabbitmq-server-functional.yml
+++ b/tests/test-rabbitmq-server-functional.yml
@ -14,13 +14,38 @@
 # limitations under the License.

 - name: Run functional tests
-  hosts: "{{ rabbitmq_host_group }}"
+  hosts: rabbitmq_all
+  any_errors_fatal: true
  user: root
  gather_facts: true
  become: true
-  vars_files:
-    - test-vars.yml
  tasks:
+
+    - name: Gather variables for each operating system
+      include_vars: "{{ item }}"
+      with_first_found:
+        - "{{ playbook_dir }}/../vars/{{ ansible_distribution | lower }}-{{ ansible_distribution_version | lower }}.yml"
+        - "{{ playbook_dir }}/../vars/{{ ansible_distribution | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+        - "{{ playbook_dir }}/../vars/{{ ansible_os_family | lower }}-{{ ansible_distribution_major_version | lower }}.yml"
+        - "{{ playbook_dir }}/../vars/{{ ansible_distribution | lower }}.yml"
+        - "{{ playbook_dir }}/../vars/{{ ansible_os_family | lower }}.yml"
+
+    - name: Set rabbitmq_package_url
+      set_fact:
+        rabbitmq_package_url: "{{ _rabbitmq_package_url }}"
+
+    - name: Set rabbitmq_package_version
+      set_fact:
+        rabbitmq_package_version: "{{ _rabbitmq_package_version }}"
+
+    - name: Set rabbitmq_release_version
+      set_fact:
+        rabbitmq_release_version: "{{ _rabbitmq_release_version }}"
+
+    - name: Set rabbitmq_install_method
+      set_fact:
+        rabbitmq_install_method: "{{ _rabbitmq_install_method }}"
+
    - name: Check rabbitmq is running
      command: "pgrep -f rabbit"
      delegate_to: "{{ item }}"
@ -79,7 +104,9 @@

    - name: Ensure expected version of rabbitmq is running
      assert:
-        that: rabbitmqctl_status.stdout | search ("rabbit,\"RabbitMQ\",\"{{ rabbitmq_release_version }}\"")
+        that: rabbitmqctl_status.stdout | search ("rabbit,\"RabbitMQ\",\"{{ _rabbitmq_release_version }}\"")
+      when:
+        - "rabbitmq_install_method == 'file'"

    - name: Get the policy list
      command: rabbitmqctl -q list_policies
--- a/tests/test-rabbitmq-upgrades.sh
+++ b/tests/test-rabbitmq-upgrades.sh
@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# WARNING:
+# This file is use by all OpenStack-Ansible roles for testing purposes.
+# Any changes here will affect all OpenStack-Ansible role repositories
+# with immediate effect.
+
+# PURPOSE:
+# This script executes test Ansible playbooks required for performing
+# an upgrade test of the role.
+
+## Shell Opts ----------------------------------------------------------------
+
+set -e
+
+## Vars ----------------------------------------------------------------------
+
+export WORKING_DIR=${WORKING_DIR:-$(pwd)}
+export ROLE_NAME=${ROLE_NAME:-''}
+
+export ANSIBLE_PARAMETERS=${ANSIBLE_PARAMETERS:-""}
+export TEST_PLAYBOOK=${TEST_PLAYBOOK:-$WORKING_DIR/tests/test-upgrade-pre.yml}
+export TEST_CHECK_MODE=${TEST_CHECK_MODE:-false}
+export TEST_IDEMPOTENCE=${TEST_IDEMPOTENCE:-false}
+export COMMON_TESTS_PATH="${WORKING_DIR}/tests/common"
+
+echo "ANSIBLE_OVERRIDES: ${ANSIBLE_OVERRIDES}"
+echo "ANSIBLE_PARAMETERS: ${ANSIBLE_PARAMETERS}"
+echo "TEST_PLAYBOOK: ${TEST_PLAYBOOK}"
+echo "TEST_CHECK_MODE: ${TEST_CHECK_MODE}"
+echo "TEST_IDEMPOTENCE: ${TEST_IDEMPOTENCE}"
+
+## Functions -----------------------------------------------------------------
+
+function execute_ansible_playbook {
+
+  export ANSIBLE_CLI_PARAMETERS="${ANSIBLE_PARAMETERS} -e @${ANSIBLE_OVERRIDES}"
+  export ANSIBLE_BIN=${ANSIBLE_BIN:-"ansible-playbook"}
+  CMD_TO_EXECUTE="${ANSIBLE_BIN} ${TEST_PLAYBOOK} $@ ${ANSIBLE_CLI_PARAMETERS}"
+
+  echo "Executing: ${CMD_TO_EXECUTE}"
+  echo "With:"
+  echo "    ANSIBLE_INVENTORY: ${ANSIBLE_INVENTORY}"
+  echo "    ANSIBLE_LOG_PATH: ${ANSIBLE_LOG_PATH}"
+
+  ${CMD_TO_EXECUTE}
+
+}
+
+## Main ----------------------------------------------------------------------
+
+# Ensure that the Ansible environment is properly prepared
+source "${COMMON_TESTS_PATH}/test-ansible-env-prep.sh"
+
+# Prepare environment for the initial deploy of (previous and current) Galera
+# No upgrading or testing is done yet.
+export TEST_PLAYBOOK="${WORKING_DIR}/tests/test-upgrade-pre.yml"
+export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-install.log"
+
+# Execute the setup of previous version
+execute_ansible_playbook
+
+# Create an ansible venv matching previous branch
+source ${WORKING_DIR}/tests/common/test-create-previous-venv.sh
+
+# Prepare environment for the deploy of previous Galera:
+# No upgrading or testing is done yet.
+export TEST_PLAYBOOK="${WORKING_DIR}/tests/test-install-previous-rabbitmq.yml"
+export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-previous_rabbitmq-install.log"
+export PREVIOUS_VENV="ansible-previous"
+export ANSIBLE_BIN="${WORKING_DIR}/.tox/${PREVIOUS_VENV}/bin/ansible-playbook"
+
+# Execute the setup of previous Keystone
+execute_ansible_playbook
+# Unset previous branch overrides
+unset PREVIOUS_VENV
+unset ANSIBLE_BIN
+
+# Prepare environment for the upgrade
+export ANSIBLE_PARAMETERS="${ANSIBLE_PARAMETERS} -e rabbitmq_upgrade=yes"
+export TEST_PLAYBOOK="${WORKING_DIR}/tests/test-upgrade-post.yml"
+export ANSIBLE_LOG_PATH="${ANSIBLE_LOG_DIR}/ansible-execute-upgrade.log"
+
+# Execute the upgrade
+execute_ansible_playbook
--- a/tests/test-upgrade-post.yml
+++ b/tests/test-upgrade-post.yml
@ -0,0 +1,20 @@
+---
+# Copyright 2016, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Install RabbitMQ server
+- include: common/test-install-rabbitmq.yml
+
+# Run functional tests
+- include: test-rabbitmq-functional.yml
--- a/tests/test-upgrade-pre.yml
+++ b/tests/test-upgrade-pre.yml
@ -0,0 +1,20 @@
+---
+# Copyright 2015, Rackspace US, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Setup the host
+- include: common/test-setup-host.yml
+
+# Prepare the containers
+- include: common/test-prepare-containers.yml
--- a/tests/test-vars.yml
+++ b/tests/test-vars.yml
@ -1,46 +0,0 @@
-rabbitmq_host_group: "rabbitmq_all_other"
-rabbitmq_cookie_token: secrete
-rabbitmq_ssl_cert: /etc/rabbitmq/rabbitmq.pem
-rabbitmq_ssl_key: /etc/rabbitmq/rabbitmq.key
-
-rabbitmq_hipe_compile: True
-
-rabbitmq_old_package:
-  # NOTE(hwoarang): For openSUSE/Ubuntu, we install the old file explicitly. CentOS7 will get it from the external
-  # repo instead.
-  debian:
-    method: 'file'
-    rabbitmq_package_url: "http://www.rabbitmq.com/releases/rabbitmq-server/v3.6.11/rabbitmq-server_3.6.11-1_all.deb"
-    rabbitmq_package_sha256: "b487edba0bc5db5c745e62059bdc896866db22d584c26c78d50be3d61c8f5a18"
-    rabbitmq_release_version: "3.6.11"
-  redhat:
-    method: 'external_repo'
-    rabbitmq_package_url: "http://www.rabbitmq.com/releases/rabbitmq-server/v3.6.11/rabbitmq-server-3.6.11-1.el7.noarch.rpm"
-    rabbitmq_package_sha256: "e09272c9ed3d3f0a34f1eb74f757b36051a7d6e9c825e054cd0132e961398913"
-    rabbitmq_release_version: "3.6.11"
-  suse:
-    method: 'file'
-    rabbitmq_package_url: "http://www.rabbitmq.com/releases/rabbitmq-server/v3.6.11/rabbitmq-server-3.6.11-1.suse.noarch.rpm"
-    rabbitmq_package_sha256: "03b56ed3fde51e36e0fa6a38becaa701238ffd3bd20f45a45d78844bcc84945b"
-    rabbitmq_release_version: "3.6.11"
-
-# NOTE(hwoarang): This normally matches the _rabbitmq_release_version variable
-# in the vars/* distro files.
-rabbitmq_new_package:
-  # NOTE(hwoarang): For openSUSE we get the new version from the repo.
-  # For Ubuntu keep the default which is 'file'. For CentOS will get it
-  # from the external repo
-  debian:
-    method: 'file'
-    rabbitmq_release_version: "3.6.14"
-  redhat:
-    method: 'external_repo'
-    rabbitmq_release_version: "3.6.14"
-  # We upgrade from file to file in master, because:
-  # 1) The packages in method "repo", coming from the OpenStack SUSE repo "pike", are 3.6.11
-  #    and therefore too old.
-  # 2) We don't have proof yet that the packages from OpenStack SUSE repo "master" (version 3.6.14)
-  #    are stable enough to be used across the board (so, not only rabbitmq_server)
-  suse:
-    method: 'file'
-    rabbitmq_release_version: "3.6.14"
--- a/tests/test.yml
+++ b/tests/test.yml
@ -16,23 +16,11 @@
 # Setup the host
 - include: common/test-setup-host.yml

-# Install previous version of RabbitMQ server
- include: test-install-rabbitmq-server.yml
-  vars:
-    rabbitmq_install_method: "{{ rabbitmq_old_package[ansible_os_family | lower]['method'] }}"
-    rabbitmq_package_url: "{{ rabbitmq_old_package[ansible_os_family | lower]['rabbitmq_package_url'] }}"
-    rabbitmq_package_sha256: "{{ rabbitmq_old_package[ansible_os_family | lower]['rabbitmq_package_sha256'] }}"
+# Prepare the containers
+- include: common/test-prepare-containers.yml

-# Run tests
- include: test-rabbitmq-server-functional.yml
-  vars:
-    rabbitmq_release_version: "{{ rabbitmq_old_package[ansible_os_family | lower]['rabbitmq_release_version'] }}"
+# Install RabbitMQ server
+- include: common/test-install-rabbitmq.yml

- include: test-install-rabbitmq-server.yml
-  vars:
-    rabbitmq_install_method: "{{ rabbitmq_new_package[ansible_os_family | lower]['method'] }}"
-    rabbitmq_upgrade: true
-
- include: test-rabbitmq-server-functional.yml
-  vars:
-    rabbitmq_release_version: "{{ rabbitmq_new_package[ansible_os_family | lower]['rabbitmq_release_version'] }}"
+# Run functional tests
+- include: test-rabbitmq-functional.yml
--- a/tox.ini
+++ b/tox.ini
@ -103,6 +103,18 @@ commands =
    bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"


+[testenv:upgrade]
+deps =
+    {[testenv:ansible]deps}
+setenv =
+    {[testenv]setenv}
+    ANSIBLE_OVERRIDES={toxinidir}/tests/rabbitmq_server-overrides.yml
+    CLONE_UPGRADE_TESTS=yes
+commands =
+    bash -c "{toxinidir}/tests/tests-repo-clone.sh"
+    bash -c "{toxinidir}/tests/test-rabbitmq-upgrades.sh"
+
+
 [testenv:linters]
 deps =
    {[testenv:ansible]deps}
--- a/zuul.d/project.yaml
+++ b/zuul.d/project.yaml
@ -19,6 +19,7 @@
        - openstack-ansible-functional-centos-7
        - openstack-ansible-functional-opensuse-423
        - openstack-ansible-functional-ubuntu-xenial
+        - openstack-ansible-upgrade-ubuntu-xenial
    experimental:
      jobs:
        - openstack-ansible-integrated-deploy-aio
@ -28,3 +29,4 @@
        - openstack-ansible-functional-centos-7
        - openstack-ansible-functional-opensuse-423
        - openstack-ansible-functional-ubuntu-xenial
+        - openstack-ansible-upgrade-ubuntu-xenial