From f92de48c287136074046476b7bb9888d9af23b3d Mon Sep 17 00:00:00 2001 From: Mark Goddard Date: Tue, 10 Jul 2018 15:38:40 +0100 Subject: [PATCH] Refactor CI testing scripts This change is intended to ease the addition of new test scenarios, by splitting the existing setup_gate.sh script up into several smaller scripts. We may wish to further break these scripts up in future, depending on the requirements of new test jobs. Change-Id: Ifa442ac79a54f749c56d20f5f7383c3852d95268 --- tests/check-failure.sh | 27 +++++++++ tests/deploy-bifrost.sh | 21 +++++++ tests/deploy.sh | 28 +++++++++ tests/reconfigure.sh | 21 +++++++ tests/run.yml | 84 ++++++++++++++++++++++----- tests/test-bifrost.sh | 29 ++++++++++ tests/test-openstack.sh | 38 +++++++++++++ tests/upgrade-bifrost.sh | 22 +++++++ tests/upgrade.sh | 22 +++++++ tools/setup_gate.sh | 120 --------------------------------------- 10 files changed, 278 insertions(+), 134 deletions(-) create mode 100755 tests/check-failure.sh create mode 100755 tests/deploy-bifrost.sh create mode 100755 tests/deploy.sh create mode 100755 tests/reconfigure.sh create mode 100755 tests/test-bifrost.sh create mode 100755 tests/test-openstack.sh create mode 100755 tests/upgrade-bifrost.sh create mode 100755 tests/upgrade.sh diff --git a/tests/check-failure.sh b/tests/check-failure.sh new file mode 100755 index 0000000000..6689d86a68 --- /dev/null +++ b/tests/check-failure.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +check_failure() { + # All docker container's status are created, restarting, running, removing, + # paused, exited and dead. Containers without running status are treated as + # failure. removing is added in docker 1.13, just ignore it now. + failed_containers=$(sudo docker ps -a --format "{{.Names}}" \ + --filter status=created \ + --filter status=restarting \ + --filter status=paused \ + --filter status=exited \ + --filter status=dead) + + if [[ -n "$failed_containers" ]]; then + exit 1; + fi +} + + +check_failure diff --git a/tests/deploy-bifrost.sh b/tests/deploy-bifrost.sh new file mode 100755 index 0000000000..f45e40c4bb --- /dev/null +++ b/tests/deploy-bifrost.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +function deploy_bifrost { + RAW_INVENTORY=/etc/kolla/inventory + + # TODO(mgoddard): run prechecks. + # Deploy the bifrost container. + # TODO(mgoddard): add pull action when we have a local registry service in + # CI. + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost +} + + +deploy_bifrost diff --git a/tests/deploy.sh b/tests/deploy.sh new file mode 100755 index 0000000000..c06ef46534 --- /dev/null +++ b/tests/deploy.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +function deploy { + RAW_INVENTORY=/etc/kolla/inventory + + # Create dummy interface for neutron + ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all + + #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there + sudo chmod -R 777 /etc/kolla + # Actually do the deployment + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/deploy-prechecks + # TODO(jeffrey4l): add pull action when we have a local registry + # service in CI + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy +} + + +deploy diff --git a/tests/reconfigure.sh b/tests/reconfigure.sh new file mode 100755 index 0000000000..f8b2c04f40 --- /dev/null +++ b/tests/reconfigure.sh @@ -0,0 +1,21 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +function reconfigure { + RAW_INVENTORY=/etc/kolla/inventory + + # TODO(jeffrey4l): make some configure file change and + # trigger a real reconfigure + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/reconfigure-prechecks + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/reconfigure + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure +} + + +reconfigure diff --git a/tests/run.yml b/tests/run.yml index 39aed9a44c..04554b16eb 100644 --- a/tests/run.yml +++ b/tests/run.yml @@ -145,19 +145,75 @@ when: scenario == "ceph" delegate_to: "primary" - - shell: - cmd: | - set -e - set -x - export BASE_DISTRO={{ base_distro }} - export INSTALL_TYPE={{ install_type }} - export NODEPOOL_TARBALLS_MIRROR=http://{{ zuul_site_mirror_fqdn }}:8080/tarballs - export BUILD_IMAGE={{ need_build_image }} - export KOLLA_SRC_DIR={{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla - export ACTION={{ scenario }} - - tools/setup_gate.sh + - name: Run setup_gate.sh script + shell: + cmd: tools/setup_gate.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + BASE_DISTRO: "{{ base_distro }}" + INSTALL_TYPE: "{{ install_type }}" + NODEPOOL_TARBALLS_MIRROR: "http://{{ zuul_site_mirror_fqdn }}:8080/tarballs" + BUILD_IMAGE: "{{ need_build_image }}" + KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla" + ACTION: "{{ scenario }}" + + - block: + - name: Run deploy.sh script + shell: + cmd: tests/deploy.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + ACTION: "{{ scenario }}" + + - name: Run test-openstack.sh script + shell: + cmd: tests/test-openstack.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + ACTION: "{{ scenario }}" + + - name: Run reconfigure.sh script + shell: + cmd: tests/reconfigure.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + ACTION: "{{ scenario }}" + + - name: Run upgrade.sh script + shell: + cmd: tests/upgrade.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + environment: + ACTION: "{{ scenario }}" + when: scenario != "bifrost" + + - block: + - name: Run deploy-bifrost.sh script + shell: + cmd: tests/deploy-bifrost.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + + - name: Run test-bifrost.sh script + shell: + cmd: tests/test-bifrost.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + + - name: Run upgrade-bifrost.sh script + shell: + cmd: tests/upgrade-bifrost.sh + executable: /bin/bash + chdir: "{{ kolla_ansible_src_dir }}" + when: scenario == "bifrost" + + - name: Run check-failure.sh script + shell: + cmd: tests/check-failure.sh executable: /bin/bash chdir: "{{ kolla_ansible_src_dir }}" - delegate_to: "primary" - environment: '{{ zuul | zuul_legacy_vars }}' diff --git a/tests/test-bifrost.sh b/tests/test-bifrost.sh new file mode 100755 index 0000000000..94c71f482f --- /dev/null +++ b/tests/test-bifrost.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +function test_bifrost { + # TODO(mgoddard): More testing, deploy bare metal nodes. + # TODO(mgoddard): Use openstackclient when clouds.yaml works. See + # https://bugs.launchpad.net/bifrost/+bug/1754070. + attempts=0 + while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do + attempts=$((attempts + 1)) + if [[ $attempts -gt 6 ]]; then + echo "Timed out waiting for ironic conductor to become active" + exit 1 + fi + sleep 10 + done + sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list" + sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node" + sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node" +} + + +test_bifrost diff --git a/tests/test-openstack.sh b/tests/test-openstack.sh new file mode 100755 index 0000000000..f6cee78841 --- /dev/null +++ b/tests/test-openstack.sh @@ -0,0 +1,38 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +function test_openstack { + # Wait for service ready + sleep 15 + . /etc/kolla/admin-openrc.sh + # TODO(Jeffrey4l): Restart the memcached container to cleanup all cache. + # Remove this after this bug is fixed + # https://bugs.launchpad.net/oslo.cache/+bug/1590779 + sudo docker restart memcached + nova --debug service-list + openstack --debug network agent list + tools/init-runonce + nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test + + nova --debug list + # If the status is not ACTIVE, print info and exit 1 + nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}' + if echo $ACTION | grep -q "ceph"; then + openstack volume create --size 2 test_volume + openstack server add volume kolla_boot_test test_volume --device /dev/vdb + fi + if echo $ACTION | grep -q "zun"; then + openstack --debug appcontainer service list + openstack --debug appcontainer host list + # TODO(hongbin): Run a Zun container and assert the container becomes + # Running + fi +} + +test_openstack diff --git a/tests/upgrade-bifrost.sh b/tests/upgrade-bifrost.sh new file mode 100755 index 0000000000..d6e9f5d248 --- /dev/null +++ b/tests/upgrade-bifrost.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +function upgrade_bifrost { + RAW_INVENTORY=/etc/kolla/inventory + + # TODO(mgoddard): run prechecks. + # TODO(mgoddard): add pull action when we have a local registry service in + # CI. + # TODO(mgoddard): make some configuration file changes and trigger a real + # upgrade. + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/upgrade-bifrost +} + + +upgrade_bifrost diff --git a/tests/upgrade.sh b/tests/upgrade.sh new file mode 100755 index 0000000000..d3ebba63d1 --- /dev/null +++ b/tests/upgrade.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +set -o xtrace +set -o errexit + +# Enable unbuffered output for Ansible in Jenkins. +export PYTHONUNBUFFERED=1 + + +function upgrade { + RAW_INVENTORY=/etc/kolla/inventory + + # TODO(jeffrey4l): need run a real upgrade + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/upgrade-prechecks + # TODO(mgoddard): add pull action when we have a local registry service in + # CI. + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade + tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade +} + + +upgrade diff --git a/tools/setup_gate.sh b/tools/setup_gate.sh index a6cb4e391f..9cdbf73e76 100755 --- a/tools/setup_gate.sh +++ b/tools/setup_gate.sh @@ -122,118 +122,6 @@ function prepare_images { popd } -function sanity_check { - # Wait for service ready - sleep 15 - . /etc/kolla/admin-openrc.sh - # TODO(Jeffrey4l): Restart the memcached container to cleanup all cache. - # Remove this after this bug is fixed - # https://bugs.launchpad.net/oslo.cache/+bug/1590779 - sudo docker restart memcached - nova --debug service-list - openstack --debug network agent list - tools/init-runonce - nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test - - nova --debug list - # If the status is not ACTIVE, print info and exit 1 - nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}' - if echo $ACTION | grep -q "ceph"; then - openstack volume create --size 2 test_volume - openstack server add volume kolla_boot_test test_volume --device /dev/vdb - fi - if echo $ACTION | grep -q "zun"; then - openstack --debug appcontainer service list - openstack --debug appcontainer host list - # TODO(hongbin): Run a Zun container and assert the container becomes - # Running - fi -} - -function test_openstack { - # Create dummy interface for neutron - ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all - - #TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there - sudo chmod -R 777 /etc/kolla - # Actually do the deployment - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1 - # TODO(jeffrey4l): add pull action when we have a local registry - # service in CI - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy - - # Test OpenStack Environment - # TODO: use kolla-ansible check when it's ready - - sanity_check - - # TODO(jeffrey4l): make some configure file change and - # trigger a real reconfigure - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/reconfigure - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure - # TODO(jeffrey4l): need run a real upgrade - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade - - # run prechecks again - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2 -} - -function sanity_check_bifrost { - # TODO(mgoddard): More testing, deploy bare metal nodes. - # TODO(mgoddard): Use openstackclient when clouds.yaml works. See - # https://bugs.launchpad.net/bifrost/+bug/1754070. - attempts=0 - while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do - attempts=$((attempts + 1)) - if [[ $attempts -gt 6 ]]; then - echo "Timed out waiting for ironic conductor to become active" - exit 1 - fi - sleep 10 - done - sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list" - sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node" - sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node" -} - -function test_bifrost { - # TODO(mgoddard): run prechecks. - - # Deploy the bifrost container. - # TODO(mgoddard): add pull action when we have a local registry service in - # CI. - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost - - # Test Bifrost Environment - sanity_check_bifrost - - # TODO(mgoddard): make some configuration file changes and trigger a real - # reconfigure. - tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost2 - - # TODO(mgoddard): perform an upgrade. -} - -check_failure() { - # All docker container's status are created, restarting, running, removing, - # paused, exited and dead. Containers without running status are treated as - # failure. removing is added in docker 1.13, just ignore it now. - failed_containers=$(sudo docker ps -a --format "{{.Names}}" \ - --filter status=created \ - --filter status=restarting \ - --filter status=paused \ - --filter status=exited \ - --filter status=dead) - - if [[ -n "$failed_containers" ]]; then - exit 1; - fi -} - - clone_repos setup_ansible @@ -242,11 +130,3 @@ setup_node tools/kolla-ansible -i ${RAW_INVENTORY} -e ansible_user=$USER bootstrap-servers > /tmp/logs/ansible/bootstrap-servers prepare_images - -if [[ $ACTION != bifrost ]]; then - test_openstack -else - test_bifrost -fi - -check_failure