Refactor CI testing scripts
This change is intended to ease the addition of new test scenarios, by splitting the existing setup_gate.sh script up into several smaller scripts. We may wish to further break these scripts up in future, depending on the requirements of new test jobs. Change-Id: Ifa442ac79a54f749c56d20f5f7383c3852d95268
This commit is contained in:
Mark Goddard
parent
f941d47dcb
commit
f92de48c28
|
|
@ -0,0 +1,27 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
check_failure() {
|
||||
# All docker container's status are created, restarting, running, removing,
|
||||
# paused, exited and dead. Containers without running status are treated as
|
||||
# failure. removing is added in docker 1.13, just ignore it now.
|
||||
failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
|
||||
--filter status=created \
|
||||
--filter status=restarting \
|
||||
--filter status=paused \
|
||||
--filter status=exited \
|
||||
--filter status=dead)
|
||||
|
||||
if [[ -n "$failed_containers" ]]; then
|
||||
exit 1;
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
check_failure
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
function deploy_bifrost {
|
||||
RAW_INVENTORY=/etc/kolla/inventory
|
||||
|
||||
# TODO(mgoddard): run prechecks.
|
||||
# Deploy the bifrost container.
|
||||
# TODO(mgoddard): add pull action when we have a local registry service in
|
||||
# CI.
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
|
||||
}
|
||||
|
||||
|
||||
deploy_bifrost
|
||||
|
|
@ -0,0 +1,28 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
function deploy {
|
||||
RAW_INVENTORY=/etc/kolla/inventory
|
||||
|
||||
# Create dummy interface for neutron
|
||||
ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all
|
||||
|
||||
#TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
|
||||
sudo chmod -R 777 /etc/kolla
|
||||
# Actually do the deployment
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/deploy-prechecks
|
||||
# TODO(jeffrey4l): add pull action when we have a local registry
|
||||
# service in CI
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy
|
||||
}
|
||||
|
||||
|
||||
deploy
|
||||
|
|
@ -0,0 +1,21 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
function reconfigure {
|
||||
RAW_INVENTORY=/etc/kolla/inventory
|
||||
|
||||
# TODO(jeffrey4l): make some configure file change and
|
||||
# trigger a real reconfigure
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/reconfigure-prechecks
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/reconfigure
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure
|
||||
}
|
||||
|
||||
|
||||
reconfigure
|
||||
|
|
@ -145,19 +145,75 @@
|
|||
when: scenario == "ceph"
|
||||
delegate_to: "primary"
|
||||
|
||||
- shell:
|
||||
cmd: |
|
||||
set -e
|
||||
set -x
|
||||
export BASE_DISTRO={{ base_distro }}
|
||||
export INSTALL_TYPE={{ install_type }}
|
||||
export NODEPOOL_TARBALLS_MIRROR=http://{{ zuul_site_mirror_fqdn }}:8080/tarballs
|
||||
export BUILD_IMAGE={{ need_build_image }}
|
||||
export KOLLA_SRC_DIR={{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla
|
||||
export ACTION={{ scenario }}
|
||||
|
||||
tools/setup_gate.sh
|
||||
- name: Run setup_gate.sh script
|
||||
shell:
|
||||
cmd: tools/setup_gate.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
BASE_DISTRO: "{{ base_distro }}"
|
||||
INSTALL_TYPE: "{{ install_type }}"
|
||||
NODEPOOL_TARBALLS_MIRROR: "http://{{ zuul_site_mirror_fqdn }}:8080/tarballs"
|
||||
BUILD_IMAGE: "{{ need_build_image }}"
|
||||
KOLLA_SRC_DIR: "{{ ansible_env.HOME }}/src/git.openstack.org/openstack/kolla"
|
||||
ACTION: "{{ scenario }}"
|
||||
|
||||
- block:
|
||||
- name: Run deploy.sh script
|
||||
shell:
|
||||
cmd: tests/deploy.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
ACTION: "{{ scenario }}"
|
||||
|
||||
- name: Run test-openstack.sh script
|
||||
shell:
|
||||
cmd: tests/test-openstack.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
ACTION: "{{ scenario }}"
|
||||
|
||||
- name: Run reconfigure.sh script
|
||||
shell:
|
||||
cmd: tests/reconfigure.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
ACTION: "{{ scenario }}"
|
||||
|
||||
- name: Run upgrade.sh script
|
||||
shell:
|
||||
cmd: tests/upgrade.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
environment:
|
||||
ACTION: "{{ scenario }}"
|
||||
when: scenario != "bifrost"
|
||||
|
||||
- block:
|
||||
- name: Run deploy-bifrost.sh script
|
||||
shell:
|
||||
cmd: tests/deploy-bifrost.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
|
||||
- name: Run test-bifrost.sh script
|
||||
shell:
|
||||
cmd: tests/test-bifrost.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
|
||||
- name: Run upgrade-bifrost.sh script
|
||||
shell:
|
||||
cmd: tests/upgrade-bifrost.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
when: scenario == "bifrost"
|
||||
|
||||
- name: Run check-failure.sh script
|
||||
shell:
|
||||
cmd: tests/check-failure.sh
|
||||
executable: /bin/bash
|
||||
chdir: "{{ kolla_ansible_src_dir }}"
|
||||
delegate_to: "primary"
|
||||
environment: '{{ zuul | zuul_legacy_vars }}'
|
||||
|
|
|
|||
|
|
@ -0,0 +1,29 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
function test_bifrost {
|
||||
# TODO(mgoddard): More testing, deploy bare metal nodes.
|
||||
# TODO(mgoddard): Use openstackclient when clouds.yaml works. See
|
||||
# https://bugs.launchpad.net/bifrost/+bug/1754070.
|
||||
attempts=0
|
||||
while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
|
||||
attempts=$((attempts + 1))
|
||||
if [[ $attempts -gt 6 ]]; then
|
||||
echo "Timed out waiting for ironic conductor to become active"
|
||||
exit 1
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
|
||||
sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
|
||||
sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
|
||||
}
|
||||
|
||||
|
||||
test_bifrost
|
||||
|
|
@ -0,0 +1,38 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
function test_openstack {
|
||||
# Wait for service ready
|
||||
sleep 15
|
||||
. /etc/kolla/admin-openrc.sh
|
||||
# TODO(Jeffrey4l): Restart the memcached container to cleanup all cache.
|
||||
# Remove this after this bug is fixed
|
||||
# https://bugs.launchpad.net/oslo.cache/+bug/1590779
|
||||
sudo docker restart memcached
|
||||
nova --debug service-list
|
||||
openstack --debug network agent list
|
||||
tools/init-runonce
|
||||
nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test
|
||||
|
||||
nova --debug list
|
||||
# If the status is not ACTIVE, print info and exit 1
|
||||
nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}'
|
||||
if echo $ACTION | grep -q "ceph"; then
|
||||
openstack volume create --size 2 test_volume
|
||||
openstack server add volume kolla_boot_test test_volume --device /dev/vdb
|
||||
fi
|
||||
if echo $ACTION | grep -q "zun"; then
|
||||
openstack --debug appcontainer service list
|
||||
openstack --debug appcontainer host list
|
||||
# TODO(hongbin): Run a Zun container and assert the container becomes
|
||||
# Running
|
||||
fi
|
||||
}
|
||||
|
||||
test_openstack
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
function upgrade_bifrost {
|
||||
RAW_INVENTORY=/etc/kolla/inventory
|
||||
|
||||
# TODO(mgoddard): run prechecks.
|
||||
# TODO(mgoddard): add pull action when we have a local registry service in
|
||||
# CI.
|
||||
# TODO(mgoddard): make some configuration file changes and trigger a real
|
||||
# upgrade.
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/upgrade-bifrost
|
||||
}
|
||||
|
||||
|
||||
upgrade_bifrost
|
||||
|
|
@ -0,0 +1,22 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -o xtrace
|
||||
set -o errexit
|
||||
|
||||
# Enable unbuffered output for Ansible in Jenkins.
|
||||
export PYTHONUNBUFFERED=1
|
||||
|
||||
|
||||
function upgrade {
|
||||
RAW_INVENTORY=/etc/kolla/inventory
|
||||
|
||||
# TODO(jeffrey4l): need run a real upgrade
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/upgrade-prechecks
|
||||
# TODO(mgoddard): add pull action when we have a local registry service in
|
||||
# CI.
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade
|
||||
}
|
||||
|
||||
|
||||
upgrade
|
||||
|
|
@ -122,118 +122,6 @@ function prepare_images {
|
|||
popd
|
||||
}
|
||||
|
||||
function sanity_check {
|
||||
# Wait for service ready
|
||||
sleep 15
|
||||
. /etc/kolla/admin-openrc.sh
|
||||
# TODO(Jeffrey4l): Restart the memcached container to cleanup all cache.
|
||||
# Remove this after this bug is fixed
|
||||
# https://bugs.launchpad.net/oslo.cache/+bug/1590779
|
||||
sudo docker restart memcached
|
||||
nova --debug service-list
|
||||
openstack --debug network agent list
|
||||
tools/init-runonce
|
||||
nova --debug boot --poll --image $(openstack image list | awk '/cirros/ {print $2}') --nic net-id=$(openstack network list | awk '/demo-net/ {print $2}') --flavor 1 kolla_boot_test
|
||||
|
||||
nova --debug list
|
||||
# If the status is not ACTIVE, print info and exit 1
|
||||
nova --debug show kolla_boot_test | awk '{buf=buf"\n"$0} $2=="status" && $4!="ACTIVE" {failed="yes"}; END {if (failed=="yes") {print buf; exit 1}}'
|
||||
if echo $ACTION | grep -q "ceph"; then
|
||||
openstack volume create --size 2 test_volume
|
||||
openstack server add volume kolla_boot_test test_volume --device /dev/vdb
|
||||
fi
|
||||
if echo $ACTION | grep -q "zun"; then
|
||||
openstack --debug appcontainer service list
|
||||
openstack --debug appcontainer host list
|
||||
# TODO(hongbin): Run a Zun container and assert the container becomes
|
||||
# Running
|
||||
fi
|
||||
}
|
||||
|
||||
function test_openstack {
|
||||
# Create dummy interface for neutron
|
||||
ansible -m shell -i ${RAW_INVENTORY} -b -a "ip l a fake_interface type dummy" all
|
||||
|
||||
#TODO(inc0): Post-deploy complains that /etc/kolla is not writable. Probably we need to include become there
|
||||
sudo chmod -R 777 /etc/kolla
|
||||
# Actually do the deployment
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks1
|
||||
# TODO(jeffrey4l): add pull action when we have a local registry
|
||||
# service in CI
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy > /tmp/logs/ansible/deploy
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv post-deploy > /tmp/logs/ansible/post-deploy
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-deploy
|
||||
|
||||
# Test OpenStack Environment
|
||||
# TODO: use kolla-ansible check when it's ready
|
||||
|
||||
sanity_check
|
||||
|
||||
# TODO(jeffrey4l): make some configure file change and
|
||||
# trigger a real reconfigure
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv reconfigure > /tmp/logs/ansible/reconfigure
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-reconfigure
|
||||
# TODO(jeffrey4l): need run a real upgrade
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv upgrade > /tmp/logs/ansible/upgrade
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv check > /tmp/logs/ansible/check-upgrade
|
||||
|
||||
# run prechecks again
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv prechecks > /tmp/logs/ansible/prechecks2
|
||||
}
|
||||
|
||||
function sanity_check_bifrost {
|
||||
# TODO(mgoddard): More testing, deploy bare metal nodes.
|
||||
# TODO(mgoddard): Use openstackclient when clouds.yaml works. See
|
||||
# https://bugs.launchpad.net/bifrost/+bug/1754070.
|
||||
attempts=0
|
||||
while [[ $(sudo docker exec bifrost_deploy bash -c "source env-vars && ironic driver-list" | wc -l) -le 4 ]]; do
|
||||
attempts=$((attempts + 1))
|
||||
if [[ $attempts -gt 6 ]]; then
|
||||
echo "Timed out waiting for ironic conductor to become active"
|
||||
exit 1
|
||||
fi
|
||||
sleep 10
|
||||
done
|
||||
sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-list"
|
||||
sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-create --driver ipmi --name test-node"
|
||||
sudo docker exec bifrost_deploy bash -c "source env-vars && ironic node-delete test-node"
|
||||
}
|
||||
|
||||
function test_bifrost {
|
||||
# TODO(mgoddard): run prechecks.
|
||||
|
||||
# Deploy the bifrost container.
|
||||
# TODO(mgoddard): add pull action when we have a local registry service in
|
||||
# CI.
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost
|
||||
|
||||
# Test Bifrost Environment
|
||||
sanity_check_bifrost
|
||||
|
||||
# TODO(mgoddard): make some configuration file changes and trigger a real
|
||||
# reconfigure.
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -vvv deploy-bifrost > /tmp/logs/ansible/deploy-bifrost2
|
||||
|
||||
# TODO(mgoddard): perform an upgrade.
|
||||
}
|
||||
|
||||
check_failure() {
|
||||
# All docker container's status are created, restarting, running, removing,
|
||||
# paused, exited and dead. Containers without running status are treated as
|
||||
# failure. removing is added in docker 1.13, just ignore it now.
|
||||
failed_containers=$(sudo docker ps -a --format "{{.Names}}" \
|
||||
--filter status=created \
|
||||
--filter status=restarting \
|
||||
--filter status=paused \
|
||||
--filter status=exited \
|
||||
--filter status=dead)
|
||||
|
||||
if [[ -n "$failed_containers" ]]; then
|
||||
exit 1;
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
|
||||
clone_repos
|
||||
setup_ansible
|
||||
|
|
@ -242,11 +130,3 @@ setup_node
|
|||
|
||||
tools/kolla-ansible -i ${RAW_INVENTORY} -e ansible_user=$USER bootstrap-servers > /tmp/logs/ansible/bootstrap-servers
|
||||
prepare_images
|
||||
|
||||
if [[ $ACTION != bifrost ]]; then
|
||||
test_openstack
|
||||
else
|
||||
test_bifrost
|
||||
fi
|
||||
|
||||
check_failure
|
||||
|
|
|
|||
Loading…
Reference in New Issue