Rewrite smoke jobs to one using simple deployment script
Change-Id: Ib2f76e416fbbe243693bdd7eb170385294cb0986
This commit is contained in:
parent
8910b7c873
commit
86ad6d50e9
|
@ -6,7 +6,6 @@
|
|||
# The script expects fuel-ccp cloned into fuel-ccp/ and fuel-ccp-installer
|
||||
# cloned into fuel-ccp-installer/ directory (e.g. by Jenkins SCM plugin).
|
||||
|
||||
|
||||
# CONFIGURATION:
|
||||
######################################################
|
||||
: ${FUEL_DEVOPS_ENV_NAME:="fuel-ccp-${COMPONENT}-${VERSION}-deployment"}
|
||||
|
@ -21,7 +20,13 @@ export REGISTRY_IP=`ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'`
|
|||
export REGISTRY_PORT=5000
|
||||
JENKINS_GID=`getent group jenkins | cut -d":" -f3`
|
||||
REGISTRY_HASH=`docker inspect --format "{{.Id}}" registry`
|
||||
export DOCKER_REGISTRY_HOST="registry.mcp.fuel-infra.org"
|
||||
export DOCKER_REGISTRY="${DOCKER_REGISTRY_HOST}:6000"
|
||||
export DOCKER_REGISTRY_IP="`getent hosts ${DOCKER_REGISTRY_HOST} | awk '{print $1}'`"
|
||||
export IMAGES_NAMESPACE="ccp"
|
||||
export REGISTRY_NAMESPACE="mcp"
|
||||
|
||||
function prepare_k8s_env {
|
||||
# Prepare K8s env:
|
||||
source "${FUEL_DEVOPS_INSTALLATION_DIR}"/bin/activate
|
||||
NEED_TO_SETUP_ENV=false
|
||||
|
@ -48,14 +53,12 @@ export FUEL_DEVOPS_INSTALLATION_DIR=${FUEL_DEVOPS_INSTALLATION_DIR}
|
|||
export CUSTOM_YAML='hyperkube_image_repo: "${HYPERKUBE_REPO}"
|
||||
hyperkube_image_tag: "${HYPERKUBE_TAG}"
|
||||
kube_version: "${HYPERKUBE_VERSION}"
|
||||
docker_options: "--insecure-registry=${REGISTRY_IP}:${REGISTRY_PORT}"'
|
||||
|
||||
docker_options: "--insecure-registry=${REGISTRY_IP}:${REGISTRY_PORT} --insecure-registry=DOCKER_REGISTRY"'
|
||||
echo "Running on \${NODE_NAME}: \${ENV_NAME}"
|
||||
source \${FUEL_DEVOPS_INSTALLATION_DIR}/bin/activate
|
||||
echo "Use image:"
|
||||
ls -la \${IMAGE_PATH}
|
||||
env
|
||||
|
||||
pushd fuel-ccp-installer
|
||||
bash -ex "./utils/jenkins/run_k8s_deploy_test.sh"
|
||||
popd
|
||||
|
@ -72,27 +75,17 @@ EOF
|
|||
# Resume from snapshot to deploy CCP later on in the script:
|
||||
dos.py resume "${FUEL_DEVOPS_ENV_NAME}"
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
# Get IP address of first node in the cluster:
|
||||
ADMIN_IP=$(ENV_NAME=${FUEL_DEVOPS_ENV_NAME} python fuel-ccp-installer/utils/jenkins/env.py get_slaves_ips | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+" | head -1)
|
||||
SSH_COMMAND="sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@${ADMIN_IP}"
|
||||
SCP_COMMAND="sshpass -p vagrant scp -o StrictHostKeyChecking=no"
|
||||
|
||||
# remove old key:
|
||||
ssh-keygen -R "${ADMIN_IP}"
|
||||
|
||||
# Let's wait 5 sec to all vms up
|
||||
sleep 5
|
||||
|
||||
# Store info about Jenkins job on VM:
|
||||
echo "${BUILD_TAG}" | ${SSH_COMMAND} "tee -a JENKINS_INFO.TXT"
|
||||
function fix_restored_env {
|
||||
|
||||
# After restore snapshot ntp service doesn't work and need restart on all nodes
|
||||
${SSH_COMMAND} "sudo service ntp restart"
|
||||
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo service ntp restart"
|
||||
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo service ntp restart"
|
||||
|
||||
|
||||
# Dirty hack for workaround network problems on CI envs.
|
||||
# When we deploy env some time after (few minutes) it change resolv.conf into broken one
|
||||
# From this reason after bring up env we restart network and and kill dhclient,
|
||||
|
@ -114,16 +107,14 @@ ${SSH_COMMAND} "sudo ./fix_dns.sh"
|
|||
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo ./fix_dns.sh"
|
||||
${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo ./fix_dns.sh"
|
||||
|
||||
# Change registry ip address to slave and set tag
|
||||
sed -i 's/127.0.0.1:31500/'${REGISTRY_IP}':'${REGISTRY_PORT}'/g' fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml
|
||||
cat >> fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml << EOF
|
||||
images:
|
||||
tag: "${ZUUL_CHANGE}"
|
||||
EOF
|
||||
}
|
||||
|
||||
function copy_code_to_vm {
|
||||
# Prepare env on "admin" VM:
|
||||
if [ ${COMPONENT} == "full" ]; then
|
||||
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
|
||||
elif [ ${COMPONENT} == "smoke" ]; then
|
||||
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
|
||||
else
|
||||
${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/
|
||||
# set +x is just for security reasons to avoid publishing internal IP
|
||||
|
@ -132,23 +123,299 @@ else
|
|||
set -x
|
||||
${SCP_COMMAND} -r containers/openstack/ vagrant@"${ADMIN_IP}":/tmp/ccp-repos
|
||||
fi
|
||||
}
|
||||
|
||||
|
||||
function prepare_ccp_config {
|
||||
cat > ccp.yml << EOF
|
||||
builder:
|
||||
push: True
|
||||
workers: 1
|
||||
registry:
|
||||
address: "${REGISTRY_IP}:${REGISTRY_PORT}"
|
||||
timeout: 1500
|
||||
replicas:
|
||||
database: 3
|
||||
rpc: 3
|
||||
repositories:
|
||||
path: /tmp/ccp-repos
|
||||
skip_empty: True
|
||||
entrypoint_repo_name: entrypoint
|
||||
repos:
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-debian-base
|
||||
name: debian-base
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-entrypoint
|
||||
name: entrypoint
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-etcd
|
||||
name: etcd
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-galera
|
||||
name: galera
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-glance
|
||||
name: glance
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-heat
|
||||
name: heat
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-horizon
|
||||
name: horizon
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-keystone
|
||||
name: keystone
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-memcached
|
||||
name: memcached
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-neutron
|
||||
name: neutron
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-nova
|
||||
name: nova
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-openstack-base
|
||||
name: openstack-base
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-rabbitmq
|
||||
name: rabbitmq
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-ceph
|
||||
name: ceph
|
||||
- git_url: https://git.openstack.org/openstack/fuel-ccp-ironic
|
||||
name: ironic
|
||||
configs:
|
||||
private_interface: ens3
|
||||
nodes:
|
||||
node[1-3]:
|
||||
roles:
|
||||
- controller-net-bridge
|
||||
node1:
|
||||
roles:
|
||||
- db
|
||||
- messaging
|
||||
- openvswitch
|
||||
- controller-net-host
|
||||
node[2-3]:
|
||||
roles:
|
||||
- db
|
||||
- messaging
|
||||
- openvswitch
|
||||
- compute
|
||||
sources:
|
||||
openstack/cinder:
|
||||
git_url: https://git.openstack.org/openstack/cinder.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/ironic:
|
||||
git_url: https://git.openstack.org/openstack/ironic.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/murano:
|
||||
git_url: https://git.openstack.org/openstack/murano.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/murano-dashboard:
|
||||
git_url: https://git.openstack.org/openstack/murano-dashboard.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/sahara:
|
||||
git_url: https://git.openstack.org/openstack/sahara.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/sahara-dashboard:
|
||||
git_url: https://git.openstack.org/openstack/sahara-dashboard.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/keystone:
|
||||
git_url: https://git.openstack.org/openstack/keystone.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/horizon:
|
||||
git_url: https://git.openstack.org/openstack/horizon.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/nova:
|
||||
git_url: https://git.openstack.org/openstack/nova.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/neutron:
|
||||
git_url: https://git.openstack.org/openstack/neutron.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/heat:
|
||||
git_url: https://git.openstack.org/openstack/heat.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/glance:
|
||||
git_url: https://git.openstack.org/openstack/glance.git
|
||||
git_ref: ${BRANCH}
|
||||
openstack/requirements:
|
||||
git_url: https://git.openstack.org/openstack/requirements.git
|
||||
git_ref: ${BRANCH}
|
||||
services:
|
||||
database:
|
||||
service_def: galera
|
||||
rpc:
|
||||
service_def: rabbitmq
|
||||
notifications:
|
||||
service_def: rabbitmq
|
||||
roles:
|
||||
db:
|
||||
- database
|
||||
messaging:
|
||||
- rpc
|
||||
- notifications
|
||||
controller-net-host:
|
||||
- neutron-dhcp-agent
|
||||
- neutron-l3-agent
|
||||
- neutron-metadata-agent
|
||||
controller-net-bridge:
|
||||
- etcd
|
||||
- glance-api
|
||||
- glance-registry
|
||||
- heat-api-cfn
|
||||
- heat-api
|
||||
- heat-engine
|
||||
- horizon
|
||||
- keystone
|
||||
- memcached
|
||||
- neutron-server
|
||||
- nova-api
|
||||
- nova-conductor
|
||||
- nova-consoleauth
|
||||
- nova-novncproxy
|
||||
- nova-scheduler
|
||||
- rabbitmq
|
||||
compute:
|
||||
- nova-compute
|
||||
- nova-libvirt
|
||||
openvswitch:
|
||||
- neutron-openvswitch-agent
|
||||
- openvswitch-db
|
||||
- openvswitch-vswitchd
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
ccp_wait_for_deployment_to_finish () {
|
||||
cnt=0
|
||||
until [[ `${SSH_COMMAND} ccp status -s -f value -c status` == "ok" ]]; do
|
||||
echo "Waiting for OpenStack deployment to finish..."
|
||||
sleep 5
|
||||
cnt=$((cnt + 1))
|
||||
if [ ${cnt} -eq $1 ]; then
|
||||
echo "Max time exceeded"
|
||||
${SSH_COMMAND} ccp status
|
||||
${SSH_COMMAND} fuel-ccp/tools/diagnostic-snapshot.sh -n ccp -c ccp.yml
|
||||
${SCP_COMMAND} vagrant@"${ADMIN_IP}":/tmp/ccp-diag/*.tar.gz .
|
||||
return 1
|
||||
fi
|
||||
done
|
||||
echo "...................................."
|
||||
echo "Jobs and pods in namespace: ccp"
|
||||
${SSH_COMMAND} kubectl --namespace ccp get jobs
|
||||
${SSH_COMMAND} kubectl --namespace ccp get pods
|
||||
echo "openrc file: openrc-ccp"
|
||||
${SSH_COMMAND} cat openrc-ccp
|
||||
echo "...................................."
|
||||
}
|
||||
|
||||
function ccp_install {
|
||||
${SSH_COMMAND} "sudo -H pip install -r fuel-ccp/requirements.txt"
|
||||
${SSH_COMMAND} "sudo -H pip install fuel-ccp/"
|
||||
}
|
||||
|
||||
|
||||
function deploy_ccp {
|
||||
pwd
|
||||
${SCP_COMMAND} ccp.yml vagrant@"${ADMIN_IP}":~/
|
||||
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build -c etcd memcached rabbitmq galera"
|
||||
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy -c etcd memcached database"
|
||||
ccp_wait_for_deployment_to_finish 50
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build -c keystone"
|
||||
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy -c keystone"
|
||||
ccp_wait_for_deployment_to_finish 30
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build"
|
||||
${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy"
|
||||
ccp_wait_for_deployment_to_finish 200
|
||||
if [ $? -ne 0 ]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
}
|
||||
|
||||
prepare_k8s_env
|
||||
|
||||
|
||||
# Get IP address of first node in the cluster:
|
||||
ADMIN_IP=$(ENV_NAME=${FUEL_DEVOPS_ENV_NAME} python fuel-ccp-installer/utils/jenkins/env.py get_slaves_ips | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+" | head -1)
|
||||
SSH_COMMAND="sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@${ADMIN_IP}"
|
||||
SCP_COMMAND="sshpass -p vagrant scp -o StrictHostKeyChecking=no"
|
||||
|
||||
# remove old key:
|
||||
ssh-keygen -R "${ADMIN_IP}"
|
||||
|
||||
# Let's wait 5 sec to all vms up
|
||||
sleep 5
|
||||
|
||||
# Store info about Jenkins job on VM:
|
||||
echo "${BUILD_TAG}" | ${SSH_COMMAND} "tee -a JENKINS_INFO.TXT"
|
||||
|
||||
fix_restored_env
|
||||
copy_code_to_vm
|
||||
|
||||
|
||||
|
||||
# set version of openstack
|
||||
if [ "${VERSION}" == "master" ];then
|
||||
export IMAGES_TAG="latest"
|
||||
export BRANCH="master"
|
||||
elif [ "${VERSION}" == "ocata" ];then
|
||||
export IMAGES_TAG="ocata"
|
||||
export BRANCH="stable/ocata"
|
||||
elif [ "${VERSION}" == "newton" ]; then
|
||||
export IMAGES_TAG="newton"
|
||||
export BRANCH="stable/newton"
|
||||
else
|
||||
export IMAGES_TAG="mitaka"
|
||||
export BRANCH="stable/mitaka"
|
||||
fi
|
||||
|
||||
prepare_ccp_config
|
||||
|
||||
ccp_install
|
||||
|
||||
if [ ${COMPONENT} == "smoke" ]; then
|
||||
set +e
|
||||
# Run CCP deployment and OpenStack tests:
|
||||
${SSH_COMMAND} "pushd fuel-ccp && APT_CACHE_SERVER=http://${APT_CACHE_SERVER_IP}:${APT_CACHE_SERVER_PORT} tox -e multi-deploy -- --openstack-version ${VERSION} --number-of-envs 1 -d"
|
||||
deploy_ccp
|
||||
DEPLOY_STATUS=$?
|
||||
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" "echo ${DOCKER_REGISTRY_IP} ${DOCKER_REGISTRY_HOST} |sudo tee -a /etc/hosts"
|
||||
set -e
|
||||
|
||||
docker exec "${REGISTRY_HASH}" chgrp -R "${JENKINS_GID}" /var/lib/registry
|
||||
docker exec "${REGISTRY_HASH}" chmod -R g+w /var/lib/registry
|
||||
#set tag dependent from test result
|
||||
if [[ "${DEPLOY_STATUS}" == 0 ]]; then
|
||||
DOCKER_TAG="${IMAGES_TAG}"
|
||||
else
|
||||
DOCKER_TAG="${IMAGES_TAG}-unstable"
|
||||
fi
|
||||
|
||||
IMG=`sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@${ADMIN_IP} docker images --format "{{.Repository}}" | awk -F'/' -v search=/${IMAGES_NAMESPACE}/ '$0 ~ search {print $3}'`
|
||||
|
||||
# we need docker config file to authentication in remote repository
|
||||
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" mkdir -p /home/vagrant/.docker/
|
||||
sshpass -p vagrant scp -o StrictHostKeyChecking=no /home/jenkins/.docker/config.json vagrant@"${ADMIN_IP}":~/.docker/
|
||||
|
||||
for f in ${IMG}; do
|
||||
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" \
|
||||
"docker tag ${REGISTRY_IP}:${REGISTRY_PORT}/${IMAGES_NAMESPACE}/$f:latest ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:${DOCKER_TAG} && docker push ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:${DOCKER_TAG}"
|
||||
if [ "${IMAGES_TAG}" == "ocata" ]; then
|
||||
sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" \
|
||||
"docker tag ${REGISTRY_IP}:${REGISTRY_PORT}/${IMAGES_NAMESPACE}/${f}:latest ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:latest \
|
||||
&& docker push ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:latest"
|
||||
fi
|
||||
done
|
||||
else
|
||||
set +e
|
||||
# Run CCP deployment and OpenStack tests:
|
||||
deploy_ccp
|
||||
DEPLOY_STATUS=$?
|
||||
echo "Deploy status: ${DEPLOY_STATUS}"
|
||||
set -e
|
||||
fi
|
||||
|
||||
set +e
|
||||
# download snapshot if exists
|
||||
${SCP_COMMAND} vagrant@"${ADMIN_IP}":/tmp/ccp-diag/*.tar.gz .
|
||||
|
||||
# remove images from local registry
|
||||
for i in `curl http://${REGISTRY_IP}:${REGISTRY_PORT}/v2/_catalog | jq -r '.repositories[]'`; do
|
||||
REGISTRY_DATA_DIR=/home/jenkins/registry/data/docker/registry/v2/ /home/jenkins/registry/delete_docker_registry_image.py --image "${i}":"${ZUUL_CHANGE}"
|
||||
done
|
||||
set -e
|
||||
|
||||
|
||||
# Revert to fresh to decrease image size
|
||||
dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}"
|
||||
|
||||
|
|
|
@ -60,7 +60,10 @@
|
|||
version:
|
||||
- master
|
||||
- newton
|
||||
- mitaka
|
||||
- ocata
|
||||
component:
|
||||
- smoke
|
||||
- full
|
||||
- debian-base
|
||||
- etcd
|
||||
|
|
|
@ -197,8 +197,10 @@ projects:
|
|||
post:
|
||||
- update-jenkins-jobs
|
||||
periodic:
|
||||
- 'build-newton-images-smoke'
|
||||
- 'build-ocata-images-smoke'
|
||||
- 'fuel-ccp-smoke-master-deployment'
|
||||
- 'fuel-ccp-smoke-newton-deployment'
|
||||
- 'fuel-ccp-smoke-ocata-deployment'
|
||||
- 'fuel-ccp-smoke-mitaka-deployment'
|
||||
|
||||
jobs:
|
||||
- name: ^(fuel-ccp).*(newton-deployment)
|
||||
|
|
Loading…
Reference in New Issue