From 690d41fec82a00596e314f017bf082ca47c072cf Mon Sep 17 00:00:00 2001 From: Artur Zarzycki Date: Mon, 16 Jan 2017 13:54:13 +0100 Subject: [PATCH] Add simple deployment debug job with upload images to external registry Add simple deployment debug job which is not removing env after finishing, also it can be run manually on pointed node. Just to debug some special cases. Morover this job send images to external registry located on host. Change-Id: Ia7b350b2e19f33b37ea922aa55f71336751ea8ef --- .../builders/fuel-ccp-debug-deployment.sh | 124 ++++++++++++++++++ jenkins-tp/jobs/fuel-ccp-debug-deploy.yaml | 48 +++++++ 2 files changed, 172 insertions(+) create mode 100644 jenkins-tp/jobs/builders/fuel-ccp-debug-deployment.sh create mode 100644 jenkins-tp/jobs/fuel-ccp-debug-deploy.yaml diff --git a/jenkins-tp/jobs/builders/fuel-ccp-debug-deployment.sh b/jenkins-tp/jobs/builders/fuel-ccp-debug-deployment.sh new file mode 100644 index 0000000..fac3f2e --- /dev/null +++ b/jenkins-tp/jobs/builders/fuel-ccp-debug-deployment.sh @@ -0,0 +1,124 @@ +#!/bin/bash -ex + +# Deploys CCP and runs simple, built-in OpenStack tests. +# Kubernetes cluster is expected to be pre-deployed and snapshoted - if not +# this script will ensure this. +# The script expects fuel-ccp cloned into fuel-ccp/ and fuel-ccp-installer +# cloned into fuel-ccp-installer/ directory (e.g. by Jenkins SCM plugin). + + +# CONFIGURATION: +###################################################### +: ${FUEL_DEVOPS_ENV_NAME:="fuel-ccp-debug-${VERSION}-deployment"} +FUEL_DEVOPS_SNAPSHOT_NAME="fresh" +FUEL_DEVOPS_INSTALLATION_DIR="/home/jenkins/venv-fuel-devops-3.0" +HYPERKUBE_REPO="quay.io/coreos/hyperkube" +HYPERKUBE_TAG="v1.5.1_coreos.0" +HYPERKUBE_VERSION="v1.5.1" +export APT_CACHE_SERVER_IP="`getent hosts cache-scc.ng.mirantis.net| awk '{print $1}'`" +export APT_CACHE_SERVER_PORT="3142" +export REGISTRY_IP=`ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'` + +# Prepare K8s env: +source "${FUEL_DEVOPS_INSTALLATION_DIR}"/bin/activate +NEED_TO_SETUP_ENV=false +dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" || NEED_TO_SETUP_ENV=true +dos.py resume "${FUEL_DEVOPS_ENV_NAME}" || NEED_TO_SETUP_ENV=true +if [ "${NEED_TO_SETUP_ENV}" = "true" ]; then + echo "Couldn't revert fuel-devops snapshot, recreating environment." + + # Try to delete old envs to ensure clean host: + dos.py destroy "${FUEL_DEVOPS_ENV_NAME}" || true + dos.py erase "${FUEL_DEVOPS_ENV_NAME}" || true + + # Create kargo deployment script: + cat > k8s_deploy.sh << EOF +#!/bin/bash -ex + +export ENV_NAME=${FUEL_DEVOPS_ENV_NAME} +export IMAGE_PATH="/home/jenkins/workspace/cloud-images/default-image.qcow2" +export DONT_DESTROY_ON_SUCCESS=1 +export DEPLOY_METHOD="kargo" +export SLAVES_COUNT="3" +export WORKSPACE="/home/jenkins/workspace" +export FUEL_DEVOPS_INSTALLATION_DIR=${FUEL_DEVOPS_INSTALLATION_DIR} +export CUSTOM_YAML='hyperkube_image_repo: "${HYPERKUBE_REPO}" +hyperkube_image_tag: "${HYPERKUBE_TAG}" +kube_version: "${HYPERKUBE_VERSION}" +docker_options: "--insecure-registry=${REGISTRY_IP}' + +echo "Running on \${NODE_NAME}: \${ENV_NAME}" +source \${FUEL_DEVOPS_INSTALLATION_DIR}/bin/activate +echo "Use image:" +ls -la \${IMAGE_PATH} +env + +pushd fuel-ccp-installer +bash -ex "./utils/jenkins/run_k8s_deploy_test.sh" +popd +EOF + chmod +x k8s_deploy.sh + + # Now deploy the cluster: + ./k8s_deploy.sh + + # Create fresh snapshot: + dos.py suspend "${FUEL_DEVOPS_ENV_NAME}" + dos.py snapshot "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" + + # Resume from snapshot to deploy CCP later on in the script: + dos.py resume "${FUEL_DEVOPS_ENV_NAME}" +fi + + +# Get IP address of first node in the cluster: +ADMIN_IP=$(ENV_NAME=${FUEL_DEVOPS_ENV_NAME} python fuel-ccp-installer/utils/jenkins/env.py get_slaves_ips | grep -o "[0-9]\+\.[0-9]\+\.[0-9]\+\.[0-9]\+" | head -1) +SSH_COMMAND="sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@${ADMIN_IP}" +SCP_COMMAND="sshpass -p vagrant scp -o StrictHostKeyChecking=no" + +# remove old key: +ssh-keygen -R "${ADMIN_IP}" + +# Store info about Jenkins job on VM: +echo "${BUILD_TAG}" | ${SSH_COMMAND} "tee -a JENKINS_INFO.TXT" + +# After restore snapshot ntp service doesn't work and need restart on all nodes +${SSH_COMMAND} "sudo service ntp restart" +${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo service ntp restart" +${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo service ntp restart" + +# Dirty hack for workaround network problems on CI envs. +# When we deploy env some time after (few minutes) it change resolv.conf into broken one +# From this reason after bring up env we restart network and and kill dhclient, +# we also restart docker and kubelet to make sure that all net-host containers are in good shape + +cat > fix_dns.sh << EOF +sudo service networking restart +sudo pkill -9 dhclient +sudo service kubelet restart +sudo service docker restart +EOF + +chmod +x fix_dns.sh + +${SCP_COMMAND} fix_dns.sh vagrant@"${ADMIN_IP}":~/ +${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node2:~/" +${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node3:~/" +${SSH_COMMAND} "sudo ./fix_dns.sh" +${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo ./fix_dns.sh" +${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo ./fix_dns.sh" + +sed -i 's/127.0.0.1:31500/'${REGISTRY_IP}'/g' fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml +sed -i '/builder:/a\ tag:'${ZUUL_CHANGE} fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml + +${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/ + + +# Run CCP deployment and OpenStack tests: +${SSH_COMMAND} "pushd fuel-ccp && APT_CACHE_SERVER=http://${APT_CACHE_SERVER_IP}:${APT_CACHE_SERVER_PORT} tox -e multi-deploy -- --openstack-version ${VERSION} --number-of-envs 1 -d" + +# Revert to fresh to decrease image size +dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" + +# Clean-up (snapshot should remain for next jobs): +dos.py destroy "${FUEL_DEVOPS_ENV_NAME}" diff --git a/jenkins-tp/jobs/fuel-ccp-debug-deploy.yaml b/jenkins-tp/jobs/fuel-ccp-debug-deploy.yaml new file mode 100644 index 0000000..cff700b --- /dev/null +++ b/jenkins-tp/jobs/fuel-ccp-debug-deploy.yaml @@ -0,0 +1,48 @@ +- job-template: + name: fuel-ccp-debug-{version}-deployment + description: | + Deploy CCP and run basic OpenStack tests (uses snapshot for K8s cluster) + node: standard + builders: + - inject: + properties-content: | + COMPONENT=full + VERSION={version} + - shell: + !include-raw-escape 'builders/fuel-ccp-debug-deployment.sh' + concurrent: true + properties: + - heavy-job: + weight: 2 + - throttle: + option: project + max-per-node: 1 + - least-load: + disabled: False + scm: + - openstack: + scm-basedir: 'fuel-ccp' + scm-branch: 'master' + scm-repo: 'fuel-ccp' + - openstack: + scm-basedir: 'fuel-ccp-installer' + scm-branch: 'master' + scm-repo: 'fuel-ccp-installer' + wrappers: + - ng-cleanup + - fuel-ccp-ci-jenkins + - timeout: + timeout: 120 + fail: true + parameters: + - node: + name: NODE + allowed-multiselect: true + +- project: + name: Deploy CCP in debug mode without remove snapshots and with selected node + version: + - master + - newton + jobs: + - 'fuel-ccp-debug-{version}-deployment'