diff --git a/jenkins-tp/jobs/builders/fuel-ccp-simple-deployment.sh b/jenkins-tp/jobs/builders/fuel-ccp-simple-deployment.sh old mode 100644 new mode 100755 index b15aed2..995ef1e --- a/jenkins-tp/jobs/builders/fuel-ccp-simple-deployment.sh +++ b/jenkins-tp/jobs/builders/fuel-ccp-simple-deployment.sh @@ -6,7 +6,6 @@ # The script expects fuel-ccp cloned into fuel-ccp/ and fuel-ccp-installer # cloned into fuel-ccp-installer/ directory (e.g. by Jenkins SCM plugin). - # CONFIGURATION: ###################################################### : ${FUEL_DEVOPS_ENV_NAME:="fuel-ccp-${COMPONENT}-${VERSION}-deployment"} @@ -21,21 +20,27 @@ export REGISTRY_IP=`ip -4 addr show eth0 | grep -oP '(?<=inet\s)\d+(\.\d+){3}'` export REGISTRY_PORT=5000 JENKINS_GID=`getent group jenkins | cut -d":" -f3` REGISTRY_HASH=`docker inspect --format "{{.Id}}" registry` +export DOCKER_REGISTRY_HOST="registry.mcp.fuel-infra.org" +export DOCKER_REGISTRY="${DOCKER_REGISTRY_HOST}:6000" +export DOCKER_REGISTRY_IP="`getent hosts ${DOCKER_REGISTRY_HOST} | awk '{print $1}'`" +export IMAGES_NAMESPACE="ccp" +export REGISTRY_NAMESPACE="mcp" -# Prepare K8s env: -source "${FUEL_DEVOPS_INSTALLATION_DIR}"/bin/activate -NEED_TO_SETUP_ENV=false -dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" || NEED_TO_SETUP_ENV=true -dos.py resume "${FUEL_DEVOPS_ENV_NAME}" || NEED_TO_SETUP_ENV=true -if [ "${NEED_TO_SETUP_ENV}" = "true" ]; then - echo "Couldn't revert fuel-devops snapshot, recreating environment." +function prepare_k8s_env { + # Prepare K8s env: + source "${FUEL_DEVOPS_INSTALLATION_DIR}"/bin/activate + NEED_TO_SETUP_ENV=false + dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" || NEED_TO_SETUP_ENV=true + dos.py resume "${FUEL_DEVOPS_ENV_NAME}" || NEED_TO_SETUP_ENV=true + if [ "${NEED_TO_SETUP_ENV}" = "true" ]; then + echo "Couldn't revert fuel-devops snapshot, recreating environment." - # Try to delete old envs to ensure clean host: - dos.py destroy "${FUEL_DEVOPS_ENV_NAME}" || true - dos.py erase "${FUEL_DEVOPS_ENV_NAME}" || true + # Try to delete old envs to ensure clean host: + dos.py destroy "${FUEL_DEVOPS_ENV_NAME}" || true + dos.py erase "${FUEL_DEVOPS_ENV_NAME}" || true - # Create kargo deployment script: - cat > k8s_deploy.sh << EOF + # Create kargo deployment script: + cat > k8s_deploy.sh << EOF #!/bin/bash -ex export ENV_NAME=${FUEL_DEVOPS_ENV_NAME} @@ -48,30 +53,282 @@ export FUEL_DEVOPS_INSTALLATION_DIR=${FUEL_DEVOPS_INSTALLATION_DIR} export CUSTOM_YAML='hyperkube_image_repo: "${HYPERKUBE_REPO}" hyperkube_image_tag: "${HYPERKUBE_TAG}" kube_version: "${HYPERKUBE_VERSION}" -docker_options: "--insecure-registry=${REGISTRY_IP}:${REGISTRY_PORT}"' - +docker_options: "--insecure-registry=${REGISTRY_IP}:${REGISTRY_PORT} --insecure-registry=DOCKER_REGISTRY"' echo "Running on \${NODE_NAME}: \${ENV_NAME}" source \${FUEL_DEVOPS_INSTALLATION_DIR}/bin/activate echo "Use image:" ls -la \${IMAGE_PATH} env - pushd fuel-ccp-installer bash -ex "./utils/jenkins/run_k8s_deploy_test.sh" popd EOF - chmod +x k8s_deploy.sh + chmod +x k8s_deploy.sh - # Now deploy the cluster: - ./k8s_deploy.sh + # Now deploy the cluster: + ./k8s_deploy.sh - # Create fresh snapshot: - dos.py suspend "${FUEL_DEVOPS_ENV_NAME}" - dos.py snapshot "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" + # Create fresh snapshot: + dos.py suspend "${FUEL_DEVOPS_ENV_NAME}" + dos.py snapshot "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" - # Resume from snapshot to deploy CCP later on in the script: - dos.py resume "${FUEL_DEVOPS_ENV_NAME}" -fi + # Resume from snapshot to deploy CCP later on in the script: + dos.py resume "${FUEL_DEVOPS_ENV_NAME}" + fi +} + + +function fix_restored_env { + + # After restore snapshot ntp service doesn't work and need restart on all nodes + ${SSH_COMMAND} "sudo service ntp restart" + ${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo service ntp restart" + ${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo service ntp restart" + + + # Dirty hack for workaround network problems on CI envs. + # When we deploy env some time after (few minutes) it change resolv.conf into broken one + # From this reason after bring up env we restart network and and kill dhclient, + # we also restart docker and kubelet to make sure that all net-host containers are in good shape + + cat > fix_dns.sh << EOF + sudo service networking restart + sudo pkill -9 dhclient + sudo service kubelet restart + sudo service docker restart +EOF + + chmod +x fix_dns.sh + + ${SCP_COMMAND} fix_dns.sh vagrant@"${ADMIN_IP}":~/ + ${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node2:~/" + ${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node3:~/" + ${SSH_COMMAND} "sudo ./fix_dns.sh" + ${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo ./fix_dns.sh" + ${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo ./fix_dns.sh" + +} + +function copy_code_to_vm { + # Prepare env on "admin" VM: + if [ ${COMPONENT} == "full" ]; then + ${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/ + elif [ ${COMPONENT} == "smoke" ]; then + ${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/ + else + ${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/ + # set +x is just for security reasons to avoid publishing internal IP + set +x + getent hosts zuul.mcp.fuel-infra.org | ${SSH_COMMAND} "sudo tee -a /etc/hosts" + set -x + ${SCP_COMMAND} -r containers/openstack/ vagrant@"${ADMIN_IP}":/tmp/ccp-repos + fi +} + + +function prepare_ccp_config { +cat > ccp.yml << EOF +builder: + push: True + workers: 1 +registry: + address: "${REGISTRY_IP}:${REGISTRY_PORT}" + timeout: 1500 +replicas: + database: 3 + rpc: 3 +repositories: + path: /tmp/ccp-repos + skip_empty: True + entrypoint_repo_name: entrypoint + repos: + - git_url: https://git.openstack.org/openstack/fuel-ccp-debian-base + name: debian-base + - git_url: https://git.openstack.org/openstack/fuel-ccp-entrypoint + name: entrypoint + - git_url: https://git.openstack.org/openstack/fuel-ccp-etcd + name: etcd + - git_url: https://git.openstack.org/openstack/fuel-ccp-galera + name: galera + - git_url: https://git.openstack.org/openstack/fuel-ccp-glance + name: glance + - git_url: https://git.openstack.org/openstack/fuel-ccp-heat + name: heat + - git_url: https://git.openstack.org/openstack/fuel-ccp-horizon + name: horizon + - git_url: https://git.openstack.org/openstack/fuel-ccp-keystone + name: keystone + - git_url: https://git.openstack.org/openstack/fuel-ccp-memcached + name: memcached + - git_url: https://git.openstack.org/openstack/fuel-ccp-neutron + name: neutron + - git_url: https://git.openstack.org/openstack/fuel-ccp-nova + name: nova + - git_url: https://git.openstack.org/openstack/fuel-ccp-openstack-base + name: openstack-base + - git_url: https://git.openstack.org/openstack/fuel-ccp-rabbitmq + name: rabbitmq + - git_url: https://git.openstack.org/openstack/fuel-ccp-ceph + name: ceph + - git_url: https://git.openstack.org/openstack/fuel-ccp-ironic + name: ironic +configs: + private_interface: ens3 +nodes: + node[1-3]: + roles: + - controller-net-bridge + node1: + roles: + - db + - messaging + - openvswitch + - controller-net-host + node[2-3]: + roles: + - db + - messaging + - openvswitch + - compute +sources: + openstack/cinder: + git_url: https://git.openstack.org/openstack/cinder.git + git_ref: ${BRANCH} + openstack/ironic: + git_url: https://git.openstack.org/openstack/ironic.git + git_ref: ${BRANCH} + openstack/murano: + git_url: https://git.openstack.org/openstack/murano.git + git_ref: ${BRANCH} + openstack/murano-dashboard: + git_url: https://git.openstack.org/openstack/murano-dashboard.git + git_ref: ${BRANCH} + openstack/sahara: + git_url: https://git.openstack.org/openstack/sahara.git + git_ref: ${BRANCH} + openstack/sahara-dashboard: + git_url: https://git.openstack.org/openstack/sahara-dashboard.git + git_ref: ${BRANCH} + openstack/keystone: + git_url: https://git.openstack.org/openstack/keystone.git + git_ref: ${BRANCH} + openstack/horizon: + git_url: https://git.openstack.org/openstack/horizon.git + git_ref: ${BRANCH} + openstack/nova: + git_url: https://git.openstack.org/openstack/nova.git + git_ref: ${BRANCH} + openstack/neutron: + git_url: https://git.openstack.org/openstack/neutron.git + git_ref: ${BRANCH} + openstack/heat: + git_url: https://git.openstack.org/openstack/heat.git + git_ref: ${BRANCH} + openstack/glance: + git_url: https://git.openstack.org/openstack/glance.git + git_ref: ${BRANCH} + openstack/requirements: + git_url: https://git.openstack.org/openstack/requirements.git + git_ref: ${BRANCH} +services: + database: + service_def: galera + rpc: + service_def: rabbitmq + notifications: + service_def: rabbitmq +roles: + db: + - database + messaging: + - rpc + - notifications + controller-net-host: + - neutron-dhcp-agent + - neutron-l3-agent + - neutron-metadata-agent + controller-net-bridge: + - etcd + - glance-api + - glance-registry + - heat-api-cfn + - heat-api + - heat-engine + - horizon + - keystone + - memcached + - neutron-server + - nova-api + - nova-conductor + - nova-consoleauth + - nova-novncproxy + - nova-scheduler + - rabbitmq + compute: + - nova-compute + - nova-libvirt + openvswitch: + - neutron-openvswitch-agent + - openvswitch-db + - openvswitch-vswitchd + +EOF +} + +ccp_wait_for_deployment_to_finish () { + cnt=0 + until [[ `${SSH_COMMAND} ccp status -s -f value -c status` == "ok" ]]; do + echo "Waiting for OpenStack deployment to finish..." + sleep 5 + cnt=$((cnt + 1)) + if [ ${cnt} -eq $1 ]; then + echo "Max time exceeded" + ${SSH_COMMAND} ccp status + ${SSH_COMMAND} fuel-ccp/tools/diagnostic-snapshot.sh -n ccp -c ccp.yml + ${SCP_COMMAND} vagrant@"${ADMIN_IP}":/tmp/ccp-diag/*.tar.gz . + return 1 + fi + done + echo "...................................." + echo "Jobs and pods in namespace: ccp" + ${SSH_COMMAND} kubectl --namespace ccp get jobs + ${SSH_COMMAND} kubectl --namespace ccp get pods + echo "openrc file: openrc-ccp" + ${SSH_COMMAND} cat openrc-ccp + echo "...................................." +} + +function ccp_install { + ${SSH_COMMAND} "sudo -H pip install -r fuel-ccp/requirements.txt" + ${SSH_COMMAND} "sudo -H pip install fuel-ccp/" +} + + +function deploy_ccp { + pwd + ${SCP_COMMAND} ccp.yml vagrant@"${ADMIN_IP}":~/ + ${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build -c etcd memcached rabbitmq galera" + ${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy -c etcd memcached database" + ccp_wait_for_deployment_to_finish 50 + if [ $? -ne 0 ]; then + return 1 + fi + ${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build -c keystone" + ${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy -c keystone" + ccp_wait_for_deployment_to_finish 30 + if [ $? -ne 0 ]; then + return 1 + fi + ${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml build" + ${SSH_COMMAND} "ccp -vvv --debug --config-file ~/ccp.yml deploy" + ccp_wait_for_deployment_to_finish 200 + if [ $? -ne 0 ]; then + return 1 + fi + +} + +prepare_k8s_env # Get IP address of first node in the cluster: @@ -88,67 +345,77 @@ sleep 5 # Store info about Jenkins job on VM: echo "${BUILD_TAG}" | ${SSH_COMMAND} "tee -a JENKINS_INFO.TXT" -# After restore snapshot ntp service doesn't work and need restart on all nodes -${SSH_COMMAND} "sudo service ntp restart" -${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo service ntp restart" -${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo service ntp restart" +fix_restored_env +copy_code_to_vm -# Dirty hack for workaround network problems on CI envs. -# When we deploy env some time after (few minutes) it change resolv.conf into broken one -# From this reason after bring up env we restart network and and kill dhclient, -# we also restart docker and kubelet to make sure that all net-host containers are in good shape -cat > fix_dns.sh << EOF -sudo service networking restart -sudo pkill -9 dhclient -sudo service kubelet restart -sudo service docker restart -EOF -chmod +x fix_dns.sh - -${SCP_COMMAND} fix_dns.sh vagrant@"${ADMIN_IP}":~/ -${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node2:~/" -${SSH_COMMAND} "scp -o StrictHostKeyChecking=no fix_dns.sh vagrant@node3:~/" -${SSH_COMMAND} "sudo ./fix_dns.sh" -${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node2 sudo ./fix_dns.sh" -${SSH_COMMAND} "ssh -o StrictHostKeyChecking=no node3 sudo ./fix_dns.sh" - -# Change registry ip address to slave and set tag -sed -i 's/127.0.0.1:31500/'${REGISTRY_IP}':'${REGISTRY_PORT}'/g' fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml -cat >> fuel-ccp/tools/ccp-multi-deploy/config/ccp-configs-common.yaml << EOF -images: - tag: "${ZUUL_CHANGE}" -EOF - -# Prepare env on "admin" VM: -if [ ${COMPONENT} == "full" ];then - ${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/ +# set version of openstack +if [ "${VERSION}" == "master" ];then + export IMAGES_TAG="latest" + export BRANCH="master" +elif [ "${VERSION}" == "ocata" ];then + export IMAGES_TAG="ocata" + export BRANCH="stable/ocata" +elif [ "${VERSION}" == "newton" ]; then + export IMAGES_TAG="newton" + export BRANCH="stable/newton" else - ${SCP_COMMAND} -r fuel-ccp/ vagrant@"${ADMIN_IP}":~/ - # set +x is just for security reasons to avoid publishing internal IP - set +x - getent hosts zuul.mcp.fuel-infra.org | ${SSH_COMMAND} "sudo tee -a /etc/hosts" - set -x - ${SCP_COMMAND} -r containers/openstack/ vagrant@"${ADMIN_IP}":/tmp/ccp-repos + export IMAGES_TAG="mitaka" + export BRANCH="stable/mitaka" fi -# Run CCP deployment and OpenStack tests: -${SSH_COMMAND} "pushd fuel-ccp && APT_CACHE_SERVER=http://${APT_CACHE_SERVER_IP}:${APT_CACHE_SERVER_PORT} tox -e multi-deploy -- --openstack-version ${VERSION} --number-of-envs 1 -d" +prepare_ccp_config -docker exec "${REGISTRY_HASH}" chgrp -R "${JENKINS_GID}" /var/lib/registry -docker exec "${REGISTRY_HASH}" chmod -R g+w /var/lib/registry +ccp_install + +if [ ${COMPONENT} == "smoke" ]; then + set +e + # Run CCP deployment and OpenStack tests: + deploy_ccp + DEPLOY_STATUS=$? + sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" "echo ${DOCKER_REGISTRY_IP} ${DOCKER_REGISTRY_HOST} |sudo tee -a /etc/hosts" + set -e + + #set tag dependent from test result + if [[ "${DEPLOY_STATUS}" == 0 ]]; then + DOCKER_TAG="${IMAGES_TAG}" + else + DOCKER_TAG="${IMAGES_TAG}-unstable" + fi + + IMG=`sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@${ADMIN_IP} docker images --format "{{.Repository}}" | awk -F'/' -v search=/${IMAGES_NAMESPACE}/ '$0 ~ search {print $3}'` + + # we need docker config file to authentication in remote repository + sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" mkdir -p /home/vagrant/.docker/ + sshpass -p vagrant scp -o StrictHostKeyChecking=no /home/jenkins/.docker/config.json vagrant@"${ADMIN_IP}":~/.docker/ + + for f in ${IMG}; do + sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" \ + "docker tag ${REGISTRY_IP}:${REGISTRY_PORT}/${IMAGES_NAMESPACE}/$f:latest ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:${DOCKER_TAG} && docker push ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:${DOCKER_TAG}" + if [ "${IMAGES_TAG}" == "ocata" ]; then + sshpass -p vagrant ssh -o StrictHostKeyChecking=no vagrant@"${ADMIN_IP}" \ + "docker tag ${REGISTRY_IP}:${REGISTRY_PORT}/${IMAGES_NAMESPACE}/${f}:latest ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:latest \ + && docker push ${DOCKER_REGISTRY}/${REGISTRY_NAMESPACE}/${f}:latest" + fi + done +else + set +e + # Run CCP deployment and OpenStack tests: + deploy_ccp + DEPLOY_STATUS=$? + echo "Deploy status: ${DEPLOY_STATUS}" + set -e +fi set +e -# download snapshot if exists -${SCP_COMMAND} vagrant@"${ADMIN_IP}":/tmp/ccp-diag/*.tar.gz . - # remove images from local registry for i in `curl http://${REGISTRY_IP}:${REGISTRY_PORT}/v2/_catalog | jq -r '.repositories[]'`; do - REGISTRY_DATA_DIR=/home/jenkins/registry/data/docker/registry/v2/ /home/jenkins/registry/delete_docker_registry_image.py --image "${i}":"${ZUUL_CHANGE}" + REGISTRY_DATA_DIR=/home/jenkins/registry/data/docker/registry/v2/ /home/jenkins/registry/delete_docker_registry_image.py --image "${i}":"${ZUUL_CHANGE}" done set -e + # Revert to fresh to decrease image size dos.py revert "${FUEL_DEVOPS_ENV_NAME}" "${FUEL_DEVOPS_SNAPSHOT_NAME}" diff --git a/jenkins-tp/jobs/fuel-ccp-simple-deployment.yaml b/jenkins-tp/jobs/fuel-ccp-simple-deployment.yaml index fa88850..b5621b1 100644 --- a/jenkins-tp/jobs/fuel-ccp-simple-deployment.yaml +++ b/jenkins-tp/jobs/fuel-ccp-simple-deployment.yaml @@ -60,7 +60,10 @@ version: - master - newton + - mitaka + - ocata component: + - smoke - full - debian-base - etcd diff --git a/zuul-tp/layout.yaml b/zuul-tp/layout.yaml index de3d959..77099cd 100644 --- a/zuul-tp/layout.yaml +++ b/zuul-tp/layout.yaml @@ -197,8 +197,10 @@ projects: post: - update-jenkins-jobs periodic: - - 'build-newton-images-smoke' - - 'build-ocata-images-smoke' + - 'fuel-ccp-smoke-master-deployment' + - 'fuel-ccp-smoke-newton-deployment' + - 'fuel-ccp-smoke-ocata-deployment' + - 'fuel-ccp-smoke-mitaka-deployment' jobs: - name: ^(fuel-ccp).*(newton-deployment)