From 43d623ec0c1120c12656f152c51e352bf9c6e614 Mon Sep 17 00:00:00 2001
From: yatin <ykarel@redhat.com>
Date: Mon, 23 Oct 2017 17:56:38 +0530
Subject: [PATCH] Add zuul v3 jobs in stable/ocata

Change-Id: Id2212fd5cb9caee538664308540c75b7bdaddbbb
(cherry picked from commit 839884593e6f6dabaebe401b013465c836fefc84)
---
 .zuul.yaml                                  |  20 ++++
 magnum/tests/contrib/post_test_hook.sh      |   8 +-
 playbooks/magnum-buildimages-base.yaml      | 102 ++++++++++++++++++++
 playbooks/magnum-functional-base.yaml       |  82 ++++++++++++++++
 playbooks/magnum-tox-migration.yaml         |  85 ++++++++++++++++
 playbooks/post/upload-images.yaml           |  21 ++++
 playbooks/post/upload-logs-tox.yaml         |  67 +++++++++++++
 playbooks/post/upload-logs.yaml             |  14 +++
 playbooks/pre/prepare-workspace-images.yaml |  12 +++
 playbooks/pre/prepare-workspace.yaml        |  23 +++++
 10 files changed, 430 insertions(+), 4 deletions(-)
 create mode 100644 .zuul.yaml
 create mode 100644 playbooks/magnum-buildimages-base.yaml
 create mode 100644 playbooks/magnum-functional-base.yaml
 create mode 100644 playbooks/magnum-tox-migration.yaml
 create mode 100644 playbooks/post/upload-images.yaml
 create mode 100644 playbooks/post/upload-logs-tox.yaml
 create mode 100644 playbooks/post/upload-logs.yaml
 create mode 100644 playbooks/pre/prepare-workspace-images.yaml
 create mode 100644 playbooks/pre/prepare-workspace.yaml

diff --git a/.zuul.yaml b/.zuul.yaml
new file mode 100644
index 0000000000..a3f203fe26
--- /dev/null
+++ b/.zuul.yaml
@@ -0,0 +1,20 @@
+- project:
+    name: openstack/magnum
+    check:
+      jobs:
+        - magnum-functional-api
+        - magnum-functional-k8s
+        - magnum-functional-swarm-mode
+    gate:
+      jobs:
+        - magnum-functional-api
+    experimental:
+      jobs:
+        - magnum-functional-dcos
+        - magnum-functional-mesos
+        - magnum-functional-swarm
+        - magnum-functional-k8s-ironic
+        - magnum-functional-swarm-ironic
+        - magnum-functional-k8s-multinode
+        - magnum-functional-swarm-mode-multinode
+        - magnum-non-functional-tox-migration
diff --git a/magnum/tests/contrib/post_test_hook.sh b/magnum/tests/contrib/post_test_hook.sh
index d0f69921b4..b0c4325dce 100755
--- a/magnum/tests/contrib/post_test_hook.sh
+++ b/magnum/tests/contrib/post_test_hook.sh
@@ -148,7 +148,7 @@ constraints="-c $REQUIREMENTS_DIR/upper-constraints.txt"
 sudo -H pip install $constraints -U -r requirements.txt -r test-requirements.txt
 
 export MAGNUM_DIR="$BASE/new/magnum"
-sudo chown -R jenkins:stack $MAGNUM_DIR
+sudo chown -R $USER:stack $MAGNUM_DIR
 
 # Run functional tests
 # Currently we support functional-api, functional-k8s, will support swarm,
@@ -170,7 +170,7 @@ if [[ "api" == "$coe" ]]; then
     source $BASE/new/devstack/functions
     echo "TEMPEST_SERVICES+=,magnum" >> $localrc_path
     pushd $BASE/new/tempest
-    sudo chown -R jenkins:stack $BASE/new/tempest
+    sudo chown -R $USER:stack $BASE/new/tempest
 
     # Set demo credentials
     source $BASE/new/devstack/accrc/demo/demo
@@ -193,7 +193,7 @@ if [[ "api" == "$coe" ]]; then
     export MAGNUM_TESTS=${MAGNUM_TESTS:-'magnum.tests.functional.api.v1'}
 
     echo "Running tempest magnum test suites"
-    sudo -H -u jenkins tox -eall-plugin -- $MAGNUM_TESTS --concurrency=$MAGNUM_TEMPEST_CONCURRENCY
+    sudo -H -u $USER tox -eall-plugin -- $MAGNUM_TESTS --concurrency=$MAGNUM_TEMPEST_CONCURRENCY
 else
     # Get admin credentials
     pushd ../devstack
@@ -203,7 +203,7 @@ else
     create_test_data $coe $special
 
     target="${coe}${special}"
-    sudo -E -H -u jenkins tox -e functional-"$target" -- --concurrency=1
+    sudo -E -H -u $USER tox -e functional-"$target" -- --concurrency=1
 fi
 EXIT_CODE=$?
 
diff --git a/playbooks/magnum-buildimages-base.yaml b/playbooks/magnum-buildimages-base.yaml
new file mode 100644
index 0000000000..f8aea51e66
--- /dev/null
+++ b/playbooks/magnum-buildimages-base.yaml
@@ -0,0 +1,102 @@
+- hosts: primary
+  tasks:
+
+    - shell:
+        cmd: |
+          set -u
+          set -e
+          set -x
+          cd ~
+
+          if [[ "{{ image_name }}" =~ ^(ubuntu-mesos|centos-dcos)$ ]]; then
+              EXTRA_PROJECTS="openstack/tripleo-image-elements openstack/heat-templates"
+          else
+              EXTRA_PROJECTS=""
+          fi
+
+          /usr/zuul-env/bin/zuul-cloner --cache-dir /opt/git \
+            git://git.openstack.org \
+            openstack/diskimage-builder \
+            openstack/dib-utils \
+            openstack/magnum $EXTRA_PROJECTS
+
+          virtualenv env
+
+          ./env/bin/pip install $(pwd)/openstack/dib-utils
+          ./env/bin/pip install $(pwd)/openstack/diskimage-builder
+
+          # TODO(pabelanger): Remove once we migrated to bindep
+          ./openstack/diskimage-builder/tests/install_test_deps.sh
+
+          # activate the virtualenv so that any tools run by dib run
+          # using the python inside it
+          set +u
+          source ./env/bin/activate
+          set -u
+
+          DIB_ELEMENTS=./openstack/diskimage-builder/diskimage_builder/elements
+
+          if [ "{{ image_name }}" == "ubuntu-mesos" ]; then
+              TRIPLEO_ELEMENTS=./openstack/tripleo-image-elements/elements
+              HEAT_ELEMENTS=./openstack/heat-templates/hot/software-config/elements
+              MESOS_ELEMENTS=./openstack/magnum/magnum/drivers/mesos_ubuntu_v1/image
+              export ELEMENTS_PATH=$TRIPLEO_ELEMENTS:$HEAT_ELEMENTS:$MESOS_ELEMENTS
+
+              $MESOS_ELEMENTS/install_imagebuild_deps.sh
+
+              export DIB_RELEASE=trusty
+
+              export DIB_IMAGE_SIZE=2.2
+
+              disk-image-create ubuntu vm docker mesos \
+                os-collect-config os-refresh-config os-apply-config \
+                heat-config heat-config-script -o $WORKSPACE/{{ image_name }}.qcow2
+
+              $MESOS_ELEMENTS/validate_image.sh $WORKSPACE/{{ image_name }}.qcow2
+          elif [ "{{ image_name }}" == "centos-dcos" ]; then
+              DCOS_ELEMENTS=./openstack/magnum/contrib/drivers/dcos_centos_v1/image
+              TRIPLEO_ELEMENTS=./openstack/tripleo-image-elements/elements
+              HEAT_ELEMENTS=./openstack/heat-templates/hot/software-config/elements
+              # Order matters, we need the docker elements from DCOS_ELEMENTS to be used first
+              export ELEMENTS_PATH=$DCOS_ELEMENTS:$DIB_ELEMENTS:$TRIPLEO_ELEMENTS:$HEAT_ELEMENTS
+
+              $DCOS_ELEMENTS/install_imagebuild_deps.sh
+
+              export DIB_IMAGE_SIZE=3.0
+
+              export FS_TYPE=xfs
+
+              curl -O https://downloads.dcos.io/dcos/stable/commit/e64024af95b62c632c90b9063ed06296fcf38ea5/dcos_generate_config.sh
+              export DCOS_GENERATE_CONFIG_SRC=`pwd`/dcos_generate_config.sh
+
+              disk-image-create \
+                centos7 vm docker dcos selinux-permissive \
+                os-collect-config os-refresh-config os-apply-config \
+                heat-config heat-config-script \
+                -o $WORKSPACE/{{ image_name }}.qcow2
+
+              #TODO: Add size validation
+          else
+              MAGNUM_ELEMENTS=./openstack/magnum/magnum/drivers/common/image
+              export ELEMENTS_PATH=$DIB_ELEMENTS:$MAGNUM_ELEMENTS
+              $MAGNUM_ELEMENTS/fedora-atomic/install_imagebuild_deps.sh
+
+              export DIB_RELEASE="25"
+
+              export DIB_IMAGE_SIZE=2.5
+
+              export FEDORA_ATOMIC_TREE_URL="https://kojipkgs.fedoraproject.org/atomic/25/"
+              export FEDORA_ATOMIC_TREE_REF="8b15e9b988b4b02f4cb8b39bdd63d182ab7004a8926ecdac6314ee5c7ffa646b"
+
+              disk-image-create -x -o $WORKSPACE/{{ image_name }}-dib fedora-atomic
+
+              # validate image
+              $MAGNUM_ELEMENTS/fedora-atomic/validate_atomic_image.sh $WORKSPACE/{{ image_name }}-dib.qcow2
+          fi
+
+          set +u
+          deactivate
+          set -u
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/magnum-functional-base.yaml b/playbooks/magnum-functional-base.yaml
new file mode 100644
index 0000000000..8a61276dd7
--- /dev/null
+++ b/playbooks/magnum-functional-base.yaml
@@ -0,0 +1,82 @@
+- hosts: primary
+  tasks:
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          export PYTHONUNBUFFERED=true
+
+          if [ "{{ neutron }}" -eq 1 ] ; then
+              export DEVSTACK_GATE_NEUTRON=1
+          fi
+
+          export DEVSTACK_GATE_TEMPEST=1
+          if [ "{{ tempest }}" -eq 0 ] ; then
+              # Do not run any tempest tests
+              export DEVSTACK_GATE_TEMPEST_NOTESTS=1
+          fi
+
+          if [ "{{ branch_override }}" != "default" ] ; then
+              export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE
+          fi
+
+          export PROJECTS="openstack/magnum $PROJECTS"
+          export PROJECTS="openstack/python-magnumclient $PROJECTS"
+          export PROJECTS="openstack/diskimage-builder $PROJECTS"
+
+          if [ "{{ multinode }}" -eq 1 ] ; then
+              export DEVSTACK_GATE_TOPOLOGY="multinode"
+              export DEVSTACK_SUBNODE_CONFIG+=$'\n'"disable_service tempest"
+          fi
+
+          if [ "{{ ironic }}" -eq 1 ] ; then
+              export PROJECTS="openstack/ironic $PROJECTS"
+              export PROJECTS="openstack/ironic-lib $PROJECTS"
+              export PROJECTS="openstack/ironic-python-agent $PROJECTS"
+              export PROJECTS="openstack/python-ironicclient $PROJECTS"
+              export PROJECTS="openstack/pyghmi $PROJECTS"
+              export PROJECTS="openstack/virtualbmc $PROJECTS"
+              export MAGNUM_GATE_SPECIAL="-ironic"
+          fi
+
+          if [ "{{ horizon }}" -eq 0 ] ; then
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service horizon"
+          else
+              export DEVSTACK_GATE_HORIZON=1
+          fi
+          if [ "{{ swift }}" -eq 0 ] ; then
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-account"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-container"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-object"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service s-proxy"
+          fi
+          if [ "{{ ceilometer }}" -eq 0 ] ; then
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acentral"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-acompute"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-evaluator"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-alarm-notifier"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-api"
+              export DEVSTACK_LOCAL_CONFIG+=$'\n'"disable_service ceilometer-collector"
+          fi
+
+          # Keep localrc to be able to set some vars in post_test_hook
+          export KEEP_LOCALRC=1
+
+          function gate_hook {
+              cd /opt/stack/new/magnum/
+              ./magnum/tests/contrib/gate_hook.sh {{ coe }} $MAGNUM_GATE_SPECIAL
+          }
+          export -f gate_hook
+
+          function post_test_hook {
+              source $BASE/new/devstack/accrc/admin/admin
+              cd /opt/stack/new/magnum/
+              ./magnum/tests/contrib/post_test_hook.sh {{ coe }} $MAGNUM_GATE_SPECIAL
+          }
+          export -f post_test_hook
+
+          cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh
+          ./safe-devstack-vm-gate-wrap.sh
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/magnum-tox-migration.yaml b/playbooks/magnum-tox-migration.yaml
new file mode 100644
index 0000000000..bd415b753d
--- /dev/null
+++ b/playbooks/magnum-tox-migration.yaml
@@ -0,0 +1,85 @@
+- hosts: primary
+  tasks:
+
+    - name: Ensure workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          CLONEMAP=`mktemp`
+          REQS_DIR=`mktemp -d`
+          function cleanup {
+              mkdir -p $WORKSPACE
+              rm -rf $CLONEMAP $REQS_DIR
+          }
+          trap cleanup EXIT
+          cat > $CLONEMAP << EOF
+          clonemap:
+            - name: $ZUUL_PROJECT
+              dest: .
+          EOF
+          # zuul cloner works poorly if there are 2 names that are the
+          # same in here.
+          if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then
+          cat >> $CLONEMAP << EOF
+            - name: openstack/requirements
+              dest: $REQS_DIR
+          EOF
+          fi
+          /usr/zuul-env/bin/zuul-cloner -m $CLONEMAP --cache-dir /opt/git \
+              git://git.openstack.org $ZUUL_PROJECT openstack/requirements
+          # REQS_DIR is not set for openstack/requirements and there is also
+          # no need to copy in this case.
+          if [[ "$ZUUL_PROJECT" != "openstack/requirements" ]]; then
+              cp $REQS_DIR/upper-constraints.txt ./
+          fi
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          if [ -x tools/test-setup.sh ] ; then
+            tools/test-setup.sh
+          fi
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          set -x
+          sudo rm -f /etc/sudoers.d/zuul
+          # Prove that general sudo access is actually revoked
+          ! sudo -n true
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: /usr/local/jenkins/slave_scripts/run-tox.sh migration
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
+
+    - shell:
+        cmd: |
+          OUT=`git ls-files --other --exclude-standard --directory`
+          if [ -z "$OUT" ]; then
+              echo "No extra files created during test."
+              exit 0
+          else
+              echo "The following un-ignored files were created during the test:"
+              echo "$OUT"
+              exit 0  # TODO: change to 1 to fail tests.
+          fi
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/post/upload-images.yaml b/playbooks/post/upload-images.yaml
new file mode 100644
index 0000000000..3ffc164be2
--- /dev/null
+++ b/playbooks/post/upload-images.yaml
@@ -0,0 +1,21 @@
+- hosts: primary
+  tasks:
+
+    - name: Ensure artifacts directory exists
+      file:
+        path: '{{ zuul.executor.work_root }}/artifacts'
+        state: directory
+      delegate_to: localhost
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.work_root }}/artifacts/images'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/*.qcow2
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/playbooks/post/upload-logs-tox.yaml b/playbooks/post/upload-logs-tox.yaml
new file mode 100644
index 0000000000..68fbdf81bf
--- /dev/null
+++ b/playbooks/post/upload-logs-tox.yaml
@@ -0,0 +1,67 @@
+- hosts: primary
+  tasks:
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=**/*nose_results.html
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=**/*testr_results.html.gz
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/.testrepository/tmp*
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=**/*testrepository.subunit.gz
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
+
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}/tox'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/.tox/*/log/*
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/playbooks/post/upload-logs.yaml b/playbooks/post/upload-logs.yaml
new file mode 100644
index 0000000000..baf8760865
--- /dev/null
+++ b/playbooks/post/upload-logs.yaml
@@ -0,0 +1,14 @@
+- hosts: primary
+  tasks:
+    - name: Copy files from {{ ansible_user_dir }}/workspace/ on node
+      synchronize:
+        src: '{{ ansible_user_dir }}/workspace/'
+        dest: '{{ zuul.executor.log_root }}'
+        mode: pull
+        copy_links: true
+        verify_host: true
+        rsync_opts:
+          - --include=/logs/**
+          - --include=*/
+          - --exclude=*
+          - --prune-empty-dirs
diff --git a/playbooks/pre/prepare-workspace-images.yaml b/playbooks/pre/prepare-workspace-images.yaml
new file mode 100644
index 0000000000..0fe2d13d86
--- /dev/null
+++ b/playbooks/pre/prepare-workspace-images.yaml
@@ -0,0 +1,12 @@
+- hosts: all
+  tasks:
+
+    - name: Ensure legacy workspace directory
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: /usr/local/jenkins/slave_scripts/install-distro-packages.sh
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'
diff --git a/playbooks/pre/prepare-workspace.yaml b/playbooks/pre/prepare-workspace.yaml
new file mode 100644
index 0000000000..412a04a755
--- /dev/null
+++ b/playbooks/pre/prepare-workspace.yaml
@@ -0,0 +1,23 @@
+- hosts: all
+  name: magnum-prepare-workspace
+  tasks:
+    - name: Ensure workspace directory exists
+      file:
+        path: '{{ ansible_user_dir }}/workspace'
+        state: directory
+
+    - shell:
+        cmd: |
+          set -e
+          set -x
+          cat > clonemap.yaml << EOF
+          clonemap:
+            - name: openstack-infra/devstack-gate
+              dest: devstack-gate
+          EOF
+          /usr/zuul-env/bin/zuul-cloner -m clonemap.yaml --cache-dir /opt/git \
+              git://git.openstack.org \
+              openstack-infra/devstack-gate
+        executable: /bin/bash
+        chdir: '{{ ansible_user_dir }}/workspace'
+      environment: '{{ zuul | zuul_legacy_vars }}'