From 1aa15783fabae7e4979e7808705550a60e18affa Mon Sep 17 00:00:00 2001 From: Vitaly Gridnev Date: Wed, 29 Mar 2017 13:56:17 +0400 Subject: [PATCH] implement a builder for scenario jobs this commit implements base builder for sahara's scenario jobs. we are going to implement several new scenarios in future, and so implementing this builder to make our life easier. Change-Id: I2496a8fb5b4e5e7520ae99916de0dcef7467618e --- jenkins/jobs/projects.yaml | 31 ++----- jenkins/jobs/sahara.yaml | 171 +++++++++++++++++-------------------- zuul/layout.yaml | 16 ++-- 3 files changed, 94 insertions(+), 124 deletions(-) diff --git a/jenkins/jobs/projects.yaml b/jenkins/jobs/projects.yaml index bda2f715af..bbdc147a6a 100644 --- a/jenkins/jobs/projects.yaml +++ b/jenkins/jobs/projects.yaml @@ -12989,12 +12989,6 @@ - pypi-jobs - openstack-client-publish-jobs - openstack-releasenotes-jobs - - '{pipeline}-{name}-dsvm-scenario-{network}-{node}{suffix}': - pipeline: gate - node: ubuntu-xenial - suffix: '' - branch-override: default - network: neutron - '{pipeline}-tempest-dsvm-neutron-src-python-saharaclient-{node}{suffix}': pipeline: gate node: @@ -13573,31 +13567,28 @@ - ubuntu-xenial suffix: '-nv' branch-override: default - - '{pipeline}-{name}-dsvm-scenario-{network}-{node}{suffix}': + - '{pipeline}-sahara-dsvm-scenario-{network}-{node}{suffix}': pipeline: gate node: ubuntu-xenial suffix: '' - branch-override: default network: neutron - - '{pipeline}-sahara-dsvm-multinode-{node-release}{suffix}': + branch-override: default + - '{pipeline}-sahara-dsvm-scenario-multinode-{plugin}-{node-release}{suffix}': pipeline: gate node-release: ubuntu-xenial node: ubuntu-xenial-2-node suffix: '-nv' branch-override: default + plugin: spark # only for mitaka branch - - '{pipeline}-{name}-dsvm-scenario-{network}-{node}{suffix}': + - '{pipeline}-sahara-dsvm-scenario-{network}-{node}{suffix}': pipeline: gate node: ubuntu-trusty suffix: '-nv' branch-override: default - network: nova - - '{pipeline}-{name}-dsvm-scenario-{network}-{node}{suffix}': - pipeline: gate - node: ubuntu-trusty - suffix: '-nv' - branch-override: default - network: neutron + network: + - nova + - neutron - api-ref-jobs: service: data-processing - '{pipeline}-grenade-dsvm-sahara-{node}{suffix}': @@ -13682,12 +13673,6 @@ - gate-{name}-pylint-{node}: node: ubuntu-xenial - openstack-releasenotes-jobs - - '{pipeline}-{name}-dsvm-scenario-{network}-{node}{suffix}': - pipeline: gate - node: ubuntu-xenial - suffix: '' - branch-override: default - network: neutron - project: name: scalpels diff --git a/jenkins/jobs/sahara.yaml b/jenkins/jobs/sahara.yaml index db6a940294..7e81f66215 100644 --- a/jenkins/jobs/sahara.yaml +++ b/jenkins/jobs/sahara.yaml @@ -1,3 +1,58 @@ +- builder: + name: devstack-sahara + + builders: + - link-logs + - net-info + - devstack-checkout + - shell: | + #!/bin/bash -xe + export PYTHONUNBUFFERED=true + export DEVSTACK_SUBNODE_CONFIG=" " + export DEVSTACK_LOCAL_CONFIG="enable_plugin sahara git://git.openstack.org/openstack/sahara" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer" + export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin heat git://git.openstack.org/openstack/heat" + export DEVSTACK_GATE_TEMPEST=0 + export DEVSTACK_GATE_SAHARA=1 + export DEVSTACK_GATE_EXERCISES=0 + export DEVSTACK_GATE_INSTALL_TESTONLY=1 + export KEEP_LOCALRC=1 + export PROJECTS="openstack/sahara-tests $PROJECTS" + export PROJECTS="openstack/sahara-image-elements openstack/sahara-extra $PROJECTS" + if [ "{network}" == "nova" ] ; then + export DEVSTACK_GATE_NEUTRON=0 + else + export DEVSTACK_GATE_NEUTRON=1 + fi + if [ "$ZUUL_PROJECT" = "openstack/python-saharaclient" ]; then + export DEVSTACK_PROJECT_FROM_GIT=python-saharaclient + export PROJECTS="openstack/python-saharaclient $PROJECTS" + fi + if [ "{multinode}" -eq 1 ] ; then + export DEVSTACK_GATE_TOPOLOGY="multinode" + fi + + export BRANCH_OVERRIDE={branch-override} + if [ "{branch-override}" != "default" ] ; then + export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE + fi + + function pre_test_hook {{ + cd /opt/stack/new/sahara-tests/tools/gate/scenario + ./pre_test_hook.sh {network} {plugin} + }} + export -f pre_test_hook + + function post_test_hook {{ + cd /opt/stack/new/sahara-tests/tools/gate/scenario + ./post_test_hook.sh {network} {plugin} + }} + export -f post_test_hook + + cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh + ./safe-devstack-vm-gate-wrap.sh + + - job-template: name: 'gate-sahara-dashboard-dsvm-integration-{node}' node: '{node}' @@ -269,7 +324,7 @@ - job-template: - name: '{pipeline}-{name}-dsvm-scenario-{network}-{node}{suffix}' + name: '{pipeline}-sahara-dsvm-scenario-{network}-{node}{suffix}' node: '{node}' wrappers: @@ -278,52 +333,32 @@ - timestamps builders: - - link-logs - - net-info - - devstack-checkout - - shell: | - #!/bin/bash -xe - export PYTHONUNBUFFERED=true - export DEVSTACK_LOCAL_CONFIG="enable_plugin sahara git://git.openstack.org/openstack/sahara" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin ceilometer git://git.openstack.org/openstack/ceilometer" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin heat git://git.openstack.org/openstack/heat" - export DEVSTACK_GATE_TEMPEST=0 - export DEVSTACK_GATE_SAHARA=1 - export DEVSTACK_GATE_EXERCISES=0 - export DEVSTACK_GATE_INSTALL_TESTONLY=1 - export KEEP_LOCALRC=1 - export PROJECTS="openstack/sahara-tests $PROJECTS" + - devstack-sahara: + network: "{network}" + multinode: 0 + plugin: "fake" + branch-override: "{branch-override}" - if [ "{name}" == "python-saharaclient" ] ; then - export PROJECTS="openstack/python-saharaclient $PROJECTS" - export DEVSTACK_PROJECT_FROM_GIT=python-saharaclient - fi + publishers: + - devstack-logs + - console-log - export BRANCH_OVERRIDE={branch-override} - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - if [ "{network}" == "nova" ] ; then - export DEVSTACK_GATE_NEUTRON=0 - else - export DEVSTACK_GATE_NEUTRON=1 - fi +- job-template: + name: '{pipeline}-sahara-dsvm-scenario-multinode-{plugin}-{node-release}{suffix}' + node: '{node}' - function pre_test_hook {{ - cd /opt/stack/new/sahara-tests/tools/gate/scenario - ./pre_test_hook.sh {network} - }} - export -f pre_test_hook + wrappers: + - build-timeout: + timeout: 130 + - timestamps - function post_test_hook {{ - cd /opt/stack/new/sahara-tests/tools/gate/scenario - ./post_test_hook.sh {network} - }} - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh + builders: + - devstack-sahara: + network: neutron + multinode: 1 + plugin: "{plugin}" + branch-override: "{branch-override}" publishers: - devstack-logs @@ -366,55 +401,3 @@ publishers: - devstack-logs - console-log - - -- job-template: - name: '{pipeline}-sahara-dsvm-multinode-{node-release}{suffix}' - node: '{node}' - - wrappers: - - build-timeout: - timeout: 130 - - timestamps - - builders: - - link-logs - - net-info - - devstack-checkout - - shell: | - #!/bin/bash -xe - export PYTHONUNBUFFERED=true - export DEVSTACK_SUBNODE_CONFIG=" " - export DEVSTACK_LOCAL_CONFIG="enable_plugin sahara git://git.openstack.org/openstack/sahara" - export DEVSTACK_LOCAL_CONFIG+=$'\n'"enable_plugin heat git://git.openstack.org/openstack/heat" - export DEVSTACK_GATE_TEMPEST=0 - export DEVSTACK_GATE_EXERCISES=0 - export DEVSTACK_GATE_INSTALL_TESTONLY=1 - export KEEP_LOCALRC=1 - export DEVSTACK_GATE_SAHARA=1 - export PROJECTS="openstack/sahara-tests $PROJECTS" - export BRANCH_OVERRIDE={branch-override} - if [ "$BRANCH_OVERRIDE" != "default" ] ; then - export OVERRIDE_ZUUL_BRANCH=$BRANCH_OVERRIDE - fi - export DEVSTACK_GATE_NEUTRON=1 - export DEVSTACK_GATE_TOPOLOGY="multinode" - - function pre_test_hook {{ - cd /opt/stack/new/sahara-tests/tools/gate/multinode - ./pre_test_hook.sh - }} - export -f pre_test_hook - - function post_test_hook {{ - cd /opt/stack/new/sahara-tests/tools/gate/multinode - ./post_test_hook.sh - }} - export -f post_test_hook - - cp devstack-gate/devstack-vm-gate-wrap.sh ./safe-devstack-vm-gate-wrap.sh - ./safe-devstack-vm-gate-wrap.sh - - publishers: - - devstack-logs - - console-log diff --git a/zuul/layout.yaml b/zuul/layout.yaml index 11ca2b85d7..0a822274c5 100755 --- a/zuul/layout.yaml +++ b/zuul/layout.yaml @@ -2923,7 +2923,7 @@ jobs: - name: ^gate-sahara-dsvm-scenario-(nova|neutron)-ubuntu-trusty-nv$ branch: ^stable/mitaka$ - - name: ^gate-sahara-dsvm-multinode-ubuntu-xenial-nv$ + - name: ^gate-sahara-dsvm-scenario-multinode-spark-ubuntu-xenial-nv$ branch: ^(?!stable/(mitaka|newton|ocata)).*$ - name: ^(gate|post)-sahara-extra-artifacts$ @@ -2956,7 +2956,7 @@ jobs: # do not run sahara scenario tests when the change is only # about documentation and/or the Tempest plugin. - - name: ^gate-(sahara|sahara-tests)-dsvm-scenario.*$ + - name: ^gate-sahara-dsvm-scenario.*$ skip-if: - project: ^openstack/sahara$ all-files-match-any: @@ -16715,14 +16715,14 @@ projects: - gate-python-saharaclient-python34 - gate-sahara-cli-ubuntu-trusty-nv - gate-sahara-cli-ubuntu-xenial-nv - - gate-python-saharaclient-dsvm-scenario-neutron-ubuntu-xenial + - gate-sahara-dsvm-scenario-neutron-ubuntu-xenial - gate-tempest-dsvm-neutron-src-python-saharaclient-ubuntu-trusty - gate-tempest-dsvm-neutron-src-python-saharaclient-ubuntu-xenial post: - python-saharaclient-coverage-ubuntu-trusty - python-saharaclient-coverage-ubuntu-xenial gate: - - gate-python-saharaclient-dsvm-scenario-neutron-ubuntu-xenial + - gate-sahara-dsvm-scenario-neutron-ubuntu-xenial - gate-tempest-dsvm-neutron-src-python-saharaclient-ubuntu-trusty - gate-tempest-dsvm-neutron-src-python-saharaclient-ubuntu-xenial experimental: @@ -17265,7 +17265,7 @@ projects: - sahara-coverage-ubuntu-trusty - sahara-coverage-ubuntu-xenial experimental: - - gate-sahara-dsvm-multinode-ubuntu-xenial-nv + - gate-sahara-dsvm-scenario-multinode-spark-ubuntu-xenial-nv - name: openstack/sahara-ci-config template: @@ -17372,14 +17372,16 @@ projects: check: - gate-sahara-tests-pylint-ubuntu-xenial - gate-tempest-dsvm-sahara-ubuntu-xenial-nv - - gate-sahara-tests-dsvm-scenario-neutron-ubuntu-xenial + - gate-sahara-dsvm-scenario-neutron-ubuntu-xenial - gate-sahara-cli-ubuntu-xenial-nv # TODO(esikachev): remove when tox -e cover working correctly - sahara-tests-coverage-ubuntu-xenial-nv gate: - - gate-sahara-tests-dsvm-scenario-neutron-ubuntu-xenial + - gate-sahara-dsvm-scenario-neutron-ubuntu-xenial post: - sahara-tests-coverage-ubuntu-xenial-nv + experimental: + - gate-sahara-dsvm-scenario-multinode-spark-ubuntu-xenial-nv - name: openstack/scalpels template: