From cf5a3200b57b4c24565dad5ae73b149802f53484 Mon Sep 17 00:00:00 2001 From: Shu Yingya Date: Tue, 27 Dec 2016 11:20:26 +0000 Subject: [PATCH] Add jobs for CDH 5.9 c7 and u14 jobs add for sahara, sahara-image-elements and sahara-tests Partially-Implements: blueprint cdh-5-9-support Depends-On: I3fc155669ded5f8cc80f9da4afbec7e225b23391 Change-Id: I3e7a19ecfbe87ed3cdda67f479b82d0bfae9d9bd --- config/devstack/local.sh | 6 +++++- config/devstack/pre_unstack.sh | 6 +++++- config/zuul/layout.yaml | 20 ++++++++++++++++++++ jenkins_job_builder/diskimage.yaml | 1 + jenkins_job_builder/sahara.yaml | 1 + slave-scripts/dib.sh | 6 ++++++ slave-scripts/functions-dib.sh | 3 +++ 7 files changed, 41 insertions(+), 2 deletions(-) diff --git a/config/devstack/local.sh b/config/devstack/local.sh index 0095a1a3..f470769f 100755 --- a/config/devstack/local.sh +++ b/config/devstack/local.sh @@ -30,8 +30,10 @@ UBUNTU_AMBARI_2_2_IMAGE_PATH=/home/ubuntu/images/ambari_2.2_u14.qcow2 UBUNTU_CDH_5_5_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.5.0_u14.qcow2 CENTOS_CDH_5_5_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.5.0_c6.6.qcow2 CENTOS7_CDH_5_5_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.5.0_c7.qcow2 -CENTOS7_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_c7.qcow2 UBUNTU_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_u14.qcow2 +CENTOS7_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_c7.qcow2 +UBUNTU_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_u14.qcow2 +CENTOS7_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_c7.qcow2 SPARK_1_6_0_IMAGE_PATH=/home/ubuntu/images/spark_1.6.0_u14.qcow2 MAPR_5_1_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.1.0.mrv2_u14.qcow2 MAPR_5_2_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.2.0.mrv2_u14.qcow2 @@ -89,6 +91,8 @@ openstack image create $(basename -s .qcow2 $CENTOS_CDH_5_5_0_IMAGE_PATH) --file openstack image create $(basename -s .qcow2 $CENTOS7_CDH_5_5_0_IMAGE_PATH) --file $CENTOS7_CDH_5_5_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.5.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="centos" openstack image create $(basename -s .qcow2 $CENTOS7_CDH_5_7_0_IMAGE_PATH) --file $CENTOS7_CDH_5_7_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.7.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="centos" openstack image create $(basename -s .qcow2 $UBUNTU_CDH_5_7_0_IMAGE_PATH) --file $UBUNTU_CDH_5_7_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.7.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu" +openstack image create $(basename -s .qcow2 $CENTOS7_CDH_5_9_0_IMAGE_PATH) --file $CENTOS7_CDH_5_9_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.9.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="centos" +openstack image create $(basename -s .qcow2 $UBUNTU_CDH_5_9_0_IMAGE_PATH) --file $UBUNTU_CDH_5_9_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_5.9.0'='True' --property '_sahara_tag_cdh'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $SPARK_1_6_0_IMAGE_PATH) --file $SPARK_1_6_0_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_spark'='True' --property '_sahara_tag_1.6.0'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $MAPR_5_1_0_MRV2_IMAGE_PATH) --file $MAPR_5_1_0_MRV2_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_5.1.0.mrv2'='True' --property '_sahara_username'="ubuntu" openstack image create $(basename -s .qcow2 $MAPR_5_2_0_MRV2_IMAGE_PATH) --file $MAPR_5_2_0_MRV2_IMAGE_PATH --disk-format qcow2 --container-format bare --property '_sahara_tag_ci'='True' --property '_sahara_tag_mapr'='True' --property '_sahara_tag_5.2.0.mrv2'='True' --property '_sahara_username'="ubuntu" diff --git a/config/devstack/pre_unstack.sh b/config/devstack/pre_unstack.sh index 18283a1d..60348766 100755 --- a/config/devstack/pre_unstack.sh +++ b/config/devstack/pre_unstack.sh @@ -22,8 +22,10 @@ UBUNTU_AMBARI_2_2_IMAGE_PATH=/home/ubuntu/images/ambari_2.2_u14.qcow2 UBUNTU_CDH_5_5_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.5.0_u14.qcow2 CENTOS_CDH_5_5_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.5.0_c6.6.qcow2 CENTOS7_CDH_5_5_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.5.0_c7.qcow2 -CENTOS7_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_c7.qcow2 UBUNTU_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_u14.qcow2 +CENTOS7_CDH_5_7_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.7.0_c7.qcow2 +UBUNTU_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_u14.qcow2 +CENTOS7_CDH_5_9_0_IMAGE_PATH=/home/ubuntu/images/cdh_5.9.0_c7.qcow2 SPARK_1_6_0_IMAGE_PATH=/home/ubuntu/images/spark_1.6.0_u14.qcow2 MAPR_5_1_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.1.0.mrv2_u14.qcow2 MAPR_5_2_0_MRV2_IMAGE_PATH=/home/ubuntu/images/mapr_5.2.0.mrv2_u14.qcow2 @@ -38,6 +40,8 @@ openstack image save --file $CENTOS_CDH_5_5_0_IMAGE_PATH $(get_id $CENTOS_CDH_5_ openstack image save --file $CENTOS7_CDH_5_5_0_IMAGE_PATH $(get_id $CENTOS7_CDH_5_5_0_IMAGE_PATH) openstack image save --file $CENTOS7_CDH_5_7_0_IMAGE_PATH $(get_id $CENTOS7_CDH_5_7_0_IMAGE_PATH) openstack image save --file $UBUNTU_CDH_5_7_0_IMAGE_PATH $(get_id $UBUNTU_CDH_5_7_0_IMAGE_PATH) +openstack image save --file $CENTOS7_CDH_5_9_0_IMAGE_PATH $(get_id $CENTOS7_CDH_5_9_0_IMAGE_PATH) +openstack image save --file $UBUNTU_CDH_5_9_0_IMAGE_PATH $(get_id $UBUNTU_CDH_5_9_0_IMAGE_PATH) openstack image save --file $SPARK_1_6_0_IMAGE_PATH $(get_id $SPARK_1_6_0_IMAGE_PATH) openstack image save --file $MAPR_5_1_0_MRV2_IMAGE_PATH $(get_id $MAPR_5_1_0_MRV2_IMAGE_PATH) openstack image save --file $MAPR_5_2_0_MRV2_IMAGE_PATH $(get_id $MAPR_5_2_0_MRV2_IMAGE_PATH) diff --git a/config/zuul/layout.yaml b/config/zuul/layout.yaml index 8eb266d5..52e73b4e 100644 --- a/config/zuul/layout.yaml +++ b/config/zuul/layout.yaml @@ -78,6 +78,8 @@ projects: - gate-sahara-cdh_5.5.0-c7 - gate-sahara-cdh_5.7.0-c7 - gate-sahara-cdh_5.7.0-u14 + - gate-sahara-cdh_5.9.0-c7 + - gate-sahara-cdh_5.9.0-u14 - gate-sahara-storm_1.0.1-u14 - gate-sahara-ambari_2.3-c7 - gate-sahara-ambari_2.4-c7 @@ -94,6 +96,8 @@ projects: - gate-sahara_tests-cdh_5.5.0-c7 - gate-sahara_tests-cdh_5.7.0-c7 - gate-sahara_tests-cdh_5.7.0-u14 + - gate-sahara_tests-cdh_5.9.0-c7 + - gate-sahara_tests-cdh_5.9.0-u14 - gate-sahara_tests-ambari_2.3-c7 - gate-sahara_tests-ambari_2.4-c7 - gate-sahara_tests-ambari_2.4-u14 @@ -129,6 +133,8 @@ projects: - dib-cdh_5.5.0-c7 - dib-cdh_5.7.0-c7 - dib-cdh_5.7.0-u14 + - dib-cdh_5.9.0-c7 + - dib-cdh_5.9.0-u14 - dib-storm_1.0.1-u14 - dib-ambari_2.2-c7 - dib-ambari_2.2-u14 @@ -141,6 +147,8 @@ projects: - dib-cdh_5.5.0-u14 - dib-cdh_5.5.0-c7 - dib-cdh_5.7.0-c7 + - dib-cdh_5.9.0-c7 + - dib-cdh_5.9.0-u14 - dib-storm_1.0.1-u14 - dib-ambari_2.2-c7 - dib-ambari_2.2-u14 @@ -221,6 +229,18 @@ jobs: - ^sahara_tests/.*.py$ - ^diskimage-create/.*$ - ^elements/.*$ + - name: ^(?!nightly).*-cdh_5.9.0-.*$ + skip-if: + - branch: ^(stable/liberty|stable/mitaka|stable/newton)$ + voting: false + files: + - ^sahara_tests/scenario/defaults/cdh-5.9.*$ + - ^sahara_tests/scenario/defaults/edp.yaml.mako$ + - ^sahara_tests/scenario/defaults/edp-examples/.*$ + - ^sahara/.*$ + - ^sahara_tests/.*.py$ + - ^diskimage-create/.*$ + - ^elements/.*$ - name: ^(?!nightly|weekly).*-vanilla_2.7.1-.*$ skip-if: - branch: stable/liberty diff --git a/jenkins_job_builder/diskimage.yaml b/jenkins_job_builder/diskimage.yaml index 2199988b..42d02b1b 100644 --- a/jenkins_job_builder/diskimage.yaml +++ b/jenkins_job_builder/diskimage.yaml @@ -20,6 +20,7 @@ - vanilla_2.7.1 - cdh_5.5.0 - cdh_5.7.0 + - cdh_5.9.0 - ambari_2.2 os: - u14 diff --git a/jenkins_job_builder/sahara.yaml b/jenkins_job_builder/sahara.yaml index 3ddef038..c46c2ba1 100644 --- a/jenkins_job_builder/sahara.yaml +++ b/jenkins_job_builder/sahara.yaml @@ -53,6 +53,7 @@ plugin: - cdh_5.5.0 - cdh_5.7.0 + - cdh_5.9.0 - ambari_2.4 - '{pipeline}-{project_name}-{plugin}-{os}-{param}': project_name: sahara_tests diff --git a/slave-scripts/dib.sh b/slave-scripts/dib.sh index 35e43080..dc183837 100755 --- a/slave-scripts/dib.sh +++ b/slave-scripts/dib.sh @@ -78,6 +78,12 @@ case $plugin in upload_image "${plugin}" "${username}" "${cdh_5_7_0_image:?}" ;; + cdh_5.9.0) + env cloudera_5_9_${os_type}_image_name="${cdh_5_9_0_image:?}" SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p cloudera -i $os_type -v 5.9 + check_error_code $? "${cdh_5_9_0_image:?}".qcow2 + upload_image "${plugin}" "${username}" "${cdh_5_9_0_image:?}" + ;; + ambari_2.2) env ambari_${os_type}_image_name="${ambari_2_2_image:?}" SIM_REPO_PATH=$WORKSPACE tox -e venv -- sahara-image-create -p ambari -i $os_type check_error_code $? "${ambari_2_2_image:?}".qcow2 diff --git a/slave-scripts/functions-dib.sh b/slave-scripts/functions-dib.sh index 5ecc2914..7442970f 100755 --- a/slave-scripts/functions-dib.sh +++ b/slave-scripts/functions-dib.sh @@ -68,6 +68,9 @@ upload_image() { cdh_5.7.0) image_properties="--property _sahara_tag_5.7.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}" ;; + cdh_5.9.0) + image_properties="--property _sahara_tag_5.9.0=True --property _sahara_tag_cdh=True --property _sahara_username=${username}" + ;; spark_1.6.0) image_properties="--property _sahara_tag_spark=True --property _sahara_tag_1.6.0=True --property _sahara_username=${username}" ;;