Added python3 jobs

Added jobs in sahara and sahara-tests for testing sahara with python3.
Jobs is non-voting

Change-Id: Ib36e1e01f89f060e02916ffdc7d1dc3cded9cf69

config/zuul/layout.yaml

@@ -71,6 +71,7 @@ projects:
     check:
       - gate-sahara-vanilla_2.7.1-u14
       - gate-sahara-spark_1.6.0-u14
+      - gate-sahara-spark_1.6.0-u14-python3
       - gate-sahara-mapr_5.1.0.mrv2-u14
       - gate-sahara-mapr_5.2.0.mrv2-u14
       - gate-sahara-cdh_5.5.0-u14
@@ -99,6 +100,7 @@ projects:
       - gate-sahara_tests-cdh_5.5.0-u14-mitaka
       - gate-sahara_tests-cdh_5.5.0-c7-mitaka
       - gate-sahara_tests-spark_1.6.0-u14-mitaka
+      - gate-sahara_tests-spark_1.6.0-u14-python3
       - gate-sahara_tests-vanilla_2.7.1-u14-mitaka
       - gate-sahara_tests-storm_1.0.1-u14
       - gate-sahara_tests-ambari_2.3-c7-mitaka
@@ -268,6 +270,10 @@ jobs:
       - ^sahara_tests/.*.py$
       - ^diskimage-create/.*$
       - ^elements/.*$
+  - name: gate-.*-spark_1.6.0-u14-python3
+    voting: false
+    skip-if:
+      - branch: ^(stable/liberty|stable/mitaka)$
   - name: gate-sahara-pep8-trunk
     voting: false
   - name: ^(?!nightly|weekly).*-sahara_tests-.*$

jenkins_job_builder/sahara.yaml

@@ -14,12 +14,12 @@
     node: stack-42 || stack-43
 
 - job-template:
-    name: '{pipeline}-sahara_tests-{plugin}-{os}-{release}'
+    name: '{pipeline}-{project_name}-{plugin}-{os}-{param}'
     defaults: global
     builders:
       - gerrit-git-prep
       - ci-config-prep
-      - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-sahara_tests.sh"
+      - shell: "FUNCTION_PATH=$WORKSPACE/sahara-ci-config/slave-scripts $WORKSPACE/sahara-ci-config/slave-scripts/gate-{project_name}.sh"
     properties:
       - zeromq-event
     publishers:
@@ -30,8 +30,7 @@
 
 - project:
     name: sahara
-    pipeline:
+    pipeline: gate
-      - gate
     project_name:
       - sahara
       - sahara_tests
@@ -40,10 +39,8 @@
       - u14
     jobs:
      - '{pipeline}-{project_name}-{plugin}-{os}':
-          plugin:
+          plugin: ambari_2.3
-             - ambari_2.3
+          os: c7
-          os:
-             - c7
      - '{pipeline}-{project_name}-{plugin}-{os}':
           plugin:
              - vanilla_2.7.1
@@ -57,38 +54,40 @@
              - cdh_5.5.0
              - cdh_5.7.0
              - ambari_2.4
-     - '{pipeline}-sahara_tests-{plugin}-{os}-{release}':
+     - '{pipeline}-{project_name}-{plugin}-{os}-{param}':
+          project_name: sahara_tests
           plugin:
             - vanilla_2.7.1
             - spark_1.6.0
           os: u14
-          release:
+          param: mitaka
-            - mitaka
+     - '{pipeline}-{project_name}-{plugin}-{os}-{param}':
-     - '{pipeline}-sahara_tests-{plugin}-{os}-{release}':
+          project_name: sahara_tests
-          plugin:
+          plugin: cdh_5.5.0
-            - cdh_5.5.0
+          param: mitaka
-          release: mitaka
+     - '{pipeline}-{project_name}-{plugin}-{os}-{param}':
-     - '{pipeline}-sahara_tests-{plugin}-{os}-{release}':
+          project_name: sahara_tests
-          plugin:
+          plugin: ambari_2.3
-            - ambari_2.3
           os: c7
-          release: mitaka
+          param: mitaka
+     - '{pipeline}-{project_name}-{plugin}-{os}-{param}':
+          project_name: sahara_tests
+          plugin: spark_1.6.0
+          os: u14
+          param: python3
 
 - project:
     name: sahara-periodic
-    pipeline:
+    pipeline: nightly
-      - nightly
+    project_name: sahara_tests
-    project_name:
-      - sahara_tests
     jobs:
      - '{pipeline}-{project_name}-{plugin}-{os}':
           plugin:
              - vanilla_2.7.1
-             -  spark_1.6.0
+             - spark_1.6.0
           os: u14
      - '{pipeline}-{project_name}-{plugin}-{os}':
-          plugin:
+          plugin: cdh_5.5.0
-             - cdh_5.5.0
           os:
             - c7
             - u14
@@ -97,11 +96,10 @@
             - ambari_2.4
             - ambari_2.3
           os: c7
-     - '{pipeline}-sahara_tests-{plugin}-{os}-{release}':
+     - '{pipeline}-{project_name}-{plugin}-{os}-{param}':
-          plugin:
+          plugin: cdh_5.5.0
-            - cdh_5.5.0
           os: u14
-          release: mitaka
+          param: mitaka
 
 - job:
     name: gate-sahara-pep8-trunk

slave-scripts/functions-common.sh

@@ -150,7 +150,7 @@ write_sahara_main_conf() {
   insert_config_value $conf_path DEFAULT min_transient_cluster_active_time 30
   insert_config_value $conf_path DEFAULT node_domain ci
   insert_config_value $conf_path DEFAULT plugins $plugin
-  insert_config_value $conf_path database connection mysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8
+  insert_config_value $conf_path database connection mysql+pymysql://sahara-citest:sahara-citest@localhost/sahara?charset=utf8
   insert_config_value $conf_path keystone_authtoken auth_uri http://$OPENSTACK_HOST:5000/v2.0/
   insert_config_value $conf_path keystone_authtoken identity_uri http://$OPENSTACK_HOST:35357/
   insert_config_value $conf_path keystone_authtoken admin_user $OS_USERNAME

slave-scripts/gate-sahara.sh

@@ -42,7 +42,19 @@ case $plugin in
        ;;
 esac
 
-sudo pip install -U -c https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt -r requirements.txt . --no-cache-dir
+pip_cmd="install -U -c https://git.openstack.org/cgit/openstack/requirements/plain/upper-constraints.txt -r requirements.txt . --no-cache-dir"
+
+case $(echo $JOB_NAME | awk -F '-' '{ print $NF }') in
+    python3)
+       sudo apt install python3-pip python3-dev -y
+       sudo pip3 $pip_cmd
+       sudo pip3 install pymysql
+       ;;
+    *)
+       sudo pip $pip_cmd
+       ;;
+esac
+
 enable_pypi
 write_sahara_main_conf "$sahara_conf_file" "$sahara_plugin"
 write_tests_conf "$cluster_name" "$image_variable_name" "$image_name" "$scenario_conf_file"