summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTelles Nobrega <tenobreg@redhat.com>2017-12-06 15:10:36 -0300
committerTelles Nobrega <tenobreg@redhat.com>2017-12-13 09:55:41 -0300
commit05085a81dbfbde23ba16334bf60119e8a73ff431 (patch)
tree06e04b16e094543b4aaf21e5f4fe87346e4f4ad3
parent589abcce315a973a5526fc22d5386ee3bb808a6e (diff)
Adding Spark 2.2.0HEADmaster
Adding newest version of spark. Change-Id: Ib2894d5d93d3ecfd17e0fb1eba4d687a97406027
Notes
Notes (review): Code-Review+2: Shu Yingya <yingya.shu@easystack.cn> Code-Review+2: Jeremy Freudberg <jeremyfreudberg@gmail.com> Workflow+1: Jeremy Freudberg <jeremyfreudberg@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Wed, 13 Dec 2017 16:54:32 +0000 Reviewed-on: https://review.openstack.org/526135 Project: openstack/sahara-image-elements Branch: refs/heads/master
-rw-r--r--diskimage-create/README.rst4
-rwxr-xr-xdiskimage-create/diskimage-create.sh22
-rwxr-xr-xelements/spark/root.d/50-download-spark3
3 files changed, 19 insertions, 10 deletions
diff --git a/diskimage-create/README.rst b/diskimage-create/README.rst
index 0c3d1e1..ca815b8 100644
--- a/diskimage-create/README.rst
+++ b/diskimage-create/README.rst
@@ -80,8 +80,8 @@ like this:
80 80
81.. sourcecode:: bash 81.. sourcecode:: bash
82 82
83 tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0] # spark standalone 83 tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0|2.2.0] # spark standalone
84 tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0] # spark on vanilla 84 tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0|2.2.0] # spark on vanilla
85 85
868. To select which MapR version to target use the '-r' commandline option like 868. To select which MapR version to target use the '-r' commandline option like
87this: 87this:
diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh
index a8edcf6..5b3693b 100755
--- a/diskimage-create/diskimage-create.sh
+++ b/diskimage-create/diskimage-create.sh
@@ -13,7 +13,7 @@ DEBUG_MODE="false"
13DIB_DEFAULT_MAPR_VERSION="5.2.0" 13DIB_DEFAULT_MAPR_VERSION="5.2.0"
14 14
15# The default version for Spark plugin 15# The default version for Spark plugin
16DIB_DEFAULT_SPARK_VERSION="2.1.0" 16DIB_DEFAULT_SPARK_VERSION="2.2.0"
17 17
18# The default version for Storm plugin 18# The default version for Storm plugin
19DIB_DEFAULT_STORM_VERSION="1.1.0" 19DIB_DEFAULT_STORM_VERSION="1.1.0"
@@ -34,7 +34,7 @@ usage() {
34 echo " [-i ubuntu|fedora|centos|centos7]" 34 echo " [-i ubuntu|fedora|centos|centos7]"
35 echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]" 35 echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
36 echo " [-r 5.1.0|5.2.0]" 36 echo " [-r 5.1.0|5.2.0]"
37 echo " [-s 1.3.1|1.6.0|2.1.0]" 37 echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
38 echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]" 38 echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
39 echo " [-f qcow2|raw]" 39 echo " [-f qcow2|raw]"
40 echo " [-d]" 40 echo " [-d]"
@@ -167,7 +167,7 @@ fi
167 167
168check_spark_version () { 168check_spark_version () {
169 case "$DIB_SPARK_VERSION" in 169 case "$DIB_SPARK_VERSION" in
170 "1.3.1" | "1.6.0" | "2.1.0");; 170 "1.3.1" | "1.6.0" | "2.1.0" | "2.2.0");;
171 "") 171 "")
172 echo "Spark version not specified" 172 echo "Spark version not specified"
173 echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used" 173 echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used"
@@ -563,16 +563,23 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
563 unset SPARK_HADOOP_DL 563 unset SPARK_HADOOP_DL
564fi 564fi
565 565
566########################## 566###########################
567# Image for Spark plugin # 567# Images for Spark plugin #
568########################## 568###########################
569 569
570if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then 570if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
571 export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce" 571 export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce"
572 export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES 572 export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
573 export DIB_SPARK_VERSION 573 export DIB_SPARK_VERSION
574 export plugin_type="spark" 574 export plugin_type="spark"
575 export DIB_CDH_VERSION="5.5" 575
576 if [ "$DIB_SPARK_VERSION" = "2.2.0" ]; then
577 export DIB_CDH_VERSION="5.11"
578 export DIB_RELEASE=${DIB_RELEASE:-xenial}
579 else
580 export DIB_RELEASE=${DIB_RELEASE:-trusty}
581 export DIB_CDH_VERSION="5.5"
582 fi
576 # Tell the cloudera element to install only hdfs 583 # Tell the cloudera element to install only hdfs
577 export DIB_CDH_HDFS_ONLY=1 584 export DIB_CDH_HDFS_ONLY=1
578 585
@@ -580,7 +587,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
580 export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"} 587 export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"}
581 588
582 # Creating Ubuntu cloud image 589 # Creating Ubuntu cloud image
583 export DIB_RELEASE=${DIB_RELEASE:-trusty}
584 image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence 590 image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
585 unset DIB_CLOUD_INIT_DATASOURCES 591 unset DIB_CLOUD_INIT_DATASOURCES
586 unset DIB_HDFS_LIB_DIR 592 unset DIB_HDFS_LIB_DIR
diff --git a/elements/spark/root.d/50-download-spark b/elements/spark/root.d/50-download-spark
index 757bcbc..eac5b47 100755
--- a/elements/spark/root.d/50-download-spark
+++ b/elements/spark/root.d/50-download-spark
@@ -18,6 +18,9 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
18 5.5) 18 5.5)
19 SPARK_HADOOP_DL=hadoop2.6 19 SPARK_HADOOP_DL=hadoop2.6
20 ;; 20 ;;
21 5.11)
22 SPARK_HADOOP_DL=hadoop2.6
23 ;;
21 *) 24 *)
22 echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported." 25 echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported."
23 echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version." 26 echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version."