openstack
/
sahara-image-elements
Code Issues Proposed changes

Adding Spark 2.2.0 

Adding newest version of spark.

Change-Id: Ib2894d5d93d3ecfd17e0fb1eba4d687a97406027
This commit is contained in:
Telles Nobrega 2017-12-06 15:10:36 -03:00
parent 589abcce31
commit 05085a81db
3 changed files with 19 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
diskimage-create/README.rst
View File

@ -80,8 +80,8 @@ like this:
.. sourcecode:: bash
tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0] # spark standalone
tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0] # spark on vanilla
tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0|2.1.0|2.2.0] # spark standalone
tox -e venv -- sahara-image-create -p vanilla -v 2.7.1 -s [1.6.0|2.1.0|2.2.0] # spark on vanilla
8. To select which MapR version to target use the '-r' commandline option like
this:

22
diskimage-create/diskimage-create.sh
View File

@ -13,7 +13,7 @@ DEBUG_MODE="false"
DIB_DEFAULT_MAPR_VERSION="5.2.0"
# The default version for Spark plugin
DIB_DEFAULT_SPARK_VERSION="2.1.0"
DIB_DEFAULT_SPARK_VERSION="2.2.0"
# The default version for Storm plugin
DIB_DEFAULT_STORM_VERSION="1.1.0"
@ -34,7 +34,7 @@ usage() {
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-r 5.1.0|5.2.0]"
echo " [-s 1.3.1|1.6.0|2.1.0]"
echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
echo " [-f qcow2|raw]"
echo " [-d]"
@ -167,7 +167,7 @@ fi
check_spark_version () {
case "$DIB_SPARK_VERSION" in
"1.3.1" | "1.6.0" | "2.1.0");;
"1.3.1" | "1.6.0" | "2.1.0" | "2.2.0");;
"")
echo "Spark version not specified"
echo "Spark ${DIB_DEFAULT_SPARK_VERSION} will be used"
@ -563,16 +563,23 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
unset SPARK_HADOOP_DL
fi
##########################
# Image for Spark plugin #
##########################
###########################
# Images for Spark plugin #
###########################
if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce"
export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
export DIB_SPARK_VERSION
export plugin_type="spark"
export DIB_CDH_VERSION="5.5"
if [ "$DIB_SPARK_VERSION" = "2.2.0" ]; then
export DIB_CDH_VERSION="5.11"
export DIB_RELEASE=${DIB_RELEASE:-xenial}
else
export DIB_RELEASE=${DIB_RELEASE:-trusty}
export DIB_CDH_VERSION="5.5"
fi
# Tell the cloudera element to install only hdfs
export DIB_CDH_HDFS_ONLY=1
@ -580,7 +587,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
export ubuntu_image_name=${ubuntu_spark_image_name:-"ubuntu_sahara_spark_latest"}
# Creating Ubuntu cloud image
export DIB_RELEASE=${DIB_RELEASE:-trusty}
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
unset DIB_CLOUD_INIT_DATASOURCES
unset DIB_HDFS_LIB_DIR

3
elements/spark/root.d/50-download-spark
View File

@ -18,6 +18,9 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
5.5)
SPARK_HADOOP_DL=hadoop2.6
;;
5.11)
SPARK_HADOOP_DL=hadoop2.6
;;
*)
echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported."
echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version."