Added support of Spark 1.6.0

Spark 1.6.0 is available now for deployment.
changes:
* using spark 1.6.0 version as default version for spark plugin
* using hadoop 2.6 for spark 1.6.0 (by setting up
  DIB_CDH_VERSION="5.4")
* we use only HDFS part of CDH for Spark plugin so we don't need to
  create symbolic links for oozie

Depends-on: I9a46a503c7e52d756c7de8c8694dbfc51f80f2be
bp: support-spark-160

Change-Id: Id3c4fb5a0cb1349824972c1be90ea0e0be64cb4b
This commit is contained in:
Michael Ionkin 2016-02-05 15:54:16 +03:00
parent e320ab37d3
commit 3626fb6429
3 changed files with 18 additions and 7 deletions

View File

@ -41,7 +41,13 @@ NOTE: Do not create all images for all plugins with the same mirrors. Different
tox -e venv -- sahara-image-create -i [ubuntu|fedora|centos|centos7]
7. If the host system is missing packages required for diskimage-create.sh, the '-u' commandline option will instruct the script to install them without prompt.
7. To select which Spark version to target use the '-s' commandline option like this:
.. sourcecode:: bash
tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0]
8. If the host system is missing packages required for diskimage-create.sh, the '-u' commandline option will instruct the script to install them without prompt.
NOTE for 4, 5, 6:

View File

@ -13,7 +13,7 @@ DEBUG_MODE="false"
DIB_DEFAULT_MAPR_VERSION="5.0.0"
# The default version for Spark plugin
DIB_DEFAULT_SPARK_VERSION="1.3.1"
DIB_DEFAULT_SPARK_VERSION="1.6.0"
# Bare metal image generation is enabled with the -b flag, it is off by default
SIE_BAREMETAL="false"
@ -31,7 +31,7 @@ usage() {
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 2|2.6|2.7.1|4|5.0|5.3|5.4]"
echo " [-r 3.1.1|4.0.1|4.0.2|5.0.0]"
echo " [-s <Spark version>]"
echo " [-s 1.3.1|1.6.0]"
echo " [-d]"
echo " [-u]"
echo " [-j openjdk|oracle-java]"
@ -545,6 +545,9 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
echo "Overriding CDH version, CDH 4 is required for this Spark version"
export DIB_CDH_VERSION="CDH4"
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cdh"
elif [ "$DIB_SPARK_VERSION" == "1.6.0" ]; then
export DIB_CDH_VERSION="5.4"
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cloudera"
else
export DIB_CDH_VERSION=$HADOOP_VERSION
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cloudera"

View File

@ -69,10 +69,12 @@ case "$DIB_CDH_VERSION" in
;;
5.4)
wget -O $dest $HADOOP_OPENSTACK_5_4_0_URL
# Create links to keep backward version support.
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
# Create links to keep backward version support.
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
fi
;;
*)
echo "Unhandled version $DIB_CDH_VERSION for hadoop-openstack.jar"