Added support of Spark 1.6.0
Spark 1.6.0 is available now for deployment. changes: * using spark 1.6.0 version as default version for spark plugin * using hadoop 2.6 for spark 1.6.0 (by setting up DIB_CDH_VERSION="5.4") * we use only HDFS part of CDH for Spark plugin so we don't need to create symbolic links for oozie Depends-on: I9a46a503c7e52d756c7de8c8694dbfc51f80f2be bp: support-spark-160 Change-Id: Id3c4fb5a0cb1349824972c1be90ea0e0be64cb4b
This commit is contained in:
parent
e320ab37d3
commit
3626fb6429
|
@ -41,7 +41,13 @@ NOTE: Do not create all images for all plugins with the same mirrors. Different
|
|||
|
||||
tox -e venv -- sahara-image-create -i [ubuntu|fedora|centos|centos7]
|
||||
|
||||
7. If the host system is missing packages required for diskimage-create.sh, the '-u' commandline option will instruct the script to install them without prompt.
|
||||
7. To select which Spark version to target use the '-s' commandline option like this:
|
||||
|
||||
.. sourcecode:: bash
|
||||
|
||||
tox -e venv -- sahara-image-create -p spark -s [1.3.1|1.6.0]
|
||||
|
||||
8. If the host system is missing packages required for diskimage-create.sh, the '-u' commandline option will instruct the script to install them without prompt.
|
||||
|
||||
NOTE for 4, 5, 6:
|
||||
|
||||
|
|
|
@ -13,7 +13,7 @@ DEBUG_MODE="false"
|
|||
DIB_DEFAULT_MAPR_VERSION="5.0.0"
|
||||
|
||||
# The default version for Spark plugin
|
||||
DIB_DEFAULT_SPARK_VERSION="1.3.1"
|
||||
DIB_DEFAULT_SPARK_VERSION="1.6.0"
|
||||
|
||||
# Bare metal image generation is enabled with the -b flag, it is off by default
|
||||
SIE_BAREMETAL="false"
|
||||
|
@ -31,7 +31,7 @@ usage() {
|
|||
echo " [-i ubuntu|fedora|centos|centos7]"
|
||||
echo " [-v 2|2.6|2.7.1|4|5.0|5.3|5.4]"
|
||||
echo " [-r 3.1.1|4.0.1|4.0.2|5.0.0]"
|
||||
echo " [-s <Spark version>]"
|
||||
echo " [-s 1.3.1|1.6.0]"
|
||||
echo " [-d]"
|
||||
echo " [-u]"
|
||||
echo " [-j openjdk|oracle-java]"
|
||||
|
@ -545,6 +545,9 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
|
|||
echo "Overriding CDH version, CDH 4 is required for this Spark version"
|
||||
export DIB_CDH_VERSION="CDH4"
|
||||
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cdh"
|
||||
elif [ "$DIB_SPARK_VERSION" == "1.6.0" ]; then
|
||||
export DIB_CDH_VERSION="5.4"
|
||||
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cloudera"
|
||||
else
|
||||
export DIB_CDH_VERSION=$HADOOP_VERSION
|
||||
ubuntu_elements_sequence="$COMMON_ELEMENTS hadoop-cloudera"
|
||||
|
|
|
@ -69,10 +69,12 @@ case "$DIB_CDH_VERSION" in
|
|||
;;
|
||||
5.4)
|
||||
wget -O $dest $HADOOP_OPENSTACK_5_4_0_URL
|
||||
# Create links to keep backward version support.
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
|
||||
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
|
||||
# Create links to keep backward version support.
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
|
||||
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "Unhandled version $DIB_CDH_VERSION for hadoop-openstack.jar"
|
||||
|
|
Loading…
Reference in New Issue