Add support to build 2.8.2 image
Also tweak Hive a bit and refer to artifacts in a new (but not totally ideal) location. Co-Authored-By: Jeremy Freudberg <jeremyfreudberg@gmail.com> Change-Id: I3a25ee8c282849911089adf6c3593b1bb50fd067
This commit is contained in:
parent
8cdff5d71f
commit
12efcf84e2
|
@ -32,7 +32,7 @@ usage() {
|
||||||
echo "Usage: $(basename $0)"
|
echo "Usage: $(basename $0)"
|
||||||
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
|
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
|
||||||
echo " [-i ubuntu|fedora|centos|centos7]"
|
echo " [-i ubuntu|fedora|centos|centos7]"
|
||||||
echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
|
echo " [-v 2.7.1|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
|
||||||
echo " [-r 5.1.0|5.2.0]"
|
echo " [-r 5.1.0|5.2.0]"
|
||||||
echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
|
echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
|
||||||
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
|
echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
|
||||||
|
@ -184,7 +184,7 @@ case "$PLUGIN" in
|
||||||
"");;
|
"");;
|
||||||
"vanilla")
|
"vanilla")
|
||||||
case "$HADOOP_VERSION" in
|
case "$HADOOP_VERSION" in
|
||||||
"" | "2.7.1");;
|
"" | "2.7.1" | "2.8.2");;
|
||||||
*)
|
*)
|
||||||
echo -e "Unknown hadoop version selected.\nAborting"
|
echo -e "Unknown hadoop version selected.\nAborting"
|
||||||
exit 1
|
exit 1
|
||||||
|
@ -474,14 +474,33 @@ image_create() {
|
||||||
esac
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
|
set_hive_version() {
|
||||||
|
if [ -z "${HIVE_VERSION:-}" ]; then
|
||||||
|
case "$DIB_HADOOP_VERSION" in
|
||||||
|
"2.7.1" )
|
||||||
|
export HIVE_VERSION="0.11.0"
|
||||||
|
;;
|
||||||
|
"2.8.2" )
|
||||||
|
export HIVE_VERSION="2.3.2"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo -e "Unknown Hadoop version, therefore cannot choose Hive version.\nAborting."
|
||||||
|
exit 1
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
#############################
|
#############################
|
||||||
# Images for Vanilla plugin #
|
# Images for Vanilla plugin #
|
||||||
#############################
|
#############################
|
||||||
|
|
||||||
if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
|
if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
|
||||||
export HIVE_VERSION=${HIVE_VERSION:-"0.11.0"}
|
|
||||||
export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"}
|
export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"}
|
||||||
|
export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"}
|
||||||
export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
|
export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
|
||||||
|
export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"}
|
||||||
export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
|
export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
|
||||||
export plugin_type="vanilla"
|
export plugin_type="vanilla"
|
||||||
|
|
||||||
|
@ -524,6 +543,15 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
|
||||||
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
||||||
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_1_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_1_latest"}
|
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_1_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_1_latest"}
|
||||||
export DIB_RELEASE=${DIB_RELEASE:-xenial}
|
export DIB_RELEASE=${DIB_RELEASE:-xenial}
|
||||||
|
set_hive_version
|
||||||
|
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
|
||||||
|
unset DIB_RELEASE
|
||||||
|
fi
|
||||||
|
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
|
||||||
|
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
|
||||||
|
export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_8_2_image_name:-"ubuntu_sahara_vanilla_hadoop_2_8_2_latest"}
|
||||||
|
export DIB_RELEASE=${DIB_RELEASE:-xenial}
|
||||||
|
set_hive_version
|
||||||
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
|
image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
|
||||||
unset DIB_RELEASE
|
unset DIB_RELEASE
|
||||||
fi
|
fi
|
||||||
|
@ -535,6 +563,13 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
|
||||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
|
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
|
||||||
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
||||||
export fedora_image_name=${fedora_vanilla_hadoop_2_7_1_image_name:-"fedora_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
|
export fedora_image_name=${fedora_vanilla_hadoop_2_7_1_image_name:-"fedora_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
|
||||||
|
set_hive_version
|
||||||
|
image_create fedora $fedora_image_name $fedora_elements_sequence
|
||||||
|
fi
|
||||||
|
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
|
||||||
|
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
|
||||||
|
export fedora_image_name=${fedora_vanilla_hadoop_2_8_2_image_name:-"fedora_sahara_vanilla_hadoop_2_8_2_latest$suffix"}
|
||||||
|
set_hive_version
|
||||||
image_create fedora $fedora_image_name $fedora_elements_sequence
|
image_create fedora $fedora_image_name $fedora_elements_sequence
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
@ -544,6 +579,7 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
|
||||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
|
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
|
||||||
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
||||||
export centos_image_name=${centos_vanilla_hadoop_2_7_1_image_name:-"centos_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
|
export centos_image_name=${centos_vanilla_hadoop_2_7_1_image_name:-"centos_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
|
||||||
|
set_hive_version
|
||||||
image_create centos $centos_image_name $centos_elements_sequence
|
image_create centos $centos_image_name $centos_elements_sequence
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
@ -553,6 +589,13 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
|
||||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
|
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.1" ]; then
|
||||||
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_1:-"2.7.1"}
|
||||||
export centos7_image_name=${centos7_vanilla_hadoop_2_7_1_image_name:-"centos7_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
|
export centos7_image_name=${centos7_vanilla_hadoop_2_7_1_image_name:-"centos7_sahara_vanilla_hadoop_2_7_1_latest$suffix"}
|
||||||
|
set_hive_version
|
||||||
|
image_create centos7 $centos7_image_name $centos7_elements_sequence
|
||||||
|
fi
|
||||||
|
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then
|
||||||
|
export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"}
|
||||||
|
export centos7_image_name=${centos7_vanilla_hadoop_2_8_2_image_name:-"centos7_sahara_vanilla_hadoop_2_8_2_latest$suffix"}
|
||||||
|
set_hive_version
|
||||||
image_create centos7 $centos7_image_name $centos7_elements_sequence
|
image_create centos7 $centos7_image_name $centos7_elements_sequence
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
|
|
|
@ -11,6 +11,9 @@ function install_hadoop_v2 {
|
||||||
"2.7.1")
|
"2.7.1")
|
||||||
hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}"
|
hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}"
|
||||||
;;
|
;;
|
||||||
|
"2.8.2")
|
||||||
|
hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}"
|
||||||
|
;;
|
||||||
*)
|
*)
|
||||||
echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
|
echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
@ -10,7 +10,12 @@ tmp_dir=/tmp/hive
|
||||||
cd $tmp_dir
|
cd $tmp_dir
|
||||||
|
|
||||||
if [ -z "${HIVE_DOWNLOAD_URL:-}" ]; then
|
if [ -z "${HIVE_DOWNLOAD_URL:-}" ]; then
|
||||||
HIVE_FILE=hive-$HIVE_VERSION-bin.tar.gz
|
# hive package has renamed to "apache-hive-*" since 0.13.0
|
||||||
|
if [[ "$HIVE_VERSION" < "0.13.0" ]]; then
|
||||||
|
HIVE_FILE=hive-$HIVE_VERSION-bin.tar.gz
|
||||||
|
else
|
||||||
|
HIVE_FILE=apache-hive-$HIVE_VERSION-bin.tar.gz
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
|
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
|
||||||
fi
|
fi
|
||||||
|
@ -21,6 +26,15 @@ mv $HIVE_DIR /opt/hive
|
||||||
rm -r $tmp_dir
|
rm -r $tmp_dir
|
||||||
chmod -R a+rw /opt/hive
|
chmod -R a+rw /opt/hive
|
||||||
|
|
||||||
|
HIVE_MASTER_VERSION=${HIVE_VERSION%.*}.0
|
||||||
|
|
||||||
|
# NOTE(jfreud): Hive 0.11.0 was released without this DB upgrade script.
|
||||||
|
# All future versions of Hive do have the script.
|
||||||
|
if [ "$HIVE_VERSION" != "0.11.0" ]; then
|
||||||
|
# replace the relative path with absolute path
|
||||||
|
sed -i "s/SOURCE hive-txn-schema-$HIVE_MASTER_VERSION.mysql.sql;/SOURCE \/opt\/hive\/scripts\/metastore\/upgrade\/mysql\/hive-txn-schema-$HIVE_MASTER_VERSION.mysql.sql;/" /opt/hive/scripts/metastore/upgrade/mysql/hive-schema-$HIVE_MASTER_VERSION.mysql.sql
|
||||||
|
fi
|
||||||
|
|
||||||
ln -s /usr/share/java/mysql-connector-java.jar /opt/hive/lib/libmysql-java.jar
|
ln -s /usr/share/java/mysql-connector-java.jar /opt/hive/lib/libmysql-java.jar
|
||||||
chown -R hadoop:hadoop /opt/hive
|
chown -R hadoop:hadoop /opt/hive
|
||||||
cat >> /home/hadoop/.bashrc <<EOF
|
cat >> /home/hadoop/.bashrc <<EOF
|
||||||
|
|
|
@ -7,7 +7,7 @@ set -eu
|
||||||
set -o pipefail
|
set -o pipefail
|
||||||
|
|
||||||
if [ -z "${HIVE_DOWNLOAD_URL:-}" ]; then
|
if [ -z "${HIVE_DOWNLOAD_URL:-}" ]; then
|
||||||
version_check=$(echo $HIVE_VERSION | sed -e '/[0-9]\.[0-9][0-9]\.[0-9]/d')
|
version_check=$(echo $HIVE_VERSION | sed -e '/[0-9]\.[0-9]\{1,2\}\.[0-9]/d')
|
||||||
if [ ! -z $version_check ]; then
|
if [ ! -z $version_check ]; then
|
||||||
echo -e "Unable to install Hive: You should specify HIVE_DOWNLOAD_URL or HIVE_VERSION.\nAborting"
|
echo -e "Unable to install Hive: You should specify HIVE_DOWNLOAD_URL or HIVE_VERSION.\nAborting"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
@ -10,7 +10,13 @@ tmp_dir=$TARGET_ROOT/tmp/hive
|
||||||
mkdir -p $tmp_dir
|
mkdir -p $tmp_dir
|
||||||
|
|
||||||
if [ -z ${HIVE_DOWNLOAD_URL:-} ]; then
|
if [ -z ${HIVE_DOWNLOAD_URL:-} ]; then
|
||||||
HIVE_DOWNLOAD_URL=http://archive.apache.org/dist/hive/hive-$HIVE_VERSION/hive-$HIVE_VERSION-bin.tar.gz
|
# hive package has renamed to "apache-hive-*" since 0.13.0
|
||||||
|
if [[ "$HIVE_VERSION" < "0.13.0" ]]; then
|
||||||
|
HIVE_FILE=hive-$HIVE_VERSION-bin.tar.gz
|
||||||
|
else
|
||||||
|
HIVE_FILE=apache-hive-$HIVE_VERSION-bin.tar.gz
|
||||||
|
fi
|
||||||
|
HIVE_DOWNLOAD_URL=http://archive.apache.org/dist/hive/hive-$HIVE_VERSION/$HIVE_FILE
|
||||||
fi
|
fi
|
||||||
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
|
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
|
||||||
cached_tar="$DIB_IMAGE_CACHE/$HIVE_FILE"
|
cached_tar="$DIB_IMAGE_CACHE/$HIVE_FILE"
|
||||||
|
|
|
@ -16,6 +16,12 @@ if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then
|
||||||
else
|
else
|
||||||
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL)
|
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL)
|
||||||
fi
|
fi
|
||||||
|
elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
|
||||||
|
if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" ]; then
|
||||||
|
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_FILE)
|
||||||
|
else
|
||||||
|
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL)
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo "Unknown Hadoop version selected. Aborting."
|
echo "Unknown Hadoop version selected. Aborting."
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
@ -11,6 +11,11 @@ if [ "$DIB_HADOOP_VERSION" == "2.7.1" ]; then
|
||||||
echo "OOZIE_HADOOP_V2_7_1_FILE and OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
|
echo "OOZIE_HADOOP_V2_7_1_FILE and OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
|
||||||
exit 1
|
exit 1
|
||||||
fi
|
fi
|
||||||
|
elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then
|
||||||
|
if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_8_2_FILE:-}" ]; then
|
||||||
|
echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
else
|
else
|
||||||
echo "Unknown Hadoop version selected. Aborting"
|
echo "Unknown Hadoop version selected. Aborting"
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
@ -12,6 +12,9 @@ mkdir -p $tmp_dir
|
||||||
if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then
|
if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then
|
||||||
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}"
|
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}"
|
||||||
OOZIE_FILE="${OOZIE_HADOOP_V2_7_1_FILE:-}"
|
OOZIE_FILE="${OOZIE_HADOOP_V2_7_1_FILE:-}"
|
||||||
|
elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
|
||||||
|
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}"
|
||||||
|
OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}"
|
||||||
else
|
else
|
||||||
echo "Unknown Hadoop version selected. Aborting."
|
echo "Unknown Hadoop version selected. Aborting."
|
||||||
exit 1
|
exit 1
|
||||||
|
|
|
@ -12,6 +12,8 @@ case "$plugin_type" in
|
||||||
case "$DIB_HADOOP_VERSION" in
|
case "$DIB_HADOOP_VERSION" in
|
||||||
"2.7.1")
|
"2.7.1")
|
||||||
hadoop="2.7.1";;
|
hadoop="2.7.1";;
|
||||||
|
"2.8.2")
|
||||||
|
hadoop="2.8.2";;
|
||||||
esac
|
esac
|
||||||
DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop.jar"
|
DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop.jar"
|
||||||
;;
|
;;
|
||||||
|
|
|
@ -6,8 +6,10 @@ case "$PLUGIN" in
|
||||||
"vanilla")
|
"vanilla")
|
||||||
# temporary don't test building fedora image
|
# temporary don't test building fedora image
|
||||||
for distro in ubuntu centos7; do
|
for distro in ubuntu centos7; do
|
||||||
# testing only 2.7.1 for speed up
|
# limit to 2 releases
|
||||||
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v 2.7.1
|
for version in 2.7.1 2.8.2; do
|
||||||
|
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version
|
||||||
|
done
|
||||||
done
|
done
|
||||||
;;
|
;;
|
||||||
"cloudera")
|
"cloudera")
|
||||||
|
|
Loading…
Reference in New Issue