From 5707f5cc5a3e551d53070d7b3564225cd8c28cf7 Mon Sep 17 00:00:00 2001 From: Zhuang Changkun Date: Mon, 16 Apr 2018 20:32:59 +0800 Subject: [PATCH] Add support to build 2.7.5 image Change-Id: Ia16692632cbe6380d4021dd031eed09362873905 --- diskimage-create/diskimage-create.sh | 29 +++++++++++++++++-- elements/hadoop/install.d/40-setup-hadoop | 3 ++ elements/oozie/install.d/50-setup-oozie | 6 ++++ elements/oozie/root.d/0-check-oozie | 5 ++++ elements/oozie/root.d/50-download-oozie | 3 ++ .../swift_hadoop/post-install.d/81-add-jar | 2 ++ tools/gate/build-images | 4 +-- 7 files changed, 48 insertions(+), 4 deletions(-) diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index 3ae9e1d7..e8441c93 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -32,7 +32,7 @@ usage() { echo "Usage: $(basename $0)" echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]" echo " [-i ubuntu|fedora|centos|centos7]" - echo " [-v 2.7.1|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]" + echo " [-v 2.7.1|2.7.5|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]" echo " [-r 5.1.0|5.2.0]" echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]" echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]" @@ -184,7 +184,7 @@ case "$PLUGIN" in "");; "vanilla") case "$HADOOP_VERSION" in - "" | "2.7.1" | "2.8.2");; + "" | "2.7.1" | "2.7.5" | "2.8.2");; *) echo -e "Unknown hadoop version selected.\nAborting" exit 1 @@ -480,6 +480,9 @@ set_hive_version() { "2.7.1" ) export HIVE_VERSION="0.11.0" ;; + "2.7.5" ) + export HIVE_VERSION="2.3.2" + ;; "2.8.2" ) export HIVE_VERSION="2.3.2" ;; @@ -498,8 +501,10 @@ set_hive_version() { if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"} + export HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.5.tar.gz"} export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"} export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"} + export OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_5_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.7.5.tar.gz"} export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"} export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib" export plugin_type="vanilla" @@ -547,6 +552,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence unset DIB_RELEASE fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then + export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} + export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_7_5_image_name:-"ubuntu_sahara_vanilla_hadoop_2_7_5_latest"} + export DIB_RELEASE=${DIB_RELEASE:-xenial} + set_hive_version + image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence + unset DIB_RELEASE + fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_8_2_image_name:-"ubuntu_sahara_vanilla_hadoop_2_8_2_latest"} @@ -566,6 +579,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then set_hive_version image_create fedora $fedora_image_name $fedora_elements_sequence fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then + export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} + export fedora_image_name=${fedora_vanilla_hadoop_2_7_5_image_name:-"fedora_sahara_vanilla_hadoop_2_7_5_latest$suffix"} + set_hive_version + image_create fedora $fedora_image_name $fedora_elements_sequence + fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} export fedora_image_name=${fedora_vanilla_hadoop_2_8_2_image_name:-"fedora_sahara_vanilla_hadoop_2_8_2_latest$suffix"} @@ -592,6 +611,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then set_hive_version image_create centos7 $centos7_image_name $centos7_elements_sequence fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.7.5" ]; then + export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_7_5:-"2.7.5"} + export centos7_image_name=${centos7_vanilla_hadoop_2_7_5_image_name:-"centos7_sahara_vanilla_hadoop_2_7_5_latest$suffix"} + set_hive_version + image_create centos7 $centos7_image_name $centos7_elements_sequence + fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.8.2" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_8_2:-"2.8.2"} export centos7_image_name=${centos7_vanilla_hadoop_2_8_2_image_name:-"centos7_sahara_vanilla_hadoop_2_8_2_latest$suffix"} diff --git a/elements/hadoop/install.d/40-setup-hadoop b/elements/hadoop/install.d/40-setup-hadoop index e74046ba..ac99d34f 100755 --- a/elements/hadoop/install.d/40-setup-hadoop +++ b/elements/hadoop/install.d/40-setup-hadoop @@ -11,6 +11,9 @@ function install_hadoop_v2 { "2.7.1") hadoop_native_libs_url="${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL}" ;; + "2.7.5") + hadoop_native_libs_url="${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL}" + ;; "2.8.2") hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}" ;; diff --git a/elements/oozie/install.d/50-setup-oozie b/elements/oozie/install.d/50-setup-oozie index bfc14886..9e45702b 100755 --- a/elements/oozie/install.d/50-setup-oozie +++ b/elements/oozie/install.d/50-setup-oozie @@ -16,6 +16,12 @@ if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then else OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL) fi +elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then + if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" ]; then + OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_FILE) + else + OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL) + fi elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" ]; then OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_FILE) diff --git a/elements/oozie/root.d/0-check-oozie b/elements/oozie/root.d/0-check-oozie index db21d2fb..1980b588 100755 --- a/elements/oozie/root.d/0-check-oozie +++ b/elements/oozie/root.d/0-check-oozie @@ -11,6 +11,11 @@ if [ "$DIB_HADOOP_VERSION" == "2.7.1" ]; then echo "OOZIE_HADOOP_V2_7_1_FILE and OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" exit 1 fi +elif [ "$DIB_HADOOP_VERSION" == "2.7.5" ]; then + if [ -z "${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_7_5_FILE:-}" ]; then + echo "OOZIE_HADOOP_V2_7_5_FILE and OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" + exit 1 + fi elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then if [ -z "${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V2_8_2_FILE:-}" ]; then echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" diff --git a/elements/oozie/root.d/50-download-oozie b/elements/oozie/root.d/50-download-oozie index 192c9d29..6033f8e9 100755 --- a/elements/oozie/root.d/50-download-oozie +++ b/elements/oozie/root.d/50-download-oozie @@ -12,6 +12,9 @@ mkdir -p $tmp_dir if [[ "$DIB_HADOOP_VERSION" == "2.7.1" ]]; then OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL:-}" OOZIE_FILE="${OOZIE_HADOOP_V2_7_1_FILE:-}" +elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then + OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL:-}" + OOZIE_FILE="${OOZIE_HADOOP_V2_7_5_FILE:-}" elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}" diff --git a/elements/swift_hadoop/post-install.d/81-add-jar b/elements/swift_hadoop/post-install.d/81-add-jar index a0659621..c3dcda8d 100755 --- a/elements/swift_hadoop/post-install.d/81-add-jar +++ b/elements/swift_hadoop/post-install.d/81-add-jar @@ -12,6 +12,8 @@ case "$plugin_type" in case "$DIB_HADOOP_VERSION" in "2.7.1") hadoop="2.7.1";; + "2.7.5") + hadoop="2.7.5";; "2.8.2") hadoop="2.8.2";; esac diff --git a/tools/gate/build-images b/tools/gate/build-images index 3cac6a62..38fd23af 100755 --- a/tools/gate/build-images +++ b/tools/gate/build-images @@ -6,8 +6,8 @@ case "$PLUGIN" in "vanilla") # temporary don't test building fedora image for distro in ubuntu centos7; do - # limit to 2 releases - for version in 2.7.1 2.8.2; do + # limit to 3 releases + for version in 2.7.1 2.7.5 2.8.2; do tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version done done