diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index a79bfd00..c07767da 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -23,7 +23,7 @@ usage() { echo "Usage: $(basename $0)" echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]" echo " [-i ubuntu|fedora|centos]" - echo " [-v 1|2|2.3|2.4|2.6|5.0|5.3|plain]" + echo " [-v 1|2|2.6|5.0|5.3|plain]" echo " [-r 3.1.1|4.0.1]" echo " [-d]" echo " [-m]" @@ -112,7 +112,7 @@ if [ -n "$BASE_IMAGE_OS" -a "$BASE_IMAGE_OS" != "ubuntu" -a "$BASE_IMAGE_OS" != fi if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then - if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.3" -a "$HADOOP_VERSION" != "2.4" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then + if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then if [ "$PLUGIN" = "cloudera" -a "$HADOOP_VERSION" != "5.0" -a "$HADOOP_VERSION" != "5.3" ]; then echo -e "Unknown hadoop version selected.\nAborting" exit 1 @@ -227,11 +227,7 @@ fi if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} export OOZIE_HADOOP_V1_DOWNLOAD_URL=${OOZIE_HADOOP_V1_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0.tar.gz"} - export OOZIE_HADOOP_V2_3_DOWNLOAD_URL=${OOZIE_HADOOP_V2_3_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.0-hadoop-2.3.0.tar.gz"} - export OOZIE_HADOOP_V2_4_DOWNLOAD_URL=${OOZIE_HADOOP_V2_4_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-juno-4.0.1-hadoop-2.4.1.tar.gz"} export OOZIE_HADOOP_V2_6_DOWNLOAD_URL=${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/oozie-4.0.1-hadoop-2.6.0.tar.gz"} - export HADOOP_V2_3_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_3_NATIVE_LIBS_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/hadoop-2.3.0-native-libs.tar.gz"} - export HADOOP_V2_4_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_4_NATIVE_LIBS_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/hadoop-native-libs-juno-2.4.1.tar.gz"} export HADOOP_V2_6_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_6_NATIVE_LIBS_DOWNLOAD_URL:-"http://sahara-files.mirantis.com/hadoop-native-libs-2.6.0.tar.gz"} export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"} export HIVE_VERSION=${HIVE_VERSION:-"0.11.0"} @@ -276,18 +272,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then disk-image-create $elements_sequence -o $ubuntu_image_name mv $ubuntu_image_name.qcow2 ../ fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.3" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_3:-"2.3.0"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_3_image_name:-"ubuntu_sahara_vanilla_hadoop_2_3_latest"} - disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name - mv $ubuntu_image_name.qcow2 ../ - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.4" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_4:-"2.4.1"} - export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_4_image_name:-"ubuntu_sahara_vanilla_hadoop_2_4_latest"} - disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name - mv $ubuntu_image_name.qcow2 ../ - fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.6" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_6:-"2.6.0"} export ubuntu_image_name=${ubuntu_vanilla_hadoop_2_6_image_name:-"ubuntu_sahara_vanilla_hadoop_2_6_latest"} @@ -306,18 +290,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then disk-image-create $elements_sequence -o $fedora_image_name mv $fedora_image_name.qcow2 ../ fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.3" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_3:-"2.3.0"} - export fedora_image_name=${fedora_vanilla_hadoop_2_3_image_name:-"fedora_sahara_vanilla_hadoop_2_3_latest$suffix"} - disk-image-create $fedora_elements_sequence -o $fedora_image_name - mv $fedora_image_name.qcow2 ../ - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.4" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_4:-"2.4.1"} - export fedora_image_name=${fedora_vanilla_hadoop_2_4_image_name:-"fedora_sahara_vanilla_hadoop_2_4_latest$suffix"} - disk-image-create $fedora_elements_sequence -o $fedora_image_name - mv $fedora_image_name.qcow2 ../ - fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.6" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_6:-"2.6.0"} export fedora_image_name=${fedora_vanilla_hadoop_2_6_image_name:-"fedora_sahara_vanilla_hadoop_2_6_latest$suffix"} @@ -340,18 +312,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then disk-image-create $elements_sequence -n -o $centos_image_name mv $centos_image_name.qcow2 ../ fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.3" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_3:-"2.3.0"} - export centos_image_name=${centos_vanilla_hadoop_2_3_image_name:-"centos_sahara_vanilla_hadoop_2_3_latest$suffix"} - disk-image-create $centos_elements_sequence -n -o $centos_image_name - mv $centos_image_name.qcow2 ../ - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.4" ]; then - export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_4:-"2.4.1"} - export centos_image_name=${centos_vanilla_hadoop_2_4_image_name:-"centos_sahara_vanilla_hadoop_2_4_latest$suffix"} - disk-image-create $centos_elements_sequence -n -o $centos_image_name - mv $centos_image_name.qcow2 ../ - fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "2.6" ]; then export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_2_6:-"2.6.0"} export centos_image_name=${centos_vanilla_hadoop_2_6_image_name:-"centos_sahara_vanilla_hadoop_2_6_latest$suffix"} diff --git a/elements/hadoop/README.md b/elements/hadoop/README.md index 7621c6b3..16edc0f0 100644 --- a/elements/hadoop/README.md +++ b/elements/hadoop/README.md @@ -6,19 +6,19 @@ HOWTO build Hadoop Native Libs + Install: *jdk >= 6*, *maven*, *cmake* and *protobuf >= 2.5.0* + Get Hadoop source code: ```sh -$ wget http://archive.apache.org/dist/hadoop/core/hadoop-2.3.0/hadoop-2.3.0-src.tar.gz +$ wget http://archive.apache.org/dist/hadoop/core/hadoop-2.6.0/hadoop-2.6.0-src.tar.gz ``` + Unpack source ```sh -$ tar xvf hadoop-2.3.0-src.tar.gz +$ tar xvf hadoop-2.6.0-src.tar.gz ``` + Build Hadoop ```sh -$ cd hadoop-2.3.0-src +$ cd hadoop-2.6.0-src $ mvn package -Pdist,native -DskipTests ``` + Create tarball with Hadoop Native Libs ```sh -$ cd hadoop-dist/target/hadoop-2.3.0/lib -$ tar -czvf hadoop-native-libs-2.3.0.tar.gz native +$ cd hadoop-dist/target/hadoop-2.6.0/lib +$ tar -czvf hadoop-native-libs-2.6.0.tar.gz native ``` diff --git a/elements/hadoop/post-install.d/40-setup-hadoop b/elements/hadoop/post-install.d/40-setup-hadoop index 68fc93dc..21a7faf5 100755 --- a/elements/hadoop/post-install.d/40-setup-hadoop +++ b/elements/hadoop/post-install.d/40-setup-hadoop @@ -80,12 +80,6 @@ EOF function install_hadoop_v2 { case "$DIB_HADOOP_VERSION" in - "2.3.0") - hadoop_native_libs_url="${HADOOP_V2_3_NATIVE_LIBS_DOWNLOAD_URL}" - ;; - "2.4.1") - hadoop_native_libs_url="${HADOOP_V2_4_NATIVE_LIBS_DOWNLOAD_URL}" - ;; "2.6.0") hadoop_native_libs_url="${HADOOP_V2_6_NATIVE_LIBS_DOWNLOAD_URL}" ;; diff --git a/elements/oozie/install.d/50-setup-oozie b/elements/oozie/install.d/50-setup-oozie index 643bb746..0a78a7ff 100755 --- a/elements/oozie/install.d/50-setup-oozie +++ b/elements/oozie/install.d/50-setup-oozie @@ -9,12 +9,6 @@ mkdir $tmp_dir if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then OOZIE_DOWNLOAD_URL="$OOZIE_HADOOP_V1_DOWNLOAD_URL" OOZIE_FILE="$OOZIE_HADOOP_V1_FILE" -elif [[ "$DIB_HADOOP_VERSION" == "2.3.0" ]]; then - OOZIE_DOWNLOAD_URL="$OOZIE_HADOOP_V2_3_DOWNLOAD_URL" - OOZIE_FILE="$OOZIE_HADOOP_V2_3_FILE" -elif [[ "$DIB_HADOOP_VERSION" == "2.4.1" ]]; then - OOZIE_DOWNLOAD_URL="$OOZIE_HADOOP_V2_4_DOWNLOAD_URL" - OOZIE_FILE="$OOZIE_HADOOP_V2_4_FILE" elif [[ "$DIB_HADOOP_VERSION" == "2.6.0" ]]; then OOZIE_DOWNLOAD_URL="$OOZIE_HADOOP_V2_6_DOWNLOAD_URL" OOZIE_FILE="$OOZIE_HADOOP_V2_6_FILE" diff --git a/elements/oozie/root.d/0-check b/elements/oozie/root.d/0-check index 7c99afb1..2e8af520 100755 --- a/elements/oozie/root.d/0-check +++ b/elements/oozie/root.d/0-check @@ -6,16 +6,6 @@ if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then echo "OOZIE_HADOOP_V1_FILE and OOZIE_HADOOP_V1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" exit 1 fi -elif [ "$DIB_HADOOP_VERSION" == "2.3.0" ]; then - if [ -z "$OOZIE_HADOOP_V2_3_DOWNLOAD_URL" -a -z "$OOZIE_HADOOP_V2_3_FILE" ]; then - echo "OOZIE_HADOOP_V2_3_FILE and OOZIE_HADOOP_V2_3_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi -elif [ "$DIB_HADOOP_VERSION" == "2.4.1" ]; then - if [ -z "$OOZIE_HADOOP_V2_4_DOWNLOAD_URL" -a -z "$OOZIE_HADOOP_V2_4_FILE" ]; then - echo "OOZIE_HADOOP_V2_4_FILE and OOZIE_HADOOP_V2_4_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" - exit 1 - fi elif [ "$DIB_HADOOP_VERSION" == "2.6.0" ]; then if [ -z "$OOZIE_HADOOP_V2_6_DOWNLOAD_URL" -a -z "$OOZIE_HADOOP_V2_6_FILE" ]; then echo "OOZIE_HADOOP_V2_6_FILE and OOZIE_HADOOP_V2_6_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"