sahara-image-elements/elements/hadoop/post-install.d/40-setup-hadoop

#!/bin/bash

# XXX: This is in post-install.d, instead of install.d, because the
# hadoop RPM claims ownership of files owned by the filesystem RPM,
# such as /usr and /bin, and installing hadoop then updating
# filesystem results in a failure. This can be moved to install.d when
# HADOOP-9777 is resolved.
#   https://issues.apache.org/jira/browse/HADOOP-9777

function download_hadoop_package {
    echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
    install-packages wget
    wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-$DIB_HADOOP_VERSION/$package"
    if [ $? -ne 0 ]; then
        echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
        exit 1
    fi
}

function find_java_home {
    JAVA_HOME=$(find $TARGET_ROOT$JAVA_TARGET_LOCATION/ -maxdepth 1 -name "jdk*")

    if [ -z "$JAVA_HOME" ]; then
        case "$distro" in
            Ubuntu )
                JAVA_HOME=$(readlink -e /usr/bin/java | sed "s:bin/java::")
            ;;
            Fedora | RedHatEnterpriseServer | CentOS )
                JAVA_HOME=$(rpm --eval '%{?java_home}')
            ;;
        esac
    fi
}

function install_hadoop_v1 {
    case "$distro" in
        Ubuntu )
            package="hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb"
        ;;
        Fedora | RedHatEnterpriseServer | CentOS )
            package="hadoop-$DIB_HADOOP_VERSION-1.x86_64.rpm"
        ;;
    esac

    download_hadoop_package

    case "$distro" in
        Ubuntu )
            dpkg -i $tmp_dir/$package
        ;;
        Fedora | RedHatEnterpriseServer | CentOS )
            if [ $(lsb_release -rs) -ge '19' ]; then
                rpm -i $tmp_dir/$package --relocate /usr=/usr --replacefiles
            else
                rpm -ivh --replacefiles $tmp_dir/$package
            fi
            chmod 755 /usr/sbin/start-*
            chmod 755 /usr/sbin/stop-*
            chmod 755 /usr/sbin/slaves.sh
            chmod 755 /usr/sbin/update-hadoop-env.sh
        ;;
    esac
    rm -r $tmp_dir

    echo "Pre-configuring Hadoop"
    find_java_home

    cat >> /home/hadoop/.bashrc <<EOF
PATH=\$PATH:/usr/sbin:$JAVA_HOME/bin
JAVA_HOME=$JAVA_HOME
HADOOP_HOME=/usr/share/hadoop/
EOF

    sed -i -e "s,export JAVA_HOME=.*,export JAVA_HOME=$JAVA_HOME," \
        -e "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=/mnt/log/hadoop/\$USER," \
        -e "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=/mnt/log/hadoop/hdfs," \
        /etc/hadoop/hadoop-env.sh
}


function install_hadoop_v2 {
    case "$DIB_HADOOP_VERSION" in
        "2.6.0")
            hadoop_native_libs_url="${HADOOP_V2_6_NATIVE_LIBS_DOWNLOAD_URL}"
        ;;
        *)
            echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
            exit 1
        ;;
    esac

    package="hadoop-$DIB_HADOOP_VERSION.tar.gz"

    download_hadoop_package

    echo "Installing hadoop"

    INSTALL_DIR="/opt"
    HADOOP_HOME="/opt/hadoop"
    mkdir -p "$INSTALL_DIR"
    tar xvf "$tmp_dir/$package" -C "$INSTALL_DIR"
    ln -s "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION" "$HADOOP_HOME"
    chown -R hadoop:hadoop "$INSTALL_DIR/hadoop-$DIB_HADOOP_VERSION"
    chown -R hadoop:hadoop "$HADOOP_HOME"
    rm -r $tmp_dir

    echo "Inject Hadoop native libs"
    rm -r "$HADOOP_HOME/lib/native"
    wget "$hadoop_native_libs_url"
    native_libs_filename=$(basename "$hadoop_native_libs_url")
    tar xvf "$native_libs_filename" -C "$HADOOP_HOME/lib"
    rm "$native_libs_filename"

    echo "Pre-configuring Hadoop"
    find_java_home

    HADOOP_PID_DIR="/var/run/hadoop"
    mkdir $HADOOP_PID_DIR
    chown hadoop:hadoop $HADOOP_PID_DIR

    cat >> /etc/profile.d/hadoop.sh <<EOF
export JAVA_HOME=$JAVA_HOME
export HADOOP_COMMON_HOME=$HADOOP_HOME
export PATH=\$PATH:$JAVA_HOME/bin:$HADOOP_HOME/bin:$HADOOP_HOME/sbin
export HADOOP_HDFS_HOME=\$HADOOP_COMMON_HOME
export HADOOP_YARN_HOME=\$HADOOP_COMMON_HOME
export HADOOP_MAPRED_HOME=\$HADOOP_COMMON_HOME
export HADOOP_PID_DIR=$HADOOP_PID_DIR
export YARN_PID_DIR=$HADOOP_PID_DIR
export HADOOP_MAPRED_PID_DIR=$HADOOP_PID_DIR
EOF

    sed -i -e "s,export JAVA_HOME=.*,export JAVA_HOME=$JAVA_HOME," \
        -e "s,\#export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=/mnt/log/hadoop/\$USER," \
        -e "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=/mnt/log/hadoop/hdfs," \
        $HADOOP_HOME/etc/hadoop/hadoop-env.sh

    sed -i -e "s,\# export JAVA_HOME=.*,export JAVA_HOME=$JAVA_HOME," \
        -e "s,YARN_LOG_DIR=.*,YARN_LOG_DIR=/mnt/log/hadoop/yarn," \
        $HADOOP_HOME/etc/hadoop/yarn-env.sh

    # enable swiftfs
    ln -s ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-openstack-${DIB_HADOOP_VERSION}.jar ${HADOOP_HOME}/share/hadoop/common/lib/
}

distro=$(lsb_release -is || :)
if [ ! "$distro" == "Fedora" -a ! "$distro" == "Ubuntu" -a ! "$distro" == "RedHatEnterpriseServer" -a ! "$distro" == "CentOS" ]; then
    echo "Unknown distro: $distro. Exiting."
    exit 1
fi

echo "Hadoop setup begins for $distro"
tmp_dir=/tmp/hadoop

echo "Creating hadoop user & group"
case "$distro" in
    Ubuntu )
        addgroup hadoop
        adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
        adduser hadoop sudo
    ;;
    Fedora | RedHatEnterpriseServer | CentOS )
        adduser -G adm,wheel hadoop
    ;;
esac

: ${JAVA_TARGET_LOCATION:="/usr/java"}

if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
    install_hadoop_v1
else
    install_hadoop_v2
fi

echo "Applying firstboot script"

RC_SCRIPT_DIR=""
if [ "$distro" == "Ubuntu" ]; then
    # File '/etc/rc.local' may not exist
    if [ -f "/etc/rc.local" ]; then
        mv /etc/rc.local /etc/rc.local.old
    fi
    RC_SCRIPT_DIR="/etc"
else
    # File '/etc/rc.d/rc.local' may not exist
    if [ -f "/etc/rc.d/rc.local" ]; then
        mv /etc/rc.d/rc.local /etc/rc.d/rc.local.old
    fi
    RC_SCRIPT_DIR="/etc/rc.d"
fi

install -D -g root -o root -m 0755 $(dirname $0)/firstboot $RC_SCRIPT_DIR/rc.local