Replace elements/hadoop/ with Fedora&Ubuntu supporting elements/hadoop_fedora

Note: this requires a DIB version that includes change I7a42409a (SHA
82eacdec) from 11 July 2013

Implements: blueprint merge-dib-hadoop-elements
Change-Id: I23d873adfc7c16e36cf26af2cd9156c2a8ec90fc
This commit is contained in:
Matthew Farrellee 2013-07-16 08:32:46 -04:00
parent 1c9c52a977
commit 40931bdb5f
13 changed files with 40 additions and 157 deletions

View File

@ -38,7 +38,7 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
.. sourcecode:: bash
DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd -o fedora_hadoop_1_1_2
In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).

View File

@ -1 +1 @@
Installs Java and Hadoop on Ubuntu
Installs Java and Hadoop, configures SSH

View File

@ -1,12 +1,30 @@
#!/bin/bash
# Common
chown root:root /mnt
mkdir -p /mnt/log/hadoop
chown hadoop:hadoop /mnt/log/hadoop
mkdir /var/run/hadoop
mkdir /run/hadoop
chown hadoop:hadoop /var/run/hadoop/
chown hadoop:hadoop /run/hadoop/
echo "Setup ubuntu's directory"
mkdir -p /home/ubuntu/.ssh
touch /home/ubuntu/.ssh/authorized_keys
chown -R ubuntu:ubuntu /home/ubuntu
mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
distro=$(lsb_release -is || :)
case "$distro" in
Ubuntu )
mkdir /run/hadoop
chown hadoop:hadoop /run/hadoop/
mkdir -p /home/ubuntu/.ssh
touch /home/ubuntu/.ssh/authorized_keys
chown -R ubuntu:ubuntu /home/ubuntu
;;
Fedora )
sleep 10
rm /etc/resolv.conf
service network restart
chown -R ec2-user:ec2-user /home/ec2-user
chown -R ec2-user:ec2-user /etc/hadoop
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
iptables -F
;;
* )
echo "Unknown distro: $distro. Exiting."
exit 1
;;
esac

View File

@ -1,43 +0,0 @@
#!/bin/bash
tmp_dir=/tmp/java
mkdir -p $TARGET_ROOT/$tmp_dir
echo "Java setup begins"
set -e
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
mkdir -p $TARGET_ROOT/$tmp_dir
script_dir=$(dirname $0)
install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
echo "Java file moved"
else
wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
if [ $? -eq 0 ]; then
echo "Java is downloading"
else
echo "Url error. Exit"
exit 1
fi
fi
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
filename=$(basename $filename)
if echo $filename | grep -q -s -F .tar.gz ; then
path=$(pwd)
cd $TARGET_ROOT/$tmp_dir
echo $TARGET_ROOT/$tmp_dir/$filename | xargs -n 1 tar -zxvf
cd $path
else
if echo $filename | grep -q -s -F .bin ; then
path=$(pwd)
cd $TARGET_ROOT/$tmp_dir
echo -e "\n" | sh $filename
rm $filename
cd $path
fi
fi
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
filename=$(basename $filename)
javaPath=/usr/lib/
mkdir -p $TARGET_ROOT/$javaPath
mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
echo "Java was installed"

View File

@ -1,37 +0,0 @@
#!/bin/bash
tmp_dir=/tmp/hadoop
mkdir -p $TARGET_ROOT/$tmp_dir
echo "Installing SSH server"
mkdir /root/.ssh
install-packages openssh-server
echo "Creating hadoop user & group"
addgroup hadoop
adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
adduser hadoop sudo
echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb"
if [ $? -ne 0 ]; then
echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nExit"
exit 1
fi
echo "Installing Hadoop"
dpkg -i $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
rm $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
echo "Pre-configuring Hadoop"
filename=$(find $TARGET_ROOT/usr/lib/ -maxdepth 1 -name "jdk*")
echo -e "HADOOP_HOME=/usr/share/hadoop/\nPATH=\$PATH:/usr/sbin/" >> $TARGET_ROOT/home/hadoop/.bashrc
echo -e "JAVA_HOME=$filename\nPATH=\$PATH:$filename/bin/" >> $TARGET_ROOT/home/hadoop/.bashrc
sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
log_dir=/mnt/log/hadoop/\$USER/
sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
log_dir=/mnt/log/hadoop/hdfs
sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh

View File

@ -1,8 +0,0 @@
#!/bin/bash
sed -i -e 's/PasswordAuthentication no/PasswordAuthentication yes/' \
-e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \
/etc/ssh/sshd_config
echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes\nGSSAPICleanupCredentials yes" >> /etc/ssh/sshd_config
sed -i -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
echo -e "StrictHostKeyChecking no" >> /etc/ssh/ssh_config

View File

@ -1,11 +1,16 @@
#!/bin/bash
set -e
source $_LIB/die
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
[ -n "$JAVA_FILE" ] || die "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
fi
[ -n "$DIB_HADOOP_VERSION" ] || die "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
if [ -z "$JAVA_FILE" ]; then
echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
exit 1
fi
fi
if [ -z "$DIB_HADOOP_VERSION" ]; then
echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
exit 1
fi
version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
if [[ -z $version_check ]]; then
echo "All variables are set, continue."
@ -13,4 +18,3 @@ else
echo "Version error. Exit"
exit 1
fi

View File

@ -1 +0,0 @@
Installs Java and Hadoop on Fedora

View File

@ -1,30 +0,0 @@
#!/bin/bash
# Common
chown root:root /mnt
mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
distro=$(lsb_release -is || :)
case "$distro" in
Ubuntu )
mkdir /run/hadoop
chown hadoop:hadoop /run/hadoop/
mkdir -p /home/ubuntu/.ssh
touch /home/ubuntu/.ssh/authorized_keys
chown -R ubuntu:ubuntu /home/ubuntu
;;
Fedora )
sleep 10
rm /etc/resolv.conf
service network restart
chown -R ec2-user:ec2-user /home/ec2-user
chown -R ec2-user:ec2-user /etc/hadoop
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
iptables -F
;;
* )
echo "Unknown distro: $distro. Exiting."
exit 1
;;
esac

View File

@ -1,20 +0,0 @@
#!/bin/bash
set -e
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
if [ -z "$JAVA_FILE" ]; then
echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
exit 1
fi
fi
if [ -z "$DIB_HADOOP_VERSION" ]; then
echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
exit 1
fi
version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
if [[ -z $version_check ]]; then
echo "All variables are set, continue."
else
echo "Version error. Exit"
exit 1
fi