Replace elements/hadoop/ with Fedora&Ubuntu supporting elements/hadoop_fedora
Note: this requires a DIB version that includes change I7a42409a (SHA 82eacdec) from 11 July 2013 Implements: blueprint merge-dib-hadoop-elements Change-Id: I23d873adfc7c16e36cf26af2cd9156c2a8ec90fc
This commit is contained in:
parent
1c9c52a977
commit
40931bdb5f
|
@ -38,7 +38,7 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
|
|||
|
||||
.. sourcecode:: bash
|
||||
|
||||
DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
|
||||
JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd -o fedora_hadoop_1_1_2
|
||||
|
||||
In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
|
||||
You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).
|
||||
|
|
|
@ -1 +1 @@
|
|||
Installs Java and Hadoop on Ubuntu
|
||||
Installs Java and Hadoop, configures SSH
|
||||
|
|
|
@ -1,12 +1,30 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Common
|
||||
chown root:root /mnt
|
||||
mkdir -p /mnt/log/hadoop
|
||||
chown hadoop:hadoop /mnt/log/hadoop
|
||||
mkdir /var/run/hadoop
|
||||
mkdir /run/hadoop
|
||||
chown hadoop:hadoop /var/run/hadoop/
|
||||
chown hadoop:hadoop /run/hadoop/
|
||||
echo "Setup ubuntu's directory"
|
||||
mkdir -p /home/ubuntu/.ssh
|
||||
touch /home/ubuntu/.ssh/authorized_keys
|
||||
chown -R ubuntu:ubuntu /home/ubuntu
|
||||
mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
|
||||
mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
|
||||
|
||||
distro=$(lsb_release -is || :)
|
||||
case "$distro" in
|
||||
Ubuntu )
|
||||
mkdir /run/hadoop
|
||||
chown hadoop:hadoop /run/hadoop/
|
||||
mkdir -p /home/ubuntu/.ssh
|
||||
touch /home/ubuntu/.ssh/authorized_keys
|
||||
chown -R ubuntu:ubuntu /home/ubuntu
|
||||
;;
|
||||
Fedora )
|
||||
sleep 10
|
||||
rm /etc/resolv.conf
|
||||
service network restart
|
||||
chown -R ec2-user:ec2-user /home/ec2-user
|
||||
chown -R ec2-user:ec2-user /etc/hadoop
|
||||
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
|
||||
iptables -F
|
||||
;;
|
||||
* )
|
||||
echo "Unknown distro: $distro. Exiting."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
|
|
@ -1,43 +0,0 @@
|
|||
#!/bin/bash
|
||||
tmp_dir=/tmp/java
|
||||
mkdir -p $TARGET_ROOT/$tmp_dir
|
||||
echo "Java setup begins"
|
||||
set -e
|
||||
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
|
||||
mkdir -p $TARGET_ROOT/$tmp_dir
|
||||
script_dir=$(dirname $0)
|
||||
install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
|
||||
echo "Java file moved"
|
||||
else
|
||||
wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "Java is downloading"
|
||||
else
|
||||
echo "Url error. Exit"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
|
||||
filename=$(basename $filename)
|
||||
|
||||
if echo $filename | grep -q -s -F .tar.gz ; then
|
||||
path=$(pwd)
|
||||
cd $TARGET_ROOT/$tmp_dir
|
||||
echo $TARGET_ROOT/$tmp_dir/$filename | xargs -n 1 tar -zxvf
|
||||
cd $path
|
||||
else
|
||||
if echo $filename | grep -q -s -F .bin ; then
|
||||
path=$(pwd)
|
||||
cd $TARGET_ROOT/$tmp_dir
|
||||
echo -e "\n" | sh $filename
|
||||
rm $filename
|
||||
cd $path
|
||||
fi
|
||||
fi
|
||||
|
||||
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
|
||||
filename=$(basename $filename)
|
||||
javaPath=/usr/lib/
|
||||
mkdir -p $TARGET_ROOT/$javaPath
|
||||
mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
|
||||
echo "Java was installed"
|
|
@ -1,37 +0,0 @@
|
|||
#!/bin/bash
|
||||
tmp_dir=/tmp/hadoop
|
||||
mkdir -p $TARGET_ROOT/$tmp_dir
|
||||
echo "Installing SSH server"
|
||||
|
||||
mkdir /root/.ssh
|
||||
install-packages openssh-server
|
||||
|
||||
echo "Creating hadoop user & group"
|
||||
|
||||
addgroup hadoop
|
||||
adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
|
||||
adduser hadoop sudo
|
||||
|
||||
echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
|
||||
|
||||
wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nExit"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Installing Hadoop"
|
||||
|
||||
dpkg -i $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
|
||||
rm $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
|
||||
|
||||
echo "Pre-configuring Hadoop"
|
||||
|
||||
filename=$(find $TARGET_ROOT/usr/lib/ -maxdepth 1 -name "jdk*")
|
||||
echo -e "HADOOP_HOME=/usr/share/hadoop/\nPATH=\$PATH:/usr/sbin/" >> $TARGET_ROOT/home/hadoop/.bashrc
|
||||
echo -e "JAVA_HOME=$filename\nPATH=\$PATH:$filename/bin/" >> $TARGET_ROOT/home/hadoop/.bashrc
|
||||
sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
|
||||
log_dir=/mnt/log/hadoop/\$USER/
|
||||
sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
|
||||
log_dir=/mnt/log/hadoop/hdfs
|
||||
sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
|
|
@ -1,8 +0,0 @@
|
|||
#!/bin/bash
|
||||
sed -i -e 's/PasswordAuthentication no/PasswordAuthentication yes/' \
|
||||
-e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \
|
||||
/etc/ssh/sshd_config
|
||||
echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes\nGSSAPICleanupCredentials yes" >> /etc/ssh/sshd_config
|
||||
|
||||
sed -i -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
|
||||
echo -e "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
|
|
@ -1,11 +1,16 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
source $_LIB/die
|
||||
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
|
||||
[ -n "$JAVA_FILE" ] || die "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
|
||||
fi
|
||||
[ -n "$DIB_HADOOP_VERSION" ] || die "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
|
||||
|
||||
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
|
||||
if [ -z "$JAVA_FILE" ]; then
|
||||
echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [ -z "$DIB_HADOOP_VERSION" ]; then
|
||||
echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
|
||||
exit 1
|
||||
fi
|
||||
version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
|
||||
if [[ -z $version_check ]]; then
|
||||
echo "All variables are set, continue."
|
||||
|
@ -13,4 +18,3 @@ else
|
|||
echo "Version error. Exit"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
|
|
|
@ -1 +0,0 @@
|
|||
Installs Java and Hadoop on Fedora
|
|
@ -1,30 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Common
|
||||
chown root:root /mnt
|
||||
mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
|
||||
mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
|
||||
|
||||
distro=$(lsb_release -is || :)
|
||||
case "$distro" in
|
||||
Ubuntu )
|
||||
mkdir /run/hadoop
|
||||
chown hadoop:hadoop /run/hadoop/
|
||||
mkdir -p /home/ubuntu/.ssh
|
||||
touch /home/ubuntu/.ssh/authorized_keys
|
||||
chown -R ubuntu:ubuntu /home/ubuntu
|
||||
;;
|
||||
Fedora )
|
||||
sleep 10
|
||||
rm /etc/resolv.conf
|
||||
service network restart
|
||||
chown -R ec2-user:ec2-user /home/ec2-user
|
||||
chown -R ec2-user:ec2-user /etc/hadoop
|
||||
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
|
||||
iptables -F
|
||||
;;
|
||||
* )
|
||||
echo "Unknown distro: $distro. Exiting."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
|
@ -1,20 +0,0 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
|
||||
if [ -z "$JAVA_FILE" ]; then
|
||||
echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [ -z "$DIB_HADOOP_VERSION" ]; then
|
||||
echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
|
||||
exit 1
|
||||
fi
|
||||
version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
|
||||
if [[ -z $version_check ]]; then
|
||||
echo "All variables are set, continue."
|
||||
else
|
||||
echo "Version error. Exit"
|
||||
exit 1
|
||||
fi
|
Loading…
Reference in New Issue