Add Hadoop element for creating Fedora cloud image for Savanna v0.2.

* ssh configuration
* java and hadoop installation from tmp directory
* setup network connection

Implements blueprint savanna-image-elements
Fixes bug: https://bugs.launchpad.net/savanna/+bug/1188442

Change-Id: I5b6c6ef3b78e29967b7869e3499f6c146d309dae
This commit is contained in:
Ivan Berezovskiy 2013-06-28 18:18:52 +04:00
parent 558e4dcd0b
commit 17b87b9876
7 changed files with 139 additions and 1 deletions

View File

@ -32,8 +32,15 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
.. sourcecode:: bash
JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-password -o hadoop_1_1_2
JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-passwd -o hadoop_1_1_2
6.2. Fedora cloud image
.. sourcecode:: bash
DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).
'DIB_IMAGE_SIZE' is parameter that specifes a volume of hard disk of instance. You need to specify it because Fedora doesn't use all available volume.
In case if you have already downloaded jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter.

View File

@ -0,0 +1 @@
Installs Java and Hadoop on Fedora

View File

@ -0,0 +1,13 @@
#!/bin/bash
sleep 10
rm /etc/resolv.conf
service network restart
chown -R ec2-user:ec2-user /home/ec2-user
chown -R ec2-user:ec2-user /etc/hadoop
mkdir -p /var/run/hadoop
chown root:root /mnt
mkdir -p /mnt/log/hadoop
chown hadoop:hadoop /var/run/hadoop
chown hadoop:hadoop /mnt/log/hadoop
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
iptables -F

View File

@ -0,0 +1,44 @@
#!/bin/bash
echo "Java setup begins"
install-packages wget
tmp_dir=/tmp/java/
mkdir -p $TARGET_ROOT/$tmp_dir
set -e
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
script_dir=$(dirname $0)
install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
filename=$JAVA_FILE
echo "Java file moved"
else
wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
if [ $? -eq 0 ]; then
echo "Java is downloading"
else
echo "Url error. Exit"
exit 1
fi
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
filename=$(basename $filename)
fi
if echo $tmp_dir/$filename | grep -q -s -F .tar.gz ; then
pushd $TARGET_ROOT/$tmp_dir
echo -e "\n" | tar -zxvf $TARGET_ROOT/$tmp_dir/$filename
popd
else
if echo $filename | grep -q -s -F .bin ; then
pushd $TARGET_ROOT/$tmp_dir
echo -e "\n" | sh $filename
rm $filename
popd
fi
fi
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
filename=$(basename $filename)
javaPath=/usr/java/
mkdir -p $TARGET_ROOT/$javaPath
mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
rm -r $tmp_dir
echo "Java was installed"

View File

@ -0,0 +1,32 @@
#!/bin/bash
echo "Hadoop setup begins"
tmp_dir=/tmp/hadoop
install-packages openssh-server
echo "Creating hadoop user & group"
adduser -G adm,wheel hadoop
echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
wget -P $tmp_dir/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm"
if [ $? -ne 0 ]; then
echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
exit 1
fi
rpm -ivh --replacefiles $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
chmod 755 /usr/sbin/start-*
chmod 755 /usr/sbin/stop-*
chmod 755 /usr/sbin/slaves.sh
chmod 755 /usr/sbin/update-hadoop-env.sh
rm $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
rm -r $tmp_dir
echo "Pre-configuring Hadoop"
filename=$(find $TARGET_ROOT/usr/java/ -maxdepth 1 -name "jdk*")
echo -e "PATH=\$PATH:/usr/sbin:$filename/bin\nJAVA_HOME=$filename" >> /home/hadoop/.bashrc
echo -e "HADOOP_HOME=/usr/share/hadoop/" >> /home/hadoop/.bashrc
sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
log_dir=/mnt/log/hadoop/\$USER/
sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
log_dir=/mnt/log/hadoop/hdfs
sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh

View File

@ -0,0 +1,21 @@
#!/bin/bash
echo "Adjusting ssh configuration"
sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
echo "UseDNS no" >> /etc/ssh/sshd_config
echo "PermitTunnel yes" >> /etc/ssh/sshd_config
echo "SyslogFacility AUTH" >> /etc/ssh/sshd_config
echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
echo "StrictModes yes" >> /etc/ssh/sshd_config
echo "RSAAuthentication yes" >> /etc/ssh/sshd_config
echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
echo "IgnoreRhosts yes" >> /etc/ssh/sshd_config
echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
echo "GSSAPIDelegateCredentials no" >> /etc/ssh/ssh_config
sed -i 's/ GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
sed -i 's/ssh_pwauth: 0/ssh_pwauth: 1/' /etc/cloud/cloud.cfg
chmod 640 /etc/sudoers
sed -i 's/Defaults requiretty/#Defaults requiretty/' /etc/sudoers
chmod 0440 /etc/sudoers

View File

@ -0,0 +1,20 @@
#!/bin/bash
set -e
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
if [ -z "$JAVA_FILE" ]; then
echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
exit 1
fi
fi
if [ -z "$DIB_HADOOP_VERSION" ]; then
echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
exit 1
fi
version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
if [[ -z $version_check ]]; then
echo "All variables are set, continue."
else
echo "Version error. Exit"
exit 1
fi