Add Hadoop element for creating Fedora cloud image for Savanna v0.2.
* ssh configuration * java and hadoop installation from tmp directory * setup network connection Implements blueprint savanna-image-elements Fixes bug: https://bugs.launchpad.net/savanna/+bug/1188442 Change-Id: I5b6c6ef3b78e29967b7869e3499f6c146d309dae
This commit is contained in:
parent
558e4dcd0b
commit
17b87b9876
|
@ -32,8 +32,15 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
|
|||
|
||||
.. sourcecode:: bash
|
||||
|
||||
JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-password -o hadoop_1_1_2
|
||||
JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-passwd -o hadoop_1_1_2
|
||||
|
||||
6.2. Fedora cloud image
|
||||
|
||||
.. sourcecode:: bash
|
||||
|
||||
DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
|
||||
|
||||
In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
|
||||
You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).
|
||||
'DIB_IMAGE_SIZE' is parameter that specifes a volume of hard disk of instance. You need to specify it because Fedora doesn't use all available volume.
|
||||
In case if you have already downloaded jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter.
|
||||
|
|
|
@ -0,0 +1 @@
|
|||
Installs Java and Hadoop on Fedora
|
|
@ -0,0 +1,13 @@
|
|||
#!/bin/bash
|
||||
sleep 10
|
||||
rm /etc/resolv.conf
|
||||
service network restart
|
||||
chown -R ec2-user:ec2-user /home/ec2-user
|
||||
chown -R ec2-user:ec2-user /etc/hadoop
|
||||
mkdir -p /var/run/hadoop
|
||||
chown root:root /mnt
|
||||
mkdir -p /mnt/log/hadoop
|
||||
chown hadoop:hadoop /var/run/hadoop
|
||||
chown hadoop:hadoop /mnt/log/hadoop
|
||||
#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
|
||||
iptables -F
|
|
@ -0,0 +1,44 @@
|
|||
#!/bin/bash
|
||||
echo "Java setup begins"
|
||||
install-packages wget
|
||||
tmp_dir=/tmp/java/
|
||||
mkdir -p $TARGET_ROOT/$tmp_dir
|
||||
set -e
|
||||
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
|
||||
script_dir=$(dirname $0)
|
||||
install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
|
||||
filename=$JAVA_FILE
|
||||
echo "Java file moved"
|
||||
else
|
||||
wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
|
||||
if [ $? -eq 0 ]; then
|
||||
echo "Java is downloading"
|
||||
else
|
||||
echo "Url error. Exit"
|
||||
exit 1
|
||||
fi
|
||||
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
|
||||
filename=$(basename $filename)
|
||||
fi
|
||||
|
||||
if echo $tmp_dir/$filename | grep -q -s -F .tar.gz ; then
|
||||
pushd $TARGET_ROOT/$tmp_dir
|
||||
echo -e "\n" | tar -zxvf $TARGET_ROOT/$tmp_dir/$filename
|
||||
popd
|
||||
else
|
||||
if echo $filename | grep -q -s -F .bin ; then
|
||||
pushd $TARGET_ROOT/$tmp_dir
|
||||
echo -e "\n" | sh $filename
|
||||
rm $filename
|
||||
popd
|
||||
fi
|
||||
fi
|
||||
|
||||
filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
|
||||
filename=$(basename $filename)
|
||||
javaPath=/usr/java/
|
||||
mkdir -p $TARGET_ROOT/$javaPath
|
||||
mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
|
||||
rm -r $tmp_dir
|
||||
echo "Java was installed"
|
||||
|
|
@ -0,0 +1,32 @@
|
|||
#!/bin/bash
|
||||
echo "Hadoop setup begins"
|
||||
tmp_dir=/tmp/hadoop
|
||||
|
||||
install-packages openssh-server
|
||||
echo "Creating hadoop user & group"
|
||||
adduser -G adm,wheel hadoop
|
||||
|
||||
echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
|
||||
wget -P $tmp_dir/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
rpm -ivh --replacefiles $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
|
||||
chmod 755 /usr/sbin/start-*
|
||||
chmod 755 /usr/sbin/stop-*
|
||||
chmod 755 /usr/sbin/slaves.sh
|
||||
chmod 755 /usr/sbin/update-hadoop-env.sh
|
||||
rm $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
|
||||
rm -r $tmp_dir
|
||||
|
||||
echo "Pre-configuring Hadoop"
|
||||
filename=$(find $TARGET_ROOT/usr/java/ -maxdepth 1 -name "jdk*")
|
||||
echo -e "PATH=\$PATH:/usr/sbin:$filename/bin\nJAVA_HOME=$filename" >> /home/hadoop/.bashrc
|
||||
echo -e "HADOOP_HOME=/usr/share/hadoop/" >> /home/hadoop/.bashrc
|
||||
sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
|
||||
log_dir=/mnt/log/hadoop/\$USER/
|
||||
sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
|
||||
log_dir=/mnt/log/hadoop/hdfs
|
||||
sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
|
|
@ -0,0 +1,21 @@
|
|||
#!/bin/bash
|
||||
echo "Adjusting ssh configuration"
|
||||
|
||||
sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
|
||||
echo "UseDNS no" >> /etc/ssh/sshd_config
|
||||
echo "PermitTunnel yes" >> /etc/ssh/sshd_config
|
||||
echo "SyslogFacility AUTH" >> /etc/ssh/sshd_config
|
||||
echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
|
||||
echo "StrictModes yes" >> /etc/ssh/sshd_config
|
||||
echo "RSAAuthentication yes" >> /etc/ssh/sshd_config
|
||||
echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
|
||||
echo "IgnoreRhosts yes" >> /etc/ssh/sshd_config
|
||||
|
||||
echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
|
||||
echo "GSSAPIDelegateCredentials no" >> /etc/ssh/ssh_config
|
||||
sed -i 's/ GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
|
||||
|
||||
sed -i 's/ssh_pwauth: 0/ssh_pwauth: 1/' /etc/cloud/cloud.cfg
|
||||
chmod 640 /etc/sudoers
|
||||
sed -i 's/Defaults requiretty/#Defaults requiretty/' /etc/sudoers
|
||||
chmod 0440 /etc/sudoers
|
|
@ -0,0 +1,20 @@
|
|||
#!/bin/bash
|
||||
set -e
|
||||
|
||||
if [ -z "$JAVA_DOWNLOAD_URL" ]; then
|
||||
if [ -z "$JAVA_FILE" ]; then
|
||||
echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
if [ -z "$DIB_HADOOP_VERSION" ]; then
|
||||
echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
|
||||
exit 1
|
||||
fi
|
||||
version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
|
||||
if [[ -z $version_check ]]; then
|
||||
echo "All variables are set, continue."
|
||||
else
|
||||
echo "Version error. Exit"
|
||||
exit 1
|
||||
fi
|
Loading…
Reference in New Issue