Add Hadoop element for creating Fedora cloud image for Savanna v0.2.0.2a1

* ssh configuration
* java and hadoop installation from tmp directory
* setup network connection

Implements blueprint savanna-image-elements
Fixes bug: https://bugs.launchpad.net/savanna/+bug/1188442

Change-Id: I5b6c6ef3b78e29967b7869e3499f6c146d309dae

7 files changed, 139 insertions, 1 deletions

diff --git a/elements/README.rst b/elements/README.rst
index 0d9c493..47a3a4f 100644
--- a/elements/README.rst
+++ b/elements/README.rst
@@ -32,8 +32,15 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
 
 .. sourcecode:: bash
 
-    JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-password -o hadoop_1_1_2
+    JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-passwd -o hadoop_1_1_2
+
+6.2. Fedora cloud image
+
+.. sourcecode:: bash
+
+    DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
 
 In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
 You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).
+'DIB_IMAGE_SIZE' is parameter that specifes a volume of hard disk of instance. You need to specify it because Fedora doesn't use all available volume.
 In case if you have already downloaded jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter.
diff --git a/elements/hadoop_fedora/README.md b/elements/hadoop_fedora/README.md
new file mode 100644
index 0000000..26e5a69
--- /dev/null
+++ b/elements/hadoop_fedora/README.md
@@ -0,0 +1 @@
+Installs Java and Hadoop on Fedora
diff --git a/elements/hadoop_fedora/first-boot.d/99-setup b/elements/hadoop_fedora/first-boot.d/99-setup
new file mode 100755
index 0000000..75a87fd
--- /dev/null
+++ b/elements/hadoop_fedora/first-boot.d/99-setup
@@ -0,0 +1,13 @@
+#!/bin/bash
+sleep 10
+rm /etc/resolv.conf
+service network restart
+chown -R ec2-user:ec2-user /home/ec2-user
+chown -R ec2-user:ec2-user /etc/hadoop
+mkdir -p /var/run/hadoop
+chown root:root /mnt
+mkdir -p /mnt/log/hadoop
+chown hadoop:hadoop /var/run/hadoop
+chown hadoop:hadoop /mnt/log/hadoop
+#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
+iptables -F
diff --git a/elements/hadoop_fedora/post-install.d/11-setup-java b/elements/hadoop_fedora/post-install.d/11-setup-java
new file mode 100755
index 0000000..0ced81d
--- /dev/null
+++ b/elements/hadoop_fedora/post-install.d/11-setup-java
@@ -0,0 +1,44 @@
+#!/bin/bash
+echo "Java setup begins"
+install-packages wget
+tmp_dir=/tmp/java/
+mkdir -p $TARGET_ROOT/$tmp_dir
+set -e
+if [ -z "$JAVA_DOWNLOAD_URL" ]; then
+   script_dir=$(dirname $0)
+   install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
+   filename=$JAVA_FILE
+   echo "Java file moved"
+else
+   wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
+   if [ $? -eq 0 ]; then
+      echo "Java is downloading"
+      else
+      echo "Url error. Exit"
+      exit 1
+   fi
+   filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
+   filename=$(basename $filename)
+fi
+
+if echo $tmp_dir/$filename | grep -q -s -F .tar.gz ; then
+   pushd $TARGET_ROOT/$tmp_dir
+   echo -e "\n" | tar -zxvf $TARGET_ROOT/$tmp_dir/$filename
+   popd
+   else
+      if echo $filename | grep -q -s -F .bin ; then
+         pushd $TARGET_ROOT/$tmp_dir
+         echo -e "\n" | sh $filename
+         rm $filename
+         popd
+      fi
+fi
+
+filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
+filename=$(basename $filename)
+javaPath=/usr/java/
+mkdir -p $TARGET_ROOT/$javaPath
+mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
+rm -r $tmp_dir
+echo "Java was installed"
+
diff --git a/elements/hadoop_fedora/post-install.d/12-setup-hadoop b/elements/hadoop_fedora/post-install.d/12-setup-hadoop
new file mode 100755
index 0000000..98068ad
--- /dev/null
+++ b/elements/hadoop_fedora/post-install.d/12-setup-hadoop
@@ -0,0 +1,32 @@
+#!/bin/bash
+echo "Hadoop setup begins"
+tmp_dir=/tmp/hadoop
+
+install-packages openssh-server
+echo "Creating hadoop user & group"
+adduser -G adm,wheel hadoop
+
+echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
+wget -P $tmp_dir/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm"
+if [ $? -ne 0 ]; then
+   echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
+   exit 1
+fi
+
+rpm -ivh --replacefiles $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
+chmod 755 /usr/sbin/start-*
+chmod 755 /usr/sbin/stop-*
+chmod 755 /usr/sbin/slaves.sh
+chmod 755 /usr/sbin/update-hadoop-env.sh
+rm $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
+rm -r $tmp_dir
+
+echo "Pre-configuring Hadoop"
+filename=$(find $TARGET_ROOT/usr/java/ -maxdepth 1 -name "jdk*")
+echo -e "PATH=\$PATH:/usr/sbin:$filename/bin\nJAVA_HOME=$filename" >> /home/hadoop/.bashrc
+echo -e "HADOOP_HOME=/usr/share/hadoop/" >> /home/hadoop/.bashrc
+sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
+log_dir=/mnt/log/hadoop/\$USER/
+sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
+log_dir=/mnt/log/hadoop/hdfs
+sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
diff --git a/elements/hadoop_fedora/post-install.d/13-connection-setup b/elements/hadoop_fedora/post-install.d/13-connection-setup
new file mode 100755
index 0000000..af0af49
--- /dev/null
+++ b/elements/hadoop_fedora/post-install.d/13-connection-setup
@@ -0,0 +1,21 @@
+#!/bin/bash
+echo "Adjusting ssh configuration"
+
+sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
+echo "UseDNS no" >> /etc/ssh/sshd_config
+echo "PermitTunnel yes" >> /etc/ssh/sshd_config
+echo "SyslogFacility AUTH" >> /etc/ssh/sshd_config
+echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
+echo "StrictModes yes" >> /etc/ssh/sshd_config
+echo "RSAAuthentication yes" >> /etc/ssh/sshd_config
+echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
+echo "IgnoreRhosts yes" >> /etc/ssh/sshd_config
+
+echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
+echo "GSSAPIDelegateCredentials no" >> /etc/ssh/ssh_config
+sed -i 's/        GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
+
+sed -i 's/ssh_pwauth:   0/ssh_pwauth:   1/' /etc/cloud/cloud.cfg
+chmod 640 /etc/sudoers
+sed -i 's/Defaults    requiretty/#Defaults    requiretty/' /etc/sudoers
+chmod 0440 /etc/sudoers
diff --git a/elements/hadoop_fedora/root.d/0-check b/elements/hadoop_fedora/root.d/0-check
new file mode 100755
index 0000000..b90bfbc
--- /dev/null
+++ b/elements/hadoop_fedora/root.d/0-check
@@ -0,0 +1,20 @@
+#!/bin/bash
+set -e
+
+if [ -z "$JAVA_DOWNLOAD_URL" ]; then
+   if [ -z "$JAVA_FILE" ]; then
+      echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
+      exit 1
+   fi
+fi
+if [ -z "$DIB_HADOOP_VERSION" ]; then
+   echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
+   exit 1
+fi
+version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
+if [[ -z $version_check ]]; then
+   echo "All variables are set, continue."
+else
+   echo "Version error. Exit"
+   exit 1
+fi