Update Hadoop element for Savanna v0.2

* add ssh configuration * java installs from tmp directory * setup ubuntu's directory for ssh connection Fixes bug#1188438 Implements blueprint savanna-image-elements Change-Id: I0a5b67a85a13534c7f69af304a4b3532920b856f
2013-06-14 16:15:01 +04:00 · 2013-06-14 16:15:01 +04:00 · 8739ad0357
parent 4c59050d04
commit 8739ad0357
6 changed files with 45 additions and 40 deletions
--- a/elements/hadoop/README.md
+++ b/elements/hadoop/README.md
@ -1 +1 @@
-Installs Java 6 and Hadoop on Ubuntu
+Installs Java and Hadoop on Ubuntu
--- a/elements/hadoop/first-boot.d/99-setup
+++ b/elements/hadoop/first-boot.d/99-setup
@ -0,0 +1,10 @@
+#!/bin/bash
+echo -e "Creating hadoop_pid_dir"
+mkdir /var/run/hadoop
+mkdir /run/hadoop
+chown hadoop:hadoop /var/run/hadoop/
+chown hadoop:hadoop /run/hadoop/
+echo "Setup ubuntu's directory"
+mkdir -p /home/ubuntu/.ssh
+touch /home/ubuntu/.ssh/authorized_keys
+chown -R ubuntu:ubuntu /home/ubuntu
--- a/elements/hadoop/install.d/70-setup-java
+++ b/elements/hadoop/install.d/70-setup-java
@ -1,13 +1,15 @@
 #!/bin/bash
+tmp_dir=/tmp/java
+mkdir -p $TARGET_ROOT/$tmp_dir
 echo "Java setup begins"
 set -e
 if [ -z "$JAVA_DOWNLOAD_URL" ]; then
-   mkdir -p $TARGET_ROOT/home/ubuntu
+   mkdir -p $TARGET_ROOT/$tmp_dir
   script_dir=$(dirname $0)
-   install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/home/ubuntu/
+   install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
   echo "Java file moved"
 else
-   wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/home/ubuntu/ $JAVA_DOWNLOAD_URL
+   wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
   if [ $? -eq 0 ]; then
      echo "Java is downloading"
      else
@ -15,31 +17,27 @@ else
      exit 1
   fi
 fi
-filename=$(find $TARGET_ROOT/home/ubuntu/ -maxdepth 1 -name "jdk*")
+filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
 filename=$(basename $filename)

 if echo $filename | grep -q -s -F .tar.gz ; then
   path=$(pwd)
-   cd $TARGET_ROOT/home/ubuntu/
-   echo $TARGET_ROOT/home/ubuntu/$filename | xargs -n 1 tar -zxvf
+   cd $TARGET_ROOT/$tmp_dir
+   echo $TARGET_ROOT/$tmp_dir/$filename | xargs -n 1 tar -zxvf
   cd $path
   else
      if echo $filename | grep -q -s -F .bin ; then
         path=$(pwd)
-         cd $TARGET_ROOT/home/ubuntu/
+         cd $TARGET_ROOT/$tmp_dir
         echo -e "\n" | sh $filename
         rm $filename
         cd $path
      fi
 fi

-filename=$(find $TARGET_ROOT/home/ubuntu/ -maxdepth 1 -type d -name "jdk*")
+filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
 filename=$(basename $filename)
 javaPath=/usr/lib/
 mkdir -p $TARGET_ROOT/$javaPath
-mv $TARGET_ROOT/home/ubuntu/$filename $TARGET_ROOT/$javaPath
-cat $TARGET_ROOT/root/.bashrc >> $TARGET_ROOT/home/ubuntu/.bashrc
-echo -e "export JAVA_HOME=$javaPath$filename\nexport PATH=\$PATH:$javaPath$filename/bin" >> $TARGET_ROOT/home/ubuntu/.bashrc
-echo -e "export JAVA_HOME=$javaPath$filename\nexport PATH=\$PATH:$javaPath$filename/bin" >> $TARGET_ROOT/root/.bashrc
-source $TARGET_ROOT/home/ubuntu/.bashrc
+mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
 echo "Java was installed"
--- a/elements/hadoop/install.d/80-setup-hadoop
+++ b/elements/hadoop/install.d/80-setup-hadoop
@ -1,7 +1,9 @@
 #!/bin/bash
-
+tmp_dir=/tmp/hadoop
+mkdir -p $TARGET_ROOT/$tmp_dir
 echo "Installing SSH server"

+mkdir /root/.ssh
 install-packages openssh-server

 echo "Creating hadoop user & group"
@ -9,11 +11,10 @@ echo "Creating hadoop user & group"
 addgroup hadoop
 adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
 adduser hadoop sudo
-echo -e "swordfish\nswordfish\n" | passwd hadoop

 echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"

-wget -P /home/ubuntu/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb"
+wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb"
 if [ $? -ne 0 ]; then
   echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nExit"
   exit 1
@ -21,15 +22,19 @@ fi

 echo "Installing Hadoop"

-dpkg -i /home/ubuntu/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
-rm /home/ubuntu/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
+dpkg -i $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
+rm $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb

 echo "Pre-configuring Hadoop"

-cat $TARGET_ROOT/home/ubuntu/.bashrc > $TARGET_ROOT/home/hadoop/.bashrc
 filename=$(find $TARGET_ROOT/usr/lib/ -maxdepth 1 -name "jdk*")
 echo -e "HADOOP_HOME=/usr/share/hadoop/\nPATH=\$PATH:/usr/sbin/" >> $TARGET_ROOT/home/hadoop/.bashrc
+echo -e "JAVA_HOME=$filename\nPATH=\$PATH:$filename/bin/" >> $TARGET_ROOT/home/hadoop/.bashrc
 sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
+log_dir=/mnt/log/hadoop/\$USER/
+sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
+log_dir=/mnt/log/hadoop/hdfs
+sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh

 echo "Generating keypair for 'hadoop' user"

@ -37,16 +42,5 @@ su -c "mkdir -m 700 /home/hadoop/.ssh" hadoop
 su -c "ssh-keygen -t rsa -P \"\" -N \"\" -f /home/hadoop/.ssh/id_rsa" hadoop
 su -c "cat /home/hadoop/.ssh/id_rsa.pub >> /home/hadoop/.ssh/authorized_keys" hadoop
 chmod 600 /home/hadoop/.ssh/authorized_keys
-chown hadoop:hadoop /var/run/

-mkdir /root/.ssh
 cp /home/hadoop/.ssh/authorized_keys /root/.ssh/
-
-echo "Adjusting ssh configuration"
-
-sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
-echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes" >> /etc/ssh/sshd_config
-
-sed -i -e 's/#   StrictHostKeyChecking ask/StrictHostKeyChecking no/' \
-       -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \
-    /etc/ssh/ssh_config
--- a/elements/hadoop/install.d/90-setup-ssh
+++ b/elements/hadoop/install.d/90-setup-ssh
@ -0,0 +1,8 @@
+#!/bin/bash
+sed -i -e 's/PasswordAuthentication no/PasswordAuthentication yes/' \
+       -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \
+    /etc/ssh/sshd_config
+echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes\nGSSAPICleanupCredentials yes" >> /etc/ssh/sshd_config
+
+sed -i -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
+echo -e "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
--- a/elements/hadoop/root.d/0-check
+++ b/elements/hadoop/root.d/0-check
@ -1,16 +1,11 @@
 #!/bin/bash
 set -e
-
+source $_LIB/die
 if [ -z "$JAVA_DOWNLOAD_URL" ]; then
-   if [ -z "$JAVA_FILE" ]; then
-      echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
-      exit 1
-   fi
-fi
-if [ -z "$DIB_HADOOP_VERSION" ]; then
-   echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
-   exit 1
+   [ -n "$JAVA_FILE" ] || die "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
 fi
+[ -n "$DIB_HADOOP_VERSION" ] || die "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
+
 version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
 if [[ -z $version_check ]]; then
   echo "All variables are set, continue."