Adding Storm and Zookeeper elements

This patch implements storm element and zookeeper.
They are both needed by storm plugin.

Change-Id: If279457b8f2b0fc18dca031afa979d0485bbbd81
Partial-implements: blueprint storm-integration

diskimage-create/diskimage-create.sh

@@ -37,7 +37,7 @@ while getopts "p:i:v:d:m" opt; do
     *)
       echo
       echo "Usage: $(basename $0)"
-      echo "         [-p vanilla|spark|hdp|cloudera]"
+      echo "         [-p vanilla|spark|hdp|cloudera|storm]"
       echo "         [-i ubuntu|fedora|centos]"
       echo "         [-v 1|2|2.3|2.4|plain]"
       echo "         [-d]"
@@ -81,7 +81,7 @@ if [ "$DEBUG_MODE" = "true" -a "$platform" != 'NAME="Ubuntu"' ]; then
   fi
 fi
 
-if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" ]; then
+if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" -a "$PLUGIN" != "storm" ]; then
   echo -e "Unknown plugin selected.\nAborting"
   exit 1
 fi
@@ -321,6 +321,30 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then
   mv $ubuntu_image_name.qcow2 ../
 fi
 
+
+##########################
+# Image for Storm plugin #
+##########################
+
+if [ -z "$PLUGIN" -o "$PLUGIN" = "storm" ]; then
+  # Ignoring image type and hadoop version options
+  echo "For storm plugin options -i and -v are ignored"
+
+  export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
+  export DIB_STORM_VERSION=${DIB_STORM_VERSION:-0.9.1}
+  export ubuntu_image_name=${ubuntu_storm_image_name:-"ubuntu_sahara_storm_latest"}
+
+  ubuntu_elements_sequence="base vm ubuntu java zookeeper storm"
+
+  if [ -n "$USE_MIRRORS" ]; then
+    mirror_element=" apt-mirror"
+    ubuntu_elements_sequence=$ubuntu_elements_sequence$mirror_element
+  fi
+
+  # Creating Ubuntu cloud image
+  disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name
+  mv $ubuntu_image_name.qcow2 ../
+fi
 #########################
 # Images for HDP plugin #
 #########################

elements/storm/element-deps

@@ -0,0 +1 @@
+zookeeper

elements/storm/install.d/60-storm

@@ -0,0 +1,76 @@
+#!/bin/bash
+# This script installs storm and its dependencies
+# More documentation on the README.md file
+
+install-packages wget unzip build-essential uuid-dev git libtool autoconf pkg-config tar
+
+tmp_dir=/tmp
+
+cd $tmp_dir
+echo "Installing ZeroMQ"
+wget http://download.zeromq.org/zeromq-2.1.7.tar.gz
+tar -xzf zeromq-2.1.7.tar.gz
+cd zeromq-2.1.7
+./configure
+make
+make install
+cd ..
+
+echo "Installing JZMQ"
+git clone https://github.com/nathanmarz/jzmq.git
+
+cd jzmq/src
+touch classdist_noinst.stamp
+CLASSPATH=.:./.:$CLASSPATH javac -d . org/zeromq/ZMQ.java org/zeromq/ZMQException.java org/zeromq/ZMQQueue.java org/zeromq/ZMQForwarder.java org/zeromq/ZMQStreamer.java
+cd ..
+
+cd $tmp_dir/jzmq
+./autogen.sh
+./configure
+make
+make install
+
+echo "Creating Storm User"
+groupadd -g 53001 storm
+mkdir -p /app/home/storm
+useradd -u 53001 -g 53001 -d /app/home/storm -s /bin/bash storm -c "Storm service account"
+chmod 700 /app/home/storm
+chage -I -1 -E -1 -m -1 -M -1 -W -1 -E -1 storm
+
+cd $tmp_dir
+echo "Downloading Storm"
+# The user is not providing his own Spark distribution package
+if [ -z "$STORM_DOWNLOAD_URL" ]; then
+# Check storm version
+    case "$DIB_STORM_VERSION" in
+      0.9.*)
+        STORM_DOWNLOAD_URL="http://archive.apache.org/dist/incubator/storm/apache-storm-$DIB_STORM_VERSION-incubating/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz"
+        ;;
+      *)
+        echo -e "WARNING: Storm version $DIB_STORM_VERSION not supported."
+        echo -e "WARNING: make sure STORM_DOWNLOAD_URL points to a compatible Storm version."
+        exit 1
+        ;;
+    esac
+fi
+
+wget $STORM_DOWNLOAD_URL
+cd /usr/local
+tar -xvf /tmp/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz
+chown -R storm:storm apache-storm-$DIB_STORM_VERSION-incubating
+ln -s apache-storm-$DIB_STORM_VERSION-incubating storm
+
+
+echo "Configuring Storm home"
+mkdir -p /app/storm
+chown -R storm:storm /app/storm
+chmod 750 /app/storm
+
+echo "Installing Supervisor Deamon"
+
+install-packages supervisor
+update-rc.d supervisor defaults
+chmod 600 /etc/supervisord.conf
+
+mkdir -p /var/log/storm
+chown -R storm:storm /var/log/storm

elements/storm/root.d/0-check

@@ -0,0 +1,5 @@
+#!/bin/bash
+if [ -z "$STORM_DOWNLOAD_URL" -a -z "$DIB_STORM_VERSION"]; then
+  echo -e "Neither DIB_STORM_VERSION nor STORM_DOWNLOAD_URL are set. Impossible to install Storm.\nAborting"
+  exit 1
+fi

elements/zookeeper/install.d/40-zookeeper

@@ -0,0 +1,19 @@
+#!/bin/bash
+# This scrip installs zookeeper
+# More documentation on the README.md file
+
+install-packages wget
+
+echo "Downloading Zookeeper"
+
+mkdir -p /opt/zookeeper
+
+cd /opt/zookeeper
+
+wget http://archive.apache.org/dist/zookeeper/stable/zookeeper-3.4.6.tar.gz
+tar -xzf zookeeper-3.4.6.tar.gz
+rm -rf zookeeper-3.4.6.tar.gz
+
+PID_PATH="/var/zookeeper"
+
+mkdir -p $PID_PATH

elements/zookeeper/post-install.d/50-zookeeper

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+# We have to chown the Zookeeper directories to make it usable for the default user
+# Unfortunately the Ubuntu user does not exists when DIB is creating the image
+# and we need to execute this code the first time the VM boots.
+
+sed -i -e 's,^exit 0$,[ -f /opt/zookeeper/firstboot.sh ] \&\& sh /opt/zookeeper/firstboot.sh; exit 0,' /etc/rc.local
+
+cat >> /opt/zookeeper/firstboot.sh <<EOF
+#!/bin/sh
+chown -R ubuntu:ubuntu /opt/zookeeper
+chown -R ubuntu:ubuntu /var/zookeeper
+rm \$0
+EOF
+