diff --git a/classdist_noinst.stamp b/classdist_noinst.stamp new file mode 100644 index 00000000..e69de29b diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index 0524a64d..f569d2bd 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -37,7 +37,7 @@ while getopts "p:i:v:d:m" opt; do *) echo echo "Usage: $(basename $0)" - echo " [-p vanilla|spark|hdp|cloudera]" + echo " [-p vanilla|spark|hdp|cloudera|storm]" echo " [-i ubuntu|fedora|centos]" echo " [-v 1|2|2.3|2.4|plain]" echo " [-d]" @@ -81,7 +81,7 @@ if [ "$DEBUG_MODE" = "true" -a "$platform" != 'NAME="Ubuntu"' ]; then fi fi -if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" ]; then +if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" -a "$PLUGIN" != "storm" ]; then echo -e "Unknown plugin selected.\nAborting" exit 1 fi @@ -321,6 +321,30 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "spark" ]; then mv $ubuntu_image_name.qcow2 ../ fi + +########################## +# Image for Storm plugin # +########################## + +if [ -z "$PLUGIN" -o "$PLUGIN" = "storm" ]; then + # Ignoring image type and hadoop version options + echo "For storm plugin options -i and -v are ignored" + + export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"} + export DIB_STORM_VERSION=${DIB_STORM_VERSION:-0.9.1} + export ubuntu_image_name=${ubuntu_storm_image_name:-"ubuntu_sahara_storm_latest"} + + ubuntu_elements_sequence="base vm ubuntu java zookeeper storm" + + if [ -n "$USE_MIRRORS" ]; then + mirror_element=" apt-mirror" + ubuntu_elements_sequence=$ubuntu_elements_sequence$mirror_element + fi + + # Creating Ubuntu cloud image + disk-image-create $ubuntu_elements_sequence -o $ubuntu_image_name + mv $ubuntu_image_name.qcow2 ../ +fi ######################### # Images for HDP plugin # ######################### diff --git a/elements/storm/element-deps b/elements/storm/element-deps new file mode 100644 index 00000000..1bfac538 --- /dev/null +++ b/elements/storm/element-deps @@ -0,0 +1 @@ +zookeeper diff --git a/elements/storm/install.d/60-storm b/elements/storm/install.d/60-storm new file mode 100755 index 00000000..3ded5749 --- /dev/null +++ b/elements/storm/install.d/60-storm @@ -0,0 +1,76 @@ +#!/bin/bash +# This script installs storm and its dependencies +# More documentation on the README.md file + +install-packages wget unzip build-essential uuid-dev git libtool autoconf pkg-config tar + +tmp_dir=/tmp + +cd $tmp_dir +echo "Installing ZeroMQ" +wget http://download.zeromq.org/zeromq-2.1.7.tar.gz +tar -xzf zeromq-2.1.7.tar.gz +cd zeromq-2.1.7 +./configure +make +make install +cd .. + +echo "Installing JZMQ" +git clone https://github.com/nathanmarz/jzmq.git + +cd jzmq/src +touch classdist_noinst.stamp +CLASSPATH=.:./.:$CLASSPATH javac -d . org/zeromq/ZMQ.java org/zeromq/ZMQException.java org/zeromq/ZMQQueue.java org/zeromq/ZMQForwarder.java org/zeromq/ZMQStreamer.java +cd .. + +cd $tmp_dir/jzmq +./autogen.sh +./configure +make +make install + +echo "Creating Storm User" +groupadd -g 53001 storm +mkdir -p /app/home/storm +useradd -u 53001 -g 53001 -d /app/home/storm -s /bin/bash storm -c "Storm service account" +chmod 700 /app/home/storm +chage -I -1 -E -1 -m -1 -M -1 -W -1 -E -1 storm + +cd $tmp_dir +echo "Downloading Storm" +# The user is not providing his own Spark distribution package +if [ -z "$STORM_DOWNLOAD_URL" ]; then +# Check storm version + case "$DIB_STORM_VERSION" in + 0.9.*) + STORM_DOWNLOAD_URL="http://archive.apache.org/dist/incubator/storm/apache-storm-$DIB_STORM_VERSION-incubating/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz" + ;; + *) + echo -e "WARNING: Storm version $DIB_STORM_VERSION not supported." + echo -e "WARNING: make sure STORM_DOWNLOAD_URL points to a compatible Storm version." + exit 1 + ;; + esac +fi + +wget $STORM_DOWNLOAD_URL +cd /usr/local +tar -xvf /tmp/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz +chown -R storm:storm apache-storm-$DIB_STORM_VERSION-incubating +ln -s apache-storm-$DIB_STORM_VERSION-incubating storm + + +echo "Configuring Storm home" +mkdir -p /app/storm +chown -R storm:storm /app/storm +chmod 750 /app/storm + +echo "Installing Supervisor Deamon" + +install-packages supervisor +update-rc.d supervisor defaults +chmod 600 /etc/supervisord.conf + +mkdir -p /var/log/storm +chown -R storm:storm /var/log/storm diff --git a/elements/storm/root.d/0-check b/elements/storm/root.d/0-check new file mode 100755 index 00000000..2e246d80 --- /dev/null +++ b/elements/storm/root.d/0-check @@ -0,0 +1,5 @@ +#!/bin/bash +if [ -z "$STORM_DOWNLOAD_URL" -a -z "$DIB_STORM_VERSION"]; then + echo -e "Neither DIB_STORM_VERSION nor STORM_DOWNLOAD_URL are set. Impossible to install Storm.\nAborting" + exit 1 +fi diff --git a/elements/zookeeper/install.d/40-zookeeper b/elements/zookeeper/install.d/40-zookeeper new file mode 100755 index 00000000..53d605e0 --- /dev/null +++ b/elements/zookeeper/install.d/40-zookeeper @@ -0,0 +1,19 @@ +#!/bin/bash +# This scrip installs zookeeper +# More documentation on the README.md file + +install-packages wget + +echo "Downloading Zookeeper" + +mkdir -p /opt/zookeeper + +cd /opt/zookeeper + +wget http://archive.apache.org/dist/zookeeper/stable/zookeeper-3.4.6.tar.gz +tar -xzf zookeeper-3.4.6.tar.gz +rm -rf zookeeper-3.4.6.tar.gz + +PID_PATH="/var/zookeeper" + +mkdir -p $PID_PATH diff --git a/elements/zookeeper/post-install.d/50-zookeeper b/elements/zookeeper/post-install.d/50-zookeeper new file mode 100755 index 00000000..77394df0 --- /dev/null +++ b/elements/zookeeper/post-install.d/50-zookeeper @@ -0,0 +1,15 @@ +#!/bin/bash + +# We have to chown the Zookeeper directories to make it usable for the default user +# Unfortunately the Ubuntu user does not exists when DIB is creating the image +# and we need to execute this code the first time the VM boots. + +sed -i -e 's,^exit 0$,[ -f /opt/zookeeper/firstboot.sh ] \&\& sh /opt/zookeeper/firstboot.sh; exit 0,' /etc/rc.local + +cat >> /opt/zookeeper/firstboot.sh <