summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMatthew Farrellee <matt@redhat.com>2013-07-16 08:32:46 -0400
committerMatthew Farrellee <matt@redhat.com>2013-07-16 08:37:36 -0400
commit40931bdb5f262359587f9991f8d0b95253b34b8e (patch)
tree4ddd92adf2c164005308e2cd013d7ff833e0d4d8
parent1c9c52a977b4c47ffca2b213182ceec20561d338 (diff)
Replace elements/hadoop/ with Fedora&Ubuntu supporting elements/hadoop_fedora0.2.1.rc1
Note: this requires a DIB version that includes change I7a42409a (SHA 82eacdec) from 11 July 2013 Implements: blueprint merge-dib-hadoop-elements Change-Id: I23d873adfc7c16e36cf26af2cd9156c2a8ec90fc
Notes
Notes (review): Verified+2: Jenkins Approved+1: Sergey Lukjanov <slukjanov@mirantis.com> Code-Review+2: Sergey Lukjanov <slukjanov@mirantis.com> Code-Review+1: Ivan Berezovskiy <iberezovskiy@mirantis.com> Submitted-by: Jenkins Submitted-at: Tue, 16 Jul 2013 15:07:42 +0000 Reviewed-on: https://review.openstack.org/37237 Project: stackforge/savanna-extra Branch: refs/heads/master
-rw-r--r--elements/README.rst2
-rw-r--r--elements/hadoop/README.md2
-rwxr-xr-xelements/hadoop/first-boot.d/99-setup38
-rwxr-xr-xelements/hadoop/install.d/20-setup-java (renamed from elements/hadoop_fedora/install.d/20-setup-java)0
-rwxr-xr-xelements/hadoop/install.d/30-setup-hadoop (renamed from elements/hadoop_fedora/install.d/30-setup-hadoop)0
-rwxr-xr-xelements/hadoop/install.d/40-setup-ssh (renamed from elements/hadoop_fedora/install.d/40-setup-ssh)0
-rwxr-xr-xelements/hadoop/install.d/70-setup-java43
-rwxr-xr-xelements/hadoop/install.d/80-setup-hadoop37
-rwxr-xr-xelements/hadoop/install.d/90-setup-ssh8
-rwxr-xr-xelements/hadoop/root.d/0-check14
-rw-r--r--elements/hadoop_fedora/README.md1
-rwxr-xr-xelements/hadoop_fedora/first-boot.d/99-setup30
-rwxr-xr-xelements/hadoop_fedora/root.d/0-check20
13 files changed, 39 insertions, 156 deletions
diff --git a/elements/README.rst b/elements/README.rst
index 721d2aa..120e8de 100644
--- a/elements/README.rst
+++ b/elements/README.rst
@@ -38,7 +38,7 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
38 38
39.. sourcecode:: bash 39.. sourcecode:: bash
40 40
41 DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2 41 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop root-passwd -o fedora_hadoop_1_1_2
42 42
43In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed. 43In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
44You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin). 44You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).
diff --git a/elements/hadoop/README.md b/elements/hadoop/README.md
index 6b82de1..cbc6ad8 100644
--- a/elements/hadoop/README.md
+++ b/elements/hadoop/README.md
@@ -1 +1 @@
Installs Java and Hadoop on Ubuntu Installs Java and Hadoop, configures SSH
diff --git a/elements/hadoop/first-boot.d/99-setup b/elements/hadoop/first-boot.d/99-setup
index 9190ef2..27a7ed2 100755
--- a/elements/hadoop/first-boot.d/99-setup
+++ b/elements/hadoop/first-boot.d/99-setup
@@ -1,12 +1,30 @@
1#!/bin/bash 1#!/bin/bash
2
3# Common
2chown root:root /mnt 4chown root:root /mnt
3mkdir -p /mnt/log/hadoop 5mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
4chown hadoop:hadoop /mnt/log/hadoop 6mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
5mkdir /var/run/hadoop 7
6mkdir /run/hadoop 8distro=$(lsb_release -is || :)
7chown hadoop:hadoop /var/run/hadoop/ 9case "$distro" in
8chown hadoop:hadoop /run/hadoop/ 10 Ubuntu )
9echo "Setup ubuntu's directory" 11 mkdir /run/hadoop
10mkdir -p /home/ubuntu/.ssh 12 chown hadoop:hadoop /run/hadoop/
11touch /home/ubuntu/.ssh/authorized_keys 13 mkdir -p /home/ubuntu/.ssh
12chown -R ubuntu:ubuntu /home/ubuntu 14 touch /home/ubuntu/.ssh/authorized_keys
15 chown -R ubuntu:ubuntu /home/ubuntu
16 ;;
17 Fedora )
18 sleep 10
19 rm /etc/resolv.conf
20 service network restart
21 chown -R ec2-user:ec2-user /home/ec2-user
22 chown -R ec2-user:ec2-user /etc/hadoop
23 #TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
24 iptables -F
25 ;;
26 * )
27 echo "Unknown distro: $distro. Exiting."
28 exit 1
29 ;;
30esac
diff --git a/elements/hadoop_fedora/install.d/20-setup-java b/elements/hadoop/install.d/20-setup-java
index 8952eec..8952eec 100755
--- a/elements/hadoop_fedora/install.d/20-setup-java
+++ b/elements/hadoop/install.d/20-setup-java
diff --git a/elements/hadoop_fedora/install.d/30-setup-hadoop b/elements/hadoop/install.d/30-setup-hadoop
index 81654f3..81654f3 100755
--- a/elements/hadoop_fedora/install.d/30-setup-hadoop
+++ b/elements/hadoop/install.d/30-setup-hadoop
diff --git a/elements/hadoop_fedora/install.d/40-setup-ssh b/elements/hadoop/install.d/40-setup-ssh
index 0a1a539..0a1a539 100755
--- a/elements/hadoop_fedora/install.d/40-setup-ssh
+++ b/elements/hadoop/install.d/40-setup-ssh
diff --git a/elements/hadoop/install.d/70-setup-java b/elements/hadoop/install.d/70-setup-java
deleted file mode 100755
index b4efb65..0000000
--- a/elements/hadoop/install.d/70-setup-java
+++ /dev/null
@@ -1,43 +0,0 @@
1#!/bin/bash
2tmp_dir=/tmp/java
3mkdir -p $TARGET_ROOT/$tmp_dir
4echo "Java setup begins"
5set -e
6if [ -z "$JAVA_DOWNLOAD_URL" ]; then
7 mkdir -p $TARGET_ROOT/$tmp_dir
8 script_dir=$(dirname $0)
9 install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
10 echo "Java file moved"
11else
12 wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
13 if [ $? -eq 0 ]; then
14 echo "Java is downloading"
15 else
16 echo "Url error. Exit"
17 exit 1
18 fi
19fi
20filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
21filename=$(basename $filename)
22
23if echo $filename | grep -q -s -F .tar.gz ; then
24 path=$(pwd)
25 cd $TARGET_ROOT/$tmp_dir
26 echo $TARGET_ROOT/$tmp_dir/$filename | xargs -n 1 tar -zxvf
27 cd $path
28 else
29 if echo $filename | grep -q -s -F .bin ; then
30 path=$(pwd)
31 cd $TARGET_ROOT/$tmp_dir
32 echo -e "\n" | sh $filename
33 rm $filename
34 cd $path
35 fi
36fi
37
38filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
39filename=$(basename $filename)
40javaPath=/usr/lib/
41mkdir -p $TARGET_ROOT/$javaPath
42mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
43echo "Java was installed"
diff --git a/elements/hadoop/install.d/80-setup-hadoop b/elements/hadoop/install.d/80-setup-hadoop
deleted file mode 100755
index c371782..0000000
--- a/elements/hadoop/install.d/80-setup-hadoop
+++ /dev/null
@@ -1,37 +0,0 @@
1#!/bin/bash
2tmp_dir=/tmp/hadoop
3mkdir -p $TARGET_ROOT/$tmp_dir
4echo "Installing SSH server"
5
6mkdir /root/.ssh
7install-packages openssh-server
8
9echo "Creating hadoop user & group"
10
11addgroup hadoop
12adduser --ingroup hadoop --disabled-password --gecos GECOS hadoop
13adduser hadoop sudo
14
15echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
16
17wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop_"$DIB_HADOOP_VERSION"-1_x86_64.deb"
18if [ $? -ne 0 ]; then
19 echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nExit"
20 exit 1
21fi
22
23echo "Installing Hadoop"
24
25dpkg -i $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
26rm $tmp_dir/hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb
27
28echo "Pre-configuring Hadoop"
29
30filename=$(find $TARGET_ROOT/usr/lib/ -maxdepth 1 -name "jdk*")
31echo -e "HADOOP_HOME=/usr/share/hadoop/\nPATH=\$PATH:/usr/sbin/" >> $TARGET_ROOT/home/hadoop/.bashrc
32echo -e "JAVA_HOME=$filename\nPATH=\$PATH:$filename/bin/" >> $TARGET_ROOT/home/hadoop/.bashrc
33sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
34log_dir=/mnt/log/hadoop/\$USER/
35sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
36log_dir=/mnt/log/hadoop/hdfs
37sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
diff --git a/elements/hadoop/install.d/90-setup-ssh b/elements/hadoop/install.d/90-setup-ssh
deleted file mode 100755
index 33e8f29..0000000
--- a/elements/hadoop/install.d/90-setup-ssh
+++ /dev/null
@@ -1,8 +0,0 @@
1#!/bin/bash
2sed -i -e 's/PasswordAuthentication no/PasswordAuthentication yes/' \
3 -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' \
4 /etc/ssh/sshd_config
5echo -e "AuthorizedKeysFile .ssh/authorized_keys\nUseDNS no\nPermitTunnel yes\nGSSAPICleanupCredentials yes" >> /etc/ssh/sshd_config
6
7sed -i -e 's/GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
8echo -e "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
diff --git a/elements/hadoop/root.d/0-check b/elements/hadoop/root.d/0-check
index 61a79e7..b90bfbc 100755
--- a/elements/hadoop/root.d/0-check
+++ b/elements/hadoop/root.d/0-check
@@ -1,11 +1,16 @@
1#!/bin/bash 1#!/bin/bash
2set -e 2set -e
3source $_LIB/die 3
4if [ -z "$JAVA_DOWNLOAD_URL" ]; then 4if [ -z "$JAVA_DOWNLOAD_URL" ]; then
5 [ -n "$JAVA_FILE" ] || die "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit" 5 if [ -z "$JAVA_FILE" ]; then
6 echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
7 exit 1
8 fi
9fi
10if [ -z "$DIB_HADOOP_VERSION" ]; then
11 echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
12 exit 1
6fi 13fi
7[ -n "$DIB_HADOOP_VERSION" ] || die "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
8
9version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d') 14version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
10if [[ -z $version_check ]]; then 15if [[ -z $version_check ]]; then
11 echo "All variables are set, continue." 16 echo "All variables are set, continue."
@@ -13,4 +18,3 @@ else
13 echo "Version error. Exit" 18 echo "Version error. Exit"
14 exit 1 19 exit 1
15fi 20fi
16
diff --git a/elements/hadoop_fedora/README.md b/elements/hadoop_fedora/README.md
deleted file mode 100644
index 26e5a69..0000000
--- a/elements/hadoop_fedora/README.md
+++ /dev/null
@@ -1 +0,0 @@
1Installs Java and Hadoop on Fedora
diff --git a/elements/hadoop_fedora/first-boot.d/99-setup b/elements/hadoop_fedora/first-boot.d/99-setup
deleted file mode 100755
index 27a7ed2..0000000
--- a/elements/hadoop_fedora/first-boot.d/99-setup
+++ /dev/null
@@ -1,30 +0,0 @@
1#!/bin/bash
2
3# Common
4chown root:root /mnt
5mkdir -p /var/run/hadoop ; chown hadoop:hadoop /var/run/hadoop
6mkdir -p /mnt/log/hadoop ; chown hadoop:hadoop /mnt/log/hadoop
7
8distro=$(lsb_release -is || :)
9case "$distro" in
10 Ubuntu )
11 mkdir /run/hadoop
12 chown hadoop:hadoop /run/hadoop/
13 mkdir -p /home/ubuntu/.ssh
14 touch /home/ubuntu/.ssh/authorized_keys
15 chown -R ubuntu:ubuntu /home/ubuntu
16 ;;
17 Fedora )
18 sleep 10
19 rm /etc/resolv.conf
20 service network restart
21 chown -R ec2-user:ec2-user /home/ec2-user
22 chown -R ec2-user:ec2-user /etc/hadoop
23 #TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
24 iptables -F
25 ;;
26 * )
27 echo "Unknown distro: $distro. Exiting."
28 exit 1
29 ;;
30esac
diff --git a/elements/hadoop_fedora/root.d/0-check b/elements/hadoop_fedora/root.d/0-check
deleted file mode 100755
index b90bfbc..0000000
--- a/elements/hadoop_fedora/root.d/0-check
+++ /dev/null
@@ -1,20 +0,0 @@
1#!/bin/bash
2set -e
3
4if [ -z "$JAVA_DOWNLOAD_URL" ]; then
5 if [ -z "$JAVA_FILE" ]; then
6 echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
7 exit 1
8 fi
9fi
10if [ -z "$DIB_HADOOP_VERSION" ]; then
11 echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
12 exit 1
13fi
14version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
15if [[ -z $version_check ]]; then
16 echo "All variables are set, continue."
17else
18 echo "Version error. Exit"
19 exit 1
20fi