summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorIvan Berezovskiy <iberezovskiy@mirantis.com>2013-06-28 18:18:52 +0400
committerIvan Berezovskiy <iberezovskiy@mirantis.com>2013-07-01 17:06:09 +0400
commit17b87b9876d759b0762f2b52d0d6ea8e36d6f90c (patch)
treed5f96a7770801235bf4131e660b660b19051b9ae
parent558e4dcd0bfc78e3c29cf48a2d064eac83e948b5 (diff)
Add Hadoop element for creating Fedora cloud image for Savanna v0.2.0.2a1
* ssh configuration * java and hadoop installation from tmp directory * setup network connection Implements blueprint savanna-image-elements Fixes bug: https://bugs.launchpad.net/savanna/+bug/1188442 Change-Id: I5b6c6ef3b78e29967b7869e3499f6c146d309dae
Notes
Notes (review): Verified+2: Jenkins Approved+1: Sergey Lukjanov <slukjanov@mirantis.com> Code-Review+2: Sergey Lukjanov <slukjanov@mirantis.com> Code-Review+1: Alexander Ignatov <aignatov@mirantis.com> Code-Review+1: Matthew Farrellee <matt@redhat.com> Submitted-by: Jenkins Submitted-at: Mon, 01 Jul 2013 15:43:51 +0000 Reviewed-on: https://review.openstack.org/34887 Project: stackforge/savanna-extra Branch: refs/heads/master
-rw-r--r--elements/README.rst9
-rw-r--r--elements/hadoop_fedora/README.md1
-rwxr-xr-xelements/hadoop_fedora/first-boot.d/99-setup13
-rwxr-xr-xelements/hadoop_fedora/post-install.d/11-setup-java44
-rwxr-xr-xelements/hadoop_fedora/post-install.d/12-setup-hadoop32
-rwxr-xr-xelements/hadoop_fedora/post-install.d/13-connection-setup21
-rwxr-xr-xelements/hadoop_fedora/root.d/0-check20
7 files changed, 139 insertions, 1 deletions
diff --git a/elements/README.rst b/elements/README.rst
index 0d9c493..47a3a4f 100644
--- a/elements/README.rst
+++ b/elements/README.rst
@@ -32,8 +32,15 @@ Steps how to create cloud image with Apache Hadoop installed using diskimage-bui
32 32
33.. sourcecode:: bash 33.. sourcecode:: bash
34 34
35 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-password -o hadoop_1_1_2 35 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_HADOOP_VERSION=1.1.2 disk-image-create base vm hadoop ubuntu root-passwd -o hadoop_1_1_2
36
376.2. Fedora cloud image
38
39.. sourcecode:: bash
40
41 DIB_HADOOP_VERSION=1.1.2 JAVA_FILE=jdk-7u21-linux-x64.tar.gz DIB_IMAGE_SIZE=10 disk-image-create base vm fedora hadoop_fedora root-passwd -o fedora_hadoop_1_1_2
36 42
37In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed. 43In this command 'DIB_HADOOP_VERSION' parameter is version of hadoop needs to be installed.
38You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin). 44You can use 'JAVA_DOWNLOAD_URL' parameter to specify download link for JDK (tarball or bin).
45'DIB_IMAGE_SIZE' is parameter that specifes a volume of hard disk of instance. You need to specify it because Fedora doesn't use all available volume.
39In case if you have already downloaded jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter. 46In case if you have already downloaded jdk package, move it to "elements/hadoop/install.d/" and use its filename as 'JAVA_FILE' parameter.
diff --git a/elements/hadoop_fedora/README.md b/elements/hadoop_fedora/README.md
new file mode 100644
index 0000000..26e5a69
--- /dev/null
+++ b/elements/hadoop_fedora/README.md
@@ -0,0 +1 @@
Installs Java and Hadoop on Fedora
diff --git a/elements/hadoop_fedora/first-boot.d/99-setup b/elements/hadoop_fedora/first-boot.d/99-setup
new file mode 100755
index 0000000..75a87fd
--- /dev/null
+++ b/elements/hadoop_fedora/first-boot.d/99-setup
@@ -0,0 +1,13 @@
1#!/bin/bash
2sleep 10
3rm /etc/resolv.conf
4service network restart
5chown -R ec2-user:ec2-user /home/ec2-user
6chown -R ec2-user:ec2-user /etc/hadoop
7mkdir -p /var/run/hadoop
8chown root:root /mnt
9mkdir -p /mnt/log/hadoop
10chown hadoop:hadoop /var/run/hadoop
11chown hadoop:hadoop /mnt/log/hadoop
12#TODO: configure iptables (https://bugs.launchpad.net/savanna/+bug/1195744)
13iptables -F
diff --git a/elements/hadoop_fedora/post-install.d/11-setup-java b/elements/hadoop_fedora/post-install.d/11-setup-java
new file mode 100755
index 0000000..0ced81d
--- /dev/null
+++ b/elements/hadoop_fedora/post-install.d/11-setup-java
@@ -0,0 +1,44 @@
1#!/bin/bash
2echo "Java setup begins"
3install-packages wget
4tmp_dir=/tmp/java/
5mkdir -p $TARGET_ROOT/$tmp_dir
6set -e
7if [ -z "$JAVA_DOWNLOAD_URL" ]; then
8 script_dir=$(dirname $0)
9 install -D -g root -o root -m 0755 $script_dir/$JAVA_FILE $TARGET_ROOT/$tmp_dir
10 filename=$JAVA_FILE
11 echo "Java file moved"
12else
13 wget --no-cookies --header "Cookie: gpw_e24=http%3A%2F%2Fwww.oracle.com" -P $TARGET_ROOT/$tmp_dir $JAVA_DOWNLOAD_URL
14 if [ $? -eq 0 ]; then
15 echo "Java is downloading"
16 else
17 echo "Url error. Exit"
18 exit 1
19 fi
20 filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -name "jdk*")
21 filename=$(basename $filename)
22fi
23
24if echo $tmp_dir/$filename | grep -q -s -F .tar.gz ; then
25 pushd $TARGET_ROOT/$tmp_dir
26 echo -e "\n" | tar -zxvf $TARGET_ROOT/$tmp_dir/$filename
27 popd
28 else
29 if echo $filename | grep -q -s -F .bin ; then
30 pushd $TARGET_ROOT/$tmp_dir
31 echo -e "\n" | sh $filename
32 rm $filename
33 popd
34 fi
35fi
36
37filename=$(find $TARGET_ROOT/$tmp_dir -maxdepth 1 -type d -name "jdk*")
38filename=$(basename $filename)
39javaPath=/usr/java/
40mkdir -p $TARGET_ROOT/$javaPath
41mv $TARGET_ROOT/$tmp_dir/$filename $TARGET_ROOT/$javaPath
42rm -r $tmp_dir
43echo "Java was installed"
44
diff --git a/elements/hadoop_fedora/post-install.d/12-setup-hadoop b/elements/hadoop_fedora/post-install.d/12-setup-hadoop
new file mode 100755
index 0000000..98068ad
--- /dev/null
+++ b/elements/hadoop_fedora/post-install.d/12-setup-hadoop
@@ -0,0 +1,32 @@
1#!/bin/bash
2echo "Hadoop setup begins"
3tmp_dir=/tmp/hadoop
4
5install-packages openssh-server
6echo "Creating hadoop user & group"
7adduser -G adm,wheel hadoop
8
9echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
10wget -P $tmp_dir/ "http://archive.apache.org/dist/hadoop/core/hadoop-"$DIB_HADOOP_VERSION"/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm"
11if [ $? -ne 0 ]; then
12 echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
13 exit 1
14fi
15
16rpm -ivh --replacefiles $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
17chmod 755 /usr/sbin/start-*
18chmod 755 /usr/sbin/stop-*
19chmod 755 /usr/sbin/slaves.sh
20chmod 755 /usr/sbin/update-hadoop-env.sh
21rm $tmp_dir/hadoop-"$DIB_HADOOP_VERSION"-1.x86_64.rpm
22rm -r $tmp_dir
23
24echo "Pre-configuring Hadoop"
25filename=$(find $TARGET_ROOT/usr/java/ -maxdepth 1 -name "jdk*")
26echo -e "PATH=\$PATH:/usr/sbin:$filename/bin\nJAVA_HOME=$filename" >> /home/hadoop/.bashrc
27echo -e "HADOOP_HOME=/usr/share/hadoop/" >> /home/hadoop/.bashrc
28sed -i "s,export JAVA_HOME=.*,export JAVA_HOME=$filename," /etc/hadoop/hadoop-env.sh
29log_dir=/mnt/log/hadoop/\$USER/
30sed -i "s,export HADOOP_LOG_DIR=.*,export HADOOP_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
31log_dir=/mnt/log/hadoop/hdfs
32sed -i "s,export HADOOP_SECURE_DN_LOG_DIR=.*,export HADOOP_SECURE_DN_LOG_DIR=$log_dir," /etc/hadoop/hadoop-env.sh
diff --git a/elements/hadoop_fedora/post-install.d/13-connection-setup b/elements/hadoop_fedora/post-install.d/13-connection-setup
new file mode 100755
index 0000000..af0af49
--- /dev/null
+++ b/elements/hadoop_fedora/post-install.d/13-connection-setup
@@ -0,0 +1,21 @@
1#!/bin/bash
2echo "Adjusting ssh configuration"
3
4sed -i 's/PasswordAuthentication no/PasswordAuthentication yes/' /etc/ssh/sshd_config
5echo "UseDNS no" >> /etc/ssh/sshd_config
6echo "PermitTunnel yes" >> /etc/ssh/sshd_config
7echo "SyslogFacility AUTH" >> /etc/ssh/sshd_config
8echo "PermitRootLogin yes" >> /etc/ssh/sshd_config
9echo "StrictModes yes" >> /etc/ssh/sshd_config
10echo "RSAAuthentication yes" >> /etc/ssh/sshd_config
11echo "PubkeyAuthentication yes" >> /etc/ssh/sshd_config
12echo "IgnoreRhosts yes" >> /etc/ssh/sshd_config
13
14echo "StrictHostKeyChecking no" >> /etc/ssh/ssh_config
15echo "GSSAPIDelegateCredentials no" >> /etc/ssh/ssh_config
16sed -i 's/ GSSAPIAuthentication yes/GSSAPIAuthentication no/' /etc/ssh/ssh_config
17
18sed -i 's/ssh_pwauth: 0/ssh_pwauth: 1/' /etc/cloud/cloud.cfg
19chmod 640 /etc/sudoers
20sed -i 's/Defaults requiretty/#Defaults requiretty/' /etc/sudoers
21chmod 0440 /etc/sudoers
diff --git a/elements/hadoop_fedora/root.d/0-check b/elements/hadoop_fedora/root.d/0-check
new file mode 100755
index 0000000..b90bfbc
--- /dev/null
+++ b/elements/hadoop_fedora/root.d/0-check
@@ -0,0 +1,20 @@
1#!/bin/bash
2set -e
3
4if [ -z "$JAVA_DOWNLOAD_URL" ]; then
5 if [ -z "$JAVA_FILE" ]; then
6 echo "JAVA_FILE and JAVA_DOWNLOAD_URL are not set. Impossible to install java. Exit"
7 exit 1
8 fi
9fi
10if [ -z "$DIB_HADOOP_VERSION" ]; then
11 echo "DIB_HADOOP_VERSION is not set. Impossible to install hadoop. Exit"
12 exit 1
13fi
14version_check=$(echo $DIB_HADOOP_VERSION | sed -e '/[0-9]\.[0-9]\.[0-9]/d')
15if [[ -z $version_check ]]; then
16 echo "All variables are set, continue."
17else
18 echo "Version error. Exit"
19 exit 1
20fi