Adds Intel Distribution Hadoop element to savanna-image-elements

Implements: blueprint idh-image-builder Change-Id: I195a4fcd9be796bd2994f95f128d4e6893cbc254
2014-02-27 12:25:58 +04:00 · 2014-02-27 12:25:58 +04:00 · e9493f8573
parent cc14bdbb2b
commit e9493f8573
8 changed files with 300 additions and 4 deletions
--- a/diskimage-create/README.rst
+++ b/diskimage-create/README.rst
@ -1,7 +1,7 @@
 Diskimage-builder script for creation cloud images
 ==================================================

-This script builds Ubuntu, Fedora, CentOS cloud images for use in Savanna. By default the all plugin are targeted, all images will be built. The '-p' option can be used to select plugin (vanilla, spark or hdp). The '-i' option can be used to select image type (ubuntu, fedora or centos). The '-v' option can be used to select hadoop version (1, 2 or plain).
+This script builds Ubuntu, Fedora, CentOS cloud images for use in Savanna. By default the all plugin are targeted, all images will be built. The '-p' option can be used to select plugin (vanilla, spark, idh or hdp). The '-i' option can be used to select image type (ubuntu, fedora or centos). The '-v' option can be used to select hadoop version (1, 2 or plain).

 NOTE: You should use Ubuntu or Fedora host OS for building images, CentOS as a host OS has not been tested well.

@ -25,7 +25,7 @@ For users:

 .. sourcecode:: bash

-  sudo bash savanna-image-elements/diskimage-create/diskimage-create.sh -p [vanilla|spark|hdp]
+  sudo bash savanna-image-elements/diskimage-create/diskimage-create.sh -p [vanilla|spark|hdp|idh]

 5. To select which hadoop version to target use the '-v' commandline option like this:

@ -44,6 +44,7 @@ NOTE for 4, 5, 6:
 For Vanilla you can create ubuntu, fedora and centos cloud image with hadoop 1.x.x and 2.x.x versions. Use environment variables 'DIB_HADOOP_VERSION_1' and 'DIB_HADOOP_VERSION_2' to change defaults.
 For Spark you can create only ubuntu image with one hadoop version. You shouldn't specify image type and hadoop version.
 For HDP you can create only centos image with hadoop 1.3.0 or 2.0 and without hadoop ('plain' image). You shouldn't specify image type.
+For IDH you can create only centos image with one hadoop version. You shouldn't specify image type and hadoop version.

 For developers:

--- a/diskimage-create/diskimage-create.sh
+++ b/diskimage-create/diskimage-create.sh
@ -15,7 +15,7 @@ while getopts "p:i:v:" opt; do
    ;;
    *)
      echo
-      echo "Usage: $(basename $0) [-p vanilla|spark|hdp] [-i ubuntu|fedora|centos] [-v 1|2|plain]"
+      echo "Usage: $(basename $0) [-p vanilla|spark|hdp|idh] [-i ubuntu|fedora|centos] [-v 1|2|plain]"
      echo "'-p' is plugin version, '-i' is image type, '-v' is hadoop version"
      echo "You shouldn't specify hadoop version and image type for spark plugin"
      echo "You shouldn't specify image type for hdp plugin"
@ -27,7 +27,7 @@ while getopts "p:i:v:" opt; do
 done

 # Checks of input
-if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" ]; then
+if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "idh" ]; then
  echo -e "Unknown plugin selected.\nAborting"
  exit 1
 fi
@ -273,6 +273,34 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "hdp" ]; then
  fi
 fi

+########################
+# Image for IDH plugin #
+########################
+
+if [ -z "$PLUGIN" -o "$PLUGIN" = "idh" ]; then
+  # Ignoring image type and hadoop version options
+  echo "For idh plugin options -i and -v are ignored"
+
+  export DIB_IMAGE_SIZE="10"
+  export BASE_IMAGE_FILE="CentOS-6.4-cloud-init.qcow2"
+  export DIB_CLOUD_IMAGES="http://savanna-files.mirantis.com"
+  export centos_image_name_idh="centos_savanna_idh_latest"
+
+  centos_elements_sequence="vm rhel hadoop-idh"
+
+  if [ "$platform" = 'NAME="Ubuntu"' ]; then
+      echo "**************************************************************"
+      echo "WARNING: As a workaround for DIB bug 1204824, you are about to"
+      echo "         create a CentOS image that has SELinux               "
+      echo "         disabled. Do not use these images in production.     "
+      echo "**************************************************************"
+      centos_elements_sequence="$centos_elements_sequence selinux-permissive"
+      centos_image_name_idh="$centos_image_name_idh.selinux-permissive"
+  fi
+
+  disk-image-create $centos_elements_sequence -o $centos_image_name_idh
+  mv $centos_image_name_idh.qcow2 ../
+fi

 popd # out of $TEMP
 rm -rf $TEMP
--- a/elements/hadoop-idh/README.md
+++ b/elements/hadoop-idh/README.md
@ -0,0 +1,11 @@
+Install the Intel Distribution Hadoop
+
+The following script:
+
+savanna-image-elements/diskimage-create/diskimage-create.sh
+
+is the default script to use for creating CentOS images with IDH installed/configured.
+
+In order to create the IDH images with the diskimage-create.sh script, use the following syntax to select the "idh" plugin:
+
+diskimage-create.sh -p idh
--- a/elements/hadoop-idh/install.d/10-install-pkgs
+++ b/elements/hadoop-idh/install.d/10-install-pkgs
@ -0,0 +1,3 @@
+#!/bin/bash
+
+cat $(dirname $0)/packages | xargs install-packages
--- a/elements/hadoop-idh/install.d/packages
+++ b/elements/hadoop-idh/install.d/packages
@ -0,0 +1,195 @@
+GeoIP
+alsa-lib
+apr
+apr-util
+apr-util-ldap
+at
+atk
+augeas-libs
+avahi-libs
+bc
+cairo
+cdparanoia-libs
+cloog-ppl
+compat-readline5
+cpp
+createrepo
+cups
+cups-libs
+cvs
+db4-cxx
+db4-devel
+dbus
+deltarpm
+ed
+expect
+facter
+fontconfig
+foomatic
+foomatic-db
+foomatic-db-filesystem
+foomatic-db-ppds
+freetype
+fuse
+fuse-libs
+gcc
+gd
+gdbm-devel
+gettext
+ghostscript
+ghostscript-fonts
+glib2
+glibc
+glibc-common
+glibc-devel
+glibc-headers
+gnuplot
+gnuplot-common
+gnutls
+gstreamer
+gstreamer-plugins-base
+gstreamer-tools
+gtk2
+hicolor-icon-theme
+httpd
+httpd-tools
+iptraf
+iso-codes
+jasper-libs
+kernel-headers
+lcms-libs
+libICE
+libSM
+libX11
+libX11-common
+libXau
+libXcomposite
+libXcursor
+libXdamage
+libXext
+libXfixes
+libXfont
+libXft
+libXi
+libXinerama
+libXpm
+libXrandr
+libXrender
+libXt
+libXtst
+libXv
+libXxf86vm
+libconfuse
+libfontenc
+libgcc
+libgenders
+libgomp
+libgudev1
+libjpeg-turbo
+libmng
+libogg
+liboil
+libpng
+libselinux-ruby
+libtasn1
+libthai
+libtheora
+libtiff
+libudev
+libvisual
+libvorbis
+libxcb
+libxml2
+libxml2-python
+libxslt
+mailcap
+mailx
+man
+mesa-dri-drivers
+mesa-dri-filesystem
+mesa-dri1-drivers
+mesa-libGL
+mesa-libGLU
+mesa-private-llvm
+mpfr
+mtools
+nagios
+nagios-common
+nagios-plugins
+nginx
+ntp
+ntpdate
+openjpeg-libs
+openssl
+pango
+patch
+pax
+pciutils
+perl
+perl-CGI
+perl-ExtUtils-MakeMaker
+perl-ExtUtils-ParseXS
+perl-Module-Pluggable
+perl-Pod-Escapes
+perl-Pod-Simple
+perl-Test-Harness
+perl-Test-Simple
+perl-Time-HiRes
+perl-devel
+perl-libs
+perl-version
+phonon-backend-gstreamer
+php
+php-cli
+php-common
+php-gd
+php-xml
+pixman
+pkgconfig
+poppler
+poppler-data
+poppler-utils
+portreserve
+ppl
+puppet
+puppet-server
+python-deltarpm
+qt
+qt-sqlite
+qt-x11
+qt3
+redhat-lsb
+redhat-lsb-compat
+redhat-lsb-core
+redhat-lsb-graphics
+redhat-lsb-printing
+ruby
+ruby-augeas
+ruby-irb
+ruby-libs
+ruby-rdoc
+ruby-shadow
+rubygem-daemons
+rubygem-fastthread
+rubygem-gem_plugin
+rubygem-mongrel
+rubygem-rake
+rubygems
+sdparm
+shared-mime-info
+syslinux
+sysstat
+tcl
+tftp-server
+time
+tmpwatch
+unzip
+urw-fonts
+virt-what
+vsftpd
+xinetd
+xml-common
+xorg-x11-font-utils
+xz
+xz-lzma-compat
+zip
--- a/elements/hadoop-idh/pre-install.d/02-prepare-repo
+++ b/elements/hadoop-idh/pre-install.d/02-prepare-repo
@ -0,0 +1,12 @@
+#/bin/bash
+
+mkdir /root/old-repo
+mv /etc/yum.repos.d/* /root/old-repo
+install -D -g root -o root -m 644 $(dirname $0)/Centos-Vault.repo /etc/yum.repos.d/
+
+install-packages wget createrepo
+mkdir -p /var/repo/intel
+wget -P /var/repo/intel --no-parent -r -l inf --content-disposition -nH --cut-dirs=5 http://repo1.intelhadoop.com:3424/evaluation/en/RHEL/2.5.1/rpm/
+createrepo /var/repo/intel
+
+install -D -g root -o root -m 644 $(dirname $0)/Intel.repo /etc/yum.repos.d/
--- a/elements/hadoop-idh/pre-install.d/Centos-Vault.repo
+++ b/elements/hadoop-idh/pre-install.d/Centos-Vault.repo
@ -0,0 +1,34 @@
+[C6.4-base]
+name=CentOS-6.4 - Base
+baseurl=http://vault.centos.org/6.4/os/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+enabled=1
+
+[C6.4-updates]
+name=CentOS-6.4 - Updates
+baseurl=http://vault.centos.org/6.4/updates/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+enabled=1
+
+[C6.4-extras]
+name=CentOS-6.4 - Extras
+baseurl=http://vault.centos.org/6.4/extras/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+enabled=1
+
+[C6.4-contrib]
+name=CentOS-6.4 - Contrib
+baseurl=http://vault.centos.org/6.4/contrib/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+enabled=1
+
+[C6.4-centosplus]
+name=CentOS-6.4 - CentOSPlus
+baseurl=http://vault.centos.org/6.4/centosplus/$basearch/
+gpgcheck=1
+gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-6
+enabled=1
--- a/elements/hadoop-idh/pre-install.d/Intel.repo
+++ b/elements/hadoop-idh/pre-install.d/Intel.repo
@ -0,0 +1,12 @@
+[intel]
+name = Intel-Distribution-for-Apache-Hadoop
+baseurl = file:///var/repo/intel
+proxy = _none_
+gpgcheck = 0
+metadata_expire = 300
+fastestmirror_enabled = 1
+failovermethod = roundrobin
+priority = 1
+enabled = 1
+retries = 5
+timeout = 10