diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index b7f3abb1..a79bfd00 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -23,7 +23,7 @@ usage() { echo "Usage: $(basename $0)" echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]" echo " [-i ubuntu|fedora|centos]" - echo " [-v 1|2|2.3|2.4|2.6|plain]" + echo " [-v 1|2|2.3|2.4|2.6|5.0|5.3|plain]" echo " [-r 3.1.1|4.0.1]" echo " [-d]" echo " [-m]" @@ -113,8 +113,10 @@ fi if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.3" -a "$HADOOP_VERSION" != "2.4" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then - echo -e "Unknown hadoop version selected.\nAborting" - exit 1 + if [ "$PLUGIN" = "cloudera" -a "$HADOOP_VERSION" != "5.0" -a "$HADOOP_VERSION" != "5.3" ]; then + echo -e "Unknown hadoop version selected.\nAborting" + exit 1 + fi fi fi @@ -495,43 +497,86 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "hdp" ]; then unset BASE_IMAGE_FILE DIB_IMAGE_SIZE DIB_CLOUD_IMAGES fi +######################### +# Images for CDH plugin # +######################### + if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then - echo "For cloudera plugin option -v is ignored" export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"} if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - cloudera_ubuntu_image_name=${cloudera_ubuntu_image_name:-ubuntu_sahara_cloudera_latest} - cloudera_elements_sequence="base vm ubuntu hadoop-cloudera" + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then + cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0} + cloudera_elements_sequence="base vm ubuntu hadoop-cloudera" - if [ -n "$USE_MIRRORS" ]; then - [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror" + if [ -n "$USE_MIRRORS" ]; then + [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror" + fi + + # Cloudera supports only 12.04 Ubuntu + export DIB_CDH_VERSION="5.0" + export DIB_RELEASE="precise" + disk-image-create $cloudera_elements_sequence -o $cloudera_5_0_ubuntu_image_name + mv $cloudera_5_0_ubuntu_image_name.qcow2 ../ + unset DIB_CDH_VERSION DIB_RELEASE fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then + cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0} + cloudera_elements_sequence="base vm ubuntu hadoop-cloudera" - # Cloudera supports only 12.04 Ubuntu - export DIB_RELEASE="precise" + if [ -n "$USE_MIRRORS" ]; then + [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror" + fi - disk-image-create $cloudera_elements_sequence -o $cloudera_ubuntu_image_name - mv $cloudera_ubuntu_image_name.qcow2 ../ - unset DIB_RELEASE + # Cloudera supports only 12.04 Ubuntu + export DIB_CDH_VERSION="5.3" + export DIB_RELEASE="precise" + disk-image-create $cloudera_elements_sequence -o $cloudera_5_3_ubuntu_image_name + mv $cloudera_5_3_ubuntu_image_name.qcow2 ../ + unset DIB_CDH_VERSION DIB_RELEASE + fi fi if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then - # CentOS cloud image: - # - Disable including 'base' element for CentOS - # - Export link and filename for CentOS cloud image to download - export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2" - export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then + # CentOS cloud image: + # - Disable including 'base' element for CentOS + # - Export link and filename for CentOS cloud image to download + export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2" + export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" + export DIB_CDH_VERSION="5.0" - cloudera_centos_image_name=${cloudera_centos_image_name:-centos_sahara_cloudera_latest} - cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive" + cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0} + cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive" - if [ -n "$USE_MIRRORS"]; then - [ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror" + if [ -n "$USE_MIRRORS"]; then + [ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror" + fi + + disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_0_centos_image_name + mv $cloudera_5_0_centos_image_name.qcow2 ../ + + unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then + # CentOS cloud image: + # - Disable including 'base' element for CentOS + # - Export link and filename for CentOS cloud image to download + export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2" + export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" + export DIB_CDH_VERSION="5.3" - disk-image-create $cloudera_elements_sequence -n -o $cloudera_centos_image_name - mv $cloudera_centos_image_name.qcow2 ../ + cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0} + cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive" - unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES + if [ -n "$USE_MIRRORS"]; then + [ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror" + fi + + disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_3_centos_image_name + mv $cloudera_5_3_centos_image_name.qcow2 ../ + + unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION + fi fi unset EXTJS_DOWNLOAD_URL fi diff --git a/elements/hadoop-cloudera/install.d/50-install-cloudera b/elements/hadoop-cloudera/install.d/50-install-cloudera index 7e5964e1..f1c9f0cf 100755 --- a/elements/hadoop-cloudera/install.d/50-install-cloudera +++ b/elements/hadoop-cloudera/install.d/50-install-cloudera @@ -12,7 +12,6 @@ install-packages \ cloudera-manager-daemons \ cloudera-manager-server \ cloudera-manager-server-db-2 \ - flume-ng \ hadoop-hdfs-datanode \ hadoop-hdfs-namenode \ hadoop-hdfs-secondarynamenode \ @@ -21,25 +20,32 @@ install-packages \ hadoop-yarn-nodemanager \ hadoop-yarn-resourcemanager \ hbase \ - hbase-solr \ hive-hcatalog \ hive-metastore \ hive-server2 \ hive-webhcat-server \ hue \ - impala \ - impala-server \ - impala-state-store \ - impala-catalog \ ntp \ oozie \ oracle-j2sdk1.7 \ - sentry \ - solr-server \ - spark-history-server \ - sqoop2 \ + spark-core \ zookeeper +if [ $DIB_CDH_VERSION != "5.0" ]; then + # CDH5.0 does not have below packages. + install-packages \ + flume-ng \ + hbase-solr \ + impala \ + impala-server \ + impala-state-store \ + impala-catalog \ + sentry \ + solr-server \ + spark-history-server \ + sqoop2 +fi + if [ $(lsb_release -is) = 'Ubuntu' ]; then unset RUNLEVEL fi diff --git a/elements/hadoop-cloudera/post-install.d/70-turn-services-off b/elements/hadoop-cloudera/post-install.d/70-turn-services-off index 701c16c1..a1c419db 100755 --- a/elements/hadoop-cloudera/post-install.d/70-turn-services-off +++ b/elements/hadoop-cloudera/post-install.d/70-turn-services-off @@ -16,9 +16,7 @@ hive-server2 \ hive-webhcat-server \ hue \ oozie \ -postgresql \ -solr-server \ -spark-history-server +postgresql do if [ $(lsb_release -is) = 'Ubuntu' ]; then update-rc.d -f $i remove @@ -27,6 +25,21 @@ do fi done +if [ $DIB_CDH_VERSION != "5.0" ]; then + for i in impala-catalog \ + impala-server \ + impala-state-store \ + solr-server \ + spark-history-server + do + if [ $(lsb_release -is) = 'Ubuntu' ]; then + update-rc.d -f $i remove + else + chkconfig $i off + fi + done +fi + if [ $(lsb_release -is) = 'CentOS' ]; then chkconfig iptables off chkconfig ip6tables off diff --git a/elements/hadoop-cloudera/pre-install.d/10-add-mirror b/elements/hadoop-cloudera/pre-install.d/10-add-mirror index a183509f..b4a57737 100755 --- a/elements/hadoop-cloudera/pre-install.d/10-add-mirror +++ b/elements/hadoop-cloudera/pre-install.d/10-add-mirror @@ -2,23 +2,92 @@ distro=$(lsb_release -is) -case $distro in - Ubuntu) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list - # Cloudera repositories - wget -O /etc/apt/sources.list.d/cdh5.list http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/cloudera.list - wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - +function add_repo_5_0_0 { + case $distro in + Ubuntu) + # Add repository with postgresql package (it's dependency of cloudera packages) + # Base image doesn't contain this repo + echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list - wget -O /etc/apt/sources.list.d/cm5.list http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/cloudera.list - wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - + # Cloudera repositories + echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list + echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list + + wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - + + echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list + echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list + + wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - + + apt-get update + ;; + CentOS) + echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo + echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo + + echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo + echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo + yum clean all + ;; + esac +} + +function add_repo_5_3_0 { + case $distro in + Ubuntu) + # Add repository with postgresql package (it's dependency of cloudera packages) + # Base image doesn't contain this repo + echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list + + # Cloudera repositories + echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list + echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list + + wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - + + echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list + echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list + + wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - + + apt-get update + ;; + CentOS) + echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo + echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo + + echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo + echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo + yum clean all + ;; + esac +} + +# Call version-specific script to install the desired version of CDH +if [[ $DIB_CDH_VERSION == "5.0" ]]; then + echo "Installing CDH Version $DIB_CDH_VERSION..." + add_repo_5_0_0 +else + if [[ $DIB_CDH_VERSION == "5.3" ]]; then + echo "Installing CDH Version $DIB_CDH_VERSION..." + add_repo_5_3_0 + else + echo "Invalid CDH Version specified, exiting install." + exit 1 + fi +fi - apt-get update - ;; - CentOS) - wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo - wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cm5/redhat/6/x86_64/cm/cloudera-manager.repo - ;; -esac