From 6900b9dcbe30748ce6e550bc7701b0dcdf6af877 Mon Sep 17 00:00:00 2001 From: Ken Chen Date: Sun, 18 Jan 2015 20:42:16 +0800 Subject: [PATCH] Add different version builders support for Cloudera plugin We add 5.0 and 5.3 versions support Cloudera plugin. They will only pull packages from 5.0.0 and 5.3.0 repositories depending on the version. 5.0.0 will pull less packages than 5.3.0, for in CDH5.0 plugin we does not support as many services as in CDH5.3 plugin. implements bp: cdh-version-management Change-Id: I720c90aa0f0174148c3ed1eab7880e924ac6835a --- diskimage-create/diskimage-create.sh | 95 +++++++++++----- .../install.d/50-install-cloudera | 26 +++-- .../post-install.d/70-turn-services-off | 19 +++- .../pre-install.d/10-add-mirror | 103 +++++++++++++++--- 4 files changed, 188 insertions(+), 55 deletions(-) diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index 36e632c9..70b2be1f 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -23,7 +23,7 @@ usage() { echo "Usage: $(basename $0)" echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]" echo " [-i ubuntu|fedora|centos]" - echo " [-v 1|2|2.3|2.4|2.6|plain]" + echo " [-v 1|2|2.3|2.4|2.6|5.0|5.3|plain]" echo " [-r 3.1.1|4.0.1]" echo " [-d]" echo " [-m]" @@ -113,8 +113,10 @@ fi if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.3" -a "$HADOOP_VERSION" != "2.4" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then - echo -e "Unknown hadoop version selected.\nAborting" - exit 1 + if [ "$PLUGIN" = "cloudera" -a "$HADOOP_VERSION" != "5.0" -a "$HADOOP_VERSION" != "5.3" ]; then + echo -e "Unknown hadoop version selected.\nAborting" + exit 1 + fi fi fi @@ -493,43 +495,86 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "hdp" ]; then unset BASE_IMAGE_FILE DIB_IMAGE_SIZE DIB_CLOUD_IMAGES fi +######################### +# Images for CDH plugin # +######################### + if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then - echo "For cloudera plugin option -v is ignored" export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"} if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - cloudera_ubuntu_image_name=${cloudera_ubuntu_image_name:-ubuntu_sahara_cloudera_latest} - cloudera_elements_sequence="base vm ubuntu hadoop-cloudera" + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then + cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0} + cloudera_elements_sequence="base vm ubuntu hadoop-cloudera" - if [ -n "$USE_MIRRORS" ]; then - [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror" + if [ -n "$USE_MIRRORS" ]; then + [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror" + fi + + # Cloudera supports only 12.04 Ubuntu + export DIB_CDH_VERSION="5.0" + export DIB_RELEASE="precise" + disk-image-create $cloudera_elements_sequence -o $cloudera_5_0_ubuntu_image_name + mv $cloudera_5_0_ubuntu_image_name.qcow2 ../ + unset DIB_CDH_VERSION DIB_RELEASE fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then + cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0} + cloudera_elements_sequence="base vm ubuntu hadoop-cloudera" - # Cloudera supports only 12.04 Ubuntu - export DIB_RELEASE="precise" + if [ -n "$USE_MIRRORS" ]; then + [ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror" + fi - disk-image-create $cloudera_elements_sequence -o $cloudera_ubuntu_image_name - mv $cloudera_ubuntu_image_name.qcow2 ../ - unset DIB_RELEASE + # Cloudera supports only 12.04 Ubuntu + export DIB_CDH_VERSION="5.3" + export DIB_RELEASE="precise" + disk-image-create $cloudera_elements_sequence -o $cloudera_5_3_ubuntu_image_name + mv $cloudera_5_3_ubuntu_image_name.qcow2 ../ + unset DIB_CDH_VERSION DIB_RELEASE + fi fi if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then - # CentOS cloud image: - # - Disable including 'base' element for CentOS - # - Export link and filename for CentOS cloud image to download - export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2" - export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then + # CentOS cloud image: + # - Disable including 'base' element for CentOS + # - Export link and filename for CentOS cloud image to download + export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2" + export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" + export DIB_CDH_VERSION="5.0" - cloudera_centos_image_name=${cloudera_centos_image_name:-centos_sahara_cloudera_latest} - cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive" + cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0} + cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive" - if [ -n "$USE_MIRRORS"]; then - [ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror" + if [ -n "$USE_MIRRORS"]; then + [ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror" + fi + + disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_0_centos_image_name + mv $cloudera_5_0_centos_image_name.qcow2 ../ + + unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then + # CentOS cloud image: + # - Disable including 'base' element for CentOS + # - Export link and filename for CentOS cloud image to download + export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2" + export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com" + export DIB_CDH_VERSION="5.3" - disk-image-create $cloudera_elements_sequence -n -o $cloudera_centos_image_name - mv $cloudera_centos_image_name.qcow2 ../ + cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0} + cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive" - unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES + if [ -n "$USE_MIRRORS"]; then + [ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror" + fi + + disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_3_centos_image_name + mv $cloudera_5_3_centos_image_name.qcow2 ../ + + unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION + fi fi unset EXTJS_DOWNLOAD_URL fi diff --git a/elements/hadoop-cloudera/install.d/50-install-cloudera b/elements/hadoop-cloudera/install.d/50-install-cloudera index 7e5964e1..f1c9f0cf 100755 --- a/elements/hadoop-cloudera/install.d/50-install-cloudera +++ b/elements/hadoop-cloudera/install.d/50-install-cloudera @@ -12,7 +12,6 @@ install-packages \ cloudera-manager-daemons \ cloudera-manager-server \ cloudera-manager-server-db-2 \ - flume-ng \ hadoop-hdfs-datanode \ hadoop-hdfs-namenode \ hadoop-hdfs-secondarynamenode \ @@ -21,25 +20,32 @@ install-packages \ hadoop-yarn-nodemanager \ hadoop-yarn-resourcemanager \ hbase \ - hbase-solr \ hive-hcatalog \ hive-metastore \ hive-server2 \ hive-webhcat-server \ hue \ - impala \ - impala-server \ - impala-state-store \ - impala-catalog \ ntp \ oozie \ oracle-j2sdk1.7 \ - sentry \ - solr-server \ - spark-history-server \ - sqoop2 \ + spark-core \ zookeeper +if [ $DIB_CDH_VERSION != "5.0" ]; then + # CDH5.0 does not have below packages. + install-packages \ + flume-ng \ + hbase-solr \ + impala \ + impala-server \ + impala-state-store \ + impala-catalog \ + sentry \ + solr-server \ + spark-history-server \ + sqoop2 +fi + if [ $(lsb_release -is) = 'Ubuntu' ]; then unset RUNLEVEL fi diff --git a/elements/hadoop-cloudera/post-install.d/70-turn-services-off b/elements/hadoop-cloudera/post-install.d/70-turn-services-off index 701c16c1..a1c419db 100755 --- a/elements/hadoop-cloudera/post-install.d/70-turn-services-off +++ b/elements/hadoop-cloudera/post-install.d/70-turn-services-off @@ -16,9 +16,7 @@ hive-server2 \ hive-webhcat-server \ hue \ oozie \ -postgresql \ -solr-server \ -spark-history-server +postgresql do if [ $(lsb_release -is) = 'Ubuntu' ]; then update-rc.d -f $i remove @@ -27,6 +25,21 @@ do fi done +if [ $DIB_CDH_VERSION != "5.0" ]; then + for i in impala-catalog \ + impala-server \ + impala-state-store \ + solr-server \ + spark-history-server + do + if [ $(lsb_release -is) = 'Ubuntu' ]; then + update-rc.d -f $i remove + else + chkconfig $i off + fi + done +fi + if [ $(lsb_release -is) = 'CentOS' ]; then chkconfig iptables off chkconfig ip6tables off diff --git a/elements/hadoop-cloudera/pre-install.d/10-add-mirror b/elements/hadoop-cloudera/pre-install.d/10-add-mirror index a183509f..b4a57737 100755 --- a/elements/hadoop-cloudera/pre-install.d/10-add-mirror +++ b/elements/hadoop-cloudera/pre-install.d/10-add-mirror @@ -2,23 +2,92 @@ distro=$(lsb_release -is) -case $distro in - Ubuntu) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list - # Cloudera repositories - wget -O /etc/apt/sources.list.d/cdh5.list http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/cloudera.list - wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - +function add_repo_5_0_0 { + case $distro in + Ubuntu) + # Add repository with postgresql package (it's dependency of cloudera packages) + # Base image doesn't contain this repo + echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list - wget -O /etc/apt/sources.list.d/cm5.list http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/cloudera.list - wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - + # Cloudera repositories + echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list + echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list + + wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - + + echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list + echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list + + wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - + + apt-get update + ;; + CentOS) + echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo + echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo + + echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo + echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo + yum clean all + ;; + esac +} + +function add_repo_5_3_0 { + case $distro in + Ubuntu) + # Add repository with postgresql package (it's dependency of cloudera packages) + # Base image doesn't contain this repo + echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list + + # Cloudera repositories + echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list + echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list + + wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - + + echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list + echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list + + wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - + + apt-get update + ;; + CentOS) + echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo + echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo + + echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo + echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo + yum clean all + ;; + esac +} + +# Call version-specific script to install the desired version of CDH +if [[ $DIB_CDH_VERSION == "5.0" ]]; then + echo "Installing CDH Version $DIB_CDH_VERSION..." + add_repo_5_0_0 +else + if [[ $DIB_CDH_VERSION == "5.3" ]]; then + echo "Installing CDH Version $DIB_CDH_VERSION..." + add_repo_5_3_0 + else + echo "Invalid CDH Version specified, exiting install." + exit 1 + fi +fi - apt-get update - ;; - CentOS) - wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo - wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cm5/redhat/6/x86_64/cm/cloudera-manager.repo - ;; -esac