openstack
/
sahara-image-elements
Code Issues Proposed changes

Add different version builders support for Cloudera plugin 

We add 5.0 and 5.3 versions support Cloudera plugin. They will only pull packages
from 5.0.0 and 5.3.0 repositories depending on the version. 5.0.0 will pull less
packages than 5.3.0, for in CDH5.0 plugin we does not support as many services as
in CDH5.3 plugin.

implements bp: cdh-version-management
Change-Id: I720c90aa0f0174148c3ed1eab7880e924ac6835a
This commit is contained in:
Ken Chen 2015-01-18 20:42:16 +08:00
parent 3e13a0cadc
commit 6900b9dcbe
4 changed files with 188 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

95
diskimage-create/diskimage-create.sh
View File

@ -23,7 +23,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]"
echo " [-i ubuntu|fedora|centos]"
echo " [-v 1|2|2.3|2.4|2.6|plain]"
echo " [-v 1|2|2.3|2.4|2.6|5.0|5.3|plain]"
echo " [-r 3.1.1|4.0.1]"
echo " [-d]"
echo " [-m]"
@ -113,8 +113,10 @@ fi
if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then
if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.3" -a "$HADOOP_VERSION" != "2.4" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then
echo -e "Unknown hadoop version selected.\nAborting"
exit 1
if [ "$PLUGIN" = "cloudera" -a "$HADOOP_VERSION" != "5.0" -a "$HADOOP_VERSION" != "5.3" ]; then
echo -e "Unknown hadoop version selected.\nAborting"
exit 1
fi
fi
fi
@ -493,43 +495,86 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "hdp" ]; then
unset BASE_IMAGE_FILE DIB_IMAGE_SIZE DIB_CLOUD_IMAGES
fi
#########################
# Images for CDH plugin #
#########################
if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
echo "For cloudera plugin option -v is ignored"
export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"}
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
cloudera_ubuntu_image_name=${cloudera_ubuntu_image_name:-ubuntu_sahara_cloudera_latest}
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0}
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
if [ -n "$USE_MIRRORS" ]; then
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
if [ -n "$USE_MIRRORS" ]; then
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
fi
# Cloudera supports only 12.04 Ubuntu
export DIB_CDH_VERSION="5.0"
export DIB_RELEASE="precise"
disk-image-create $cloudera_elements_sequence -o $cloudera_5_0_ubuntu_image_name
mv $cloudera_5_0_ubuntu_image_name.qcow2 ../
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0}
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
# Cloudera supports only 12.04 Ubuntu
export DIB_RELEASE="precise"
if [ -n "$USE_MIRRORS" ]; then
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
fi
disk-image-create $cloudera_elements_sequence -o $cloudera_ubuntu_image_name
mv $cloudera_ubuntu_image_name.qcow2 ../
unset DIB_RELEASE
# Cloudera supports only 12.04 Ubuntu
export DIB_CDH_VERSION="5.3"
export DIB_RELEASE="precise"
disk-image-create $cloudera_elements_sequence -o $cloudera_5_3_ubuntu_image_name
mv $cloudera_5_3_ubuntu_image_name.qcow2 ../
unset DIB_CDH_VERSION DIB_RELEASE
fi
fi
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then
# CentOS cloud image:
# - Disable including 'base' element for CentOS
# - Export link and filename for CentOS cloud image to download
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
# CentOS cloud image:
# - Disable including 'base' element for CentOS
# - Export link and filename for CentOS cloud image to download
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
export DIB_CDH_VERSION="5.0"
cloudera_centos_image_name=${cloudera_centos_image_name:-centos_sahara_cloudera_latest}
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0}
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
if [ -n "$USE_MIRRORS"]; then
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
if [ -n "$USE_MIRRORS"]; then
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
fi
disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_0_centos_image_name
mv $cloudera_5_0_centos_image_name.qcow2 ../
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
# CentOS cloud image:
# - Disable including 'base' element for CentOS
# - Export link and filename for CentOS cloud image to download
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
export DIB_CDH_VERSION="5.3"
disk-image-create $cloudera_elements_sequence -n -o $cloudera_centos_image_name
mv $cloudera_centos_image_name.qcow2 ../
cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0}
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES
if [ -n "$USE_MIRRORS"]; then
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
fi
disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_3_centos_image_name
mv $cloudera_5_3_centos_image_name.qcow2 ../
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION
fi
fi
unset EXTJS_DOWNLOAD_URL
fi

26
elements/hadoop-cloudera/install.d/50-install-cloudera
View File

@ -12,7 +12,6 @@ install-packages \
cloudera-manager-daemons \
cloudera-manager-server \
cloudera-manager-server-db-2 \
flume-ng \
hadoop-hdfs-datanode \
hadoop-hdfs-namenode \
hadoop-hdfs-secondarynamenode \
@ -21,25 +20,32 @@ install-packages \
hadoop-yarn-nodemanager \
hadoop-yarn-resourcemanager \
hbase \
hbase-solr \
hive-hcatalog \
hive-metastore \
hive-server2 \
hive-webhcat-server \
hue \
impala \
impala-server \
impala-state-store \
impala-catalog \
ntp \
oozie \
oracle-j2sdk1.7 \
sentry \
solr-server \
spark-history-server \
sqoop2 \
spark-core \
zookeeper
if [ $DIB_CDH_VERSION != "5.0" ]; then
# CDH5.0 does not have below packages.
install-packages \
flume-ng \
hbase-solr \
impala \
impala-server \
impala-state-store \
impala-catalog \
sentry \
solr-server \
spark-history-server \
sqoop2
fi
if [ $(lsb_release -is) = 'Ubuntu' ]; then
unset RUNLEVEL
fi

19
elements/hadoop-cloudera/post-install.d/70-turn-services-off
View File

@ -16,9 +16,7 @@ hive-server2 \
hive-webhcat-server \
hue \
oozie \
postgresql \
solr-server \
spark-history-server
postgresql
do
if [ $(lsb_release -is) = 'Ubuntu' ]; then
update-rc.d -f $i remove
@ -27,6 +25,21 @@ do
fi
done
if [ $DIB_CDH_VERSION != "5.0" ]; then
for i in impala-catalog \
impala-server \
impala-state-store \
solr-server \
spark-history-server
do
if [ $(lsb_release -is) = 'Ubuntu' ]; then
update-rc.d -f $i remove
else
chkconfig $i off
fi
done
fi
if [ $(lsb_release -is) = 'CentOS' ]; then
chkconfig iptables off
chkconfig ip6tables off

103
elements/hadoop-cloudera/pre-install.d/10-add-mirror
View File

@ -2,23 +2,92 @@
distro=$(lsb_release -is)
case $distro in
Ubuntu)
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
wget -O /etc/apt/sources.list.d/cdh5.list http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/cloudera.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
function add_repo_5_0_0 {
case $distro in
Ubuntu)
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
wget -O /etc/apt/sources.list.d/cm5.list http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/cloudera.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
apt-get update
;;
CentOS)
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
yum clean all
;;
esac
}
function add_repo_5_3_0 {
case $distro in
Ubuntu)
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
apt-get update
;;
CentOS)
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
yum clean all
;;
esac
}
# Call version-specific script to install the desired version of CDH
if [[ $DIB_CDH_VERSION == "5.0" ]]; then
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_0_0
else
if [[ $DIB_CDH_VERSION == "5.3" ]]; then
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_3_0
else
echo "Invalid CDH Version specified, exiting install."
exit 1
fi
fi
apt-get update
;;
CentOS)
wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo
wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cm5/redhat/6/x86_64/cm/cloudera-manager.repo
;;
esac