Add support to create CDH 5.11 images

Change-Id: Ibd61ca89bb24e369cca5d06e338edef2f0c6f6d6
This commit is contained in:
Evgeny Sikachov 2017-07-21 07:18:44 +00:00 committed by Jeremy Freudberg
parent 3425012a84
commit 8f2caee358
5 changed files with 115 additions and 6 deletions

View File

@ -32,7 +32,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 2.7.1|5.5|5.7|5.9|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-v 2.7.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-r 5.1.0|5.2.0]"
echo " [-s 1.3.1|1.6.0|2.1.0]"
echo " [-t 0.9.2|1.0.1|1.1.0]"
@ -188,7 +188,7 @@ case "$PLUGIN" in
esac
case "$HADOOP_VERSION" in
"" | "5.5" | "5.7" | "5.9");;
"" | "5.5" | "5.7" | "5.9" | "5.11");;
*)
echo -e "Unknown hadoop version selected.\nAborting"
exit 1
@ -196,8 +196,8 @@ case "$PLUGIN" in
esac
if [ "$BASE_IMAGE_OS" = "centos7" ]; then
if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" -a ! "$HADOOP_VERSION" = "5.7" -a ! "$HADOOP_VERSION" = "5.9" ]; then
echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5, CDH 5.7 and CDH 5.9"
if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" -a ! "$HADOOP_VERSION" = "5.7" -a ! "$HADOOP_VERSION" = "5.9" -a ! "$HADOOP_VERSION" = "5.11" ]; then
echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5 or higher version"
exit 1
fi
fi
@ -672,6 +672,15 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
image_create ubuntu $cloudera_5_9_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.11" ]; then
export DIB_CDH_VERSION="5.11"
export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0}
cloudera_5_11_ubuntu_image_name=${cloudera_5_11_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION}
export DIB_RELEASE="xenial"
image_create ubuntu $cloudera_5_11_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE DIB_CDH_MINOR_VERSION
fi
fi
if [ "$BASE_IMAGE_OS" = "centos" ]; then
@ -712,6 +721,15 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
cloudera_5_9_centos7_image_name=${cloudera_5_9_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION}
image_create centos7 $cloudera_5_9_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.11" ]; then
export DIB_CDH_VERSION="5.11"
export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0}
cloudera_5_11_centos7_image_name=${cloudera_5_11_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION}
image_create centos7 $cloudera_5_11_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_CDH_MINOR_VERSION
fi
fi

View File

@ -60,6 +60,9 @@ case "$DIB_CDH_VERSION" in
5.9)
HADOOP_OPENSTACK_5_9_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar"
;;
5.11)
HADOOP_OPENSTACK_5_11_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar"
;;
esac
HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar"
@ -92,6 +95,15 @@ case "$DIB_CDH_VERSION" in
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
fi
;;
5.11)
wget -O $dest $HADOOP_OPENSTACK_5_11_x_URL
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
# Create links to keep backward version support.
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
fi
;;
*)
echo "Unhandled version $DIB_CDH_VERSION for hadoop-openstack.jar"
exit 1

View File

@ -231,6 +231,81 @@ function add_repo_5_9_x {
esac
}
function add_repo_5_11_x {
case $DISTRO_NAME in
ubuntu )
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ xenial universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list
echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh/archive.key | apt-key add -
echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list
echo "deb-src http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/xenial/amd64/cm/archive.key | apt-key add -
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/cloudera.list
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/archive.key | apt-key add -
# add Kafka repository
echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/ xenial-kafka2.2.0 contrib' >> /etc/apt/sources.list
wget -qO - https://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/archive.key | apt-key add -
#change repository priority
echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin
apt-get update
;;
centos | rhel | centos7 | rhel7 )
case $DISTRO_NAME in
centos | rhel )
releasever=6
;;
centos7 | rhel7 )
releasever=7
;;
esac
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo
echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo
echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo
RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/)"
if [ "$RETURN_CODE" == "404" ]; then
echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_VERSION/" >> /etc/yum.repos.d/kms.repo
else
echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo
fi
echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo
echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo
echo "name=Cloudera's Distribution for kafka, Version 2.2.0" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "baseurl=http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/2.2.0/" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "gpgkey = http://archive.cloudera.com/kafka/redhat/$releasever/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo
echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo
yum clean all
;;
esac
}
DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0}
# Call version-specific script to install the desired version of CDH
@ -247,6 +322,10 @@ case "$DIB_CDH_VERSION" in
echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..."
add_repo_5_9_x
;;
5.11)
echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..."
add_repo_5_11_x
;;
*)
echo "Invalid CDH Version : $DIB_CDH_VERSION specified, exiting install."
exit 1

View File

@ -17,7 +17,7 @@ case "$plugin_type" in
;;
"cloudera" | "spark")
case "$DIB_CDH_VERSION" in
"5.4" | "5.5" | "5.7" | "5.9")
"5.4" | "5.5" | "5.7" | "5.9" | "5.11")
hadoop="2.6.0";;
esac
case "$plugin_type" in

View File

@ -17,7 +17,7 @@ case "$PLUGIN" in
for distro in ubuntu centos7; do
# NOTE(vgridnev): Number of versions are growing; testing only 2 latest
# releases of CDH
for version in 5.7 5.9; do
for version in 5.9 5.11; do
tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version
find . -maxdepth 1 -name '*cloudera*.qcow2' -delete
sudo rm -rf ~/.cache/image-create