Merge "Remove some codes of older version builder"

This commit is contained in:
Jenkins 2017-04-21 10:06:01 +00:00 committed by Gerrit Code Review
commit b9cf0fc089
6 changed files with 5 additions and 230 deletions

View File

@ -17,7 +17,7 @@ Environment variables the script accepts are 'DIB_HADOOP_VERSION_2_7_1',
'[ubuntu|fedora|centos|centos7]_vanilla_hadoop_2_7_1_image_name',
'ubuntu_spark_image_name', 'ubuntu_storm_image_name',
'ambari_[ubuntu|centos|centos7]_image_name',
'cloudera_[5_0|5_3|5_4|5_5|5_7|5_9]_[ubuntu|centos]_image_name',
'cloudera_[5_5|5_7|5_9]_[ubuntu|centos]_image_name',
'mapr_[ubuntu|centos|centos7]_image_name',
'plain_[ubuntu|fedora|centos|centos7]_image_name'.
@ -59,14 +59,14 @@ like this:
.. sourcecode:: bash
tox -e venv -- sahara-image-create -v [2.7.1|4|5.0|5.3|5.4|5.5|5.7|5.9]
tox -e venv -- sahara-image-create -v [2.7.1|5.5|5.7|5.9]
Also, if you are planning to select which ambari version to target use the
'-v' commandline option like this:
.. sourcecode:: bash
tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0]
tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0|2.4.2.0]
6. To select which operating system to target use the '-i' commandline option
like this:

View File

@ -32,7 +32,7 @@ usage() {
echo "Usage: $(basename $0)"
echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
echo " [-i ubuntu|fedora|centos|centos7]"
echo " [-v 2.7.1|5.5|5.7|5.9|2.2.0.0|2.2.1.0]"
echo " [-v 2.7.1|5.5|5.7|5.9|2.2.0.0|2.2.1.0|2.4.2.0]"
echo " [-r 5.1.0|5.2.0]"
echo " [-s 1.3.1|1.6.0]"
echo " [-t 0.9.2|1.0.1]"
@ -651,33 +651,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
cloudera_elements_sequence="hadoop-cloudera swift_hadoop kdc"
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
export DIB_CDH_VERSION="5.0"
cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0}
# Cloudera supports only 12.04 Ubuntu
export DIB_RELEASE="precise"
image_create ubuntu $cloudera_5_0_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
export DIB_CDH_VERSION="5.3"
cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0}
# Cloudera supports only 12.04 Ubuntu
export DIB_RELEASE="precise"
image_create ubuntu $cloudera_5_3_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.4" ]; then
export DIB_CDH_VERSION="5.4"
cloudera_5_4_ubuntu_image_name=${cloudera_5_4_ubuntu_image_name:-ubuntu_sahara_cloudera_5_4_0}
# Cloudera supports only 12.04 Ubuntu
export DIB_RELEASE="precise"
image_create ubuntu $cloudera_5_4_ubuntu_image_name $cloudera_elements_sequence
unset DIB_CDH_VERSION DIB_RELEASE
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then
export DIB_CDH_VERSION="5.5"
cloudera_5_5_ubuntu_image_name=${cloudera_5_5_ubuntu_image_name:-ubuntu_sahara_cloudera_5_5_0}
@ -709,30 +682,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then
centos_cloudera_elements_sequence="selinux-permissive disable-firewall nc"
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
export DIB_CDH_VERSION="5.0"
cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0}
image_create centos $cloudera_5_0_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence
unset DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
export DIB_CDH_VERSION="5.3"
cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0}
image_create centos $cloudera_5_3_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence
unset DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.4" ]; then
export DIB_CDH_VERSION="5.4"
cloudera_5_4_centos_image_name=${cloudera_5_4_centos_image_name:-centos_sahara_cloudera_5_4_0}
image_create centos $cloudera_5_4_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence
unset DIB_CDH_VERSION
fi
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then
export DIB_CDH_VERSION="5.5"

View File

@ -34,7 +34,6 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
zookeeper
if [ $DIB_CDH_VERSION \> "5.0" ]; then
# CDH5.0 does not have below packages.
install-packages \
flume-ng \
hadoop-kms \
@ -50,12 +49,7 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
solr-doc \
search \
spark-history-server \
sqoop2
fi
if [ $DIB_CDH_VERSION \> "5.4" ]; then
# we support Kafka from CDH 5.5
install-packages \
sqoop2 \
kafka \
kafka-server
fi
@ -72,27 +66,9 @@ case "$DIB_CDH_VERSION" in
;;
esac
HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar"
HADOOP_OPENSTACK_5_4_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.4.0/hadoop-openstack-2.6.0-cdh5.4.0.jar"
HADOOP_OPENSTACK_5_3_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.5.0-cdh5.3.0/hadoop-openstack-2.5.0-cdh5.3.0.jar"
HADOOP_OPENSTACK_5_0_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.3.0-cdh5.0.0/hadoop-openstack-2.3.0-cdh5.0.0.jar"
dest=/usr/lib/hadoop-mapreduce/hadoop-openstack.jar
case "$DIB_CDH_VERSION" in
5.0)
wget -O $dest $HADOOP_OPENSTACK_5_0_0_URL
;;
5.3)
wget -O $dest $HADOOP_OPENSTACK_5_3_0_URL
;;
5.4)
wget -O $dest $HADOOP_OPENSTACK_5_4_0_URL
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then
# Create links to keep backward version support.
ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
fi
;;
5.5)
wget -O $dest $HADOOP_OPENSTACK_5_5_0_URL
if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then

View File

@ -6,129 +6,6 @@ fi
set -eu
set -o pipefail
function add_repo_5_0_0 {
case $DISTRO_NAME in
ubuntu )
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
apt-get update
;;
centos | rhel )
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
yum clean all
;;
esac
}
function add_repo_5_3_0 {
case $DISTRO_NAME in
ubuntu )
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/cloudera.list
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/archive.key | apt-key add -
apt-get update
;;
centos | rhel )
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
wget -O /etc/yum.repos.d/kms.repo http://archive.cloudera.com/navigator-keytrustee5/redhat/6/x86_64/navigator-keytrustee/navigator-keytrustee5.repo
yum clean all
;;
esac
}
function add_repo_5_4_0 {
case $DISTRO_NAME in
ubuntu )
# Add repository with postgresql package (it's dependency of cloudera packages)
# Base image doesn't contain this repo
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
# Cloudera repositories
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' > /etc/apt/sources.list.d/cdh5.list
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' > /etc/apt/sources.list.d/cm5.list
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' >> /etc/apt/sources.list.d/cm5.list
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/cloudera.list
wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/archive.key | apt-key add -
apt-get update
;;
centos | rhel )
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.4.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.4.0/' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
wget -O /etc/yum.repos.d/kms.repo http://archive.cloudera.com/navigator-keytrustee5/redhat/6/x86_64/navigator-keytrustee/navigator-keytrustee5.repo
yum clean all
;;
esac
}
function add_repo_5_5_0 {
case $DISTRO_NAME in
ubuntu )
@ -358,18 +235,6 @@ DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0}
# Call version-specific script to install the desired version of CDH
case "$DIB_CDH_VERSION" in
5.0)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_0_0
;;
5.3)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_3_0
;;
5.4)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_4_0
;;
5.5)
echo "Installing CDH Version $DIB_CDH_VERSION..."
add_repo_5_5_0

View File

@ -15,18 +15,9 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
# INFO on hadoop versions: http://spark.apache.org/docs/latest/hadoop-third-party-distributions.html
if [ -z "${SPARK_HADOOP_DL:-}" ]; then
case "${DIB_CDH_VERSION:-}" in
5.0)
SPARK_HADOOP_DL=hadoop2.3
;;
5.3)
SPARK_HADOOP_DL=hadoop2.4
;;
5.4)
SPARK_HADOOP_DL=hadoop2.6
;;
CDH4)
SPARK_HADOOP_DL=cdh4
;;
*)
echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported."
echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version."

View File

@ -17,12 +17,6 @@ case "$plugin_type" in
;;
"cloudera" | "spark")
case "$DIB_CDH_VERSION" in
"CDH4")
hadoop="2.2.0";;
"5.0")
hadoop="2.3.0";;
"5.3")
hadoop="2.5.0";;
"5.4" | "5.5" | "5.7" | "5.9")
hadoop="2.6.0";;
esac