Merge "Add different version builders support for Cloudera plugin"
This commit is contained in:
Jenkins
Gerrit Code Review
commit
e315186c93
|
|
@ -23,7 +23,7 @@ usage() {
|
|||
echo "Usage: $(basename $0)"
|
||||
echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]"
|
||||
echo " [-i ubuntu|fedora|centos]"
|
||||
echo " [-v 1|2|2.3|2.4|2.6|plain]"
|
||||
echo " [-v 1|2|2.3|2.4|2.6|5.0|5.3|plain]"
|
||||
echo " [-r 3.1.1|4.0.1]"
|
||||
echo " [-d]"
|
||||
echo " [-m]"
|
||||
|
|
@ -113,8 +113,10 @@ fi
|
|||
|
||||
if [ -n "$HADOOP_VERSION" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2" -a "$HADOOP_VERSION" != "plain" ]; then
|
||||
if [ "$PLUGIN" = "vanilla" -a "$HADOOP_VERSION" != "1" -a "$HADOOP_VERSION" != "2.3" -a "$HADOOP_VERSION" != "2.4" -a "$HADOOP_VERSION" != "2.6" -a "$HADOOP_VERSION" != "plain" ]; then
|
||||
echo -e "Unknown hadoop version selected.\nAborting"
|
||||
exit 1
|
||||
if [ "$PLUGIN" = "cloudera" -a "$HADOOP_VERSION" != "5.0" -a "$HADOOP_VERSION" != "5.3" ]; then
|
||||
echo -e "Unknown hadoop version selected.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
|
|
@ -495,43 +497,86 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "hdp" ]; then
|
|||
unset BASE_IMAGE_FILE DIB_IMAGE_SIZE DIB_CLOUD_IMAGES
|
||||
fi
|
||||
|
||||
#########################
|
||||
# Images for CDH plugin #
|
||||
#########################
|
||||
|
||||
if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
|
||||
echo "For cloudera plugin option -v is ignored"
|
||||
export EXTJS_DOWNLOAD_URL=${EXTJS_DOWNLOAD_URL:-"http://extjs.com/deploy/ext-2.2.zip"}
|
||||
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
|
||||
cloudera_ubuntu_image_name=${cloudera_ubuntu_image_name:-ubuntu_sahara_cloudera_latest}
|
||||
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
|
||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
|
||||
cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0}
|
||||
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
|
||||
|
||||
if [ -n "$USE_MIRRORS" ]; then
|
||||
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
|
||||
if [ -n "$USE_MIRRORS" ]; then
|
||||
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
|
||||
fi
|
||||
|
||||
# Cloudera supports only 12.04 Ubuntu
|
||||
export DIB_CDH_VERSION="5.0"
|
||||
export DIB_RELEASE="precise"
|
||||
disk-image-create $cloudera_elements_sequence -o $cloudera_5_0_ubuntu_image_name
|
||||
mv $cloudera_5_0_ubuntu_image_name.qcow2 ../
|
||||
unset DIB_CDH_VERSION DIB_RELEASE
|
||||
fi
|
||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
|
||||
cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0}
|
||||
cloudera_elements_sequence="base vm ubuntu hadoop-cloudera"
|
||||
|
||||
# Cloudera supports only 12.04 Ubuntu
|
||||
export DIB_RELEASE="precise"
|
||||
if [ -n "$USE_MIRRORS" ]; then
|
||||
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$ubuntu_elements_sequence apt-mirror"
|
||||
fi
|
||||
|
||||
disk-image-create $cloudera_elements_sequence -o $cloudera_ubuntu_image_name
|
||||
mv $cloudera_ubuntu_image_name.qcow2 ../
|
||||
unset DIB_RELEASE
|
||||
# Cloudera supports only 12.04 Ubuntu
|
||||
export DIB_CDH_VERSION="5.3"
|
||||
export DIB_RELEASE="precise"
|
||||
disk-image-create $cloudera_elements_sequence -o $cloudera_5_3_ubuntu_image_name
|
||||
mv $cloudera_5_3_ubuntu_image_name.qcow2 ../
|
||||
unset DIB_CDH_VERSION DIB_RELEASE
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then
|
||||
# CentOS cloud image:
|
||||
# - Disable including 'base' element for CentOS
|
||||
# - Export link and filename for CentOS cloud image to download
|
||||
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
|
||||
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
|
||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then
|
||||
# CentOS cloud image:
|
||||
# - Disable including 'base' element for CentOS
|
||||
# - Export link and filename for CentOS cloud image to download
|
||||
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
|
||||
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
|
||||
export DIB_CDH_VERSION="5.0"
|
||||
|
||||
cloudera_centos_image_name=${cloudera_centos_image_name:-centos_sahara_cloudera_latest}
|
||||
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
|
||||
cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0}
|
||||
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
|
||||
|
||||
if [ -n "$USE_MIRRORS"]; then
|
||||
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
|
||||
if [ -n "$USE_MIRRORS"]; then
|
||||
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
|
||||
fi
|
||||
|
||||
disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_0_centos_image_name
|
||||
mv $cloudera_5_0_centos_image_name.qcow2 ../
|
||||
|
||||
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION
|
||||
fi
|
||||
if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then
|
||||
# CentOS cloud image:
|
||||
# - Disable including 'base' element for CentOS
|
||||
# - Export link and filename for CentOS cloud image to download
|
||||
export BASE_IMAGE_FILE="CentOS-6.6-cloud-init-20141118.qcow2"
|
||||
export DIB_CLOUD_IMAGES="http://sahara-files.mirantis.com"
|
||||
export DIB_CDH_VERSION="5.3"
|
||||
|
||||
disk-image-create $cloudera_elements_sequence -n -o $cloudera_centos_image_name
|
||||
mv $cloudera_centos_image_name.qcow2 ../
|
||||
cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0}
|
||||
cloudera_elements_sequence="base vm rhel hadoop-cloudera redhat-lsb selinux-permissive"
|
||||
|
||||
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES
|
||||
if [ -n "$USE_MIRRORS"]; then
|
||||
[ -n "$CENTOS_MIRROR" ] && cloudera_elements_sequence="$cloudera_elements_sequence centos-mirror"
|
||||
fi
|
||||
|
||||
disk-image-create $cloudera_elements_sequence -n -o $cloudera_5_3_centos_image_name
|
||||
mv $cloudera_5_3_centos_image_name.qcow2 ../
|
||||
|
||||
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES DIB_CDH_VERSION
|
||||
fi
|
||||
fi
|
||||
unset EXTJS_DOWNLOAD_URL
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -12,7 +12,6 @@ install-packages \
|
|||
cloudera-manager-daemons \
|
||||
cloudera-manager-server \
|
||||
cloudera-manager-server-db-2 \
|
||||
flume-ng \
|
||||
hadoop-hdfs-datanode \
|
||||
hadoop-hdfs-namenode \
|
||||
hadoop-hdfs-secondarynamenode \
|
||||
|
|
@ -21,25 +20,32 @@ install-packages \
|
|||
hadoop-yarn-nodemanager \
|
||||
hadoop-yarn-resourcemanager \
|
||||
hbase \
|
||||
hbase-solr \
|
||||
hive-hcatalog \
|
||||
hive-metastore \
|
||||
hive-server2 \
|
||||
hive-webhcat-server \
|
||||
hue \
|
||||
impala \
|
||||
impala-server \
|
||||
impala-state-store \
|
||||
impala-catalog \
|
||||
ntp \
|
||||
oozie \
|
||||
oracle-j2sdk1.7 \
|
||||
sentry \
|
||||
solr-server \
|
||||
spark-history-server \
|
||||
sqoop2 \
|
||||
spark-core \
|
||||
zookeeper
|
||||
|
||||
if [ $DIB_CDH_VERSION != "5.0" ]; then
|
||||
# CDH5.0 does not have below packages.
|
||||
install-packages \
|
||||
flume-ng \
|
||||
hbase-solr \
|
||||
impala \
|
||||
impala-server \
|
||||
impala-state-store \
|
||||
impala-catalog \
|
||||
sentry \
|
||||
solr-server \
|
||||
spark-history-server \
|
||||
sqoop2
|
||||
fi
|
||||
|
||||
if [ $(lsb_release -is) = 'Ubuntu' ]; then
|
||||
unset RUNLEVEL
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -16,9 +16,7 @@ hive-server2 \
|
|||
hive-webhcat-server \
|
||||
hue \
|
||||
oozie \
|
||||
postgresql \
|
||||
solr-server \
|
||||
spark-history-server
|
||||
postgresql
|
||||
do
|
||||
if [ $(lsb_release -is) = 'Ubuntu' ]; then
|
||||
update-rc.d -f $i remove
|
||||
|
|
@ -27,6 +25,21 @@ do
|
|||
fi
|
||||
done
|
||||
|
||||
if [ $DIB_CDH_VERSION != "5.0" ]; then
|
||||
for i in impala-catalog \
|
||||
impala-server \
|
||||
impala-state-store \
|
||||
solr-server \
|
||||
spark-history-server
|
||||
do
|
||||
if [ $(lsb_release -is) = 'Ubuntu' ]; then
|
||||
update-rc.d -f $i remove
|
||||
else
|
||||
chkconfig $i off
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
if [ $(lsb_release -is) = 'CentOS' ]; then
|
||||
chkconfig iptables off
|
||||
chkconfig ip6tables off
|
||||
|
|
|
|||
|
|
@ -2,23 +2,92 @@
|
|||
|
||||
distro=$(lsb_release -is)
|
||||
|
||||
case $distro in
|
||||
Ubuntu)
|
||||
# Add repository with postgresql package (it's dependency of cloudera packages)
|
||||
# Base image doesn't contain this repo
|
||||
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
|
||||
|
||||
# Cloudera repositories
|
||||
wget -O /etc/apt/sources.list.d/cdh5.list http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/cloudera.list
|
||||
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
|
||||
function add_repo_5_0_0 {
|
||||
case $distro in
|
||||
Ubuntu)
|
||||
# Add repository with postgresql package (it's dependency of cloudera packages)
|
||||
# Base image doesn't contain this repo
|
||||
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
|
||||
|
||||
wget -O /etc/apt/sources.list.d/cm5.list http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/cloudera.list
|
||||
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
|
||||
# Cloudera repositories
|
||||
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list
|
||||
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
|
||||
|
||||
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
|
||||
|
||||
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list
|
||||
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list
|
||||
|
||||
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
|
||||
|
||||
apt-get update
|
||||
;;
|
||||
CentOS)
|
||||
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
|
||||
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
yum clean all
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
function add_repo_5_3_0 {
|
||||
case $distro in
|
||||
Ubuntu)
|
||||
# Add repository with postgresql package (it's dependency of cloudera packages)
|
||||
# Base image doesn't contain this repo
|
||||
echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list
|
||||
|
||||
# Cloudera repositories
|
||||
echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list
|
||||
echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list
|
||||
|
||||
wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add -
|
||||
|
||||
echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list
|
||||
echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list
|
||||
|
||||
wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add -
|
||||
|
||||
apt-get update
|
||||
;;
|
||||
CentOS)
|
||||
echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
|
||||
|
||||
echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
|
||||
yum clean all
|
||||
;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Call version-specific script to install the desired version of CDH
|
||||
if [[ $DIB_CDH_VERSION == "5.0" ]]; then
|
||||
echo "Installing CDH Version $DIB_CDH_VERSION..."
|
||||
add_repo_5_0_0
|
||||
else
|
||||
if [[ $DIB_CDH_VERSION == "5.3" ]]; then
|
||||
echo "Installing CDH Version $DIB_CDH_VERSION..."
|
||||
add_repo_5_3_0
|
||||
else
|
||||
echo "Invalid CDH Version specified, exiting install."
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
apt-get update
|
||||
;;
|
||||
CentOS)
|
||||
wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cdh5/redhat/6/x86_64/cdh/cloudera-cdh5.repo
|
||||
wget -P /etc/yum.repos.d/ http://archive-primary.cloudera.com/cm5/redhat/6/x86_64/cm/cloudera-manager.repo
|
||||
;;
|
||||
esac
|
||||
|
|
|
|||
Loading…
Reference in New Issue