From 20126fbde6aff522d39218a9b19643fecd04a48a Mon Sep 17 00:00:00 2001 From: Shu Yingya Date: Wed, 12 Apr 2017 20:32:40 +0800 Subject: [PATCH] Remove some codes of older version builder Building CDH image under version 5.5.0 is no longer support. Remove these useless code. Also, adding the ambari usage info in sahara-image-create command. Change-Id: I6fffe25ee9daf651355611be675137babb67e2a8 --- diskimage-create/README.rst | 6 +- diskimage-create/diskimage-create.sh | 53 +------ .../install.d/50-install-cloudera | 26 +--- .../pre-install.d/10-add-mirror | 135 ------------------ elements/spark/root.d/50-download-spark | 9 -- .../swift_hadoop/post-install.d/81-add-jar | 6 - 6 files changed, 5 insertions(+), 230 deletions(-) diff --git a/diskimage-create/README.rst b/diskimage-create/README.rst index 1bcca846..45a1b319 100644 --- a/diskimage-create/README.rst +++ b/diskimage-create/README.rst @@ -17,7 +17,7 @@ Environment variables the script accepts are 'DIB_HADOOP_VERSION_2_7_1', '[ubuntu|fedora|centos|centos7]_vanilla_hadoop_2_7_1_image_name', 'ubuntu_spark_image_name', 'ubuntu_storm_image_name', 'ambari_[ubuntu|centos|centos7]_image_name', -'cloudera_[5_0|5_3|5_4|5_5|5_7|5_9]_[ubuntu|centos]_image_name', +'cloudera_[5_5|5_7|5_9]_[ubuntu|centos]_image_name', 'mapr_[ubuntu|centos|centos7]_image_name', 'plain_[ubuntu|fedora|centos|centos7]_image_name'. @@ -59,14 +59,14 @@ like this: .. sourcecode:: bash - tox -e venv -- sahara-image-create -v [2.7.1|4|5.0|5.3|5.4|5.5|5.7|5.9] + tox -e venv -- sahara-image-create -v [2.7.1|5.5|5.7|5.9] Also, if you are planning to select which ambari version to target use the '-v' commandline option like this: .. sourcecode:: bash - tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0] + tox -e venv -- sahara-image-create -v [2.2.0.0|2.2.1.0|2.4.2.0] 6. To select which operating system to target use the '-i' commandline option like this: diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index 58a203d4..e3074eea 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -32,7 +32,7 @@ usage() { echo "Usage: $(basename $0)" echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]" echo " [-i ubuntu|fedora|centos|centos7]" - echo " [-v 2.7.1|5.5|5.7|5.9|2.2.0.0|2.2.1.0]" + echo " [-v 2.7.1|5.5|5.7|5.9|2.2.0.0|2.2.1.0|2.4.2.0]" echo " [-r 5.1.0|5.2.0]" echo " [-s 1.3.1|1.6.0]" echo " [-t 0.9.2|1.0.1]" @@ -651,33 +651,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then cloudera_elements_sequence="hadoop-cloudera swift_hadoop kdc" if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then - export DIB_CDH_VERSION="5.0" - cloudera_5_0_ubuntu_image_name=${cloudera_5_0_ubuntu_image_name:-ubuntu_sahara_cloudera_5_0_0} - - # Cloudera supports only 12.04 Ubuntu - export DIB_RELEASE="precise" - image_create ubuntu $cloudera_5_0_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then - export DIB_CDH_VERSION="5.3" - cloudera_5_3_ubuntu_image_name=${cloudera_5_3_ubuntu_image_name:-ubuntu_sahara_cloudera_5_3_0} - - # Cloudera supports only 12.04 Ubuntu - export DIB_RELEASE="precise" - image_create ubuntu $cloudera_5_3_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.4" ]; then - export DIB_CDH_VERSION="5.4" - cloudera_5_4_ubuntu_image_name=${cloudera_5_4_ubuntu_image_name:-ubuntu_sahara_cloudera_5_4_0} - - # Cloudera supports only 12.04 Ubuntu - export DIB_RELEASE="precise" - image_create ubuntu $cloudera_5_4_ubuntu_image_name $cloudera_elements_sequence - unset DIB_CDH_VERSION DIB_RELEASE - fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then export DIB_CDH_VERSION="5.5" cloudera_5_5_ubuntu_image_name=${cloudera_5_5_ubuntu_image_name:-ubuntu_sahara_cloudera_5_5_0} @@ -709,30 +682,6 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then centos_cloudera_elements_sequence="selinux-permissive disable-firewall nc" - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then - export DIB_CDH_VERSION="5.0" - - cloudera_5_0_centos_image_name=${cloudera_5_0_centos_image_name:-centos_sahara_cloudera_5_0_0} - image_create centos $cloudera_5_0_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence - - unset DIB_CDH_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.3" ]; then - export DIB_CDH_VERSION="5.3" - - cloudera_5_3_centos_image_name=${cloudera_5_3_centos_image_name:-centos_sahara_cloudera_5_3_0} - image_create centos $cloudera_5_3_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence - - unset DIB_CDH_VERSION - fi - if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.4" ]; then - export DIB_CDH_VERSION="5.4" - - cloudera_5_4_centos_image_name=${cloudera_5_4_centos_image_name:-centos_sahara_cloudera_5_4_0} - image_create centos $cloudera_5_4_centos_image_name $cloudera_elements_sequence $centos_cloudera_elements_sequence - - unset DIB_CDH_VERSION - fi if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.5" ]; then export DIB_CDH_VERSION="5.5" diff --git a/elements/hadoop-cloudera/install.d/50-install-cloudera b/elements/hadoop-cloudera/install.d/50-install-cloudera index 531ef75e..928e713c 100755 --- a/elements/hadoop-cloudera/install.d/50-install-cloudera +++ b/elements/hadoop-cloudera/install.d/50-install-cloudera @@ -34,7 +34,6 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then zookeeper if [ $DIB_CDH_VERSION \> "5.0" ]; then - # CDH5.0 does not have below packages. install-packages \ flume-ng \ hadoop-kms \ @@ -50,12 +49,7 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then solr-doc \ search \ spark-history-server \ - sqoop2 - fi - - if [ $DIB_CDH_VERSION \> "5.4" ]; then - # we support Kafka from CDH 5.5 - install-packages \ + sqoop2 \ kafka \ kafka-server fi @@ -72,27 +66,9 @@ case "$DIB_CDH_VERSION" in ;; esac HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar" -HADOOP_OPENSTACK_5_4_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.4.0/hadoop-openstack-2.6.0-cdh5.4.0.jar" -HADOOP_OPENSTACK_5_3_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.5.0-cdh5.3.0/hadoop-openstack-2.5.0-cdh5.3.0.jar" -HADOOP_OPENSTACK_5_0_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.3.0-cdh5.0.0/hadoop-openstack-2.3.0-cdh5.0.0.jar" dest=/usr/lib/hadoop-mapreduce/hadoop-openstack.jar case "$DIB_CDH_VERSION" in - 5.0) - wget -O $dest $HADOOP_OPENSTACK_5_0_0_URL - ;; - 5.3) - wget -O $dest $HADOOP_OPENSTACK_5_3_0_URL - ;; - 5.4) - wget -O $dest $HADOOP_OPENSTACK_5_4_0_URL - if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz - fi - ;; 5.5) wget -O $dest $HADOOP_OPENSTACK_5_5_0_URL if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then diff --git a/elements/hadoop-cloudera/pre-install.d/10-add-mirror b/elements/hadoop-cloudera/pre-install.d/10-add-mirror index ac61d578..aafbc7e1 100755 --- a/elements/hadoop-cloudera/pre-install.d/10-add-mirror +++ b/elements/hadoop-cloudera/pre-install.d/10-add-mirror @@ -6,129 +6,6 @@ fi set -eu set -o pipefail -function add_repo_5_0_0 { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' > /etc/apt/sources.list.d/cdh5.list - echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.0.0 contrib' >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - - - echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' > /etc/apt/sources.list.d/cm5.list - echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.0.0 contrib' >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - - - apt-get update - ;; - centos | rhel ) - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.0.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.0.0/' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - yum clean all - ;; - esac -} - -function add_repo_5_3_0 { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' > /etc/apt/sources.list.d/cdh5.list - echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.3.0 contrib' >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - - - echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' > /etc/apt/sources.list.d/cm5.list - echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.3.0 contrib' >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/archive.key | apt-key add - - - apt-get update - ;; - centos | rhel ) - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.3.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.3.0/' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - wget -O /etc/yum.repos.d/kms.repo http://archive.cloudera.com/navigator-keytrustee5/redhat/6/x86_64/navigator-keytrustee/navigator-keytrustee5.repo - - yum clean all - ;; - esac -} - -function add_repo_5_4_0 { - case $DISTRO_NAME in - ubuntu ) - # Add repository with postgresql package (it's dependency of cloudera packages) - # Base image doesn't contain this repo - echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ precise universe multiverse main' >> /etc/apt/sources.list - - # Cloudera repositories - echo 'deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' > /etc/apt/sources.list.d/cdh5.list - echo 'deb-src http://archive.cloudera.com/cdh5/ubuntu/precise/amd64/cdh precise-cdh5.4.0 contrib' >> /etc/apt/sources.list.d/cdh5.list - - wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/precise/amd64/cdh/archive.key | apt-key add - - - echo 'deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' > /etc/apt/sources.list.d/cm5.list - echo 'deb-src http://archive.cloudera.com/cm5/ubuntu/precise/amd64/cm precise-cm5.4.0 contrib' >> /etc/apt/sources.list.d/cm5.list - - wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/precise/amd64/cm/archive.key | apt-key add - - - wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/cloudera.list - wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/precise/amd64/navigator-keytrustee/archive.key | apt-key add - - - apt-get update - ;; - centos | rhel ) - echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo - echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/5.4.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/6/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo - - echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo - echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'baseurl=http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/5.4.0/' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/6/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo - - wget -O /etc/yum.repos.d/kms.repo http://archive.cloudera.com/navigator-keytrustee5/redhat/6/x86_64/navigator-keytrustee/navigator-keytrustee5.repo - - yum clean all - ;; - esac -} - function add_repo_5_5_0 { case $DISTRO_NAME in ubuntu ) @@ -337,18 +214,6 @@ DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$DIB_CDH_VERSION.0} # Call version-specific script to install the desired version of CDH case "$DIB_CDH_VERSION" in - 5.0) - echo "Installing CDH Version $DIB_CDH_VERSION..." - add_repo_5_0_0 - ;; - 5.3) - echo "Installing CDH Version $DIB_CDH_VERSION..." - add_repo_5_3_0 - ;; - 5.4) - echo "Installing CDH Version $DIB_CDH_VERSION..." - add_repo_5_4_0 - ;; 5.5) echo "Installing CDH Version $DIB_CDH_VERSION..." add_repo_5_5_0 diff --git a/elements/spark/root.d/50-download-spark b/elements/spark/root.d/50-download-spark index b0cc7879..2080dbab 100755 --- a/elements/spark/root.d/50-download-spark +++ b/elements/spark/root.d/50-download-spark @@ -15,18 +15,9 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then # INFO on hadoop versions: http://spark.apache.org/docs/latest/hadoop-third-party-distributions.html if [ -z "${SPARK_HADOOP_DL:-}" ]; then case "${DIB_CDH_VERSION:-}" in - 5.0) - SPARK_HADOOP_DL=hadoop2.3 - ;; - 5.3) - SPARK_HADOOP_DL=hadoop2.4 - ;; 5.4) SPARK_HADOOP_DL=hadoop2.6 ;; - CDH4) - SPARK_HADOOP_DL=cdh4 - ;; *) echo "WARNING: Cloudera CDH $DIB_CDH_VERSION not supported." echo "WARNING: use the SPARK_DOWNLOAD_URL variable to install a custom Spark version." diff --git a/elements/swift_hadoop/post-install.d/81-add-jar b/elements/swift_hadoop/post-install.d/81-add-jar index 22778c8f..627af793 100755 --- a/elements/swift_hadoop/post-install.d/81-add-jar +++ b/elements/swift_hadoop/post-install.d/81-add-jar @@ -17,12 +17,6 @@ case "$plugin_type" in ;; "cloudera" | "spark") case "$DIB_CDH_VERSION" in - "CDH4") - hadoop="2.2.0";; - "5.0") - hadoop="2.3.0";; - "5.3") - hadoop="2.5.0";; "5.4" | "5.5" | "5.7" | "5.9") hadoop="2.6.0";; esac