diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh index d7be8430..c0310af6 100755 --- a/diskimage-create/diskimage-create.sh +++ b/diskimage-create/diskimage-create.sh @@ -29,7 +29,7 @@ usage() { echo "Usage: $(basename $0)" echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]" echo " [-i ubuntu|fedora|centos|centos7]" - echo " [-v 2.6|2.7.1|4|5.0|5.3|5.4|5.5|2.2.0.0|2.2.1.0]" + echo " [-v 2.6|2.7.1|4|5.0|5.3|5.4|5.5|5.7|2.2.0.0|2.2.1.0]" echo " [-r 5.0.0|5.1.0]" echo " [-s 1.3.1|1.6.0]" echo " [-d]" @@ -166,18 +166,23 @@ case "$PLUGIN" in esac case "$HADOOP_VERSION" in - "" | "5.0" | "5.3" | "5.4" | "5.5");; + "" | "5.0" | "5.3" | "5.4" | "5.5" | "5.7");; *) echo -e "Unknown hadoop version selected.\nAborting" exit 1 ;; esac + if [ "$BASE_IMAGE_OS" = "centos7" ]; then - if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" ]; then - echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5" + if [ ! -z "$HADOOP_VERSION" -a ! "$HADOOP_VERSION" = "5.5" -a ! "$HADOOP_VERSION" = "5.7" ]; then + echo -e "Unsupported version combination, Centos 7 can only be used with CDH 5.5 and CDH 5.7" exit 1 fi fi + + if [ -n "$DIB_CDH_MINOR_VERSION" ]; then + echo -e "Continuing image building with custom CDH version: \"$DIB_CDH_MINOR_VERSION\".\n" + fi ;; "spark") case "$BASE_IMAGE_OS" in @@ -637,6 +642,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then export plugin_type="cloudera" export DIB_HDFS_LIB_DIR="/usr/lib/hadoop-mapreduce" + if [ -n "$DIB_CDH_MINOR_VERSION" ]; then + # cut minor version number, e.g. from 5.7.1 to 5.7 + # this is needed if user specified minor version but didn't specify + # hadoop version by '-v' parameter + HADOOP_VERSION=${DIB_CDH_MINOR_VERSION%.*} + fi + export DIB_CDH_MINOR_VERSION=${DIB_CDH_MINOR_VERSION:-$HADOOP_VERSION.0} + cloudera_elements_sequence="hadoop-cloudera swift_hadoop" if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.0" ]; then @@ -675,6 +688,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then image_create ubuntu $cloudera_5_5_ubuntu_image_name $cloudera_elements_sequence unset DIB_CDH_VERSION DIB_RELEASE fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then + cloudera_5_7_ubuntu_image_name=${cloudera_5_7_ubuntu_image_name:-ubuntu_sahara_cloudera_$DIB_CDH_MINOR_VERSION} + + export DIB_CDH_VERSION="5.7" + export DIB_RELEASE="trusty" + image_create ubuntu $cloudera_5_7_ubuntu_image_name $cloudera_elements_sequence + unset DIB_CDH_VERSION DIB_RELEASE + fi fi if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then @@ -721,10 +742,19 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then cloudera_5_5_centos7_image_name=${cloudera_5_5_centos7_image_name:-centos7_sahara_cloudera_5_5_0} image_create centos7 $cloudera_5_5_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence + unset DIB_CDH_VERSION + fi + if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "5.7" ]; then + export DIB_CDH_VERSION="5.7" + + cloudera_5_7_centos7_image_name=${cloudera_5_7_centos7_image_name:-centos7_sahara_cloudera_$DIB_CDH_MINOR_VERSION} + image_create centos7 $cloudera_5_7_centos7_image_name $cloudera_elements_sequence $centos7_cloudera_elements_sequence + unset DIB_CDH_VERSION fi fi + unset DIB_CDH_MINOR_VERSION unset DIB_HDFS_LIB_DIR unset DIB_MIN_TMPFS unset plugin_type diff --git a/elements/hadoop-cloudera/README.rst b/elements/hadoop-cloudera/README.rst index 2a3ac067..412cf2a4 100644 --- a/elements/hadoop-cloudera/README.rst +++ b/elements/hadoop-cloudera/README.rst @@ -28,3 +28,7 @@ DIB_CDH_HDFS_ONLY :Description: If set will install only the namenode and datanode packages with their dependencies. +DIB_CDH_MINOR_VERSION + :Required: No + :Description: If set will install minor version of CDH. Available minor + versions are 5.7.x. diff --git a/elements/hadoop-cloudera/install.d/50-install-cloudera b/elements/hadoop-cloudera/install.d/50-install-cloudera index 04b3ea36..2b6424e2 100755 --- a/elements/hadoop-cloudera/install.d/50-install-cloudera +++ b/elements/hadoop-cloudera/install.d/50-install-cloudera @@ -55,6 +55,11 @@ if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then fi fi +case "$DIB_CDH_VERSION" in + 5.7) + HADOOP_OPENSTACK_5_7_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$DIB_CDH_MINOR_VERSION/hadoop-openstack-2.6.0-cdh$DIB_CDH_MINOR_VERSION.jar" + ;; +esac HADOOP_OPENSTACK_5_5_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.5.0/hadoop-openstack-2.6.0-cdh5.5.0.jar" HADOOP_OPENSTACK_5_4_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh5.4.0/hadoop-openstack-2.6.0-cdh5.4.0.jar" HADOOP_OPENSTACK_5_3_0_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.5.0-cdh5.3.0/hadoop-openstack-2.5.0-cdh5.3.0.jar" @@ -79,10 +84,21 @@ case "$DIB_CDH_VERSION" in ;; 5.5) wget -O $dest $HADOOP_OPENSTACK_5_5_0_URL - # Create links to keep backward version support. - ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz - ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz + if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then + # Create links to keep backward version support. + ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz + ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz + ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz + fi + ;; + 5.7) + wget -O $dest $HADOOP_OPENSTACK_5_7_x_URL + if [ -z "${DIB_CDH_HDFS_ONLY:-}" ]; then + # Create links to keep backward version support. + ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz + ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz + ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz + fi ;; *) echo "Unhandled version $DIB_CDH_VERSION for hadoop-openstack.jar" diff --git a/elements/hadoop-cloudera/pre-install.d/10-add-mirror b/elements/hadoop-cloudera/pre-install.d/10-add-mirror index ee35bf2d..74e8a063 100755 --- a/elements/hadoop-cloudera/pre-install.d/10-add-mirror +++ b/elements/hadoop-cloudera/pre-install.d/10-add-mirror @@ -156,22 +156,88 @@ function add_repo_5_5_0 { apt-get update ;; centos | rhel | centos7 | rhel7 ) + case $DISTRO_NAME in + centos | rhel ) + releasever=6 + ;; + centos7 | rhel7 ) + releasever=7 + ;; + esac echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/5.5.0/' >> /etc/yum.repos.d/cloudera-cdh5.repo - echo 'gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-cdh5.repo + echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/5.5.0/" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/5.5.0/' >> /etc/yum.repos.d/cloudera-manager.repo - echo 'gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/cloudera-manager.repo + echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/5.5.0/" >> /etc/yum.repos.d/cloudera-manager.repo + echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo - echo 'baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/5.5.0/' >> /etc/yum.repos.d/kms.repo - echo 'gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera' >> /etc/yum.repos.d/kms.repo + echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/5.5.0/" >> /etc/yum.repos.d/kms.repo + echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo + + yum clean all + ;; + esac +} + +function add_repo_5_7_x { + case $DISTRO_NAME in + ubuntu ) + # Add repository with postgresql package (it's dependency of cloudera packages) + # Base image doesn't contain this repo + echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ trusty universe multiverse main' >> /etc/apt/sources.list + + # Cloudera repositories + echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cdh5.list + echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh trusty-cdh$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cdh5.list + + wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/trusty/amd64/cdh/archive.key | apt-key add - + + echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" > /etc/apt/sources.list.d/cm5.list + echo "deb-src http://archive.cloudera.com/cm5/ubuntu/trusty/amd64/cm trusty-cm$DIB_CDH_MINOR_VERSION contrib" >> /etc/apt/sources.list.d/cm5.list + + wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/trusty/amd64/cm/archive.key | apt-key add - + + wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/cloudera.list + wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/trusty/amd64/navigator-keytrustee/archive.key | apt-key add - + + #change repository priority + echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin + + apt-get update + ;; + centos | rhel | centos7 | rhel7 ) + case $DISTRO_NAME in + centos | rhel ) + releasever=6 + ;; + centos7 | rhel7 ) + releasever=7 + ;; + esac + echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo + echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo "baseurl=http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/$releasever/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo + + echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo + echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo + echo "baseurl=http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/cloudera-manager.repo + echo "gpgkey = http://archive.cloudera.com/cm5/redhat/$releasever/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo + echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo + + echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo + echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo + echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/$DIB_CDH_MINOR_VERSION/" >> /etc/yum.repos.d/kms.repo + echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/$releasever/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo yum clean all @@ -197,6 +263,10 @@ case "$DIB_CDH_VERSION" in echo "Installing CDH Version $DIB_CDH_VERSION..." add_repo_5_5_0 ;; + 5.7) + echo "Installing CDH Version $DIB_CDH_MINOR_VERSION..." + add_repo_5_7_x + ;; *) echo "Invalid CDH Version : $DIB_CDH_VERSION specified, exiting install." exit 1 diff --git a/elements/swift_hadoop/post-install.d/81-add-jar b/elements/swift_hadoop/post-install.d/81-add-jar index c7be2318..f60042ec 100755 --- a/elements/swift_hadoop/post-install.d/81-add-jar +++ b/elements/swift_hadoop/post-install.d/81-add-jar @@ -25,12 +25,12 @@ case "$plugin_type" in hadoop="2.3.0";; "5.3") hadoop="2.5.0";; - "5.4" | "5.5") + "5.4" | "5.5" | "5.7") hadoop="2.6.0";; esac case "$plugin_type" in "cloudera") - DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop-cdh$DIB_CDH_VERSION.0.jar";; + DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop-cdh$DIB_CDH_MINOR_VERSION.jar";; "spark") DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack.jar";; esac diff --git a/tools/gate/build-images b/tools/gate/build-images index 03f18e2b..21252e8d 100755 --- a/tools/gate/build-images +++ b/tools/gate/build-images @@ -7,19 +7,16 @@ case "$PLUGIN" in # manually call each distro for cloudera, so we can cleanup # cache and generated images from the previous run # (saving space a bit) - for distro in ubuntu centos; do + for distro in ubuntu centos centos7; do # NOTE(vgridnev): Number of versions are growing; testing only 2 latest # releases of CDH - for version in 5.4 5.5; do + # NOTE(mionkin) centos7 is available only for CDH 5.5 and 5.7 + for version in 5.5 5.7; do tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version find . -maxdepth 1 -name '*cloudera*.qcow2' -delete sudo rm -rf ~/.cache/image-create done done - # NOTE(mionkin) centos7 is available only for cloudera 5.5 - tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i centos7 -v 5.5 - find . -maxdepth 1 -name '*cloudera*.qcow2' -delete - sudo rm -rf ~/.cache/image-create ;; *) tox -e venv -- sahara-image-create -x -u -p $PLUGIN