Merge "Image generation for CDH 5.11.0"

2017-12-07 13:53:23 +00:00 · 2017-12-07 13:53:23 +00:00 · ff794e9390
parent 77d0bf8f7e 03c6ebaecf
commit ff794e9390
15 changed files with 422 additions and 0 deletions
--- a/releasenotes/notes/cdh_5_11_0_image_generation_validation-6334ef6d04950935.yaml
+++ b/releasenotes/notes/cdh_5_11_0_image_generation_validation-6334ef6d04950935.yaml
@ -0,0 +1,4 @@
+---
+features:
+  - Enables the creation and validation of CDH 5.11.0 images using the
+    new image generation process where libguestfs replaces the use of DIB.
--- a/sahara/plugins/cdh/v5_11_0/images.py
+++ b/sahara/plugins/cdh/v5_11_0/images.py
@ -0,0 +1,43 @@
+# Copyright (c) 2016 Red Hat, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from sahara.plugins import images
+from sahara.plugins import utils as plugin_utils
+
+
+_validator = images.SaharaImageValidator.from_yaml(
+    'plugins/cdh/v5_11_0/resources/images/image.yaml',
+    resource_roots=['plugins/cdh/v5_11_0/resources/images'])
+
+
+def get_image_arguments():
+    return _validator.get_argument_list()
+
+
+def pack_image(remote, test_only=False, image_arguments=None):
+    _validator.validate(remote, test_only=test_only,
+                        image_arguments=image_arguments)
+
+
+def validate_images(cluster, test_only=False, image_arguments=None):
+    image_arguments = get_image_arguments()
+    if not test_only:
+        instances = plugin_utils.get_instances(cluster)
+    else:
+        instances = plugin_utils.get_instances(cluster)[0]
+    for instance in instances:
+        with instance.remote() as r:
+            _validator.validate(r, test_only=test_only,
+                                image_arguments=image_arguments)
--- a/sahara/plugins/cdh/v5_11_0/resources/images/centos/disable_firewall
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/centos/disable_firewall
@ -0,0 +1,20 @@
+#!/bin/bash
+
+check=$(systemctl --no-pager list-unit-files iptables.service | grep 'enabled' | wc -l)
+
+if [ $check -eq 1 ]; then
+    if [ $test_only -eq 0 ]; then
+        if type -p systemctl && [[ "$(systemctl --no-pager list-unit-files firewalld)" =~ 'enabled' ]]; then
+            systemctl disable firewalld
+        fi
+
+        if type -p service; then
+            service ip6tables save
+            service iptables save
+            chkconfig ip6tables off
+            chkconfig iptables off
+        fi
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/centos/selinux_permissive
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/centos/selinux_permissive
@ -0,0 +1,13 @@
+#!/bin/bash
+
+check=$(cat /etc/selinux/config | grep "SELINUX=permissive" | wc -l)
+echo $check
+
+if [ $check -eq  0 ]; then
+    if [ $test_only -eq 0 ]; then
+        echo "SELINUX=permissive" > /etc/selinux/config
+        echo "SELINUXTYPE=targeted" >> /etc/selinux/config
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/centos/turn_off_services
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/centos/turn_off_services
@ -0,0 +1,33 @@
+#!/bin/bash
+
+check=$(systemctl --no-pager list-unit-files oozie.service | grep 'enabled' | wc -l)
+
+if [ "$check" != "on" ]; then
+    if [ $test_only -eq 0 ]; then
+        for i in cloudera-scm-agent \
+        cloudera-scm-server \
+        cloudera-scm-server-db \
+        hadoop-hdfs-datanode \
+        hadoop-hdfs-namenode \
+        hadoop-hdfs-secondarynamenode \
+        hadoop-mapreduce-historyserver \
+        hadoop-yarn-nodemanager \
+        hadoop-yarn-resourcemanager \
+        hive-metastore \
+        hive-server2 \
+        hive-webhcat-server \
+        hue \
+        oozie \
+        postgresql \
+        impala-catalog \
+        impala-server \
+        impala-state-store \
+        solr-server \
+        spark-history-server
+        do
+            chkconfig $i off
+        done
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/centos/wget_repo
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/centos/wget_repo
@ -0,0 +1,33 @@
+#!/bin/bash
+
+if [ ! -f /etc/yum.repos.d/cloudera-cdh5.repo ]; then
+    if [ $test_only -eq 0 ]; then
+        echo '[cloudera-cdh5]' > /etc/yum.repos.d/cloudera-cdh5.repo
+        echo "name=Cloudera's Distribution for Hadoop, Version 5" >> /etc/yum.repos.d/cloudera-cdh5.repo
+        echo "baseurl=http://archive.cloudera.com/cdh5/redhat/7/x86_64/cdh/$plugin_version/" >> /etc/yum.repos.d/cloudera-cdh5.repo
+        echo "gpgkey = http://archive.cloudera.com/cdh5/redhat/7/x86_64/cdh/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-cdh5.repo
+        echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-cdh5.repo
+
+        echo '[cloudera-manager]' > /etc/yum.repos.d/cloudera-manager.repo
+        echo 'name=Cloudera Manager' >> /etc/yum.repos.d/cloudera-manager.repo
+        echo "baseurl=http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/$plugin_version/" >> /etc/yum.repos.d/cloudera-manager.repo
+        echo "gpgkey = http://archive.cloudera.com/cm5/redhat/7/x86_64/cm/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-manager.repo
+        echo 'gpgcheck = 1' >> /etc/yum.repos.d/cloudera-manager.repo
+
+        echo '[navigator-keytrustee]' > /etc/yum.repos.d/kms.repo
+        echo "name=Cloudera's Distribution for navigator-Keytrustee, Version 5" >> /etc/yum.repos.d/kms.repo
+        echo "baseurl=http://archive.cloudera.com/navigator-keytrustee5/redhat/7/x86_64/navigator-keytrustee/$plugin_version/" >> /etc/yum.repos.d/kms.repo
+        echo "gpgkey = http://archive.cloudera.com/navigator-keytrustee5/redhat/7/x86_64/navigator-keytrustee/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/kms.repo
+        echo 'gpgcheck = 1' >> /etc/yum.repos.d/kms.repo
+
+        echo "[cloudera-kafka]" > /etc/yum.repos.d/cloudera-kafka.repo
+        echo "name=Cloudera's Distribution for kafka, Version 2.0.2" >> /etc/yum.repos.d/cloudera-kafka.repo
+        echo "baseurl=http://archive.cloudera.com/kafka/redhat/7/x86_64/kafka/2.0.2/" >> /etc/yum.repos.d/cloudera-kafka.repo
+        echo "gpgkey = http://archive.cloudera.com/kafka/redhat/7/x86_64/kafka/RPM-GPG-KEY-cloudera" >> /etc/yum.repos.d/cloudera-kafka.repo
+        echo "gpgcheck = 1" >> /etc/yum.repos.d/cloudera-kafka.repo
+
+        yum clean all
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/common/add_jar
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/common/add_jar
@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+hadoop="2.6.0"
+
+HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop-cdh$plugin_version.jar"
+
+HADOOP_SWIFT_BUILD_LOCATION="http://tarballs.openstack.org/sahara/dist/hadoop-openstack/master"
+SWIFT_LIB_URI="$HADOOP_SWIFT_BUILD_LOCATION/hadoop-openstack-${hadoop}.jar"
+HADOOP_SWIFT_JAR_NAME="hadoop-openstack.jar"
+
+if [ ! -f $hdfs_lib_dir/$HADOOP_SWIFT_JAR_NAME ]; then
+    if [ $test_only -eq 0 ]; then
+        wget -O $hdfs_lib_dir/$HADOOP_SWIFT_JAR_NAME $SWIFT_LIB_URI
+
+        if [ $? -ne 0 ]; then
+            echo -e "Could not download Swift Hadoop FS implementation.\nAborting"
+            exit 1
+        fi
+
+        chmod 0644 $hdfs_lib_dir/$HADOOP_SWIFT_JAR_NAME
+    else
+        exit 0
+    fi
+fi
+
--- a/sahara/plugins/cdh/v5_11_0/resources/images/common/install_cloudera
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/common/install_cloudera
@ -0,0 +1,17 @@
+#!/bin/bash
+
+HADOOP_OPENSTACK_5_9_x_URL="https://repository.cloudera.com/artifactory/repo/org/apache/hadoop/hadoop-openstack/2.6.0-cdh$plugin_version/hadoop-openstack-2.6.0-cdh$plugin_version.jar"
+
+dest=/usr/lib/hadoop-mapreduce/hadoop-openstack.jar
+
+if [ ! -L "/usr/lib/oozie/oozie-sharelib-yarn.tar.gz" ]; then
+    if [ $test_only -eq 0 ]; then
+        wget -O $dest $HADOOP_OPENSTACK_5_9_x_URL
+
+        ln -s /usr/lib/oozie/oozie-sharelib-yarn /usr/lib/oozie/oozie-sharelib-yarn.tar.gz
+        ln -s /usr/lib/oozie/oozie-sharelib-mr1 /usr/lib/oozie/oozie-sharelib-mr1.tar.gz
+        ln -s /usr/lib/oozie/oozie-sharelib-yarn.tar.gz /usr/lib/oozie/oozie-sharelib.tar.gz
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/common/install_extjs
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/common/install_extjs
@ -0,0 +1,22 @@
+#!/bin/bash
+
+EXTJS_DESTINATION_DIR="/var/lib/oozie"
+EXTJS_DOWNLOAD_URL="http://tarballs.openstack.org/sahara/dist/common-artifacts/ext-2.2.zip"
+
+extjs_archive=/tmp/$(basename $EXTJS_DOWNLOAD_URL)
+
+if [ ! -n "$EXTJS_DESTINATION_DIR" ]; then
+    if [ $test_only -eq 0 ]; then
+        wget -O $extjs_archive $EXTJS_DOWNLOAD_URL
+
+        mkdir -p $EXTJS_DESTINATION_DIR
+        if [ -z "${EXTJS_NO_UNPACK:-}" ]; then
+            unzip -d "$EXTJS_DESTINATION_DIR" $extjs_archive
+            rm -f $extjs_archive
+        else
+            mv $extjs_archive $EXTJS_DESTINATION_DIR
+        fi
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/common/unlimited_security_artifacts
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/common/unlimited_security_artifacts
@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ ! -n /tmp/UnlimitedPolicy/ ]; then
+    if [ $test_only -eq 0 ]; then
+        mkdir /tmp/UnlimitedPolicy/
+        wget $unlimited_security_location/local_policy.jar -O /tmp/UnlimitedPolicy/local_policy.jar
+        wget $unlimited_security_location/US_export_policy.jar -O /tmp/UnlimitedPolicy/US_export_policy.jar
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/image.yaml
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/image.yaml
@ -0,0 +1,131 @@
+arguments:
+  java_distro:
+    default: cloudera-jdk
+    description: The distribution of Java to install. Defaults to openjdk.
+    choices:
+      - openjdk
+      - oracle-java
+      - cloudera-jdk
+  plugin_version:
+    default: 5.11.0
+    description: The distribution of CDH to install. Defaults to 5.11.0.
+    hidden: True
+    required: False
+    choices:
+      - 5.11.0
+      - 5.11.1
+      - 5.11.2
+  hdfs_lib_dir:
+    default: /usr/lib/hadoop-mapreduce
+    description: The path to HDFS_LIB_DIR. Default to /usr/lib/hadoop-mapreduce
+    required: False
+  unlimited_security_location:
+    default: http://sahara-files.mirantis.com/kerberos-artifacts/
+    description: Place where UnlimitedSecurity polices are located
+    required: False
+
+validators:
+  - script: common/install_extjs
+  - os_case:
+      - centos:
+          - package: wget
+          - script:
+              centos/wget_repo:
+                env_vars: [plugin_version]
+      - ubuntu:
+          - script:
+              ubuntu/wget_repo:
+                env_vars: [plugin_version]
+  - argument_case:
+          argument_name: java_distro
+          cases:
+            openjdk:
+              - any:
+                - all:
+                  - package: java-1.8.0-openjdk-devel
+                - all:
+                  - package: java-1.7.0-openjdk-devel
+            cloudera-jdk:
+              - all:
+                - package: oracle-j2sdk1.7
+  - package: ntp
+  - package:
+    - cloudera-manager-agent
+    - cloudera-manager-daemons
+    - cloudera-manager-server
+    - cloudera-manager-server-db-2
+  - package:
+    - hadoop-hdfs-namenode
+    - hadoop-hdfs-datanode
+    - hadoop-hdfs-secondarynamenode
+    - hadoop-mapreduce
+    - hadoop-mapreduce-historyserver
+    - hadoop-yarn-nodemanager
+    - hadoop-yarn-resourcemanager
+  - package:
+    - hbase
+    - hbase-solr
+  - package:
+    - hive-hcatalog
+    - hive-metastore
+    - hive-server2
+    - hive-webhcat-server
+    - hue
+  - package:
+    - oozie
+    - spark-core
+  - os_case:
+    - centos:
+      - package: spark-history-server
+  - package: zookeeper
+  - package: unzip
+  - package: flume-ng
+  - package: hadoop-kms
+  - package:
+    - impala
+    - impala-server
+    - impala-state-store
+    - impala-catalog
+    - impala-shell
+  - package: keytrustee-keyprovider
+  - package:
+    - sentry
+    - solr-server
+    - solr-doc
+    - search
+    - sqoop2
+  - package:
+    - kafka
+    - kafka-server
+  - script:
+      common/install_cloudera:
+        env_vars: [plugin_version]
+  - os_case:
+    - centos:
+      - script: centos/turn_off_services
+    - ubuntu:
+      - script: ubuntu/turn_off_services
+  - script:
+      common/add_jar:
+        env_vars: [plugin_version, hdfs_lib_dir]
+  - script:
+      common/unlimited_security_artifacts:
+        env_vars: [unlimited_security_location]
+  - os_case:
+    - centos:
+      - package:
+        - krb5-server
+        - krb5-libs
+        - krb5-workstation
+        - rng-tools
+      - package: iptables-services
+      - script: centos/selinux_permissive
+      - script: centos/disable_firewall
+      - package: nmap-ncat
+    - ubuntu:
+      - package:
+        - krb5-admin-server
+        - libpam-krb5
+        - ldap-utils
+        - krb5-user
+        - rng-tools
--- a/sahara/plugins/cdh/v5_11_0/resources/images/ubuntu/turn_off_services
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/ubuntu/turn_off_services
@ -0,0 +1,33 @@
+#!/bin/bash
+
+check=$(systemctl --no-pager list-unit-files oozie.service | grep 'enabled' | wc -l)
+
+if [ "$check" != "on" ]; then
+    if [ $test_only -eq 0 ]; then
+        for i in cloudera-scm-agent \
+        cloudera-scm-server \
+        cloudera-scm-server-db \
+        hadoop-hdfs-datanode \
+        hadoop-hdfs-namenode \
+        hadoop-hdfs-secondarynamenode \
+        hadoop-mapreduce-historyserver \
+        hadoop-yarn-nodemanager \
+        hadoop-yarn-resourcemanager \
+        hive-metastore \
+        hive-server2 \
+        hive-webhcat-server \
+        hue \
+        oozie \
+        postgresql \
+        impala-catalog \
+        impala-server \
+        impala-state-store \
+        solr-server \
+        spark-history-server
+        do
+            update-rc.d -f $i remove
+        done
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/resources/images/ubuntu/wget_repo
+++ b/sahara/plugins/cdh/v5_11_0/resources/images/ubuntu/wget_repo
@ -0,0 +1,34 @@
+#!/bin/bash
+
+if [ ! -f /etc/apt/sources.list.d/cdh5.list ]; then
+    if [ $test_only -eq 0 ]; then
+        # Add repository with postgresql package (it's dependency of cloudera packages)
+        # Base image doesn't contain this repo
+        echo -e 'deb http://nova.clouds.archive.ubuntu.com/ubuntu/ xenial universe multiverse main' >> /etc/apt/sources.list
+
+        # Cloudera repositories
+        echo "deb [arch=amd64] http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$plugin_version contrib" > /etc/apt/sources.list.d/cdh5.list
+        echo "deb-src http://archive.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh xenial-cdh$plugin_version contrib" >> /etc/apt/sources.list.d/cdh5.list
+
+        wget -qO - http://archive-primary.cloudera.com/cdh5/ubuntu/xenial/amd64/cdh/archive.key | apt-key add -
+
+        echo "deb [arch=amd64] http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$plugin_version contrib" > /etc/apt/sources.list.d/cm5.list
+        echo "deb-src http://archive.cloudera.com/cm5/ubuntu/xenial/amd64/cm xenial-cm$plugin_version contrib" >> /etc/apt/sources.list.d/cm5.list
+
+        wget -qO - http://archive-primary.cloudera.com/cm5/ubuntu/xenial/amd64/cm/archive.key | apt-key add -
+
+        wget -O /etc/apt/sources.list.d/kms.list http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/cloudera.list
+        wget -qO - http://archive.cloudera.com/navigator-keytrustee5/ubuntu/xenial/amd64/navigator-keytrustee/archive.key | apt-key add -
+
+        # add Kafka repository
+        echo -e 'deb http://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/ xenial-kafka2.2.0 contrib' >> /etc/apt/sources.list
+        wget -qO - https://archive.cloudera.com/kafka/ubuntu/xenial/amd64/kafka/archive.key | apt-key add -
+
+        #change repository priority
+        echo -e 'Package: zookeeper\nPin: origin "archive.cloudera.com"\nPin-Priority: 1001' > /etc/apt/preferences.d/cloudera-pin
+
+        apt-get update
+    else
+        exit 0
+    fi
+fi
--- a/sahara/plugins/cdh/v5_11_0/versionhandler.py
+++ b/sahara/plugins/cdh/v5_11_0/versionhandler.py
@ -19,6 +19,7 @@ from sahara.plugins.cdh.v5_11_0 import cloudera_utils
 from sahara.plugins.cdh.v5_11_0 import config_helper
 from sahara.plugins.cdh.v5_11_0 import deploy
 from sahara.plugins.cdh.v5_11_0 import edp_engine
+from sahara.plugins.cdh.v5_11_0 import images
 from sahara.plugins.cdh.v5_11_0 import plugin_utils
 from sahara.plugins.cdh.v5_11_0 import validation

@ -32,4 +33,5 @@ class VersionHandler(avm.BaseVersionHandler):
        self.plugin_utils = plugin_utils.PluginUtilsV5110()
        self.deploy = deploy
        self.edp_engine = edp_engine
+        self.images = images
        self.validation = validation.ValidatorV5110()
--- a/tools/gate/build-images
+++ b/tools/gate/build-images
@ -67,6 +67,7 @@ case "$PLUGIN" in
    "cloudera")
        build_images "cdh" "5.7.0" "centos7"
        build_images "cdh" "5.9.0" "centos7"
+        build_images "cdh" "5.11.0" "centos7"
        ;;
    "ambari")
        build_images "ambari" "2.4" "centos7"