summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xdiskimage-create/diskimage-create.sh29
-rwxr-xr-xelements/hadoop/install.d/40-setup-hadoop3
-rwxr-xr-xelements/oozie/install.d/50-setup-oozie6
-rwxr-xr-xelements/oozie/post-install.d/90-oozie-war10
-rwxr-xr-xelements/oozie/root.d/0-check-oozie5
-rwxr-xr-xelements/oozie/root.d/50-download-oozie3
-rwxr-xr-xelements/swift_hadoop/post-install.d/81-add-jar2
-rwxr-xr-xtools/gate/build-images4
8 files changed, 55 insertions, 7 deletions
diff --git a/diskimage-create/diskimage-create.sh b/diskimage-create/diskimage-create.sh
index 6d1976e..2d54d42 100755
--- a/diskimage-create/diskimage-create.sh
+++ b/diskimage-create/diskimage-create.sh
@@ -32,7 +32,7 @@ usage() {
32 echo "Usage: $(basename $0)" 32 echo "Usage: $(basename $0)"
33 echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]" 33 echo " [-p vanilla|spark|cloudera|storm|mapr|ambari|plain]"
34 echo " [-i ubuntu|fedora|centos7]" 34 echo " [-i ubuntu|fedora|centos7]"
35 echo " [-v 2.7.1|2.7.5|2.8.2|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]" 35 echo " [-v 2.7.1|2.7.5|2.8.2|3.0.1|5.5|5.7|5.9|5.11|2.2.0.0|2.2.1.0|2.4.2.0]"
36 echo " [-r 5.1.0|5.2.0]" 36 echo " [-r 5.1.0|5.2.0]"
37 echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]" 37 echo " [-s 1.3.1|1.6.0|2.1.0|2.2.0]"
38 echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]" 38 echo " [-t 0.9.2|1.0.1|1.1.0|1.1.1]"
@@ -170,7 +170,7 @@ case "$PLUGIN" in
170 "");; 170 "");;
171 "vanilla") 171 "vanilla")
172 case "$HADOOP_VERSION" in 172 case "$HADOOP_VERSION" in
173 "" | "2.7.1" | "2.7.5" | "2.8.2");; 173 "" | "2.7.1" | "2.7.5" | "2.8.2" | "3.0.1");;
174 *) 174 *)
175 echo -e "Unknown hadoop version selected.\nAborting" 175 echo -e "Unknown hadoop version selected.\nAborting"
176 exit 1 176 exit 1
@@ -447,6 +447,9 @@ set_hive_version() {
447 "2.8.2" ) 447 "2.8.2" )
448 export HIVE_VERSION="2.3.2" 448 export HIVE_VERSION="2.3.2"
449 ;; 449 ;;
450 "3.0.1" )
451 export HIVE_VERSION="3.0.0"
452 ;;
450 *) 453 *)
451 echo -e "Unknown Hadoop version, therefore cannot choose Hive version.\nAborting." 454 echo -e "Unknown Hadoop version, therefore cannot choose Hive version.\nAborting."
452 exit 1 455 exit 1
@@ -464,9 +467,11 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
464 export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"} 467 export HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.1.tar.gz"}
465 export HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.5.tar.gz"} 468 export HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_7_5_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.7.5.tar.gz"}
466 export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"} 469 export HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-2.8.2.tar.gz"}
470 export HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL=${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL:-"https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-native-libs-3.0.1.tar.gz"}
467 export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"} 471 export OOZIE_HADOOP_V2_7_1_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.2.0-hadoop-2.7.1.tar.gz"}
468 export OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_5_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.7.5.tar.gz"} 472 export OOZIE_HADOOP_V2_7_5_DOWNLOAD_URL=${OOZIE_HADOOP_V2_7_5_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.7.5.tar.gz"}
469 export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"} 473 export OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL=${OOZIE_HADOOP_V2_8_2_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-4.3.0-hadoop-2.8.2.tar.gz"}
474 export OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL=${OOZIE_HADOOP_V3_0_1_FILE:-"https://tarballs.openstack.org/sahara-extra/dist/oozie/oozie-5.0.0-hadoop-3.0.1.tar.gz"}
470 export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib" 475 export DIB_HDFS_LIB_DIR="/opt/hadoop/share/hadoop/tools/lib"
471 export plugin_type="vanilla" 476 export plugin_type="vanilla"
472 477
@@ -527,6 +532,14 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
527 image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence 532 image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
528 unset DIB_RELEASE 533 unset DIB_RELEASE
529 fi 534 fi
535 if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then
536 export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"}
537 export ubuntu_image_name=${ubuntu_vanilla_hadoop_3_0_1_image_name:-"ubuntu_sahara_vanilla_hadoop_3_0_1_latest"}
538 export DIB_RELEASE=${DIB_RELEASE:-xenial}
539 set_hive_version
540 image_create ubuntu $ubuntu_image_name $ubuntu_elements_sequence
541 unset DIB_RELEASE
542 fi
530 unset DIB_CLOUD_INIT_DATASOURCES 543 unset DIB_CLOUD_INIT_DATASOURCES
531 fi 544 fi
532 545
@@ -550,6 +563,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
550 set_hive_version 563 set_hive_version
551 image_create fedora $fedora_image_name $fedora_elements_sequence 564 image_create fedora $fedora_image_name $fedora_elements_sequence
552 fi 565 fi
566 if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then
567 export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"}
568 export fedora_image_name=${fedora_vanilla_hadoop_3_0_1_image_name:-"fedora_sahara_vanilla_hadoop_3_0_1_latest$suffix"}
569 set_hive_version
570 image_create fedora $fedora_image_name $fedora_elements_sequence
571 fi
553 fi 572 fi
554 573
555 # CentOS 7 cloud image 574 # CentOS 7 cloud image
@@ -572,6 +591,12 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "vanilla" ]; then
572 set_hive_version 591 set_hive_version
573 image_create centos7 $centos7_image_name $centos7_elements_sequence 592 image_create centos7 $centos7_image_name $centos7_elements_sequence
574 fi 593 fi
594 if [ -z "$HADOOP_VERSION" -o "$HADOOP_VERSION" = "3.0.1" ]; then
595 export DIB_HADOOP_VERSION=${DIB_HADOOP_VERSION_3_0_1:-"3.0.1"}
596 export centos7_image_name=${centos7_vanilla_hadoop_3_0_1_image_name:-"centos7_sahara_vanilla_hadoop_3_0_1_latest$suffix"}
597 set_hive_version
598 image_create centos7 $centos7_image_name $centos7_elements_sequence
599 fi
575 fi 600 fi
576 601
577 unset plugin_type 602 unset plugin_type
diff --git a/elements/hadoop/install.d/40-setup-hadoop b/elements/hadoop/install.d/40-setup-hadoop
index e483f1c..5384a27 100755
--- a/elements/hadoop/install.d/40-setup-hadoop
+++ b/elements/hadoop/install.d/40-setup-hadoop
@@ -17,6 +17,9 @@ function install_hadoop_v2 {
17 "2.8.2") 17 "2.8.2")
18 hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}" 18 hadoop_native_libs_url="${HADOOP_V2_8_2_NATIVE_LIBS_DOWNLOAD_URL}"
19 ;; 19 ;;
20 "3.0.1")
21 hadoop_native_libs_url="${HADOOP_V3_0_1_NATIVE_LIBS_DOWNLOAD_URL}"
22 ;;
20 *) 23 *)
21 echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION" 24 echo "Invalid DIB_HADOOP_VERSION: $DIB_HADOOP_VERSION"
22 exit 1 25 exit 1
diff --git a/elements/oozie/install.d/50-setup-oozie b/elements/oozie/install.d/50-setup-oozie
index 9e45702..7ad6a76 100755
--- a/elements/oozie/install.d/50-setup-oozie
+++ b/elements/oozie/install.d/50-setup-oozie
@@ -28,6 +28,12 @@ elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
28 else 28 else
29 OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL) 29 OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL)
30 fi 30 fi
31elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then
32 if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" ]; then
33 OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_FILE)
34 else
35 OOZIE_FILE=$(basename $OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL)
36 fi
31else 37else
32 echo "Unknown Hadoop version selected. Aborting." 38 echo "Unknown Hadoop version selected. Aborting."
33 exit 1 39 exit 1
diff --git a/elements/oozie/post-install.d/90-oozie-war b/elements/oozie/post-install.d/90-oozie-war
index abd955c..c3da6a4 100755
--- a/elements/oozie/post-install.d/90-oozie-war
+++ b/elements/oozie/post-install.d/90-oozie-war
@@ -7,8 +7,12 @@ set -eu
7set -o pipefail 7set -o pipefail
8 8
9ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar 9ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/libtools/mysql.jar
10ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar 10if [ "$DIB_HADOOP_VERSION" = "3.0.1" ]; then
11 11 ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/embedded-oozie-server/dependency/mysql.jar
12/opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools 12 /opt/oozie/bin/oozie-setup.sh -d /opt/oozie/libtools
13else
14 ln -s /usr/share/java/mysql-connector-java.jar /opt/oozie/oozie-server/lib/mysql.jar
15 /opt/oozie/bin/oozie-setup.sh prepare-war -d /opt/oozie/libtools
16fi
13 17
14chown -R hadoop:hadoop /opt/oozie 18chown -R hadoop:hadoop /opt/oozie
diff --git a/elements/oozie/root.d/0-check-oozie b/elements/oozie/root.d/0-check-oozie
index 1980b58..5174608 100755
--- a/elements/oozie/root.d/0-check-oozie
+++ b/elements/oozie/root.d/0-check-oozie
@@ -21,6 +21,11 @@ elif [ "$DIB_HADOOP_VERSION" == "2.8.2" ]; then
21 echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit" 21 echo "OOZIE_HADOOP_V2_8_2_FILE and OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
22 exit 1 22 exit 1
23 fi 23 fi
24elif [ "$DIB_HADOOP_VERSION" == "3.0.1" ]; then
25 if [ -z "${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}" -a -z "${OOZIE_HADOOP_V3_0_1_FILE:-}" ]; then
26 echo "OOZIE_HADOOP_V3_0_1_FILE and OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL are not set. Impossible to install Oozie. Exit"
27 exit 1
28 fi
24else 29else
25 echo "Unknown Hadoop version selected. Aborting" 30 echo "Unknown Hadoop version selected. Aborting"
26 exit 1 31 exit 1
diff --git a/elements/oozie/root.d/50-download-oozie b/elements/oozie/root.d/50-download-oozie
index 6033f8e..63da988 100755
--- a/elements/oozie/root.d/50-download-oozie
+++ b/elements/oozie/root.d/50-download-oozie
@@ -18,6 +18,9 @@ elif [[ "$DIB_HADOOP_VERSION" == "2.7.5" ]]; then
18elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then 18elif [[ "$DIB_HADOOP_VERSION" == "2.8.2" ]]; then
19 OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}" 19 OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_8_2_DOWNLOAD_URL:-}"
20 OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}" 20 OOZIE_FILE="${OOZIE_HADOOP_V2_8_2_FILE:-}"
21elif [[ "$DIB_HADOOP_VERSION" == "3.0.1" ]]; then
22 OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V3_0_1_DOWNLOAD_URL:-}"
23 OOZIE_FILE="${OOZIE_HADOOP_V3_0_1_FILE:-}"
21else 24else
22 echo "Unknown Hadoop version selected. Aborting." 25 echo "Unknown Hadoop version selected. Aborting."
23 exit 1 26 exit 1
diff --git a/elements/swift_hadoop/post-install.d/81-add-jar b/elements/swift_hadoop/post-install.d/81-add-jar
index c3dcda8..f4dc56c 100755
--- a/elements/swift_hadoop/post-install.d/81-add-jar
+++ b/elements/swift_hadoop/post-install.d/81-add-jar
@@ -16,6 +16,8 @@ case "$plugin_type" in
16 hadoop="2.7.5";; 16 hadoop="2.7.5";;
17 "2.8.2") 17 "2.8.2")
18 hadoop="2.8.2";; 18 hadoop="2.8.2";;
19 "3.0.1")
20 hadoop="3.0.1";;
19 esac 21 esac
20 DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop.jar" 22 DIB_HADOOP_SWIFT_JAR_NAME="hadoop-openstack-$hadoop.jar"
21 ;; 23 ;;
diff --git a/tools/gate/build-images b/tools/gate/build-images
index 45e0091..44afadb 100755
--- a/tools/gate/build-images
+++ b/tools/gate/build-images
@@ -34,8 +34,8 @@ case "$PLUGIN" in
34 "vanilla") 34 "vanilla")
35 # temporary don't test building fedora image 35 # temporary don't test building fedora image
36 for distro in ubuntu centos7; do 36 for distro in ubuntu centos7; do
37 # limit to 3 releases 37 # limit to 4 releases
38 for version in 2.7.1 2.7.5 2.8.2; do 38 for version in 2.7.1 2.7.5 2.8.2 3.0.1; do
39 tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version 39 tox -e venv -- sahara-image-create -x -u -p $PLUGIN -i $distro -v $version
40 done 40 done
41 done 41 done