From 2eedcec728b4f93e4cb226c5137159dcb3ddbfa3 Mon Sep 17 00:00:00 2001 From: Shu Yingya Date: Fri, 12 Jan 2018 10:55:55 +0800 Subject: [PATCH] Add support to build hadoop-2.8.2 related package This patch include some modification to build hadoop 2.8.2 required packages. 1. Modify the hadoop-swiftfs/pom.xml to fix dependency error while building hadoop-openstack.jar. It is caused by change in hadoop upstream [1],[2] 2. Add hadoop version 2.8.2 into sahara-extra/tools/ to build related packages 3. make some changes on build-oozie.sh so that we don't need to replace hadoop.version in pom.xml. we can specify hadoop.version directly in the command. 4. Apply a workaround patch to oozie 4.3.0 to solve WebUI issue [3] 5. Insert a dependency into oozie/sharelib [1]: https://github.com/apache/hadoop/commit/fa440a39b7883183b9cf43c056908ca7bf68e6a1 [2]: https://github.com/apache/hadoop/commit/12aa184479675d6c9bd36fd8451f605ee9505b47 [3]: https://issues.apache.org/jira/browse/OOZIE-2533 Change-Id: I186cf5e01907cfab4a34507be464a0da170ebb5d --- hadoop-swiftfs/pom.xml | 6 + tools/build-all-artifacts.sh | 4 +- tools/build-hadoop-openstack.sh | 2 +- tools/build-oozie.sh | 24 +++- tools/oozie_webUI.patch | 204 ++++++++++++++++++++++++++++++++ 5 files changed, 234 insertions(+), 6 deletions(-) create mode 100644 tools/oozie_webUI.patch diff --git a/hadoop-swiftfs/pom.xml b/hadoop-swiftfs/pom.xml index 2c096e0..3426228 100644 --- a/hadoop-swiftfs/pom.xml +++ b/hadoop-swiftfs/pom.xml @@ -143,6 +143,12 @@ 4.2.5 compile + + commons-httpclient + commons-httpclient + 3.1 + compile + diff --git a/tools/build-all-artifacts.sh b/tools/build-all-artifacts.sh index 9e9fd34..77984b8 100755 --- a/tools/build-all-artifacts.sh +++ b/tools/build-all-artifacts.sh @@ -6,14 +6,14 @@ export BRANCH=$1 export BRANCH=${BRANCH:-master} echo "Building all needed hadoop-openstack versions" -hadoop_versions=(2.2.0 2.3.0 2.5.0 2.6.0 2.7.1) +hadoop_versions=(2.2.0 2.3.0 2.5.0 2.6.0 2.7.1 2.8.2) for ver in ${hadoop_versions[*]} do ./tools/build-hadoop-openstack.sh $BRANCH ${ver} done echo "Building all oozie versions" -hadoop_oozie_versions=(2.7.1) +hadoop_oozie_versions=(2.7.1 2.8.2) for ver in ${hadoop_oozie_versions[*]} do ./tools/build-oozie.sh ${ver} diff --git a/tools/build-hadoop-openstack.sh b/tools/build-hadoop-openstack.sh index 469a088..d4b222f 100755 --- a/tools/build-hadoop-openstack.sh +++ b/tools/build-hadoop-openstack.sh @@ -15,7 +15,7 @@ fi BRANCH=${1} HADOOP_VERSION=${2} case "${HADOOP_VERSION}" in - "2.2.0" | "2.3.0" | "2.5.0" | "2.6.0" | "2.7.1") + "2.2.0" | "2.3.0" | "2.5.0" | "2.6.0" | "2.7.1" | "2.8.2") EXTRA_ARGS="-P hadoop2" ;; esac diff --git a/tools/build-oozie.sh b/tools/build-oozie.sh index 9e03175..ad107c5 100755 --- a/tools/build-oozie.sh +++ b/tools/build-oozie.sh @@ -16,10 +16,13 @@ PLUGIN_VERSION=${1} case "${PLUGIN_VERSION}" in "2.7.1") OOZIE_VERSION="4.2.0" - PREV_OOZIE_HADOOP_VERSION="2.3.0" HADOOP_VERSION="2.7.1" BUILD_ARGS="-Puber -P hadoop-2" ;; + "2.8.2") + OOZIE_VERSION="4.3.0" + HADOOP_VERSION="2.8.2" + BUILD_ARGS="-Puber -P hadoop-2" esac echo "Install required packages" @@ -33,13 +36,28 @@ tar xzf oozie-${OOZIE_VERSION}.tar.gz echo "Build Oozie" pushd oozie-${OOZIE_VERSION} -find . -name pom.xml | xargs sed -ri "s/${PREV_OOZIE_HADOOP_VERSION}/${HADOOP_VERSION}/g" if [ "${OOZIE_VERSION}" = "4.2.0" ]; then # see https://issues.apache.org/jira/browse/OOZIE-2417 mv pom.xml pom.xml.orig xmlstarlet ed -P -N N="http://maven.apache.org/POM/4.0.0" -d "/N:project/N:repositories/N:repository[N:url='http://repository.codehaus.org/']" pom.xml.orig >pom.xml fi -./bin/mkdistro.sh assembly:single ${BUILD_ARGS} -DjavaVersion=1.7 -DtargetJavaVersion=1.7 -DskipTests +if [ "${OOZIE_VERSION}" = "4.3.0" ]; then + # see https://issues.apache.org/jira/browse/OOZIE-2533 + patch -p0 < ./../tools/oozie_webUI.patch + + # add commons-httpclient as a dependency to sharelib/oozie + pushd sharelib/oozie + mv pom.xml pom.xml.orig + xmlstarlet ed -N N="http://maven.apache.org/POM/4.0.0" --subnode "/N:project/N:dependencies" -t elem -n dependency -v '' pom.xml.orig > pom.xml.tmp + xmlstarlet ed -P -N N="http://maven.apache.org/POM/4.0.0" \ + --subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n groupId -v commons-httpclient \ + --subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n artifactId -v commons-httpclient \ + --subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n version -v 3.1 \ + --subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n scope -v compile pom.xml.tmp > pom.xml + popd +fi + +./bin/mkdistro.sh assembly:single ${BUILD_ARGS} -Dhadoop.version=${HADOOP_VERSION} -DjavaVersion=1.8 -DtargetJavaVersion=1.8 -DskipTests mkdir -p ./../dist/oozie/ mv distro/target/oozie-${OOZIE_VERSION}-distro.tar.gz ./../dist/oozie/oozie-${OOZIE_VERSION}-hadoop-${HADOOP_VERSION}.tar.gz popd diff --git a/tools/oozie_webUI.patch b/tools/oozie_webUI.patch new file mode 100644 index 0000000..7b8da75 --- /dev/null +++ b/tools/oozie_webUI.patch @@ -0,0 +1,204 @@ +diff --git distro/src/main/tomcat/ssl-web.xml distro/src/main/tomcat/ssl-web.xml +--- distro/src/main/tomcat/ssl-web.xml ++++ distro/src/main/tomcat/ssl-web.xml +@@ -202,7 +202,7 @@ + + + +- index.jsp ++ index.html + + + +@@ -242,7 +242,7 @@ + + + authenticationfilter +- /index.jsp ++ /index.html + + + +diff --git webapp/src/main/webapp/WEB-INF/web.xml webapp/src/main/webapp/WEB-INF/web.xml +--- webapp/src/main/webapp/WEB-INF/web.xml ++++ webapp/src/main/webapp/WEB-INF/web.xml +@@ -202,7 +202,7 @@ + + + +- index.jsp ++ index.html + + + +@@ -242,7 +242,7 @@ + + + authenticationfilter +- /index.jsp ++ /index.html + + + +diff --git webapp/src/main/webapp/index.html webapp/src/main/webapp/index.html +--- /dev/null ++++ webapp/src/main/webapp/index.html +@@ -0,0 +1,72 @@ ++ ++ ++ ++ ++ Oozie Web Console ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++ ++
++
++ ++ ++ ++
++
++
++
++ ++ +diff --git webapp/src/main/webapp/index.jsp webapp/src/main/webapp/index.jsp +--- webapp/src/main/webapp/index.jsp ++++ /dev/null +@@ -1,81 +0,0 @@ +- +- +- +- +- Oozie Web Console +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- +- <%@ page +- import="org.apache.oozie.sla.service.SLAService" +- import="org.apache.oozie.service.InstrumentationService" +- import="org.apache.oozie.service.MetricsInstrumentationService" +- %> +- <% +- boolean isSLAServiceEnabled = SLAService.isEnabled(); +- boolean isInstrumentationServiceEnabled = InstrumentationService.isEnabled(); +- boolean isMetricsInstrumentationServiceEnabled = MetricsInstrumentationService.isEnabled(); +- %> +-
+-
+- +- +- +-
+-
+-
+-
+- +- +\ No newline at end of file