Add support to build hadoop-2.8.2 related package

This patch include some modification to build hadoop 2.8.2
required packages.
1. Modify the hadoop-swiftfs/pom.xml to fix dependency error
   while building hadoop-openstack.jar. It is caused by change
   in hadoop upstream [1],[2]
2. Add hadoop version 2.8.2 into sahara-extra/tools/ to build
   related packages
3. make some changes on build-oozie.sh so that we don't need
   to replace hadoop.version in pom.xml. we can specify
   hadoop.version directly in the command.
4. Apply a workaround patch to oozie 4.3.0 to solve WebUI issue [3]
5. Insert a dependency into oozie/sharelib

[1]: fa440a39b7
[2]: 12aa184479
[3]: https://issues.apache.org/jira/browse/OOZIE-2533

Change-Id: I186cf5e01907cfab4a34507be464a0da170ebb5d
This commit is contained in:
Shu Yingya 2018-01-12 10:55:55 +08:00
parent 558f100a37
commit 2eedcec728
5 changed files with 234 additions and 6 deletions

View File

@ -143,6 +143,12 @@
<version>4.2.5</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>commons-httpclient</groupId>
<artifactId>commons-httpclient</artifactId>
<version>3.1</version>
<scope>compile</scope>
</dependency>
<!-- Used for loading test resources and converting a File to byte[] -->
<dependency>

View File

@ -6,14 +6,14 @@ export BRANCH=$1
export BRANCH=${BRANCH:-master}
echo "Building all needed hadoop-openstack versions"
hadoop_versions=(2.2.0 2.3.0 2.5.0 2.6.0 2.7.1)
hadoop_versions=(2.2.0 2.3.0 2.5.0 2.6.0 2.7.1 2.8.2)
for ver in ${hadoop_versions[*]}
do
./tools/build-hadoop-openstack.sh $BRANCH ${ver}
done
echo "Building all oozie versions"
hadoop_oozie_versions=(2.7.1)
hadoop_oozie_versions=(2.7.1 2.8.2)
for ver in ${hadoop_oozie_versions[*]}
do
./tools/build-oozie.sh ${ver}

View File

@ -15,7 +15,7 @@ fi
BRANCH=${1}
HADOOP_VERSION=${2}
case "${HADOOP_VERSION}" in
"2.2.0" | "2.3.0" | "2.5.0" | "2.6.0" | "2.7.1")
"2.2.0" | "2.3.0" | "2.5.0" | "2.6.0" | "2.7.1" | "2.8.2")
EXTRA_ARGS="-P hadoop2"
;;
esac

View File

@ -16,10 +16,13 @@ PLUGIN_VERSION=${1}
case "${PLUGIN_VERSION}" in
"2.7.1")
OOZIE_VERSION="4.2.0"
PREV_OOZIE_HADOOP_VERSION="2.3.0"
HADOOP_VERSION="2.7.1"
BUILD_ARGS="-Puber -P hadoop-2"
;;
"2.8.2")
OOZIE_VERSION="4.3.0"
HADOOP_VERSION="2.8.2"
BUILD_ARGS="-Puber -P hadoop-2"
esac
echo "Install required packages"
@ -33,13 +36,28 @@ tar xzf oozie-${OOZIE_VERSION}.tar.gz
echo "Build Oozie"
pushd oozie-${OOZIE_VERSION}
find . -name pom.xml | xargs sed -ri "s/${PREV_OOZIE_HADOOP_VERSION}/${HADOOP_VERSION}/g"
if [ "${OOZIE_VERSION}" = "4.2.0" ]; then
# see https://issues.apache.org/jira/browse/OOZIE-2417
mv pom.xml pom.xml.orig
xmlstarlet ed -P -N N="http://maven.apache.org/POM/4.0.0" -d "/N:project/N:repositories/N:repository[N:url='http://repository.codehaus.org/']" pom.xml.orig >pom.xml
fi
./bin/mkdistro.sh assembly:single ${BUILD_ARGS} -DjavaVersion=1.7 -DtargetJavaVersion=1.7 -DskipTests
if [ "${OOZIE_VERSION}" = "4.3.0" ]; then
# see https://issues.apache.org/jira/browse/OOZIE-2533
patch -p0 < ./../tools/oozie_webUI.patch
# add commons-httpclient as a dependency to sharelib/oozie
pushd sharelib/oozie
mv pom.xml pom.xml.orig
xmlstarlet ed -N N="http://maven.apache.org/POM/4.0.0" --subnode "/N:project/N:dependencies" -t elem -n dependency -v '' pom.xml.orig > pom.xml.tmp
xmlstarlet ed -P -N N="http://maven.apache.org/POM/4.0.0" \
--subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n groupId -v commons-httpclient \
--subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n artifactId -v commons-httpclient \
--subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n version -v 3.1 \
--subnode "/N:project/N:dependencies/N:dependency[last()]" -t elem -n scope -v compile pom.xml.tmp > pom.xml
popd
fi
./bin/mkdistro.sh assembly:single ${BUILD_ARGS} -Dhadoop.version=${HADOOP_VERSION} -DjavaVersion=1.8 -DtargetJavaVersion=1.8 -DskipTests
mkdir -p ./../dist/oozie/
mv distro/target/oozie-${OOZIE_VERSION}-distro.tar.gz ./../dist/oozie/oozie-${OOZIE_VERSION}-hadoop-${HADOOP_VERSION}.tar.gz
popd

204
tools/oozie_webUI.patch Normal file
View File

@ -0,0 +1,204 @@
diff --git distro/src/main/tomcat/ssl-web.xml distro/src/main/tomcat/ssl-web.xml
--- distro/src/main/tomcat/ssl-web.xml
+++ distro/src/main/tomcat/ssl-web.xml
@@ -202,7 +202,7 @@
<!-- welcome-file -->
<welcome-file-list>
- <welcome-file>index.jsp</welcome-file>
+ <welcome-file>index.html</welcome-file>
</welcome-file-list>
<filter>
@@ -242,7 +242,7 @@
<filter-mapping>
<filter-name>authenticationfilter</filter-name>
- <url-pattern>/index.jsp</url-pattern>
+ <url-pattern>/index.html</url-pattern>
</filter-mapping>
<filter-mapping>
diff --git webapp/src/main/webapp/WEB-INF/web.xml webapp/src/main/webapp/WEB-INF/web.xml
--- webapp/src/main/webapp/WEB-INF/web.xml
+++ webapp/src/main/webapp/WEB-INF/web.xml
@@ -202,7 +202,7 @@
<!-- welcome-file -->
<welcome-file-list>
- <welcome-file>index.jsp</welcome-file>
+ <welcome-file>index.html</welcome-file>
</welcome-file-list>
<filter>
@@ -242,7 +242,7 @@
<filter-mapping>
<filter-name>authenticationfilter</filter-name>
- <url-pattern>/index.jsp</url-pattern>
+ <url-pattern>/index.html</url-pattern>
</filter-mapping>
<filter-mapping>
diff --git webapp/src/main/webapp/index.html webapp/src/main/webapp/index.html
--- /dev/null
+++ webapp/src/main/webapp/index.html
@@ -0,0 +1,72 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<html>
+ <head>
+ <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
+ <title>Oozie Web Console</title>
+ <link rel="stylesheet" type="text/css" href="ext-2.2/resources/css/ext-all.css"/>
+ <link rel="stylesheet" type="text/css" href="ext-2.2/resources/css/xtheme-default.css"/>
+ <link rel="stylesheet" type="text/css" href="oozie-console.css"/>
+
+ <!-- jquery needs to be before extjs -->
+ <script type="text/javascript" charset="utf8" src="console/sla/js/table/jquery-1.8.3.min.js"></script>
+ <link rel="stylesheet" type="text/css" href="console/sla/css/jquery.dataTables.css">
+ <script type="text/javascript" src="console/sla/js/table/jquery.dataTables.min.js"></script>
+ <script type="text/javascript" src="console/sla/js/table/jquery-ui-1.10.3.custom.min.js"></script>
+ <script type="text/javascript" src="console/sla/js/table/jquery-ui-timepicker-addon.js"></script>
+ <script type="text/javascript" src="console/sla/js/graph/jquery.flot.min.js"></script>
+ <script type="text/javascript" src="console/sla/js/oozie-sla.js"></script>
+
+ <script type="text/javascript" src="ext-2.2/adapter/ext/ext-base.js"></script>
+ <script type="text/javascript" src="ext-2.2/ext-all.js"></script>
+ <script type="text/javascript" src="ext-2.2/examples/grid/RowExpander.js"></script>
+ <script type="text/javascript" src="json2.js"></script>
+ <script type="text/javascript" src="oozie-console.js"></script>
+
+ </head>
+ <body>
+ <div id="dependencies" style="display:none;color:red">
+ <p><strong>Oozie web console is disabled.</strong></p>
+ <p>To enable Oozie web console install the Ext JS library.</p>
+ <p>Refer to <a href="./docs/DG_QuickStart.html">Oozie Quick Start</a> documentation for details.</p>
+ <hr />
+ </div>
+ <!-- LIBS -->
+
+ <div id="Header" style="padding:2">
+ <img src="./oozie_50x.png" height="16" width="70"/>
+ <a href="./docs/index.html" target="bottom">Documentation</a>
+ </div>
+ <div id="oozie-body" style="padding:2">
+ <div class="x-tab-panel-header x-unselectable x-tab-strip-top" style="width:1048">
+ <span style="font-family:tahoma,arial,helvetica; font-size:11px;font-weight: bold; color: #15428B;">
+ <script type="text/javascript">
+ var msg = "Oozie Web Console";
+ var isSLAServiceEnabled = "true";
+ var isInstrumentationServiceEnabled = "true";
+ var isMetricsInstrumentationServiceEnabled = "true";
+ document.title = msg;
+ document.write(msg);
+ </script>
+ </span>
+ </div>
+ <div id="oozie-console"></div>
+ <div id="info"> </div>
+ </div>
+ </body>
+</html>
diff --git webapp/src/main/webapp/index.jsp webapp/src/main/webapp/index.jsp
--- webapp/src/main/webapp/index.jsp
+++ /dev/null
@@ -1,81 +0,0 @@
-<!--
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements. See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership. The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License.
--->
-<html>
- <head>
- <meta http-equiv="Content-Type" content="text/html; charset=utf-8">
- <title>Oozie Web Console</title>
- <link rel="stylesheet" type="text/css" href="ext-2.2/resources/css/ext-all.css"/>
- <link rel="stylesheet" type="text/css" href="ext-2.2/resources/css/xtheme-default.css"/>
- <link rel="stylesheet" type="text/css" href="oozie-console.css"/>
-
- <!-- jquery needs to be before extjs -->
- <script type="text/javascript" charset="utf8" src="console/sla/js/table/jquery-1.8.3.min.js"></script>
- <link rel="stylesheet" type="text/css" href="console/sla/css/jquery.dataTables.css">
- <script type="text/javascript" src="console/sla/js/table/jquery.dataTables.min.js"></script>
- <script type="text/javascript" src="console/sla/js/table/jquery-ui-1.10.3.custom.min.js"></script>
- <script type="text/javascript" src="console/sla/js/table/jquery-ui-timepicker-addon.js"></script>
- <script type="text/javascript" src="console/sla/js/graph/jquery.flot.min.js"></script>
- <script type="text/javascript" src="console/sla/js/oozie-sla.js"></script>
-
- <script type="text/javascript" src="ext-2.2/adapter/ext/ext-base.js"></script>
- <script type="text/javascript" src="ext-2.2/ext-all.js"></script>
- <script type="text/javascript" src="ext-2.2/examples/grid/RowExpander.js"></script>
- <script type="text/javascript" src="oozie-console.js"></script>
-
- </head>
- <body>
- <div id="dependencies" style="display:none;color:red">
- <p><strong>Oozie web console is disabled.</strong></p>
- <p>To enable Oozie web console install the Ext JS library.</p>
- <p>Refer to <a href="./docs/DG_QuickStart.html">Oozie Quick Start</a> documentation for details.</p>
- <hr />
- </div>
- <!-- LIBS -->
-
- <div id="Header" style="padding:2">
- <img src="./oozie_50x.png" height="16" width="70"/>
- <a href="./docs/index.html" target="bottom">Documentation</a>
- </div>
- <%@ page
- import="org.apache.oozie.sla.service.SLAService"
- import="org.apache.oozie.service.InstrumentationService"
- import="org.apache.oozie.service.MetricsInstrumentationService"
- %>
- <%
- boolean isSLAServiceEnabled = SLAService.isEnabled();
- boolean isInstrumentationServiceEnabled = InstrumentationService.isEnabled();
- boolean isMetricsInstrumentationServiceEnabled = MetricsInstrumentationService.isEnabled();
- %>
- <div id="oozie-body" style="padding:2">
- <div class="x-tab-panel-header x-unselectable x-tab-strip-top" style="width:1048">
- <span style="font-family:tahoma,arial,helvetica; font-size:11px;font-weight: bold; color: #15428B;">
- <script type="text/javascript">
- var msg = "Oozie Web Console";
- var isSLAServiceEnabled = "<%=isSLAServiceEnabled%>";
- var isInstrumentationServiceEnabled = "<%=isInstrumentationServiceEnabled%>";
- var isMetricsInstrumentationServiceEnabled = "<%=isMetricsInstrumentationServiceEnabled%>";
- document.title = msg;
- document.write(msg);
- </script>
- </span>
- </div>
- <div id="oozie-console"></div>
- <div id="info"> </div>
- </div>
- </body>
-</html>
\ No newline at end of file