Merge "Add a SparkWrapper class"
This commit is contained in:
commit
03fcf5eb48
|
@ -0,0 +1,14 @@
|
||||||
|
===========================================
|
||||||
|
Sources for main function wrapper for Spark
|
||||||
|
===========================================
|
||||||
|
|
||||||
|
The Hadoop configuration for a Spark job must be modified if
|
||||||
|
the Spark job is going to access Swift paths. Specifically,
|
||||||
|
the Hadoop configuration must contain values that allow
|
||||||
|
the job to authenticate to the Swift service.
|
||||||
|
|
||||||
|
This wrapper adds a specified xml file to the default Hadoop
|
||||||
|
Configuration resource list and then calls the specified
|
||||||
|
main class. Any necessary Hadoop configuration values can
|
||||||
|
be added to the xml file. This allows the main class to
|
||||||
|
be run and access Swift paths without alteration.
|
|
@ -0,0 +1,54 @@
|
||||||
|
<?xml version="1.0" encoding="UTF-8"?>
|
||||||
|
<!--
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License. See accompanying LICENSE file.
|
||||||
|
-->
|
||||||
|
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||||
|
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||||
|
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
|
||||||
|
<modelVersion>4.0.0</modelVersion>
|
||||||
|
|
||||||
|
<groupId>org.openstack.sahara.edp</groupId>
|
||||||
|
<artifactId>edp-spark-wrapper</artifactId>
|
||||||
|
<version>1.0.0-SNAPSHOT</version>
|
||||||
|
<name>EDP Wrapper for Spark</name>
|
||||||
|
<packaging>jar</packaging>
|
||||||
|
|
||||||
|
<properties>
|
||||||
|
<file.encoding>UTF-8</file.encoding>
|
||||||
|
<downloadSources>true</downloadSources>
|
||||||
|
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||||
|
</properties>
|
||||||
|
|
||||||
|
<build>
|
||||||
|
<plugins>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-compiler-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<source>1.6</source>
|
||||||
|
<target>1.6</target>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
<plugin>
|
||||||
|
<groupId>org.apache.maven.plugins</groupId>
|
||||||
|
<artifactId>maven-checkstyle-plugin</artifactId>
|
||||||
|
<configuration>
|
||||||
|
<configLocation>file://${basedir}/../hadoop-swiftfs/checkstyle.xml</configLocation>
|
||||||
|
<failOnViolation>false</failOnViolation>
|
||||||
|
<format>xml</format>
|
||||||
|
<format>html</format>
|
||||||
|
</configuration>
|
||||||
|
</plugin>
|
||||||
|
</plugins>
|
||||||
|
</build>
|
||||||
|
</project>
|
|
@ -0,0 +1,22 @@
|
||||||
|
package org.openstack.sahara.edp;
|
||||||
|
|
||||||
|
import java.lang.reflect.InvocationTargetException;
|
||||||
|
import java.lang.reflect.Method;
|
||||||
|
import java.security.Permission;
|
||||||
|
import java.util.Arrays;
|
||||||
|
|
||||||
|
public class SparkWrapper {
|
||||||
|
|
||||||
|
public static void main(String[] args) throws Throwable {
|
||||||
|
|
||||||
|
Class<?> configClass
|
||||||
|
= Class.forName("org.apache.hadoop.conf.Configuration");
|
||||||
|
Method method = configClass.getMethod("addDefaultResource", String.class);
|
||||||
|
method.invoke(null, args[0]);
|
||||||
|
|
||||||
|
Class<?> mainClass = Class.forName(args[1]);
|
||||||
|
Method mainMethod = mainClass.getMethod("main", String[].class);
|
||||||
|
String[] newArgs = Arrays.copyOfRange(args, 2, args.length);
|
||||||
|
mainMethod.invoke(null, (Object) newArgs);
|
||||||
|
}
|
||||||
|
}
|
Loading…
Reference in New Issue