diff --git a/edp-adapt-for-spark/README.rst b/edp-adapt-for-spark/README.rst new file mode 100644 index 0000000..27d88b9 --- /dev/null +++ b/edp-adapt-for-spark/README.rst @@ -0,0 +1,14 @@ +=========================================== +Sources for main function wrapper for Spark +=========================================== + +The Hadoop configuration for a Spark job must be modified if +the Spark job is going to access Swift paths. Specifically, +the Hadoop configuration must contain values that allow +the job to authenticate to the Swift service. + +This wrapper adds a specified xml file to the default Hadoop +Configuration resource list and then calls the specified +main class. Any necessary Hadoop configuration values can +be added to the xml file. This allows the main class to +be run and access Swift paths without alteration. diff --git a/edp-adapt-for-spark/pom.xml b/edp-adapt-for-spark/pom.xml new file mode 100644 index 0000000..7041909 --- /dev/null +++ b/edp-adapt-for-spark/pom.xml @@ -0,0 +1,54 @@ + + + + 4.0.0 + + org.openstack.sahara.edp + edp-spark-wrapper + 1.0.0-SNAPSHOT + EDP Wrapper for Spark + jar + + + UTF-8 + true + UTF-8 + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + 1.6 + 1.6 + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + file://${basedir}/../hadoop-swiftfs/checkstyle.xml + false + xml + html + + + + + diff --git a/edp-adapt-for-spark/src/main/java/org/openstack/sahara/edp/SparkWrapper.java b/edp-adapt-for-spark/src/main/java/org/openstack/sahara/edp/SparkWrapper.java new file mode 100644 index 0000000..6377043 --- /dev/null +++ b/edp-adapt-for-spark/src/main/java/org/openstack/sahara/edp/SparkWrapper.java @@ -0,0 +1,22 @@ +package org.openstack.sahara.edp; + +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.security.Permission; +import java.util.Arrays; + +public class SparkWrapper { + + public static void main(String[] args) throws Throwable { + + Class configClass + = Class.forName("org.apache.hadoop.conf.Configuration"); + Method method = configClass.getMethod("addDefaultResource", String.class); + method.invoke(null, args[0]); + + Class mainClass = Class.forName(args[1]); + Method mainMethod = mainClass.getMethod("main", String[].class); + String[] newArgs = Arrays.copyOfRange(args, 2, args.length); + mainMethod.invoke(null, (Object) newArgs); + } +}