Host some patched jars in common artifacts
Host these jars, and also provide tooling to recreate them. These are needed for proper S3-as-a-datasource capability. Change-Id: Ied75211044b8f51623d87d36aa0588632645eea3
This commit is contained in:
parent
c118e53a8e
commit
d4e774245b
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,17 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
|
||||||
|
# First, install jdk, maven, git, etc.
|
||||||
|
|
||||||
|
for version in 2.7.1 2.7.3 2.7.5; do
|
||||||
|
git clone git://git.apache.org/hadoop.git -b branch-$version --depth 1
|
||||||
|
cd hadoop/hadoop-tools/hadoop-aws/
|
||||||
|
git apply ../../../path-style-access.patch
|
||||||
|
mvn package -Dhadoop.version=$version
|
||||||
|
mv target/hadoop-aws-$version.jar ../../../
|
||||||
|
cd ../../../
|
||||||
|
rm -rf hadoop
|
||||||
|
done
|
||||||
|
|
||||||
|
ls hadoop-aws-*.jar
|
|
@ -0,0 +1,48 @@
|
||||||
|
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
|
||||||
|
index 3486dfb..ad19d00 100644
|
||||||
|
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
|
||||||
|
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
|
||||||
|
@@ -35,6 +35,7 @@
|
||||||
|
|
||||||
|
//use a custom endpoint?
|
||||||
|
public static final String ENDPOINT = "fs.s3a.endpoint";
|
||||||
|
+ public static final String PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
|
||||||
|
//connect to s3 through a proxy server?
|
||||||
|
public static final String PROXY_HOST = "fs.s3a.proxy.host";
|
||||||
|
public static final String PROXY_PORT = "fs.s3a.proxy.port";
|
||||||
|
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
|
||||||
|
index 91a606c..17b272a 100644
|
||||||
|
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
|
||||||
|
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
|
||||||
|
@@ -40,6 +40,7 @@
|
||||||
|
|
||||||
|
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||||
|
import com.amazonaws.services.s3.AmazonS3Client;
|
||||||
|
+import com.amazonaws.services.s3.S3ClientOptions;
|
||||||
|
import com.amazonaws.services.s3.model.CannedAccessControlList;
|
||||||
|
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
|
||||||
|
import com.amazonaws.services.s3.model.ListObjectsRequest;
|
||||||
|
@@ -244,6 +245,8 @@ public void initialize(URI name, Configuration conf) throws IOException {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
+ enablePathStyleAccessIfRequired(conf);
|
||||||
|
+
|
||||||
|
maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
|
||||||
|
partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
|
||||||
|
multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD,
|
||||||
|
@@ -314,6 +317,14 @@ public void initialize(URI name, Configuration conf) throws IOException {
|
||||||
|
setConf(conf);
|
||||||
|
}
|
||||||
|
|
||||||
|
+ private void enablePathStyleAccessIfRequired(Configuration conf) {
|
||||||
|
+ final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
|
||||||
|
+ if (pathStyleAccess) {
|
||||||
|
+ LOG.debug("Enabling path style access!");
|
||||||
|
+ s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
|
||||||
|
+ }
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
/**
|
||||||
|
* Return the protocol scheme for the FileSystem.
|
||||||
|
*
|
Loading…
Reference in New Issue