Host some patched jars in common artifacts

Host these jars, and also provide tooling to recreate them.

These are needed for proper S3-as-a-datasource capability.

Change-Id: Ied75211044b8f51623d87d36aa0588632645eea3
(cherry picked from commit d4e774245b)
This commit is contained in:
Jeremy Freudberg 2018-06-22 10:47:27 -04:00 committed by Luigi Toscano
parent 786ed09b58
commit abacf8f2d4
5 changed files with 65 additions and 0 deletions

Binary file not shown.

Binary file not shown.

Binary file not shown.

View File

@ -0,0 +1,17 @@
#!/bin/bash
set -e
# First, install jdk, maven, git, etc.
for version in 2.7.1 2.7.3 2.7.5; do
git clone git://git.apache.org/hadoop.git -b branch-$version --depth 1
cd hadoop/hadoop-tools/hadoop-aws/
git apply ../../../path-style-access.patch
mvn package -Dhadoop.version=$version
mv target/hadoop-aws-$version.jar ../../../
cd ../../../
rm -rf hadoop
done
ls hadoop-aws-*.jar

View File

@ -0,0 +1,48 @@
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 3486dfb..ad19d00 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -35,6 +35,7 @@
//use a custom endpoint?
public static final String ENDPOINT = "fs.s3a.endpoint";
+ public static final String PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
//connect to s3 through a proxy server?
public static final String PROXY_HOST = "fs.s3a.proxy.host";
public static final String PROXY_PORT = "fs.s3a.proxy.port";
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 91a606c..17b272a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -40,6 +40,7 @@
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
@@ -244,6 +245,8 @@ public void initialize(URI name, Configuration conf) throws IOException {
}
}
+ enablePathStyleAccessIfRequired(conf);
+
maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD,
@@ -314,6 +317,14 @@ public void initialize(URI name, Configuration conf) throws IOException {
setConf(conf);
}
+ private void enablePathStyleAccessIfRequired(Configuration conf) {
+ final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
+ if (pathStyleAccess) {
+ LOG.debug("Enabling path style access!");
+ s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+ }
+ }
+
/**
* Return the protocol scheme for the FileSystem.
*