Host some patched jars in common artifacts
Host these jars, and also provide tooling to recreate them.
These are needed for proper S3-as-a-datasource capability.
Change-Id: Ied75211044b8f51623d87d36aa0588632645eea3
(cherry picked from commit d4e774245b
)
This commit is contained in:
parent
786ed09b58
commit
abacf8f2d4
Binary file not shown.
Binary file not shown.
Binary file not shown.
|
@ -0,0 +1,17 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
|
||||
# First, install jdk, maven, git, etc.
|
||||
|
||||
for version in 2.7.1 2.7.3 2.7.5; do
|
||||
git clone git://git.apache.org/hadoop.git -b branch-$version --depth 1
|
||||
cd hadoop/hadoop-tools/hadoop-aws/
|
||||
git apply ../../../path-style-access.patch
|
||||
mvn package -Dhadoop.version=$version
|
||||
mv target/hadoop-aws-$version.jar ../../../
|
||||
cd ../../../
|
||||
rm -rf hadoop
|
||||
done
|
||||
|
||||
ls hadoop-aws-*.jar
|
|
@ -0,0 +1,48 @@
|
|||
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
|
||||
index 3486dfb..ad19d00 100644
|
||||
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
|
||||
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
|
||||
@@ -35,6 +35,7 @@
|
||||
|
||||
//use a custom endpoint?
|
||||
public static final String ENDPOINT = "fs.s3a.endpoint";
|
||||
+ public static final String PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
|
||||
//connect to s3 through a proxy server?
|
||||
public static final String PROXY_HOST = "fs.s3a.proxy.host";
|
||||
public static final String PROXY_PORT = "fs.s3a.proxy.port";
|
||||
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
|
||||
index 91a606c..17b272a 100644
|
||||
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
|
||||
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
|
||||
@@ -40,6 +40,7 @@
|
||||
|
||||
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
|
||||
import com.amazonaws.services.s3.AmazonS3Client;
|
||||
+import com.amazonaws.services.s3.S3ClientOptions;
|
||||
import com.amazonaws.services.s3.model.CannedAccessControlList;
|
||||
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
|
||||
import com.amazonaws.services.s3.model.ListObjectsRequest;
|
||||
@@ -244,6 +245,8 @@ public void initialize(URI name, Configuration conf) throws IOException {
|
||||
}
|
||||
}
|
||||
|
||||
+ enablePathStyleAccessIfRequired(conf);
|
||||
+
|
||||
maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
|
||||
partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
|
||||
multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD,
|
||||
@@ -314,6 +317,14 @@ public void initialize(URI name, Configuration conf) throws IOException {
|
||||
setConf(conf);
|
||||
}
|
||||
|
||||
+ private void enablePathStyleAccessIfRequired(Configuration conf) {
|
||||
+ final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
|
||||
+ if (pathStyleAccess) {
|
||||
+ LOG.debug("Enabling path style access!");
|
||||
+ s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
|
||||
+ }
|
||||
+ }
|
||||
+
|
||||
/**
|
||||
* Return the protocol scheme for the FileSystem.
|
||||
*
|
Loading…
Reference in New Issue