summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJeremy Freudberg <jeremyfreudberg@gmail.com>2018-06-22 10:47:27 -0400
committerLuigi Toscano <ltoscano@redhat.com>2018-08-04 00:01:38 +0200
commitabacf8f2d43b0b5d2769849d48e78120c58fe163 (patch)
tree587ace35c581d76854ab5c91a4df2bf6d8e714b3
parent786ed09b58d6e1ec787f66b6d83d965874d01370 (diff)
Host some patched jars in common artifacts9.2.0
Host these jars, and also provide tooling to recreate them. These are needed for proper S3-as-a-datasource capability. Change-Id: Ied75211044b8f51623d87d36aa0588632645eea3 (cherry picked from commit d4e774245b6c1ce337b4bbc28ea877086b6f0f83)
Notes
Notes (review): Code-Review+2: Telles Mota Vidal Nóbrega <tenobreg@redhat.com> Workflow+1: Telles Mota Vidal Nóbrega <tenobreg@redhat.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Mon, 06 Aug 2018 17:28:19 +0000 Reviewed-on: https://review.openstack.org/588672 Project: openstack/sahara-extra Branch: refs/heads/stable/rocky
-rw-r--r--common-artifacts/hadoop-aws-2.7.1.jarbin0 -> 103460 bytes
-rw-r--r--common-artifacts/hadoop-aws-2.7.3.jarbin0 -> 126637 bytes
-rw-r--r--common-artifacts/hadoop-aws-2.7.5.jarbin0 -> 126862 bytes
-rwxr-xr-xtools/generate-patched-hadoop-aws.sh17
-rw-r--r--tools/path-style-access.patch48
5 files changed, 65 insertions, 0 deletions
diff --git a/common-artifacts/hadoop-aws-2.7.1.jar b/common-artifacts/hadoop-aws-2.7.1.jar
new file mode 100644
index 0000000..f729e8b
--- /dev/null
+++ b/common-artifacts/hadoop-aws-2.7.1.jar
Binary files differ
diff --git a/common-artifacts/hadoop-aws-2.7.3.jar b/common-artifacts/hadoop-aws-2.7.3.jar
new file mode 100644
index 0000000..0d93d7c
--- /dev/null
+++ b/common-artifacts/hadoop-aws-2.7.3.jar
Binary files differ
diff --git a/common-artifacts/hadoop-aws-2.7.5.jar b/common-artifacts/hadoop-aws-2.7.5.jar
new file mode 100644
index 0000000..8d666a2
--- /dev/null
+++ b/common-artifacts/hadoop-aws-2.7.5.jar
Binary files differ
diff --git a/tools/generate-patched-hadoop-aws.sh b/tools/generate-patched-hadoop-aws.sh
new file mode 100755
index 0000000..898e87c
--- /dev/null
+++ b/tools/generate-patched-hadoop-aws.sh
@@ -0,0 +1,17 @@
1#!/bin/bash
2
3set -e
4
5# First, install jdk, maven, git, etc.
6
7for version in 2.7.1 2.7.3 2.7.5; do
8 git clone git://git.apache.org/hadoop.git -b branch-$version --depth 1
9 cd hadoop/hadoop-tools/hadoop-aws/
10 git apply ../../../path-style-access.patch
11 mvn package -Dhadoop.version=$version
12 mv target/hadoop-aws-$version.jar ../../../
13 cd ../../../
14 rm -rf hadoop
15done
16
17ls hadoop-aws-*.jar
diff --git a/tools/path-style-access.patch b/tools/path-style-access.patch
new file mode 100644
index 0000000..87ff480
--- /dev/null
+++ b/tools/path-style-access.patch
@@ -0,0 +1,48 @@
1diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
2index 3486dfb..ad19d00 100644
3--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
4+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
5@@ -35,6 +35,7 @@
6
7 //use a custom endpoint?
8 public static final String ENDPOINT = "fs.s3a.endpoint";
9+ public static final String PATH_STYLE_ACCESS = "fs.s3a.path.style.access";
10 //connect to s3 through a proxy server?
11 public static final String PROXY_HOST = "fs.s3a.proxy.host";
12 public static final String PROXY_PORT = "fs.s3a.proxy.port";
13diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
14index 91a606c..17b272a 100644
15--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
16+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
17@@ -40,6 +40,7 @@
18
19 import com.amazonaws.auth.InstanceProfileCredentialsProvider;
20 import com.amazonaws.services.s3.AmazonS3Client;
21+import com.amazonaws.services.s3.S3ClientOptions;
22 import com.amazonaws.services.s3.model.CannedAccessControlList;
23 import com.amazonaws.services.s3.model.DeleteObjectsRequest;
24 import com.amazonaws.services.s3.model.ListObjectsRequest;
25@@ -244,6 +245,8 @@ public void initialize(URI name, Configuration conf) throws IOException {
26 }
27 }
28
29+ enablePathStyleAccessIfRequired(conf);
30+
31 maxKeys = conf.getInt(MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS);
32 partSize = conf.getLong(MULTIPART_SIZE, DEFAULT_MULTIPART_SIZE);
33 multiPartThreshold = conf.getInt(MIN_MULTIPART_THRESHOLD,
34@@ -314,6 +317,14 @@ public void initialize(URI name, Configuration conf) throws IOException {
35 setConf(conf);
36 }
37
38+ private void enablePathStyleAccessIfRequired(Configuration conf) {
39+ final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
40+ if (pathStyleAccess) {
41+ LOG.debug("Enabling path style access!");
42+ s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
43+ }
44+ }
45+
46 /**
47 * Return the protocol scheme for the FileSystem.
48 *