sahara-plugin-spark/sahara_plugin_spark/plugins/spark/resources/images/common/manipulate_s3

13 lines
597 B
Bash

#!/bin/bash
SPARK_JARS_DIR_PATH="/opt/spark/jars"
HADOOP_TOOLS_DIR_PATH="/opt/hadoop/share/hadoop/tools/lib"
HADOOP_COMMON_DIR_PATH="/opt/hadoop/share/hadoop/common/lib"
# The hadoop-aws and aws-java-sdk libraries are missing here, but we
# cannot copy them from the Hadoop folder on-disk due to
# version/patching issues
curl -sS https://tarballs.openstack.org/sahara-extra/dist/common-artifacts/hadoop-aws-2.7.3.jar -o $SPARK_JARS_DIR_PATH/hadoop-aws.jar
curl -sS https://repo1.maven.org/maven2/com/amazonaws/aws-java-sdk/1.7.4/aws-java-sdk-1.7.4.jar -o $SPARK_JARS_DIR_PATH/aws-java-sdk.jar