Start caching large-sized resources
Use the cache-url script (in the element cache-url) to download and cache resources which might be expensive (mostly because of their size) to fetch every time. As the shared cache ($DIB_IMAGE_CACHE) is available only when running the root.d elements, move the download phases to root.d scripts. Change-Id: Iec3e0f92e62c4c9542487a3c228ba8f9e884e5dd
This commit is contained in:
Pino Toscano
parent
999eb1f69e
commit
f2fac65cc4
|
|
@ -1,3 +1,4 @@
|
|||
cache-url
|
||||
java
|
||||
ssh
|
||||
sahara-version
|
||||
|
|
|
|||
|
|
@ -11,16 +11,6 @@ if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
|||
set -x
|
||||
fi
|
||||
|
||||
function download_hadoop_package {
|
||||
echo "Hadoop version $DIB_HADOOP_VERSION will be injected into image. Starting the download"
|
||||
install-packages wget
|
||||
wget -P $tmp_dir "http://archive.apache.org/dist/hadoop/core/hadoop-$DIB_HADOOP_VERSION/$package"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "Could not find Hadoop version $DIB_HADOOP_VERSION.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function install_hadoop_v1 {
|
||||
case "$DISTRO_NAME" in
|
||||
ubuntu )
|
||||
|
|
@ -31,8 +21,6 @@ function install_hadoop_v1 {
|
|||
;;
|
||||
esac
|
||||
|
||||
download_hadoop_package
|
||||
|
||||
case "$DISTRO_NAME" in
|
||||
ubuntu )
|
||||
dpkg -i $tmp_dir/$package
|
||||
|
|
@ -80,8 +68,6 @@ function install_hadoop_v2 {
|
|||
|
||||
package="hadoop-$DIB_HADOOP_VERSION.tar.gz"
|
||||
|
||||
download_hadoop_package
|
||||
|
||||
echo "Installing hadoop"
|
||||
|
||||
INSTALL_DIR="/opt"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,50 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
function download_hadoop_package {
|
||||
package=$1
|
||||
echo "Hadoop version $DIB_HADOOP_VERSION will be downloaded"
|
||||
cached_package="$DIB_IMAGE_CACHE/$package"
|
||||
$TMP_HOOKS_PATH/bin/cache-url "http://archive.apache.org/dist/hadoop/core/hadoop-$DIB_HADOOP_VERSION/$package" $cached_package
|
||||
sudo install -D -g root -o root -m 0644 $cached_package $tmp_dir
|
||||
}
|
||||
|
||||
function download_hadoop_v1 {
|
||||
case "$DISTRO_NAME" in
|
||||
ubuntu )
|
||||
package="hadoop_$DIB_HADOOP_VERSION-1_x86_64.deb"
|
||||
;;
|
||||
fedora | rhel | centos )
|
||||
package="hadoop-$DIB_HADOOP_VERSION-1.x86_64.rpm"
|
||||
;;
|
||||
esac
|
||||
|
||||
download_hadoop_package $package
|
||||
}
|
||||
|
||||
function download_hadoop_v2 {
|
||||
download_hadoop_package "hadoop-$DIB_HADOOP_VERSION.tar.gz"
|
||||
}
|
||||
|
||||
case "$DISTRO_NAME" in
|
||||
fedora | ubuntu | rhel | centos )
|
||||
;;
|
||||
*)
|
||||
echo "Unknown distro: $DISTRO_NAME. Exiting."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
tmp_dir=$TARGET_ROOT/tmp/hadoop
|
||||
mkdir -p $tmp_dir
|
||||
|
||||
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
|
||||
download_hadoop_v1
|
||||
else
|
||||
download_hadoop_v2
|
||||
fi
|
||||
|
|
@ -1,2 +1,3 @@
|
|||
cache-url
|
||||
hadoop
|
||||
mysql
|
||||
|
|
|
|||
|
|
@ -4,21 +4,16 @@ if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
|||
set -x
|
||||
fi
|
||||
|
||||
install-packages wget tar
|
||||
install-packages tar
|
||||
|
||||
tmp_dir=/tmp/hive
|
||||
mkdir -p $tmp_dir
|
||||
cd $tmp_dir
|
||||
|
||||
if [ -z "${HIVE_DOWNLOAD_URL:-}" ]; then
|
||||
HIVE_DOWNLOAD_URL=http://archive.apache.org/dist/hive/hive-$HIVE_VERSION/hive-$HIVE_VERSION-bin.tar.gz
|
||||
HIVE_FILE=hive-$HIVE_VERSION-bin.tar.gz
|
||||
else
|
||||
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
|
||||
fi
|
||||
wget $HIVE_DOWNLOAD_URL
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "Could not download hive.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
|
||||
tar xzf $HIVE_FILE
|
||||
HIVE_DIR="${HIVE_FILE%.*}"
|
||||
HIVE_DIR="${HIVE_DIR%.*}"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,18 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
tmp_dir=$TARGET_ROOT/tmp/hive
|
||||
mkdir -p $tmp_dir
|
||||
|
||||
if [ -z ${HIVE_DOWNLOAD_URL:-} ]; then
|
||||
HIVE_DOWNLOAD_URL=http://archive.apache.org/dist/hive/hive-$HIVE_VERSION/hive-$HIVE_VERSION-bin.tar.gz
|
||||
fi
|
||||
HIVE_FILE=$(basename $HIVE_DOWNLOAD_URL)
|
||||
cached_tar="$DIB_IMAGE_CACHE/$HIVE_FILE"
|
||||
$TMP_HOOKS_PATH/bin/cache-url $HIVE_DOWNLOAD_URL $cached_tar
|
||||
sudo install -D -g root -o root -m 0644 $cached_tar $tmp_dir
|
||||
|
|
@ -1 +1,2 @@
|
|||
cache-url
|
||||
sahara-version
|
||||
|
|
|
|||
|
|
@ -6,29 +6,27 @@ fi
|
|||
|
||||
echo "Oozie setup"
|
||||
|
||||
install-packages zip unzip tar wget
|
||||
install-packages zip unzip tar
|
||||
|
||||
tmp_dir=/tmp/oozie
|
||||
mkdir $tmp_dir
|
||||
|
||||
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
|
||||
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V1_DOWNLOAD_URL:-}"
|
||||
OOZIE_FILE="${OOZIE_HADOOP_V1_FILE:-}"
|
||||
if [ -z "${OOZIE_HADOOP_V1_DOWNLOAD_URL:-}" ]; then
|
||||
OOZIE_FILE=$(basename $OOZIE_HADOOP_V1_FILE)
|
||||
else
|
||||
OOZIE_FILE=$(basename $OOZIE_HADOOP_V1_DOWNLOAD_URL)
|
||||
fi
|
||||
elif [[ "$DIB_HADOOP_VERSION" == "2.6.0" ]]; then
|
||||
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-}"
|
||||
OOZIE_FILE="${OOZIE_HADOOP_V2_6_FILE:-}"
|
||||
if [ -z "${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-}" ]; then
|
||||
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_6_FILE)
|
||||
else
|
||||
OOZIE_FILE=$(basename $OOZIE_HADOOP_V2_6_DOWNLOAD_URL)
|
||||
fi
|
||||
else
|
||||
echo "Unknown Hadoop version selected. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "$OOZIE_DOWNLOAD_URL" ]; then
|
||||
wget -P $tmp_dir $OOZIE_DOWNLOAD_URL
|
||||
OOZIE_FILE=$(basename $OOZIE_DOWNLOAD_URL)
|
||||
elif [ -n "$OOZIE_FILE" ]; then
|
||||
install -D -g root -o root -m 0755 $(dirname $0)/$OOZIE_FILE $tmp_dir
|
||||
fi
|
||||
|
||||
cd $tmp_dir
|
||||
tar xzf $OOZIE_FILE
|
||||
rm $OOZIE_FILE
|
||||
|
|
@ -40,4 +38,4 @@ else
|
|||
mv * /opt/oozie/
|
||||
fi
|
||||
|
||||
rm -rf tmp_dir
|
||||
rm -rf $tmp_dir
|
||||
|
|
|
|||
|
|
@ -0,0 +1,30 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
tmp_dir=$TARGET_ROOT/tmp/oozie
|
||||
mkdir -p $tmp_dir
|
||||
|
||||
if [[ "$DIB_HADOOP_VERSION" < "2.0.0" ]]; then
|
||||
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V1_DOWNLOAD_URL:-}"
|
||||
OOZIE_FILE="${OOZIE_HADOOP_V1_FILE:-}"
|
||||
elif [[ "$DIB_HADOOP_VERSION" == "2.6.0" ]]; then
|
||||
OOZIE_DOWNLOAD_URL="${OOZIE_HADOOP_V2_6_DOWNLOAD_URL:-}"
|
||||
OOZIE_FILE="${OOZIE_HADOOP_V2_6_FILE:-}"
|
||||
else
|
||||
echo "Unknown Hadoop version selected. Aborting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ -n "${OOZIE_DOWNLOAD_URL:-}" ]; then
|
||||
OOZIE_FILE=$(basename $OOZIE_DOWNLOAD_URL)
|
||||
cached_tar="$DIB_IMAGE_CACHE/$OOZIE_FILE"
|
||||
$TMP_HOOKS_PATH/bin/cache-url "$OOZIE_DOWNLOAD_URL" $cached_tar
|
||||
sudo install -D -g root -o root -m 0755 $cached_tar $tmp_dir
|
||||
elif [ -n "$OOZIE_FILE" ]; then
|
||||
sudo install -D -g root -o root -m 0755 $(dirname $0)/$OOZIE_FILE $tmp_dir
|
||||
fi
|
||||
|
|
@ -1 +1,2 @@
|
|||
cache-url
|
||||
hadoop-cdh
|
||||
|
|
|
|||
|
|
@ -6,10 +6,9 @@ if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
|||
set -x
|
||||
fi
|
||||
|
||||
install-packages wget tar
|
||||
install-packages tar
|
||||
|
||||
tmp_dir=/tmp/spark
|
||||
mkdir -p $tmp_dir
|
||||
pushd $tmp_dir
|
||||
|
||||
# The user is not providing his own Spark distribution package
|
||||
|
|
@ -32,13 +31,6 @@ if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
|
|||
SPARK_DOWNLOAD_URL="http://archive.apache.org/dist/spark/spark-$DIB_SPARK_VERSION/spark-$DIB_SPARK_VERSION-bin-$SPARK_HADOOP_DL.tgz"
|
||||
fi
|
||||
|
||||
echo "Downloading SPARK"
|
||||
wget "$SPARK_DOWNLOAD_URL"
|
||||
if [ $? -ne 0 ]; then
|
||||
echo -e "Could not download spark.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Extracting SPARK"
|
||||
spark_file=$(basename "$SPARK_DOWNLOAD_URL")
|
||||
extract_folder=$(tar tzf $spark_file | sed -e 's@/.*@@' | uniq)
|
||||
|
|
|
|||
|
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
tmp_dir=$TARGET_ROOT/tmp/spark
|
||||
mkdir -p $tmp_dir
|
||||
|
||||
# The user is not providing his own Spark distribution package
|
||||
if [ -z "${SPARK_DOWNLOAD_URL:-}" ]; then
|
||||
# Check hadoop version
|
||||
# INFO on hadoop versions: http://spark.apache.org/docs/latest/hadoop-third-party-distributions.html
|
||||
if [ -z "${DIB_SPARK_VERSION:-}" ]; then
|
||||
case "$DIB_HADOOP_VERSION" in
|
||||
CDH4)
|
||||
DIB_SPARK_VERSION=1.0.0
|
||||
SPARK_HADOOP_DL=cdh4
|
||||
;;
|
||||
*)
|
||||
echo -e "WARNING: Hadoop version $DIB_HADOOP_VERSION not supported."
|
||||
echo -e "WARNING: make sure SPARK_DOWNLOAD_URL points to a compatible Spark version."
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
SPARK_DOWNLOAD_URL="http://archive.apache.org/dist/spark/spark-$DIB_SPARK_VERSION/spark-$DIB_SPARK_VERSION-bin-$SPARK_HADOOP_DL.tgz"
|
||||
fi
|
||||
|
||||
echo "Downloading SPARK"
|
||||
spark_file=$(basename "$SPARK_DOWNLOAD_URL")
|
||||
cached_tar="$DIB_IMAGE_CACHE/$spark_file"
|
||||
$TMP_HOOKS_PATH/bin/cache-url $SPARK_DOWNLOAD_URL $cached_tar
|
||||
sudo install -D -g root -o root -m 0755 $cached_tar $tmp_dir
|
||||
|
|
@ -1 +1,2 @@
|
|||
cache-url
|
||||
zookeeper
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ fi
|
|||
|
||||
install-packages wget unzip build-essential uuid-dev git libtool autoconf pkg-config tar
|
||||
|
||||
tmp_dir=/tmp
|
||||
tmp_dir=/tmp/storm
|
||||
|
||||
echo "Creating Storm User"
|
||||
groupadd -g 53001 storm
|
||||
|
|
@ -17,8 +17,7 @@ useradd -u 53001 -g 53001 -d /app/home/storm -s /bin/bash storm -c "Storm servic
|
|||
chmod 700 /app/home/storm
|
||||
chage -I -1 -E -1 -m -1 -M -1 -W -1 -E -1 storm
|
||||
|
||||
cd $tmp_dir
|
||||
echo "Downloading Storm"
|
||||
echo "Extracting Storm"
|
||||
# The user is not providing his own Spark distribution package
|
||||
if [ -z "${STORM_DOWNLOAD_URL:-}" ]; then
|
||||
# Check storm version
|
||||
|
|
@ -34,11 +33,12 @@ if [ -z "${STORM_DOWNLOAD_URL:-}" ]; then
|
|||
esac
|
||||
fi
|
||||
|
||||
wget $STORM_DOWNLOAD_URL
|
||||
storm_file=$(basename "$STORM_DOWNLOAD_URL")
|
||||
cd /usr/local
|
||||
tar -xvf /tmp/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz
|
||||
tar -xvf $tmp_dir/$storm_file
|
||||
chown -R storm:storm apache-storm-$DIB_STORM_VERSION-incubating
|
||||
ln -s apache-storm-$DIB_STORM_VERSION-incubating storm
|
||||
rm -r $tmp_dir
|
||||
|
||||
|
||||
echo "Configuring Storm home"
|
||||
|
|
|
|||
|
|
@ -0,0 +1,33 @@
|
|||
#!/bin/bash
|
||||
# This script installs storm and its dependencies
|
||||
# More documentation on the README.md file
|
||||
|
||||
if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then
|
||||
set -x
|
||||
fi
|
||||
set -eu
|
||||
set -o pipefail
|
||||
|
||||
tmp_dir=$TARGET_ROOT/tmp/storm
|
||||
mkdir -p $tmp_dir
|
||||
|
||||
echo "Downloading Storm"
|
||||
# The user is not providing his own Spark distribution package
|
||||
if [ -z "${STORM_DOWNLOAD_URL:-}" ]; then
|
||||
# Check storm version
|
||||
case "$DIB_STORM_VERSION" in
|
||||
0.9.*)
|
||||
STORM_DOWNLOAD_URL="http://archive.apache.org/dist/incubator/storm/apache-storm-$DIB_STORM_VERSION-incubating/apache-storm-$DIB_STORM_VERSION-incubating.tar.gz"
|
||||
;;
|
||||
*)
|
||||
echo -e "WARNING: Storm version $DIB_STORM_VERSION not supported."
|
||||
echo -e "WARNING: make sure STORM_DOWNLOAD_URL points to a compatible Storm version."
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
|
||||
storm_file=$(basename "$STORM_DOWNLOAD_URL")
|
||||
cached_tar="$DIB_IMAGE_CACHE/$storm_file"
|
||||
$TMP_HOOKS_PATH/bin/cache-url $STORM_DOWNLOAD_URL $cached_tar
|
||||
sudo install -D -g root -o root -m 0755 $cached_tar $tmp_dir
|
||||
Loading…
Reference in New Issue