Image Elements for MapR
Change-Id: I87fde2f5d5b430b935c3261d31b86a7b68ac3244 Implements: blueprint mapr-image-elements
This commit is contained in:
Artem Osadchiy
artemosadchiy
parent
351c8e41b2
commit
61823d408c
|
|
@ -27,7 +27,7 @@ NOTE: Do not create all images for all plugins with the same mirrors. Different
|
|||
|
||||
.. sourcecode:: bash
|
||||
|
||||
sudo bash sahara-image-elements/diskimage-create/diskimage-create.sh -p [vanilla|spark|hdp|cloudera|storm]
|
||||
sudo bash sahara-image-elements/diskimage-create/diskimage-create.sh -p [vanilla|spark|hdp|cloudera|storm|mapr]
|
||||
|
||||
5. To select which hadoop version to target use the '-v' commandline option like this:
|
||||
|
||||
|
|
|
|||
|
|
@ -12,20 +12,25 @@ DEBUG_MODE="false"
|
|||
# The default tag to use for the DIB repo
|
||||
DEFAULT_DIB_REPO_BRANCH="0.1.29"
|
||||
|
||||
# The default version for a MapR plugin
|
||||
DIB_DEFAULT_MAPR_VERSION="4.0.1"
|
||||
|
||||
# Default list of datasource modules for ubuntu. Workaround for bug #1375645
|
||||
export CLOUD_INIT_DATASOURCES=${DIB_CLOUD_INIT_DATASOURCES:-"NoCloud, ConfigDrive, OVF, MAAS, Ec2"}
|
||||
|
||||
usage() {
|
||||
echo
|
||||
echo "Usage: $(basename $0)"
|
||||
echo " [-p vanilla|spark|hdp|cloudera|storm]"
|
||||
echo " [-p vanilla|spark|hdp|cloudera|storm|mapr]"
|
||||
echo " [-i ubuntu|fedora|centos]"
|
||||
echo " [-v 1|2|2.3|2.4|plain]"
|
||||
echo " [-r 3.1.1|4.0.1]"
|
||||
echo " [-d]"
|
||||
echo " [-m]"
|
||||
echo " '-p' is plugin version (default: all plugins)"
|
||||
echo " '-i' is operating system of the base image (default: all supported by plugin)"
|
||||
echo " '-v' is hadoop version (default: all supported by plugin)"
|
||||
echo " '-r' is MapR Version (default: ${DIB_DEFAULT_MAPR_VERSION})"
|
||||
echo " '-d' enable debug mode, root account will have password 'hadoop'"
|
||||
echo " '-m' set the diskimage-builder repo to the master branch (default: $DEFAULT_DIB_REPO_BRANCH)"
|
||||
echo
|
||||
|
|
@ -38,7 +43,7 @@ usage() {
|
|||
exit 1
|
||||
}
|
||||
|
||||
while getopts "p:i:v:dm" opt; do
|
||||
while getopts "p:i:v:dmr:" opt; do
|
||||
case $opt in
|
||||
p)
|
||||
PLUGIN=$OPTARG
|
||||
|
|
@ -60,6 +65,9 @@ while getopts "p:i:v:dm" opt; do
|
|||
DIB_REPO_BRANCH="master"
|
||||
fi
|
||||
;;
|
||||
r)
|
||||
DIB_MAPR_VERSION=$OPTARG
|
||||
;;
|
||||
*)
|
||||
usage
|
||||
;;
|
||||
|
|
@ -93,7 +101,7 @@ if [ "$DEBUG_MODE" = "true" -a "$platform" != 'NAME="Ubuntu"' ]; then
|
|||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" -a "$PLUGIN" != "storm" ]; then
|
||||
if [ -n "$PLUGIN" -a "$PLUGIN" != "vanilla" -a "$PLUGIN" != "spark" -a "$PLUGIN" != "hdp" -a "$PLUGIN" != "cloudera" -a "$PLUGIN" != "storm" -a "$PLUGIN" != "mapr" ]; then
|
||||
echo -e "Unknown plugin selected.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
|
|
@ -120,6 +128,27 @@ if [ "$PLUGIN" = "cloudera" -a "$BASE_IMAGE_OS" = "fedora" ]; then
|
|||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$PLUGIN" = "mapr" -a "$BASE_IMAGE_OS" = "fedora" ]; then
|
||||
echo "'fedora' image type is not supported by 'mapr' plugin.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$PLUGIN" != "mapr" -a -n "$DIB_MAPR_VERSION" ]; then
|
||||
echo "'-r' parameter should be used only with 'mapr' plugin.\nAborting"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ "$PLUGIN" = "mapr" -a -z "$DIB_MAPR_VERSION" ]; then
|
||||
echo "MapR version is not specified.\n"
|
||||
echo "${DIB_DEFAULT_MAPR_VERSION} version would be used.\n"
|
||||
DIB_MAPR_VERSION=${DIB_DEFAULT_MAPR_VERSION}
|
||||
fi
|
||||
|
||||
if [ "$PLUGIN" = "mapr" -a "${DIB_MAPR_VERSION}" != "3.1.1" -a "${DIB_MAPR_VERSION}" != "4.0.1" ]; then
|
||||
echo "Unknown MapR version.\nExit"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
#################
|
||||
|
||||
if [ "$platform" = 'NAME="Ubuntu"' ]; then
|
||||
|
|
@ -485,5 +514,58 @@ if [ -z "$PLUGIN" -o "$PLUGIN" = "cloudera" ]; then
|
|||
unset EXTJS_DOWNLOAD_URL
|
||||
fi
|
||||
|
||||
##########################
|
||||
# Images for MapR plugin #
|
||||
##########################
|
||||
if [ -z "$PLUGIN" -o "$PLUGIN" = "mapr" ]; then
|
||||
echo "For mapr plugin option -v is ignored"
|
||||
export DIB_MAPR_VERSION=${DIB_MAPR_VERSION:-4.0.1}
|
||||
|
||||
export DIB_CLOUD_INIT_DATASOURCES=$CLOUD_INIT_DATASOURCES
|
||||
|
||||
export DIB_IMAGE_SIZE=${IMAGE_SIZE:-"10"}
|
||||
#MapR repository requires additional space
|
||||
export DIB_MIN_TMPFS=10
|
||||
|
||||
export JAVA_DOWNLOAD_URL=${JAVA_DOWNLOAD_URL:-"http://download.oracle.com/otn-pub/java/jdk/7u51-b13/jdk-7u51-linux-x64.tar.gz"}
|
||||
|
||||
mapr_ubuntu_elements_sequence="base vm ssh ubuntu hadoop-mapr"
|
||||
mapr_centos_elements_sequence="base vm rhel ssh hadoop-mapr redhat-lsb selinux-permissive"
|
||||
|
||||
if [ "$DEBUG_MODE" = "true" ]; then
|
||||
mapr_ubuntu_elements_sequence="$mapr_ubuntu_elements_sequence root-passwd"
|
||||
mapr_centos_elements_sequence="$mapr_centos_elements_sequence root-passwd"
|
||||
fi
|
||||
|
||||
if [ -n "$USE_MIRRORS" ]; then
|
||||
[ -n "$UBUNTU_MIRROR" ] && ubuntu_elements_sequence="$mapr_ubuntu_elements_sequence apt-mirror"
|
||||
[ -n "$CENTOS_MIRROR" ] && centos_elements_sequence="$mapr_centos_elements_sequence centos-mirror"
|
||||
fi
|
||||
|
||||
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "ubuntu" ]; then
|
||||
export DIB_RELEASE=${DIB_RELEASE:-trusty}
|
||||
|
||||
mapr_ubuntu_image_name=${mapr_ubuntu_image_name:-ubuntu_${DIB_RELEASE}_mapr_${DIB_MAPR_VERSION}_latest}
|
||||
|
||||
disk-image-create $mapr_ubuntu_elements_sequence -n -o $mapr_ubuntu_image_name
|
||||
mv $mapr_ubuntu_image_name.qcow2 ../
|
||||
|
||||
unset DIB_RELEASE
|
||||
fi
|
||||
|
||||
if [ -z "$BASE_IMAGE_OS" -o "$BASE_IMAGE_OS" = "centos" ]; then
|
||||
export BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-"CentOS-6.6-cloud-init-20141118.qcow2"}
|
||||
export DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-"http://sahara-files.mirantis.com"}
|
||||
|
||||
mapr_centos_image_name=${mapr_centos_image_name:-centos_6.5_mapr_${DIB_MAPR_VERSION}_latest}
|
||||
|
||||
disk-image-create $mapr_centos_elements_sequence -n -o $mapr_centos_image_name
|
||||
mv $mapr_centos_image_name.qcow2 ../
|
||||
|
||||
unset BASE_IMAGE_FILE DIB_CLOUD_IMAGES
|
||||
unset DIB_CLOUD_INIT_DATASOURCES
|
||||
fi
|
||||
fi
|
||||
|
||||
popd # out of $TEMP
|
||||
rm -rf $TEMP
|
||||
|
|
|
|||
|
|
@ -0,0 +1,21 @@
|
|||
Creates images with local mirrors of MapR repositories: `core <http://package.mapr.com/releases/>`_ and `ecosystem <http://package.mapr.com/releases/ecosystem-4.x/>`_.
|
||||
Installs `OpenJDK <http://http://openjdk.java.net/>`_ and `Scala <http://www.scala-lang.org/>`_.
|
||||
|
||||
In order to create the MapR images with the diskimage-create.sh script, use the following syntax to select the "MapR" plugin:
|
||||
.. sourcecode:: bash
|
||||
|
||||
sudo bash diskimage-create.sh -p mapr [-i ubuntu|centos] [-r 3.1.1|4.0.1]
|
||||
|
||||
NOTE: By default MapR 4.0.1 version will be used
|
||||
|
||||
In order to speed up image creation process you can download archives with MapR repositories and specify environment variable:
|
||||
``DIB_MAPR_CORE_DEB_REPO``, ``DIB_MAPR_CORE_RPM_REPO``, ``DIB_MAPR_ECO_DEB_REPO``, ``DIB_MAPR_ECO_RPM_REPO``
|
||||
|
||||
For example:
|
||||
.. sourcecode:: bash
|
||||
|
||||
export DIB_MAPR_CORE_DEB_REPO="file://<path-to-archive>/mapr-v4.0.1GA.deb.tgz"
|
||||
export DIB_MAPR_CORE_RPM_REPO="file://<path-to-archive>/mapr-v4.0.1GA.rpm.tgz"
|
||||
export DIB_MAPR_ECO_DEB_REPO="http://<URL>/mapr-ecosystem.deb.tgz"
|
||||
export DIB_MAPR_ECO_RPM_REPO="http://<URL>/mapr-ecosystem.rpm.tgz"
|
||||
sudo -E bash diskimage-create.sh -p mapr -r 4.0.1
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
java
|
||||
mysql
|
||||
disable-firewall
|
||||
|
|
@ -0,0 +1,159 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
echo "START: installing MapR core dependencies"
|
||||
|
||||
DISTRO=$(lsb_release -is || :)
|
||||
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
# Required for libicu48
|
||||
cat >> /etc/apt/sources.list.d/security_repo.list << EOF
|
||||
deb http://security.ubuntu.com/ubuntu precise-security main
|
||||
EOF
|
||||
# Required for libmysqlclient16
|
||||
cat >> /etc/apt/sources.list.d/security_repo.list << EOF
|
||||
deb http://security.ubuntu.com/ubuntu lucid-security main
|
||||
EOF
|
||||
apt-get update
|
||||
install-packages binutils \
|
||||
daemon \
|
||||
dpkg-dev \
|
||||
dpkg-repack \
|
||||
gcc \
|
||||
gcc-4.8 \
|
||||
gcc-doc \
|
||||
gcc-multilib \
|
||||
iputils-arping \
|
||||
libasan0 \
|
||||
libatomic1 \
|
||||
libc-dev-bin \
|
||||
libc6 \
|
||||
libc6-dev \
|
||||
libcrypt-passwdmd5-perl \
|
||||
libevent-1.4-2 \
|
||||
libevent-2.0-5 \
|
||||
libgcc-4.8-dev \
|
||||
libgomp1 \
|
||||
libgssglue1 \
|
||||
libicu48 \
|
||||
libitm1 \
|
||||
libmysqlclient-dev \
|
||||
libmysqlclient16 \
|
||||
libmysqlclient18 \
|
||||
libnfsidmap2 \
|
||||
libquadmath0 \
|
||||
libsasl2-dev \
|
||||
libsasl2-modules-gssapi-mit \
|
||||
libssl0.9.8 \
|
||||
libtirpc1 \
|
||||
libtsan0 \
|
||||
libxslt1.1 \
|
||||
linux-libc-dev \
|
||||
manpages-dev \
|
||||
mtools \
|
||||
mysql-common \
|
||||
nfs-common \
|
||||
open-iscsi \
|
||||
openjdk-6-jre \
|
||||
rpcbind \
|
||||
sdparm \
|
||||
syslinux \
|
||||
syslinux-common \
|
||||
unzip \
|
||||
watchdog \
|
||||
zlib1g-dev \
|
||||
zip
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
install-packages cdparanoia-libs \
|
||||
cups \
|
||||
cups-libs \
|
||||
createrepo \
|
||||
cvs \
|
||||
cyrus-sasl-gssapi \
|
||||
cyrus-sasl-plain \
|
||||
db4-cxx \
|
||||
db4-devel \
|
||||
foomatic \
|
||||
foomatic-db \
|
||||
foomatic-db-filesystem \
|
||||
foomatic-db-ppds \
|
||||
gdbm-devel \
|
||||
gettext \
|
||||
ghostscript \
|
||||
ghostscript-fonts \
|
||||
glibc \
|
||||
glibc-common \
|
||||
glibc-devel \
|
||||
glibc-headers \
|
||||
gstreamer \
|
||||
gstreamer-plugins-base \
|
||||
gstreamer-tools \
|
||||
hdparm \
|
||||
irqbalance \
|
||||
iso-codes \
|
||||
kernel-headers \
|
||||
lcms-libs \
|
||||
libXt \
|
||||
libXv \
|
||||
libXxf86vm \
|
||||
libgomp \
|
||||
libgssglue \
|
||||
libgudev1 \
|
||||
libicu \
|
||||
libmng \
|
||||
liboil \
|
||||
libtheora \
|
||||
libtirpc \
|
||||
libvisual \
|
||||
libxslt \
|
||||
mesa-dri-drivers \
|
||||
mesa-dri-filesystem \
|
||||
mesa-dri1-drivers \
|
||||
mesa-libGL \
|
||||
mesa-libGLU \
|
||||
mesa-private-llvm \
|
||||
mtools \
|
||||
nc \
|
||||
numactl \
|
||||
openjpeg-libs \
|
||||
patch \
|
||||
pax \
|
||||
perl-CGI \
|
||||
perl-ExtUtils-MakeMaker \
|
||||
perl-ExtUtils-ParseXS \
|
||||
perl-Test-Harness \
|
||||
perl-Test-Simple \
|
||||
perl-devel \
|
||||
phonon-backend-gstreamer \
|
||||
poppler \
|
||||
poppler-data \
|
||||
poppler-utils \
|
||||
portreserve \
|
||||
qt \
|
||||
qt-sqlite \
|
||||
qt-x11 \
|
||||
qt3 \
|
||||
redhat-lsb \
|
||||
redhat-lsb-compat \
|
||||
redhat-lsb-core \
|
||||
redhat-lsb-graphics \
|
||||
redhat-lsb-printing \
|
||||
rpcbind \
|
||||
sdparm \
|
||||
soci \
|
||||
soci-mysql \
|
||||
syslinux \
|
||||
syslinux-nonlinux \
|
||||
unzip \
|
||||
urw-fonts \
|
||||
xml-common \
|
||||
zip
|
||||
fi
|
||||
|
||||
if [ "$?" != "0" ]; then
|
||||
echo "Installation failed. Exit"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "END: installing MapR core dependencies"
|
||||
|
|
@ -0,0 +1,39 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
echo "START: installing Scala"
|
||||
|
||||
DISTRO=$(lsb_release -is || :)
|
||||
|
||||
#Current available version
|
||||
DEF_VERSION="2.11.2"
|
||||
|
||||
RETURN_CODE="$(curl -s -o /dev/null -w "%{http_code}" http://www.scala-lang.org/)"
|
||||
|
||||
if [ "$RETURN_CODE" != "200" ]; then
|
||||
echo "http://www.scala-lang.org is unreachable"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
VERSION="$(curl -s --fail http://www.scala-lang.org|grep 'scala-version'|grep -Eo '([0-9]\.?)+')"
|
||||
|
||||
if [ $? != 0 -o -z ${VERSION} ]; then
|
||||
echo "Installing default version $DEF_VERSION"
|
||||
VERSION=${DEF_VERSION}
|
||||
fi
|
||||
|
||||
PKG=scala-${VERSION}
|
||||
|
||||
URL="http://downloads.typesafe.com/scala/${VERSION}"
|
||||
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
wget -N ${URL}/${PKG}.deb
|
||||
dpkg -i ${PKG}.deb
|
||||
rm ${PKG}.deb
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
rpm -Uhv ${URL}/${PKG}.rpm
|
||||
fi
|
||||
|
||||
echo "END: installing Scala"
|
||||
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
install -D -g root -o root -m 0644 $(dirname $0)/versions.py /tmp/versions.py
|
||||
|
|
@ -0,0 +1,63 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
echo "START: installing MapR core repository"
|
||||
|
||||
install-packages curl
|
||||
|
||||
MAPR_REPO_DIR=/opt/mapr-repository/core
|
||||
|
||||
DISTRO=$(lsb_release -is || :)
|
||||
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
MAPR_REPO_URL=${DIB_MAPR_CORE_DEB_REPO:-"http://package.mapr.com/releases/v"${DIB_MAPR_VERSION}"/ubuntu/mapr-v"${DIB_MAPR_VERSION}"GA.deb.tgz"}
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
MAPR_REPO_URL=${DIB_MAPR_CORE_RPM_REPO:-"http://package.mapr.com/releases/v"${DIB_MAPR_VERSION}"/redhat/mapr-v"${DIB_MAPR_VERSION}"GA.rpm.tgz"}
|
||||
else
|
||||
echo "Unknown distribution"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Downloading MapR repository archive"
|
||||
mkdir -p ${MAPR_REPO_DIR}
|
||||
sudo curl ${MAPR_REPO_URL} -o ${MAPR_REPO_DIR}/mapr-v"${DIB_MAPR_VERSION}"GA.tgz
|
||||
|
||||
if [ $? != 0 ]; then
|
||||
echo "MapR repository archive download failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Unpacking MapR repository archive"
|
||||
tar -zxf ${MAPR_REPO_DIR}/mapr-v${DIB_MAPR_VERSION}GA.tgz -C ${MAPR_REPO_DIR}
|
||||
|
||||
echo "Removing MapR repository archive"
|
||||
rm -f ${MAPR_REPO_DIR}/mapr-v${DIB_MAPR_VERSION}GA.tgz
|
||||
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
mkdir -p "${MAPR_REPO_DIR}"/dists/binary/optional/binary-amd64
|
||||
cd ${MAPR_REPO_DIR}
|
||||
dpkg-scanpackages -m . /dev/null | gzip -9c > dists/binary/optional/binary-amd64/Packages.gz
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
createrepo "${MAPR_REPO_DIR}"
|
||||
fi
|
||||
|
||||
echo "Adding MapR repository"
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
cat >> /etc/apt/sources.list.d/maprtech.list << EOF
|
||||
deb [arch=amd64] file://${MAPR_REPO_DIR} binary optional
|
||||
EOF
|
||||
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
cat >> /etc/yum.repos.d/maprtech.repo << EOF
|
||||
[mapr-core]
|
||||
name=MapR Technologies
|
||||
baseurl=file://${MAPR_REPO_DIR}
|
||||
enabled=1
|
||||
gpgcheck=0
|
||||
protect=1
|
||||
EOF
|
||||
|
||||
fi
|
||||
|
||||
echo "END: installing MapR core repository"
|
||||
|
|
@ -0,0 +1,87 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
echo "START: installing MapR ecosystem repository"
|
||||
|
||||
MAPR_REPO_DIR=/opt/mapr-repository/ecosystem
|
||||
|
||||
DISTRO=$(lsb_release -is || :)
|
||||
|
||||
RPM_REPO=${DIB_MAPR_ECO_RPM_REPO:-}
|
||||
DEB_REPO=${DIB_MAPR_ECO_DEB_REPO:-}
|
||||
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
if [ "$DIB_MAPR_VERSION" = "3.1.1" ]; then
|
||||
MAPR_REPO_URL=${DIB_MAPR_ECO_DEB_REPO:-"http://package.mapr.com/releases/ecosystem/ubuntu/"}
|
||||
elif [ "$DIB_MAPR_VERSION" = "4.0.1" ]; then
|
||||
MAPR_REPO_URL=${DIB_MAPR_ECO_DEB_REPO:-"http://package.mapr.com/releases/ecosystem-4.x/ubuntu/"}
|
||||
fi
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
if [ "$DIB_MAPR_VERSION" = "3.1.1" ]; then
|
||||
MAPR_REPO_URL=${DIB_MAPR_ECO_RPM_REPO:-"http://package.mapr.com/releases/ecosystem/redhat/"}
|
||||
elif [ "$DIB_MAPR_VERSION" = "4.0.1" ]; then
|
||||
MAPR_REPO_URL=${DIB_MAPR_ECO_RPM_REPO:-"http://package.mapr.com/releases/ecosystem-4.x/redhat/"}
|
||||
fi
|
||||
else
|
||||
echo "Unknown distribution"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Downloading MapR repository archive"
|
||||
if [ -z ${DEB_REPO} -o -z ${RPM_REPO} ]; then
|
||||
LATEST_ARCHIVE=$(curl -s ${MAPR_REPO_URL} | grep -Po '(?<=<a href=")(mapr-ecosystem.*?\.tgz)(?<!">)' | uniq | sort | tail -n 1)
|
||||
if [ -z ${LATEST_ARCHIVE} ]; then
|
||||
echo "No repository archive found"
|
||||
exit 1
|
||||
fi
|
||||
MAPR_REPO_URL=${MAPR_REPO_URL}/${LATEST_ARCHIVE}
|
||||
fi
|
||||
|
||||
mkdir -p ${MAPR_REPO_DIR}
|
||||
|
||||
sudo curl ${MAPR_REPO_URL} -o ${MAPR_REPO_DIR}/mapr-ecosystem.tgz
|
||||
|
||||
if [ $? != 0 ]; then
|
||||
echo "MapR repository archive download failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Unpacking MapR repository archive"
|
||||
tar -zxf ${MAPR_REPO_DIR}/mapr-ecosystem.tgz -C ${MAPR_REPO_DIR}
|
||||
|
||||
echo "Removing MapR repository archive"
|
||||
rm -f ${MAPR_REPO_DIR}/mapr-ecosystem.tgz
|
||||
|
||||
echo "Removing old packages"
|
||||
SCRIPT_FILE="/tmp/versions.py"
|
||||
SPEC_FILE="/opt/mapr-repository/version-spec"
|
||||
python ${SCRIPT_FILE} --path ${MAPR_REPO_DIR} --spec-file ${SPEC_FILE} | xargs sudo rm -f || echo "Operation failed"
|
||||
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
mkdir -p "${MAPR_REPO_DIR}"/dists/binary/optional/binary-amd64
|
||||
cd ${MAPR_REPO_DIR}
|
||||
dpkg-scanpackages -m . /dev/null | gzip -9c > dists/binary/optional/binary-amd64/Packages.gz
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
createrepo "${MAPR_REPO_DIR}"
|
||||
fi
|
||||
|
||||
echo "Adding MapR repository"
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
cat >> /etc/apt/sources.list.d/maprtech.list << EOF
|
||||
deb [arch=amd64] file://${MAPR_REPO_DIR} binary optional
|
||||
EOF
|
||||
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
cat >> /etc/yum.repos.d/maprtech.repo << EOF
|
||||
[mapr-ecosystem]
|
||||
name=MapR Technologies
|
||||
baseurl=file://${MAPR_REPO_DIR}
|
||||
enabled=1
|
||||
gpgcheck=0
|
||||
protect=1
|
||||
EOF
|
||||
fi
|
||||
|
||||
echo "END: installing MapR ecosystem repository"
|
||||
|
||||
|
|
@ -0,0 +1,15 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
echo "START: downloading ExtJS library"
|
||||
|
||||
install-packages wget
|
||||
|
||||
MAPR_REPO_DIR="/opt/mapr-repository/"
|
||||
EXT_JS_URL="http://extjs.com/deploy/ext-2.2.zip"
|
||||
|
||||
mkdir -p ${MAPR_REPO_DIR}
|
||||
sudo wget ${EXT_JS_URL} -P ${MAPR_REPO_DIR}
|
||||
|
||||
echo "END: downloading ExtJS library"
|
||||
|
|
@ -0,0 +1,102 @@
|
|||
# Copyright (c) 2014, MapR Technologies
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License"); you may
|
||||
# not use this file except in compliance with the License. You may obtain
|
||||
# a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
|
||||
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
|
||||
# License for the specific language governing permissions and limitations
|
||||
# under the License.
|
||||
|
||||
import argparse
|
||||
import collections as c
|
||||
import itertools as _iter
|
||||
import operator as op
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
|
||||
spec = {
|
||||
'asynchbase': 1, 'cascading': 1, 'flume': 1, 'hbase': 1,
|
||||
'hbasethrift': 1, 'hcatalog': 1, 'hive': 2, 'hivemetastore': 2,
|
||||
'hiveserver2': 2, 'httpfs': 1, 'hue': 1, 'impala': 1, 'mahout': 1,
|
||||
'oozie': 2, 'pig': 1, 'spark': 1, 'sqoop': 0, 'sqoop2': 1, 'whirr': 1
|
||||
}
|
||||
|
||||
reg_expr = ur"mapr-([a-z]+\d?)[-_]([a-z]+)?[-_]?(\d+).(\d+).(\d+).(\d+)?"
|
||||
pattern = re.compile(reg_expr)
|
||||
|
||||
file_map = c.defaultdict(list)
|
||||
version_map = c.defaultdict(list)
|
||||
comp_map = c.defaultdict(tuple)
|
||||
|
||||
|
||||
def parse_filename(args):
|
||||
def mapper(arg):
|
||||
try:
|
||||
new_int = int(arg)
|
||||
return new_int
|
||||
except ValueError:
|
||||
return arg
|
||||
except TypeError:
|
||||
pass
|
||||
|
||||
def predicate(_type):
|
||||
return lambda x: isinstance(x, _type)
|
||||
|
||||
arg_list = filter(lambda x: op.is_not(x, None),
|
||||
map(mapper, args))
|
||||
|
||||
pair = (tuple(filter(predicate(str), arg_list)),
|
||||
filter(predicate(int), arg_list))
|
||||
|
||||
return pair
|
||||
|
||||
|
||||
def main(arg_list):
|
||||
if os.path.exists(arg_list.path):
|
||||
for f_name in os.listdir(arg_list.path):
|
||||
match = pattern.match(f_name)
|
||||
if match:
|
||||
file_map[f_name] = parse_filename(match.groups())
|
||||
|
||||
for k, g in _iter.groupby(file_map.values(),
|
||||
key=lambda i: op.getitem(i, 0)):
|
||||
for elem in g:
|
||||
version_map[k].append(op.getitem(elem, 1))
|
||||
|
||||
for (k, v) in version_map.items():
|
||||
comp_map[k] = sorted(v, reverse=True)[:spec.get(k[0], 1)]
|
||||
|
||||
version_spec = set()
|
||||
|
||||
with open(arg_list.spec_file, mode='w') as f:
|
||||
not_build_version = lambda a: len(str(a)) < 3
|
||||
for k, v in comp_map.items():
|
||||
join_versions = lambda seq: '.'.join(
|
||||
str(el) for el in filter(not_build_version, seq))
|
||||
v_str = ','.join(map(join_versions, v))
|
||||
version_spec.add('{comp:s} = {versions:s}\n'
|
||||
.format(**{'comp': k[0], 'versions': v_str}))
|
||||
f.writelines(version_spec)
|
||||
|
||||
for (f_name, comp) in file_map.items():
|
||||
if not comp[1] in comp_map[comp[0]]:
|
||||
target = os.path.join(arg_list.path, f_name)
|
||||
sys.stdout.write(str(target).strip() + '\n')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('--path', help='path to repository files')
|
||||
parser.add_argument('--spec-file', help='path to specification files')
|
||||
args = parser.parse_args()
|
||||
if not args.path or not args.spec_file:
|
||||
parser.print_help()
|
||||
exit(-1)
|
||||
main(args)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
echo "/etc/profile.d/java.sh" >> /etc/ssh/sshrc
|
||||
chmod 775 /etc/profile.d/java.sh
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
DISTRO=$(lsb_release -is || :)
|
||||
|
||||
if [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
sed '/^Defaults requiretty*/ s/^/#/' -i /etc/sudoers
|
||||
fi
|
||||
|
||||
|
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
DISTRO=$(lsb_release -is || :)
|
||||
|
||||
if [ "$DISTRO" = "Ubuntu" ]; then
|
||||
apt-get update
|
||||
elif [ "$DISTRO" = 'CentOS' -o "$DISTRO" = 'RedHatEnterpriseServer' ]; then
|
||||
yum clean all && yum repolist
|
||||
fi
|
||||
|
|
@ -0,0 +1,10 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -eux
|
||||
|
||||
echo "MapR version ${DIB_MAPR_VERSION}"
|
||||
|
||||
if [ -z ${DIB_MAPR_VERSION} ]; then
|
||||
echo "DIB_MAPR_VERSION not set.\nExit"
|
||||
exit 1
|
||||
fi
|
||||
2
tox.ini
2
tox.ini
|
|
@ -17,7 +17,7 @@ downloadcache = ~/cache/pip
|
|||
|
||||
[testenv:bashate]
|
||||
whitelist_externals = bash
|
||||
commands = bash -c "find {toxinidir} -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst \) -print0 | xargs -0 bashate -v"
|
||||
commands = bash -c "find {toxinidir} -not -wholename \*.tox/\* -and \( -name \*.sh -or -wholename \*.d/\* -and -not -name \*.md -and -not -name \*.rst -and -not -name \*.py \) -print0 | xargs -0 bashate -v"
|
||||
|
||||
[testenv:pep8]
|
||||
commands = flake8 {posargs}
|
||||
|
|
|
|||
Loading…
Reference in New Issue