openstack
/
diskimage-builder
Code Issues Proposed changes

Merge "Lock around extract-image downloads and extracts"

This commit is contained in:
Jenkins 2014-09-11 11:56:40 +00:00 committed by Gerrit Code Review
parent 9fb2c1425a ed833e628f
commit f4011996ee
1 changed files with 89 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

166
elements/redhat-common/bin/extract-image
View File

@ -14,84 +14,96 @@ CACHED_IMAGE=$4
CACHED_TAR=$DIB_IMAGE_CACHE/$BASE_IMAGE_TAR
DIB_LOCAL_IMAGE=${DIB_LOCAL_IMAGE:-""}
TAR_LOCK=$CACHED_TAR.lock
if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then
echo "Not checking freshness of cached $CACHED_TAR."
else
if [ -z "$DIB_LOCAL_IMAGE" ]; then
echo "Fetching Base Image"
# There seems to be some bad Fedora mirrors returning http 404's for the cloud image.
# If the image fails to download due to a 404 we retry once.
set +e
$TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
RV=$?
set -e
if [ "$RV" == "44" ] ; then
$TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
elif [ "$RV" != "0" ] ; then
exit 1
fi
fi
if [ ! -f $CACHED_TAR -o \
$CACHED_IMAGE -nt $CACHED_TAR ] ; then
echo "Repacking base image as tarball."
WORKING=$(mktemp -d)
EACTION="rm -r $WORKING"
trap "$EACTION" EXIT
echo "Working in $WORKING"
RAW_FILE=$(basename $BASE_IMAGE_FILE)
RAW_FILE=${RAW_FILE#.qcow2}.raw
qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $WORKING/$RAW_FILE
# F19 images have the rootfs partition on p1
# Centos7 images on p3 (p1 boot, p2 swap)
if [[ $DISTRO_NAME = "centos7" ]]; then
ROOT_PARTITON=p3
else
ROOT_PARTITON=p1
fi
# kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused
# loop device and creates it if it doesn't exist
sudo losetup -f
# XXX: Parsing stdout is dangerous, would like a better way to discover
# the device used for the image.
ROOT_LOOPDEV=$(sudo kpartx -av $WORKING/$RAW_FILE | \
awk "/loop[0-9]+$ROOT_PARTITON/ {print \$3}")
if ! timeout 5 sh -c "while ! [ -e /dev/mapper/$ROOT_LOOPDEV ]; do sleep 1; done"; then
echo "Error: Could not find /dev/mapper/$ROOT_LOOPDEV"
exit 1
fi
EACTION="sudo kpartx -d $WORKING/$RAW_FILE ; $EACTION"
trap "$EACTION" EXIT
mkdir $WORKING/mnt
sudo mount /dev/mapper/$ROOT_LOOPDEV $WORKING/mnt
EACTION="sudo umount -f $WORKING/mnt ; $EACTION"
trap "$EACTION" EXIT
# need to copy the contents of /boot into the image too, so
# mount it
if [[ $DISTRO_NAME = "centos7" ]]; then
BOOT_LOOPDEV=${ROOT_LOOPDEV/p3/p1}
sudo mount /dev/mapper/$BOOT_LOOPDEV $WORKING/mnt/boot
EACTION="sudo umount -f $WORKING/mnt/boot ; $EACTION"
trap "$EACTION" EXIT
fi
# Chroot in so that we get the correct uid/gid
sudo chroot $WORKING/mnt bin/tar -cz . > $WORKING/tmp.tar
mv $WORKING/tmp.tar $CACHED_TAR
function extract_image() {
if [ -n "$DIB_OFFLINE" -a -f "$CACHED_TAR" ] ; then
echo "Not checking freshness of cached $CACHED_TAR."
else
echo "Using cached tar from $CACHED_TAR"
if [ -z "$DIB_LOCAL_IMAGE" ]; then
echo "Fetching Base Image"
# There seems to be some bad Fedora mirrors returning http 404's for the cloud image.
# If the image fails to download due to a 404 we retry once.
set +e
$TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
RV=$?
set -e
if [ "$RV" == "44" ] ; then
$TMP_HOOKS_PATH/bin/cache-url $IMAGE_LOCATION $CACHED_IMAGE
elif [ "$RV" != "0" ] ; then
exit 1
fi
fi
if [ ! -f $CACHED_TAR -o \
$CACHED_IMAGE -nt $CACHED_TAR ] ; then
echo "Repacking base image as tarball."
WORKING=$(mktemp -d)
EACTION="rm -r $WORKING"
trap "$EACTION" EXIT
echo "Working in $WORKING"
RAW_FILE=$(basename $BASE_IMAGE_FILE)
RAW_FILE=${RAW_FILE#.qcow2}.raw
qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $WORKING/$RAW_FILE
# F19 images have the rootfs partition on p1
# Centos7 images on p3 (p1 boot, p2 swap)
if [[ $DISTRO_NAME = "centos7" ]]; then
ROOT_PARTITON=p3
else
ROOT_PARTITON=p1
fi
# kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused
# loop device and creates it if it doesn't exist
sudo losetup -f
# XXX: Parsing stdout is dangerous, would like a better way to discover
# the device used for the image.
ROOT_LOOPDEV=$(sudo kpartx -av $WORKING/$RAW_FILE | \
awk "/loop[0-9]+$ROOT_PARTITON/ {print \$3}")
if ! timeout 5 sh -c "while ! [ -e /dev/mapper/$ROOT_LOOPDEV ]; do sleep 1; done"; then
echo "Error: Could not find /dev/mapper/$ROOT_LOOPDEV"
exit 1
fi
EACTION="sudo kpartx -d $WORKING/$RAW_FILE ; $EACTION"
trap "$EACTION" EXIT
mkdir $WORKING/mnt
sudo mount /dev/mapper/$ROOT_LOOPDEV $WORKING/mnt
EACTION="sudo umount -f $WORKING/mnt ; $EACTION"
trap "$EACTION" EXIT
# need to copy the contents of /boot into the image too, so
# mount it
if [[ $DISTRO_NAME = "centos7" ]]; then
BOOT_LOOPDEV=${ROOT_LOOPDEV/p3/p1}
sudo mount /dev/mapper/$BOOT_LOOPDEV $WORKING/mnt/boot
EACTION="sudo umount -f $WORKING/mnt/boot ; $EACTION"
trap "$EACTION" EXIT
fi
# Chroot in so that we get the correct uid/gid
sudo chroot $WORKING/mnt bin/tar -cz . > $WORKING/tmp.tar
mv $WORKING/tmp.tar $CACHED_TAR
else
echo "Using cached tar from $CACHED_TAR"
fi
fi
fi
# Extract the base image (use --numeric-owner to avoid UID/GID mismatch between
# image tarball and host OS e.g. when building Fedora image on an openSUSE host)
echo "Extracting base root image from $CACHED_TAR"
sudo tar -C $TARGET_ROOT --numeric-owner -xzf $CACHED_TAR
# Extract the base image (use --numeric-owner to avoid UID/GID mismatch between
# image tarball and host OS e.g. when building Fedora image on an openSUSE host)
echo "Extracting base root image from $CACHED_TAR"
sudo tar -C $TARGET_ROOT --numeric-owner -xzf $CACHED_TAR
}
(
echo "Getting $TAR_LOCK: $(date)"
# Wait up to 20 minutes for another process to download
if ! flock -w 1200 9 ; then
echo "Did not get $TAR_LOCK: $(date)"
exit 1
fi
extract_image
) 9> $TAR_LOCK