diff --git a/.gitignore b/.gitignore index c66683a36..9c2376107 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,4 @@ *.egg-info dist *.qcow2 +*.raw diff --git a/README.md b/README.md index bc77e361d..f3667e762 100644 --- a/README.md +++ b/README.md @@ -50,8 +50,10 @@ Images are built using a chroot and bind mounted /proc /sys and /dev. The goal of the image building process is to produce blank slate machines that have all the necessary bits to fulfill a specific purpose in the running of an Openstack cloud: e.g. a nova-compute node. Images produce either a filesystem image with -a label of cloudimg-rootfs, or can be customised to produce disk images (but -will still contain a filesystem labelled cloudimg-rootfs). +a label of cloudimg-rootfs, or can be customised to produce whole disk images +(but will still contain a filesystem labelled cloudimg-rootfs). Once the file +system tree is assembled a loopback device with filesystem (or partition table +and file system) is created and the tree copied into it. An element is a particular set of code that alters how the image is built, or runs within the chroot to prepare the image. E.g. the local-config element @@ -83,7 +85,7 @@ contents can be modelled as three distinct portions: nova instance disk images, disk image cache. These would typically be stored on a dedicated partition and not overwritten when re-deploying the image. -The goal of the image building tools is to create machine images that content +The goal of the image building tools is to create machine images that contain the correct global content and are ready for 'last-mile' configuration by the nova metadata API, after which a configuration management system can take over (until the next deploy, when it all starts over from scratch). @@ -120,27 +122,27 @@ part of the process you need to customise: * inputs: $ARCH=i386|amd64|armhf $TARGET\_ROOT=/path/to/target/workarea -* cleanup.d: Perform cleanups of the root filesystem content. For instance, - temporary settings to use the image build environment HTTP proxy are removed - here in the dpkg element. Runs outside the chroot on the host environment. +* finalise.d: Perform final tuning of the root filesystem. Runs in a chroot + after the root filesystem content has been copied into the mounted + filesystem: this is an appropriate place to reset SELinux metadata, install + grub bootloaders and so on. Because this happens inside the final image, it + is important to limit operations here to only those necessary to affect the + filesystem metadata and image itself. For most operations, post-install.d + is preferred. + +* cleanup.d: Perform cleanup of the root filesystem content. For + instance, temporary settings to use the image build environment HTTP proxy + are removed here in the dpkg element. Runs outside the chroot on the host + environment. * inputs: $ARCH=i386|amd64|armhf $TARGET\_ROOT=/path/to/target/workarea -* block-device-size.d: Alter the size (in GB) of the disk image. This is useful - when a particular element will require a certain minimum (or maximum) size. - You can either error and stop the build, or adjust the size to match. - NB: Due to the current simple implementation, the last output value wins - so this should be used rarely - only one element in a mix can reliably set - a size. - - * outputs: $IMAGE\_SIZE={size\_in\_GB} - * inputs: $IMAGE\_SIZE={size\_in\_GB} - * block-device.d: customise the block device that the image will be made on - (e.g. to make partitions). + (e.g. to make partitions). Runs outside the chroot, after the target tree + has been fully populated but before the cleanup hook runs. * outputs: $IMAGE\_BLOCK\_DEVICE={path} - * inputs: $IMAGE\_BLOCK\_DEVICE={path} + * inputs: $IMAGE\_BLOCK\_DEVICE={path} $TARGET\_ROOT={path} * extra-data.d: pull in extra data from the host environment that hooks may need during image creation. This should copy any data (such as SSH keys, diff --git a/bin/disk-image-create b/bin/disk-image-create index aa1eddb55..4186ea24d 100755 --- a/bin/disk-image-create +++ b/bin/disk-image-create @@ -94,22 +94,6 @@ echo "If prompted for sudo, install sudoers.d/img-build-sudoers into /etc/sudoer ensure_nbd mk_build_dir -eval_run_d block-device-size "DIB_IMAGE_SIZE=" - -qemu-img create -f qcow2 -o preallocation=metadata $TMP_IMAGE_PATH ${DIB_IMAGE_SIZE}G - -# grab the next available /dev/nbdX and connect to it -map_nbd $TMP_IMAGE_PATH -echo "NBD Device: $NBD_DEV" - -export EXTRA_UNMOUNT="sudo qemu-nbd -d $NBD_DEV" -export IMAGE_BLOCK_DEVICE=$NBD_DEV -eval_run_d block-device "IMAGE_BLOCK_DEVICE=" - -sudo mkfs -F -t $FS_TYPE -L cloudimg-rootfs ${IMAGE_BLOCK_DEVICE} - -mount_tmp_image ${IMAGE_BLOCK_DEVICE} - create_base run_d extra-data # Run pre-install scripts. These do things that prepare the chroot for package installs @@ -119,6 +103,27 @@ do_extra_package_install run_d_in_target install run_d_in_target post-install prepare_first_boot +# Free up /mnt +unmount_image +mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built +# in kb*0.8 - underreport to get a slightly bigger device +_NEEDED_SIZE=$(sudo du --block-size=800 -x -s ${TMP_BUILD_DIR}/built | awk ' { print $1 } ') +truncate -s${_NEEDED_SIZE}K $TMP_IMAGE_PATH +LOOPDEV=$(sudo losetup --show -f $TMP_IMAGE_PATH) +export EXTRA_UNMOUNT="sudo losetup -d $LOOPDEV" +export IMAGE_BLOCK_DEVICE=$LOOPDEV +eval_run_d block-device "IMAGE_BLOCK_DEVICE=" +OPTS="" +if [ "$FS_TYPE" = "ext4" ] ; then + # Very conservative to handle images being resized a lot + OPTS="-i 4096" +fi +sudo mkfs $OPTS -t $FS_TYPE -L cloudimg-rootfs ${IMAGE_BLOCK_DEVICE} +mkdir $TMP_BUILD_DIR/mnt +sudo mount ${IMAGE_BLOCK_DEVICE} $TMP_BUILD_DIR/mnt +sudo mv -t $TMP_BUILD_DIR/mnt ${TMP_BUILD_DIR}/built/* +mount_proc_dev_sys +run_d_in_target finalise finalise_base unmount_image compress_image diff --git a/elements/fedora/bin/map-packages b/elements/fedora/bin/map-packages index c08d56cd4..4e4cf1d1c 100755 --- a/elements/fedora/bin/map-packages +++ b/elements/fedora/bin/map-packages @@ -21,6 +21,7 @@ import sys # distromatch or other rich data sources. # Debian name on the left, Fedora on the right. package_map = { + 'grub-pc': 'grub2-tools grub2', 'linux-image-generic': 'kernel', 'open-iscsi': 'iscsi-initiator-utils', 'vlan': 'vconfig', diff --git a/elements/fedora/pre-install.d/15-fedora-fixup-grub-cfg b/elements/fedora/pre-install.d/15-fedora-remove-grub similarity index 70% rename from elements/fedora/pre-install.d/15-fedora-fixup-grub-cfg rename to elements/fedora/pre-install.d/15-fedora-remove-grub index c832d23ec..8c83ab8df 100755 --- a/elements/fedora/pre-install.d/15-fedora-fixup-grub-cfg +++ b/elements/fedora/pre-install.d/15-fedora-remove-grub @@ -2,12 +2,14 @@ set -e -GRUB_CFG=/boot/grub2/grub.cfg +yum remove -y grub2-tools -[ -f "$GRUB_CFG" ] +#GRUB_CFG=/boot/grub2/grub.cfg + +#[ -f "$GRUB_CFG" ] # Update the config to have the search UUID of the image being built. # When partition staging is moved to a separate stage, this will need to happen # there. This generates a non-UUID config, which is irrelevant for booting with # hypervisor kernel + ramdisk, and fixed up by 51-grub for vm images. -GRUB_DISABLE_LINUX_UUID=true grub2-mkconfig -o $GRUB_CFG +#GRUB_DISABLE_LINUX_UUID=true grub2-mkconfig -o $GRUB_CFG diff --git a/elements/fedora/root.d/10-fedora-cloud-image b/elements/fedora/root.d/10-fedora-cloud-image index 3b39eef9f..ffc4c33d1 100755 --- a/elements/fedora/root.d/10-fedora-cloud-image +++ b/elements/fedora/root.d/10-fedora-cloud-image @@ -43,3 +43,4 @@ if [ ! -f $IMG_PATH/$BASE_IMAGE_TAR ] ; then fi # Extract the base image sudo tar -C $TARGET_ROOT -xzf $IMG_PATH/$BASE_IMAGE_TAR +sudo rmdir $TARGET_ROOT/lost+found diff --git a/elements/ubuntu/pre-install.d/00-remove-grub b/elements/ubuntu/pre-install.d/00-remove-grub new file mode 100755 index 000000000..e99f42af6 --- /dev/null +++ b/elements/ubuntu/pre-install.d/00-remove-grub @@ -0,0 +1,8 @@ +#!/bin/bash +# The grub post-kernel install hook will barf if the block device can't be +# found (as happens in a chroot). +# Temporarily remove grub, to avoid that confusion. + +set -e + +apt-get -y remove grub-pc grub2-common diff --git a/elements/ubuntu/root.d/10-cache-ubuntu-tarball b/elements/ubuntu/root.d/10-cache-ubuntu-tarball index fff8cca90..76f02b3be 100755 --- a/elements/ubuntu/root.d/10-cache-ubuntu-tarball +++ b/elements/ubuntu/root.d/10-cache-ubuntu-tarball @@ -27,3 +27,4 @@ if [ ! -f $IMG_PATH/$BASE_IMAGE_FILE ] ; then fi # Extract the base image sudo tar -C $TARGET_ROOT -xzf $IMG_PATH/$BASE_IMAGE_FILE +sudo rmdir $TARGET_ROOT/lost+found diff --git a/elements/vm/block-device.d/10-partition b/elements/vm/block-device.d/10-partition index 13a69523d..8deb69905 100755 --- a/elements/vm/block-device.d/10-partition +++ b/elements/vm/block-device.d/10-partition @@ -13,4 +13,6 @@ sudo sfdisk $IMAGE_BLOCK_DEVICE << EOF 0 0; EOF +sudo partprobe $IMAGE_BLOCK_DEVICE + echo "IMAGE_BLOCK_DEVICE=${IMAGE_BLOCK_DEVICE}p1" diff --git a/elements/vm/install.d/51-grub b/elements/vm/finalise.d/51-grub similarity index 62% rename from elements/vm/install.d/51-grub rename to elements/vm/finalise.d/51-grub index 636246e01..4db263f05 100755 --- a/elements/vm/install.d/51-grub +++ b/elements/vm/finalise.d/51-grub @@ -4,20 +4,25 @@ # different distributions gracefully. set -e +set -x -# XXX: grub-probe on the nbd0 device returns nothing - workaround, manually +install-packages grub-pc + +# XXX: grub-probe on the nbd0/loop0 device returns nothing - workaround, manually # specify modules. https://bugs.launchpad.net/ubuntu/+source/grub2/+bug/1073731 GRUBNAME=`which grub-install` || echo "trying grub2-install" if [ -z "$GRUBNAME" ]; then - GRUBNAME=`which grub2-install` + GRUBNAME="bash -x `which grub2-install`" fi if [ -z "$GRUBNAME" ]; then echo "NO grub-install or grub2-install found" exit 1 fi -BOOT_DEV=/dev/nbd0 -PART_DEV=/dev/nbd0p1 -$GRUBNAME --modules="biosdisk part_msdos" $BOOT_DEV +# FIXME: +[ -n "$IMAGE_BLOCK_DEVICE" ] +PART_DEV=$IMAGE_BLOCK_DEVICE +BOOT_DEV=$(echo $IMAGE_BLOCK_DEVICE | sed -e 's/p1//') +$GRUBNAME --target=i386-pc --modules="biosdisk part_msdos" $BOOT_DEV # This might be better factored out into a per-distro 'install-bootblock' # helper. if [ -f "/boot/grub/grub.cfg" ] ; then @@ -30,3 +35,6 @@ fi # NOTE: Updating the grub config by hand once deployed should work, its just # prepping it in a different environment that needs fiddling. sed -i "s%$PART_DEV%LABEL=cloudimg-rootfs%" $GRUB_CFG +sed -i "s%search --no-floppy --fs-uuid --set=root .*$%search --no-floppy --set=root --label cloudimg-rootfs%" $GRUB_CFG +sed -i "s%root=UUID=[A-Za-z0-9\-]*%root=LABEL=cloudimg-rootfs%" $GRUB_CFG + diff --git a/lib/common-functions b/lib/common-functions index 8cb9c3336..37cb723b0 100644 --- a/lib/common-functions +++ b/lib/common-functions @@ -16,9 +16,11 @@ function mk_build_dir () { export TMP_BUILD_DIR=$(mktemp -t -d --tmpdir=${TMP_DIR:-/tmp} image.XXXXXXXX) [ $? -eq 0 ] || die "Failed to create tmp directory" + sudo mount -t tmpfs tmpfs $TMP_BUILD_DIR + sudo chown $(id -u):$(id -g) $TMP_BUILD_DIR trap cleanup EXIT echo Building in $TMP_BUILD_DIR - export TMP_IMAGE_PATH=$TMP_BUILD_DIR/image + export TMP_IMAGE_PATH=$TMP_BUILD_DIR/image.raw export TMP_HOOKS_PATH=$TMP_BUILD_DIR/hooks } @@ -30,7 +32,7 @@ function save_image () { fi cp $TMP_IMAGE_PATH $1 - rm -r $TMP_BUILD_DIR + cleanup_dirs # All done! trap EXIT echo "Image file $1 created..." diff --git a/lib/img-functions b/lib/img-functions index f09148406..0969400ad 100644 --- a/lib/img-functions +++ b/lib/img-functions @@ -25,8 +25,8 @@ function unmount_image () { sleep 5 # oh ya don't want to forget to unmount the image sudo umount -f $TMP_BUILD_DIR/mnt || true - # having disk corruption issues; one possibility is qemu-nbd not flush dirty - # pages on disconnect? + # having disk corruption issues; one possibility is qemu-nbd not flushing + # dirty pages on disconnect? sync if [ -n "$EXTRA_UNMOUNT" ]; then $EXTRA_UNMOUNT @@ -35,6 +35,13 @@ function unmount_image () { function cleanup () { unmount_image + cleanup_dirs +} + +function cleanup_dirs () { + sudo rm -rf $TMP_BUILD_DIR/built + sudo rm -rf $TMP_BUILD_DIR/mnt + sudo umount $TMP_BUILD_DIR rm -rf $TMP_BUILD_DIR } @@ -52,14 +59,9 @@ function ensure_sudo () { sudo echo "Ensuring sudo is available" } -function mount_tmp_image () { - mkdir $TMP_BUILD_DIR/mnt - sudo mount $@ $TMP_BUILD_DIR/mnt - [ $? -eq 0 ] || die "Failed to mount image" - export TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt -} - function create_base () { + mkdir $TMP_BUILD_DIR/mnt + export TMP_MOUNT_PATH=$TMP_BUILD_DIR/mnt # Copy data in to the root. TARGET_ROOT=$TMP_MOUNT_PATH run_d root if [ -z "$(ls $TMP_MOUNT_PATH | grep -v lost+found)" ] ; then @@ -92,12 +94,14 @@ function create_base () { else echo nameserver 8.8.8.8 > $TMP_MOUNT_PATH/etc/resolv.conf fi + mount_proc_dev_sys +} +function mount_proc_dev_sys () { # supporting kernel file systems sudo mount -t proc none $TMP_MOUNT_PATH/proc sudo mount --bind /dev $TMP_MOUNT_PATH/dev sudo mount -t sysfs none $TMP_MOUNT_PATH/sys - } # Helper function to run a command inside the chroot @@ -167,7 +171,7 @@ function finalise_base () { function compress_image () { # Recreate our image to throw away unnecessary data test $IMAGE_TYPE != qcow2 && COMPRESS_IMAGE="" - qemu-img convert ${COMPRESS_IMAGE:+-c} $TMP_IMAGE_PATH -O $IMAGE_TYPE $TMP_IMAGE_PATH-new + qemu-img convert ${COMPRESS_IMAGE:+-c} -f raw $TMP_IMAGE_PATH -O $IMAGE_TYPE $TMP_IMAGE_PATH-new rm $TMP_IMAGE_PATH mv $TMP_IMAGE_PATH-new $TMP_IMAGE_PATH } diff --git a/sudoers.d/img-build-sudoers b/sudoers.d/img-build-sudoers index 2aedb15ce..f3bd4dbe8 100644 --- a/sudoers.d/img-build-sudoers +++ b/sudoers.d/img-build-sudoers @@ -24,9 +24,12 @@ ALL ALL=(root) NOPASSWD: /bin/mount -o remount\,ro\,bind /tmp/*/hooks /tmp/*/mnt ALL ALL=(root) NOPASSWD: /bin/mount -t proc none /tmp/*/mnt/proc ALL ALL=(root) NOPASSWD: /bin/mount -t sysfs none /tmp/*/mnt/sys ALL ALL=(root) NOPASSWD: /bin/mount /dev/nbd0* /tmp/*/mnt -ALL ALL=(root) NOPASSWD: /bin/mount /dev/loop*p* /tmp/*/mnt +ALL ALL=(root) NOPASSWD: /bin/mount /dev/loop* /tmp/*/mnt ALL ALL=(root) NOPASSWD: /bin/mv /tmp/*/mnt/* /tmp/*/mnt/* +ALL ALL=(root) NOPASSWD: /bin/mv -t /tmp/*/mnt /tmp/*/built/* +ALL ALL=(root) NOPASSWD: /bin/rm -* /tmp/*/mnt ALL ALL=(root) NOPASSWD: /bin/rm -* /tmp/*/mnt/* +ALL ALL=(root) NOPASSWD: /bin/rm -* /tmp/*/built ALL ALL=(root) NOPASSWD: /bin/rmdir /tmp/*/mnt/* ALL ALL=(root) NOPASSWD: /bin/tar -C /tmp/*/mnt -xzf /*/.cache/image-create/* ALL ALL=(root) NOPASSWD: /bin/umount -f /tmp/*/mnt @@ -34,9 +37,10 @@ ALL ALL=(root) NOPASSWD: /bin/umount -f /tmp/*/mnt/dev ALL ALL=(root) NOPASSWD: /bin/umount -f /tmp/*/mnt/proc ALL ALL=(root) NOPASSWD: /bin/umount -f /tmp/*/mnt/sys ALL ALL=(root) NOPASSWD: /bin/umount -f /tmp/*/mnt/tmp/in_target.d -ALL ALL=(root) NOPASSWD: /sbin/mkfs -F -t ext4 -L cloudimg-rootfs /dev/nbd0* +ALL ALL=(root) NOPASSWD: /sbin/mkfs -i 4096 -t ext4 -L cloudimg-rootfs /dev/loop* ALL ALL=(root) NOPASSWD: /sbin/modprobe nbd max_part=16 -ALL ALL=(root) NOPASSWD: /sbin/sfdisk /dev/nbd0 +ALL ALL=(root) NOPASSWD: /sbin/sfdisk /dev/nbd* +ALL ALL=(root) NOPASSWD: /sbin/sfdisk /dev/loop* ALL ALL=(root) NOPASSWD: /usr/bin/qemu-nbd -c /dev/nbd0 --cache=writeback /tmp/*/image ALL ALL=(root) NOPASSWD: /usr/bin/qemu-nbd -d /dev/nbd0 ALL ALL=(root) NOPASSWD: /usr/bin/touch /tmp/*/mnt/* @@ -45,6 +49,10 @@ ALL ALL=(root) NOPASSWD: /bin/cp -t /tmp/*/mnt/etc/ -a /tmp/*/hooks/first-boot.d ALL ALL=(root) NOPASSWD: /usr/bin/install -m 0755 -o root -g root -D */dib-run-parts /tmp/*/mnt/usr/local/bin/dib-run-parts ALL ALL=(root) SETENV: NOPASSWD: /usr/sbin/chroot /tmp/*/mnt * ALL ALL=(root) NOPASSWD: /sbin/losetup --show -r -f /tmp/*/*.raw -ALL ALL=(root) NOPASSWD: /sbin/losetup -d /dev/loop* +ALL ALL=(root) NOPASSWD: /sbin/losetup --show -f /tmp/*/*.raw ALL ALL=(root) NOPASSWD: /sbin/losetup -d /dev/loop* ALL ALL=(root) NOPASSWD: /sbin/partprobe /dev/loop* +ALL ALL=(root) NOPASSWD: /usr/bin/du --block-size=* -x -s /tmp/*/built +ALL ALL=(root) NOPASSWD: /bin/mount -t tmpfs tmpfs /tmp/image.* +ALL ALL=(root) NOPASSWD: /bin/umount /tmp/image.* +ALL ALL=(root) NOPASSWD: /bin/chown *\:* /tmp/image.*