summaryrefslogtreecommitdiff
path: root/bin/disk-image-create
blob: ad811a8fde99c19c3abde912c590a45441ce5bf8 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
#!/bin/bash
#
# Copyright 2012 Hewlett-Packard Development Company, L.P.
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may
# not use this file except in compliance with the License. You may obtain
# a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.

set -eE

# Prevent perl from complaining a lot, but also remove any unexpected side-effects
# of $LANG varying between build hosts
export LANG=C

# Store our initial environment and command line args for later
export DIB_ARGS="$@"
export DIB_ENV=$(export | grep ' DIB_.*=')

SCRIPTNAME=$(basename $0)

# this is a bit tricky to handle the case of being installed with "pip
# -e" (i.e. setuptools develop mode) and a regular install
#
# When installed normally, the scripts are installed into /usr/bin/
# and the other bits specified in "data_files" into
# /usr/share/diskimage-builder/[elements|lib|scripts] (if you're in a
# virtualenv, modulo all this with the right prefix-paths)
#
# When installed with -e, the scripts will still be installed into
# /usr/bin, but the data_files will *not* be installed.  Because the
# "diskimage_builder" python module will be linked to the source repo
# (that's the idea of develop mode) what we can do is assume the
# following:
#
# - if the python module directory has a "bin" directory, then it must
#   be the source repo and hence we have been installed via develop
#   mode.  Thus setup ourselves to use the scripts from the source
#   repo.
#
# - otherwise, try to find libraires and elements have been installed
#   into the system paths via a "normal" pip install
#
# - lastly, we might be running completely uninstalled.
#   XXX : this might cause problems at some point; we might need to "cd"
#         so python can find things, or use pip -e.
#
# This means if you want to develop your elements, then "pip -e" is
# the way to go ... your disk-image-create runs will be referencing
# the scripts from the editable source repo.  But note that changes to
# anything in bin/ will *not* be applied; those files have been
# statically copied in during install.  You'll need to iterate with
# another run of "pip install -e" if you're actually working on those
# bin/* scripts.

export SCRIPT_HOME=$(dirname $(readlink -f $0))

_DIB_PYTHON_INSTALL=$(python -c '

import inspect
import os
import sys

# this can fail if we are being run with pwd outside the source
# directory *and* have not been installed
try:
    import diskimage_builder
except ImportError:
    sys.exit(0)

print(os.path.dirname(inspect.getfile(diskimage_builder)))')

if [ -n "$_DIB_PYTHON_INSTALL"  -a -d $_DIB_PYTHON_INSTALL/../bin ]; then
    # we have been installed with "pip -e"
    export SCRIPT_HOME=$_DIB_PYTHON_INSTALL/../bin
    export _PREFIX=$SCRIPT_HOME/..
elif [ -d $SCRIPT_HOME/../share/diskimage-builder ]; then
    # we have been installed in /usr
    export _PREFIX=$SCRIPT_HOME/../share/diskimage-builder
else
    # we have not been installed in any way
    export _PREFIX=$SCRIPT_HOME/..
fi
export _LIB=$_PREFIX/lib
source $_LIB/die

IS_RAMDISK=0
if [ "$SCRIPTNAME" == "ramdisk-image-create" ]; then
  IS_RAMDISK=1
fi

function show_options () {
    echo "Usage: ${SCRIPTNAME} [OPTION]... [ELEMENT]..."
    echo
    echo "Options:"
    echo "    -a i386|amd64|armhf -- set the architecture of the image(default amd64)"
    echo "    -o imagename -- set the imagename of the output image file(default image)"
    echo "    -t qcow2,tar,vhd,docker,aci,raw -- set the image types of the output image files (default qcow2)"
    echo "       File types should be comma separated. VHD outputting requires the vhd-util"
    echo "       executable be in your PATH. ACI outputting requires the ACI_MANIFEST "
    echo "       environment variable be a path to a manifest file."
    echo "    -x -- turn on tracing (use -x -x for very detailed tracing)"
    echo "    -u -- uncompressed; do not compress the image - larger but faster"
    echo "    -c -- clear environment before starting work"
    echo "    --image-size size -- image size in GB for the created image"
    echo "    --image-cache directory -- location for cached images(default ~/.cache/image-create)"
    echo "    --max-online-resize size -- max number of filesystem blocks to support when resizing."
    echo "       Useful if you want a really large root partition when the image is deployed."
    echo "       Using a very large value may run into a known bug in resize2fs."
    echo "       Setting the value to 274877906944 will get you a 1PB root file system."
    echo "       Making this value unnecessarily large will consume extra disk space "
    echo "       on the root partition with extra file system inodes."
    echo "    --min-tmpfs size -- minimum size in GB needed in tmpfs to build the image"
    echo "    --mkfs-options -- option flags to be passed directly to mkfs."
    echo "       Options should be passed as a single string value."
    echo "    --no-tmpfs -- do not use tmpfs to speed image build"
    echo "    --offline -- do not update cached resources"
    echo "    --qemu-img-options -- option flags to be passed directly to qemu-img."
    echo "       Options need to be comma separated, and follow the key=value pattern."
    echo "    --root-label label -- label for the root filesystem.  Defaults to 'cloudimg-rootfs'."
    echo "    --ramdisk-element -- specify the main element to be used for building ramdisks."
    echo "       Defaults to 'ramdisk'.  Should be set to 'dracut-ramdisk' for platforms such"
    echo "       as RHEL and CentOS that do not package busybox."
    echo "    --install-type -- specify the default installation type. Defaults to 'source'. Set to 'package' to use package based installations by default."
    echo "    --docker-target -- specify the repo and tag to use if the output type is docker. Defaults to the value of output imagename"
    if [ "$IS_RAMDISK" == "0" ]; then
        echo "    -n skip the default inclusion of the 'base' element"
        echo "    -p package[,package,package] -- list of packages to install in the image"
    fi
    echo "    -h|--help -- display this help and exit"
    echo "    --version -- display version and exit"
    echo
    echo "ELEMENTS_PATH will allow you to specify multiple locations for the elements."
    echo
    echo "NOTE: At least one distribution root element must be specified."
    echo
    echo "NOTE: If using the VHD output format you need to have a patched version of vhd-util installed for the image"
    echo "      to be bootable. The patch is available here: https://github.com/emonty/vhd-util/blob/master/debian/patches/citrix"
    echo "      and a PPA with the patched tool is available here: https://launchpad.net/~openstack-ci-core/+archive/ubuntu/vhd-util"
    echo
    echo "Examples:"
    if [ "$IS_RAMDISK" == "0" ]; then
        echo "    ${SCRIPTNAME} -a amd64 -o ubuntu-amd64 vm ubuntu"
        echo "    export ELEMENTS_PATH=~/source/tripleo-image-elements/elements"
        echo "    ${SCRIPTNAME} -a amd64 -o fedora-amd64-heat-cfntools vm fedora heat-cfntools"
    else
        echo "    ${SCRIPTNAME} -a amd64 -o fedora-deploy deploy fedora"
        echo "    ${SCRIPTNAME} -a amd64 -o ubuntu-ramdisk ramdisk ubuntu"
    fi
}

function show_version() {
    if [ -n "$_DIB_PYTHON_INSTALL" ]; then
        python -c "from diskimage_builder import version; print(version.version_info.version_string())"
    else
        echo "diskimage-builder is not installed."
    fi
}

INSTALL_PACKAGES=""
IMAGE_TYPES=("qcow2")
COMPRESS_IMAGE="true"
export DIB_ROOT_LABEL=""
DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"}
MKFS_OPTS=""
ACI_MANIFEST=${ACI_MANIFEST:-}
DOCKER_TARGET=""
TEMP=`getopt -o a:ho:t:xucnp: -l no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"`
if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi

# Note the quotes around `$TEMP': they are essential!
eval set -- "$TEMP"

while true ; do
    case "$1" in
        -a) export ARCH=$2; shift 2 ;;
        -o) export IMAGE_NAME=$2; shift 2 ;;
        -t) IFS="," read -a IMAGE_TYPES <<< "$2"; export IMAGE_TYPES ; shift 2 ;;
        -h|--help) show_options; exit 0;;
        --version) show_version; exit 0;;
        -x) shift; export DIB_DEBUG_TRACE=$(( $DIB_DEBUG_TRACE + 1 )); set -x;;
        -u) shift; export COMPRESS_IMAGE="";;
        -c) shift ; export CLEAR_ENV=1;;
        -n) shift; export SKIP_BASE="1";;
        -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;;
        --image-size) export DIB_IMAGE_SIZE=$2; shift 2;;
        --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;;
        --max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;;
        --mkfs-options) MKFS_OPTS=$2; shift 2;;
        --min-tmpfs) export DIB_MIN_TMPFS=$2; shift 2;;
        --no-tmpfs) shift; export DIB_NO_TMPFS=1;;
        --offline) shift; export DIB_OFFLINE=1;;
        --qemu-img-options) QEMU_IMG_OPTIONS=$2; shift 2;;
        --root-label) export DIB_ROOT_LABEL=$2; shift 2;;
        --ramdisk-element) RAMDISK_ELEMENT=$2; shift 2;;
        --install-type) DIB_DEFAULT_INSTALLTYPE=$2; shift 2;;
        --docker-target) export DOCKER_TARGET=$2; shift 2 ;;
        --) shift ; break ;;
        *) echo "Internal error!" ; exit 1 ;;
    esac
done

export DIB_IMAGE_CACHE=${DIB_IMAGE_CACHE:-~/.cache/image-create}
mkdir -p $DIB_IMAGE_CACHE

if [ "$CLEAR_ENV" = "1" -a "$HOME" != "" ]; then
  echo "Re-execing to clear environment."
  echo "(note this will prevent much of the local_config element from working)"
  exec -c $0 "$@"
fi


# Display the current file/function/line in the debug output
function _ps4 {
    IFS=" " called=($(caller 0))
    local f=$(readlink -f ${called[2]})
    printf "%-80s " "$f:${called[1]}:${called[0]}"
}
export -f _ps4
export PS4='+ $(_ps4):   '


source $_LIB/img-defaults
source $_LIB/common-functions
source $_LIB/img-functions

if [ "$IS_RAMDISK" == "1" ]; then
  source $_LIB/ramdisk-defaults
  source $_LIB/ramdisk-functions
fi

# If no elements are specified theres no way we can succeed
if [ -z "$*" ]; then
    echo "ERROR: At least one distribution root element must be specified"
    exit 1
fi
arg_to_elements "$@"

if [ "${#IMAGE_TYPES[@]}" = "1" ]; then
  export IMAGE_NAME=${IMAGE_NAME%%\.${IMAGE_TYPES[0]}}
fi

# Check for required tools early on
for X in ${!IMAGE_TYPES[@]}; do
    case "${IMAGE_TYPES[$X]}" in
        qcow2)
            if [ -z "$(which qemu-img)" ]; then
                echo "qcow2 output format specified but qemu-img executable not found."
                exit 1
            fi
            ;;
        vhd)
            if [ -z "$(which vhd-util)" ]; then
                echo "vhd output format specified but no vhd-util executable found."
                exit 1
            fi
            ;;
        docker)
            if [ -z "$(which docker)" ]; then
                echo "docker output format specified but no docker executable found."
                exit 1
            fi
            if [ -z "$DOCKER_TARGET" ]; then
                echo "Please set --docker-target."
                exit 1
            fi
            ;;
    esac
done

# NOTE: fstrim is on most all recent systems. It is provided by the util-linux
# package.
if [[ -z "$(which fstrim)" ]]; then
    echo "fstrim utility is not found. This is provided by util-linux package"
    echo "Please check your PATH variable is set correctly"
    exit 1
fi

# NOTE: Tuning the rootfs uuid works only for ext filesystems.
# Rely on the below environment variable only for ext filesystems.
export DIB_IMAGE_ROOT_FS_UUID=$(uuidgen -r)
if echo "$FS_TYPE" | grep -q "^ext" && [ -z "${DIB_IMAGE_ROOT_FS_UUID}" ]; then
    echo "ext filesystem detected but no DIB_IMAGE_ROOT_FS_UUID found."
    echo "Is the uuidgen utility installed on your system?"
    exit 1
fi

# FS_TYPE isn't available until after we source img-defaults
if [ -z "$DIB_ROOT_LABEL" ]; then
    # NOTE(bnemec): XFS has a limit of 12 characters for filesystem labels
    # Not changing the default for other filesystems to maintain backwards compatibility
    if [ "$FS_TYPE" = "xfs" ]; then
        DIB_ROOT_LABEL="img-rootfs"
    else
        DIB_ROOT_LABEL="cloudimg-rootfs"
    fi
fi

# xattr support cannot be relied upon with tmpfs builds
# some kernels supoprt it, some don't
if [[ -n "${GENTOO_PROFILE}" ]]; then
    if [[ "${GENTOO_PROFILE}" =~ "hardened" ]]; then
        echo 'disabling tmpfs for gentoo hardened build'
        export DIB_NO_TMPFS=1
    fi
fi

mk_build_dir
create_base
# This variable needs to be propagated into the chroot
mkdir -p $TMP_HOOKS_PATH/environment.d
echo "export DIB_DEFAULT_INSTALLTYPE=\${DIB_DEFAULT_INSTALLTYPE:-\"${DIB_DEFAULT_INSTALLTYPE}\"}" > $TMP_HOOKS_PATH/environment.d/11-dib-install-type.bash
run_d extra-data
# Run pre-install scripts. These do things that prepare the chroot for package installs
run_d_in_target pre-install
do_extra_package_install
# Call install scripts to pull in the software users want.
run_d_in_target install
run_d_in_target post-install
# ensure we do not have a lost+found directory in the root folder
# that could cause copy to fail (it will be created again later
# when creating the file system, if it needs such directory)
if [ -e "$TMP_BUILD_DIR/mnt/lost+found" ]; then
    sudo rm -rf "$TMP_BUILD_DIR/mnt/lost+found"
fi
# Free up /mnt
unmount_image
mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built

# save xtrace state, as we want to turn it off to avoid spamming the
# logs with du output below.
xtrace=$(set +o | grep xtrace)

# temp file for holding du output
du_output=${TMP_BUILD_DIR}/du_output.tmp

if [ -n "$DIB_IMAGE_SIZE" ]; then
    du_size=$(echo "$DIB_IMAGE_SIZE" | awk '{printf("%d\n",$1 * 1024 *1024)}')
else
    set +o xtrace
    echo "Calculating image size (this may take a minute)..."
    sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
    # the last line is the total size from "-c".
    # scale this by 0.6 to create a slightly bigger image
    du_size=$(tail -n1 ${du_output} | cut -f1 | awk '{print int($1 / 0.6)}')
    $xtrace
fi

if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then
    set +o xtrace
    if [ ! -f "$du_output" ]; then
        sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output}
    fi

    du_output_show="sort -nr ${du_output} |
                     numfmt --to=iec-i --padding=7
                       --suffix=B --field=1 --from-unit=1024"

    # by default show the 10MiB and greater files & directories -- a
    # dir with lots of little files will still show up, but this helps
    # signal:noise ratio
    if [[ ${DIB_SHOW_IMAGE_USAGE_FULL:-0} == 0 ]]; then
        # numfmt will start giving a decimal place when < 10MiB
        du_output_show+="| egrep 'MiB|GiB|TiB|PiB' | grep -v '\..MiB'"
        echo "================================="
        echo "Image size report (files > 10MiB)"
        echo "================================="
    else
        echo "================="
        echo "Image size report"
        echo "================="
    fi

    eval ${du_output_show}

    echo
    echo "===== end image size report ====="
    echo

    $xtrace
fi

rm -f ${du_output}

if [ "$FS_TYPE" = "ext4" ] ; then
  # Very conservative to handle images being resized a lot
  # We set journal size to 64M so our journal is large enough when we
  # perform an FS resize.
  MKFS_OPTS="-i 4096 -J size=64 $MKFS_OPTS"
  # Grow the image size to account for the journal, only if the user
  # has not asked for a specific size.
  if [ -z "$DIB_IMAGE_SIZE" ]; then
    du_size=$(( $du_size + 65536 ))
  fi
fi

# Rounding down size so that is is a multiple of 64, works around a bug in
# qemu-img that may occur when compressing raw images that aren't a multiple
# of 64k. https://bugs.launchpad.net/ubuntu/+source/linux/+bug/1180021
du_size=$(echo "$du_size" | awk ' { if ($1 % 64 != 0) print $1 + 64 - ( $1 % 64); else print $1; } ')
truncate -s${du_size}K $TMP_IMAGE_PATH

if [ -n "$MAX_ONLINE_RESIZE" ]; then
    MKFS_OPTS="-E resize=$MAX_ONLINE_RESIZE $MKFS_OPTS"
fi

LOOPDEV=$(sudo losetup --show -f $TMP_IMAGE_PATH)
export EXTRA_UNMOUNT="detach_loopback $LOOPDEV"
export IMAGE_BLOCK_DEVICE=$LOOPDEV
eval_run_d block-device "IMAGE_BLOCK_DEVICE="
sudo mkfs $MKFS_OPTS -t $FS_TYPE -L ${DIB_ROOT_LABEL} ${IMAGE_BLOCK_DEVICE}
# Tuning the rootfs uuid works only for ext filesystems.
if echo "$FS_TYPE" | grep -q "^ext"; then
    sudo tune2fs -U ${DIB_IMAGE_ROOT_FS_UUID} ${IMAGE_BLOCK_DEVICE}
fi
mkdir $TMP_BUILD_DIR/mnt
sudo mount ${IMAGE_BLOCK_DEVICE} $TMP_BUILD_DIR/mnt
sudo mv -t $TMP_BUILD_DIR/mnt ${TMP_BUILD_DIR}/built/*
mount_proc_dev_sys
run_d_in_target finalise
finalise_base

for X in ${!IMAGE_TYPES[@]} ; do
  if [[ " tar aci " =~ "${IMAGE_TYPES[$X]}" ]]; then
    if [ "${IMAGE_TYPES[$X]}" = "aci" ]; then
      sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.aci --exclude ./sys \
               --exclude ./proc --xattrs --xattrs-include=\* \
               --transform 's,^.,rootfs,S' .
      if [ -n "$ACI_MANIFEST" ]; then
        cp $ACI_MANIFEST ${TMP_BUILD_DIR}/manifest
        sudo tar -C ${TMP_BUILD_DIR} --append -f $IMAGE_NAME.aci manifest
      else
        echo "No ACI_MANIFEST specified. An ACI_MANIFEST must be specified for"
        echo " this image to be usable."
      fi
    else
      sudo tar -C ${TMP_BUILD_DIR}/mnt -cf $IMAGE_NAME.tar --exclude ./sys \
               --exclude ./proc --xattrs --xattrs-include=\* .
    fi
    sudo chown $USER: $IMAGE_NAME.${IMAGE_TYPES[$X]}
    unset IMAGE_TYPES[$X]
  elif [ "${IMAGE_TYPES[$X]}" == "docker" ]; then
    sudo tar -C ${TMP_BUILD_DIR}/mnt -cf - --exclude ./sys \
             --exclude ./proc --xattrs --xattrs-include=\* . \
        | sudo docker import - $DOCKER_TARGET
    unset IMAGE_TYPES[$X]
  fi
done

if [[ ! $IMAGE_ELEMENT =~ no-final-image ]]; then
  # Prep filesystem by discarding all unused space
  fstrim_image
fi

# Unmount and cleanup the /mnt and /build subdirectories, to save
# space before converting the image to some other format.
unmount_image
cleanup_build_dir

if [[ (! $IMAGE_ELEMENT =~ no-final-image) && "$IS_RAMDISK" == "0" ]]; then
  has_raw_type=
  for IMAGE_TYPE in ${IMAGE_TYPES[@]} ; do
    # We have to do raw last because it is destructive
    if [ "$IMAGE_TYPE" = "raw" ]; then
      has_raw_type=1
    else
      compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE
    fi
  done
  if [ -n "$has_raw_type" ]; then
    IMAGE_TYPE="raw"
    compress_and_save_image $IMAGE_NAME.$IMAGE_TYPE
  fi
fi

# Remove the leftovers, i.e. the temporary image directory.
cleanup_image_dir

# All done!
trap EXIT