diff --git a/bin/disk-image-create b/bin/disk-image-create index b03e302b1..7afd5fcc5 100755 --- a/bin/disk-image-create +++ b/bin/disk-image-create @@ -109,6 +109,7 @@ function show_options () { echo " -x -- turn on tracing (use -x -x for very detailed tracing)" echo " -u -- uncompressed; do not compress the image - larger but faster" echo " -c -- clear environment before starting work" + echo " --checksum -- generate MD5 and SHA256 checksum files for the created image" echo " --image-size size -- image size in GB for the created image" echo " --image-cache directory -- location for cached images(default ~/.cache/image-create)" echo " --max-online-resize size -- max number of filesystem blocks to support when resizing." @@ -172,7 +173,7 @@ DIB_DEFAULT_INSTALLTYPE=${DIB_DEFAULT_INSTALLTYPE:-"source"} MKFS_OPTS="" ACI_MANIFEST=${ACI_MANIFEST:-} DOCKER_TARGET="" -TEMP=`getopt -o a:ho:t:xucnp: -l no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"` +TEMP=`getopt -o a:ho:t:xucnp: -l checksum,no-tmpfs,offline,help,version,min-tmpfs:,image-size:,image-cache:,max-online-resize:,mkfs-options:,qemu-img-options:,ramdisk-element:,root-label:,install-type:,docker-target: -n $SCRIPTNAME -- "$@"` if [ $? -ne 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi # Note the quotes around `$TEMP': they are essential! @@ -190,7 +191,8 @@ while true ; do -c) shift ; export CLEAR_ENV=1;; -n) shift; export SKIP_BASE="1";; -p) IFS="," read -a INSTALL_PACKAGES <<< "$2"; export INSTALL_PACKAGES ; shift 2 ;; - --image-size) DIB_IMAGE_SIZE=$2; shift 2;; + --checksum) shift; export DIB_CHECKSUM=1;; + --image-size) export DIB_IMAGE_SIZE=$2; shift 2;; --image-cache) export DIB_IMAGE_CACHE=$2; shift 2;; --max-online-resize) export MAX_ONLINE_RESIZE=$2; shift 2;; --mkfs-options) MKFS_OPTS=$2; shift 2;; @@ -338,26 +340,28 @@ mv $TMP_BUILD_DIR/mnt $TMP_BUILD_DIR/built # logs with du output below. xtrace=$(set +o | grep xtrace) +# temp file for holding du output +du_output=${TMP_BUILD_DIR}/du_output.tmp + if [ -n "$DIB_IMAGE_SIZE" ]; then du_size=$(echo "$DIB_IMAGE_SIZE" | awk '{printf("%d\n",$1 * 1024 *1024)}') else set +o xtrace echo "Calculating image size (this may take a minute)..." - du_output=$(sudo du -a -c -x ${TMP_BUILD_DIR}/built) + sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output} # the last line is the total size from "-c". # scale this by 0.6 to create a slightly bigger image - du_size=$(echo "$du_output" | tail -n1 | cut -f1 | \ - awk '{print int($1 / 0.6)}') + du_size=$(tail -n1 ${du_output} | cut -f1 | awk '{print int($1 / 0.6)}') $xtrace fi if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then set +o xtrace - if [ -z "$du_output" ]; then - du_output=$(sudo du -a -c -x ${TMP_BUILD_DIR}/built) + if [ ! -f "$du_output" ]; then + sudo du -a -c -x ${TMP_BUILD_DIR}/built > ${du_output} fi - du_output_show="sort -nr | + du_output_show="sort -nr ${du_output} | numfmt --to=iec-i --padding=7 --suffix=B --field=1 --from-unit=1024" @@ -376,7 +380,7 @@ if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then echo "=================" fi - eval ${du_output_show} <<< "$du_output" + eval ${du_output_show} echo echo "===== end image size report =====" @@ -385,6 +389,8 @@ if [[ "${DIB_SHOW_IMAGE_USAGE:-0}" != 0 ]]; then $xtrace fi +rm -f ${du_output} + if [ "$FS_TYPE" = "ext4" ] ; then # Very conservative to handle images being resized a lot # We set journal size to 64M so our journal is large enough when we diff --git a/bindep.txt b/bindep.txt new file mode 100644 index 000000000..4f9b42547 --- /dev/null +++ b/bindep.txt @@ -0,0 +1,2 @@ +# This is a cross-platform list tracking distribution packages needed by tests; +# see http://docs.openstack.org/infra/bindep/ for additional information. diff --git a/doc/source/conf.py b/doc/source/conf.py index 88cd76824..7577476ba 100644 --- a/doc/source/conf.py +++ b/doc/source/conf.py @@ -124,7 +124,7 @@ html_theme = 'default' #html_use_smartypants = True # Custom sidebar templates, maps document names to template names. -#html_sidebars = {} +html_sidebars = { '**': ['globaltoc.html', 'relations.html', 'sourcelink.html', 'searchbox.html'], } # Additional templates that should be rendered to pages, maps page names to # template names. diff --git a/doc/source/copyright.rst b/doc/source/copyright.rst deleted file mode 100644 index e9734d578..000000000 --- a/doc/source/copyright.rst +++ /dev/null @@ -1,20 +0,0 @@ -Copyright -========= - -Copyright 2012 Hewlett-Packard Development Company, L.P. - -Copyright (c) 2012 NTT DOCOMO, INC. - -All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); you may -not use this file except in compliance with the License. You may obtain -a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -License for the specific language governing permissions and limitations -under the License. diff --git a/doc/source/developer/developing_elements.rst b/doc/source/developer/developing_elements.rst index c4d7d5b8f..ae58c51fb 100644 --- a/doc/source/developer/developing_elements.rst +++ b/doc/source/developer/developing_elements.rst @@ -412,6 +412,11 @@ line to run it. If it should not be run as part of the default CI run, you can submit a change with it added to ``DEFAULT_SKIP_TESTS`` in that file. +Running the functional tests is time consuming. Multiple parallel +jobs can be started by specifying ``-j ``. Each of the +jobs uses a lot resources (CPU, disk space, RAM) - therefore the job +count must carefully be chosen. + python """""" diff --git a/doc/source/developer/index.rst b/doc/source/developer/index.rst index 470041bfd..cc063806b 100644 --- a/doc/source/developer/index.rst +++ b/doc/source/developer/index.rst @@ -1,5 +1,16 @@ -Developer Documentation -======================= +Developer Guide +=============== + +.. toctree:: + :maxdepth: 1 + + design + components + invocation + caches + developing_elements + dib_lint + stable_interfaces This documentation explains how to get started with creating your own disk-image-builder elements as well as some high level concepts for element @@ -13,7 +24,7 @@ To get started developing with ``diskimage-builder``, install to a $ mkdir dib $ cd dib - $ virtualenv create env + $ virtualenv env $ source env/bin/activate $ git clone https://git.openstack.org/openstack/diskimage-builder $ cd diskimage-builder @@ -24,13 +35,10 @@ and testing your changes. When you are done editing, use ``git review`` to submit changes to the upstream gerrit. -.. toctree:: - :maxdepth: 2 +Finding Work +------------ - design - components - invocation - caches - developing_elements - dib_lint - stable_interfaces +We maintain a list of low-hanging-fruit tags on launchpad: + + * `https://bugs.launchpad.net/diskimage-builder/+bugs?field.tag=low-hanging-fruit` + diff --git a/doc/source/index.rst b/doc/source/index.rst index cf2b21732..7d277349c 100644 --- a/doc/source/index.rst +++ b/doc/source/index.rst @@ -30,6 +30,15 @@ The code is available at: `__ +Issues +------ + +Issues are tracked on launchpad at: + + * `https://bugs.launchpad.net/diskimage-builder/+bugs` + + + Communication ------------- @@ -46,4 +55,3 @@ Table of Contents user_guide/index developer/index elements - copyright diff --git a/doc/source/user_guide/installation.rst b/doc/source/user_guide/installation.rst index 169c088d3..ee10e35a4 100644 --- a/doc/source/user_guide/installation.rst +++ b/doc/source/user_guide/installation.rst @@ -55,3 +55,13 @@ Installing via pip is as simple as: pip install diskimage-builder + +Package Installation +-------------------- + +On Gentoo you can emerge diskimage-builder directly. + +:: + + emerge app-emulation/diskimage-builder + diff --git a/doc/source/user_guide/supported_distros.rst b/doc/source/user_guide/supported_distros.rst index 99030adda..011c149a1 100644 --- a/doc/source/user_guide/supported_distros.rst +++ b/doc/source/user_guide/supported_distros.rst @@ -13,9 +13,9 @@ Distributions which are supported as a build host: Distributions which are supported as a target for an image: - - Centos 6, 7 - - Debian 8 ("jessie") - - Fedora 20, 21, 22 - - RHEL 6, 7 - - Ubuntu 12.04 ("precise"), 14.04 ("trusty") - - Gentoo +- Centos 6, 7 +- Debian 8 ("jessie") +- Fedora 20, 21, 22 +- RHEL 6, 7 +- Ubuntu 12.04 ("precise"), 14.04 ("trusty") +- Gentoo diff --git a/elements/cloud-init/post-install.d/20-enable-cloud-init b/elements/cloud-init/post-install.d/20-enable-cloud-init index 6368bea5a..36be728aa 100755 --- a/elements/cloud-init/post-install.d/20-enable-cloud-init +++ b/elements/cloud-init/post-install.d/20-enable-cloud-init @@ -10,6 +10,6 @@ set -o pipefail if [[ "${DISTRO_NAME}" == "gentoo" ]]; then rc-update add cloud-config default rc-update add cloud-final default - rc-update add cloud-init-local default + rc-update add cloud-init-local boot rc-update add cloud-init default fi diff --git a/elements/debian-minimal/README.rst b/elements/debian-minimal/README.rst index 04f1fbdeb..66f017218 100644 --- a/elements/debian-minimal/README.rst +++ b/elements/debian-minimal/README.rst @@ -11,16 +11,16 @@ There are two ways to configure apt-sources: and security repositories is the default. In this case you can overwrite the two environment variables to adapt the behavior: `DIB_DISTRIBUTION_MIRROR`: the mirror to use - default: http://httpredir.debian.org/debian + default: http://ftp.us.debian.org/debian `DIB_DEBIAN_COMPONENTS`: (default) `main` a comma separated list of components. For Debian this can be e.g. `main,contrib,non-free`. - Note that the default Debian series is `unstable`, and the default - mirrors for Debian can be problematic for `unstable`. Because apt - does not handle changing Packages files well across multiple out of - sync mirrors, it is recommended that you choose a single mirror of - Debian, and pass it in via `DIB_DISTRIBUTION_MIRROR`. + Note it is not recommended to use http://httpredir.debian.org/ for + `DIB_DISTRIBUTION_MIRROR` due to how unreliable it is. Be sure to + select a mirror from the official mirror list: + + https://www.debian.org/mirror/list By default only `main` component is used. If `DIB_DEBIAN_COMPONENTS` (comma separated) from the `debootstrap` diff --git a/elements/debian-minimal/environment.d/10-debian-minimal.bash b/elements/debian-minimal/environment.d/10-debian-minimal.bash index 3f2200226..13ac22fc0 100644 --- a/elements/debian-minimal/environment.d/10-debian-minimal.bash +++ b/elements/debian-minimal/environment.d/10-debian-minimal.bash @@ -1,6 +1,6 @@ export DISTRO_NAME=debian export DIB_RELEASE=${DIB_RELEASE:-stable} -export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://httpredir.debian.org/debian} +export DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://ftp.us.debian.org/debian} export DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main} export DIB_DEBIAN_COMPONENTS_WS=${DIB_DEBIAN_COMPONENTS//,/ } diff --git a/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces-udev.rules b/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces-udev.rules index 302f41bae..a8c3cf513 100644 --- a/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces-udev.rules +++ b/elements/dhcp-all-interfaces/install.d/dhcp-all-interfaces-udev.rules @@ -1 +1 @@ -SUBSYSTEM=="net", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="dhcp-interface@$name.service" +SUBSYSTEM=="net", KERNEL!="lo", ACTION=="add", TAG+="systemd", ENV{SYSTEMD_WANTS}+="dhcp-interface@$name.service" diff --git a/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service b/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service index 14c473057..112765029 100644 --- a/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service +++ b/elements/dhcp-all-interfaces/install.d/dhcp-interface@.service @@ -11,6 +11,7 @@ User=root ExecStartPre=/usr/local/sbin/dhcp-all-interfaces.sh %I ExecStart=/sbin/ifup %I RemainAfterExit=true +TimeoutStartSec=30s [Install] WantedBy=multi-user.target diff --git a/elements/growroot/README.rst b/elements/growroot/README.rst index 33a5d27e5..f0e8de585 100644 --- a/elements/growroot/README.rst +++ b/elements/growroot/README.rst @@ -9,3 +9,4 @@ This is only supported on: * ubuntu trusty or later * gentoo * fedora & centos +* suse & opensuse diff --git a/elements/growroot/pkg-map b/elements/growroot/pkg-map index 43d4ec7cf..f34ea6af5 100644 --- a/elements/growroot/pkg-map +++ b/elements/growroot/pkg-map @@ -8,6 +8,10 @@ "growpart": "cloud-utils", "e2fsprogs": "e2fsprogs" }, + "suse": { + "growpart": "growpart", + "e2fsprogs": "e2fsprogs" + }, "gentoo": { "growpart": "sys-fs/growpart", "e2fsprogs": "sys-fs/e2fsprogs" diff --git a/elements/ironic-agent/cleanup.d/99-ramdisk-create b/elements/ironic-agent/cleanup.d/99-ramdisk-create index 16229dede..5e04a18ea 100755 --- a/elements/ironic-agent/cleanup.d/99-ramdisk-create +++ b/elements/ironic-agent/cleanup.d/99-ramdisk-create @@ -11,6 +11,8 @@ set -o pipefail [ -n "$TARGET_ROOT" ] +USER=${USER:-$(whoami)} + source $_LIB/img-functions IMAGE_PATH=$(readlink -f $IMAGE_NAME) diff --git a/elements/opensuse-minimal/README.rst b/elements/opensuse-minimal/README.rst new file mode 100644 index 000000000..46df3c17c --- /dev/null +++ b/elements/opensuse-minimal/README.rst @@ -0,0 +1,28 @@ +================ +opensuse-minimal +================ + +This element will build a minimal openSUSE image. It requires 'zypper' to be +installed on the host. + +These images should be considered experimental. There are curently only x86_64 +images. + +Environment Variables +--------------------- + +DIB_RELEASE + :Required: No + :Default: 42.1 + :Description: Set the desired openSUSE release. + +DIB_OPENSUSE_MIRROR: + :Required: No + :Default: http://download.opensuse.org + :Description: To use a specific openSUSE mirror, set this variable to the + mirror URL before running bin/disk-image-create. This URL + should point to the root directory as indicated in the + http://mirrors.opensuse.org/ webpage. You normally + don't want to change that since the default setting will + pick the mirror closest to you. + :Example: ``DIB_OPENSUSE_MIRROR=http://ftp.cc.uoc.gr/mirrors/linux/opensuse/opensuse/`` diff --git a/elements/opensuse-minimal/element-deps b/elements/opensuse-minimal/element-deps new file mode 100644 index 000000000..ac6b190dd --- /dev/null +++ b/elements/opensuse-minimal/element-deps @@ -0,0 +1 @@ +zypper-minimal diff --git a/elements/opensuse-minimal/element-provides b/elements/opensuse-minimal/element-provides new file mode 100644 index 000000000..a72e04969 --- /dev/null +++ b/elements/opensuse-minimal/element-provides @@ -0,0 +1 @@ +operating-system diff --git a/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash b/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash new file mode 100644 index 000000000..58fff6d2e --- /dev/null +++ b/elements/opensuse-minimal/environment.d/10-opensuse-distro-name.bash @@ -0,0 +1,20 @@ +export DISTRO_NAME=opensuse +export DIB_RELEASE=${DIB_RELEASE:-42.1} +export DIB_OPENSUSE_MIRROR=${DIB_OPENSUSE_MIRROR:-http://download.opensuse.org} +case ${DIB_RELEASE} in + # We are using "=>" as the assignment symbol since "@" "=" etc could be used in the URI itself. + # Remember, we can't export an array in bash so we use a string instead. + # Repo format: {name}=>{uri} + # Old openSUSE releases + 13*) + ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/${DIB_RELEASE}/ " + ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/${DIB_RELEASE}/repo/oss/" + ;; + # New Leap releases + 42*) + ZYPPER_REPOS="update=>${DIB_OPENSUSE_MIRROR}/update/leap/${DIB_RELEASE}/oss/ " + ZYPPER_REPOS+="oss=>${DIB_OPENSUSE_MIRROR}/distribution/leap/${DIB_RELEASE}/repo/oss/" + ;; + *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;; +esac +export ZYPPER_REPOS diff --git a/elements/opensuse/README.rst b/elements/opensuse/README.rst index 48adc40eb..5ca0da5e4 100644 --- a/elements/opensuse/README.rst +++ b/elements/opensuse/README.rst @@ -13,6 +13,19 @@ For example, the images of openSUSE 13.2 can be found here: These images should be considered experimental. There are curently only x86_64 images. +Environment Variables +--------------------- + +DIB_RELEASE + :Required: No + :Default: 13.1 + :Description: Set the desired openSUSE release. + +DIB_CLOUD_IMAGES + :Required: No + :Default: http://download.opensuse.org/repositories/Cloud:/Images:/(openSUSE|Leap)_${DIB_RELEASE} + :Description: Set the desired URL to fetch the images from. + Notes: * There are very frequently new automated builds that include changes that @@ -21,8 +34,3 @@ Notes: point to the latest image, but will frequently change its content. The versioned one will never change content, but will frequently be deleted and replaced by a newer build with a higher version-release number. - -* Building with DIB\_EXTLINUX=1 doesn't work. It fails with: - /tmp/in\_target.d/finalise.d/51-bootloader: line 14: 16286 Segmentation fault - extlinux --install /boot/syslinux - (https://bugzilla.novell.com/show_bug.cgi?id=852856) diff --git a/elements/opensuse/element-deps b/elements/opensuse/element-deps index c0fd568db..14ee0a470 100644 --- a/elements/opensuse/element-deps +++ b/elements/opensuse/element-deps @@ -1,5 +1,4 @@ cache-url dib-run-parts -install-bin package-installs zypper diff --git a/elements/opensuse/environment.d/10-opensuse-distro-name.bash b/elements/opensuse/environment.d/10-opensuse-distro-name.bash index 22db6c1c0..38efb86fc 100644 --- a/elements/opensuse/environment.d/10-opensuse-distro-name.bash +++ b/elements/opensuse/environment.d/10-opensuse-distro-name.bash @@ -1 +1,9 @@ export DISTRO_NAME=opensuse +export DIB_RELEASE=${DIB_RELEASE:-13.1} +case ${DIB_RELEASE} in + # Old openSUSE releases + 13*) export OPENSUSE_REPO_DIR=openSUSE_${DIB_RELEASE} ;; + # New Leap releases + 42*) export OPENSUSE_REPO_DIR=openSUSE_Leap_${DIB_RELEASE} ;; + *) echo "Unsupported openSUSE release: ${DIB_RELEASE}"; exit 1 ;; +esac diff --git a/elements/opensuse/pre-install.d/00-opensuse-setup b/elements/opensuse/pre-install.d/00-opensuse-setup index ecc7efc29..b681dfeed 100755 --- a/elements/opensuse/pre-install.d/00-opensuse-setup +++ b/elements/opensuse/pre-install.d/00-opensuse-setup @@ -7,4 +7,4 @@ fi set -eu set -o pipefail -zypper ar -f http://download.opensuse.org/repositories/X11:/Bumblebee/openSUSE_13.1/X11:Bumblebee.repo +zypper ar -f http://download.opensuse.org/repositories/X11:/Bumblebee/${OPENSUSE_REPO_DIR}/X11:Bumblebee.repo diff --git a/elements/opensuse/root.d/10-opensuse-cloud-image b/elements/opensuse/root.d/10-opensuse-cloud-image index a6734b922..dc56e5366 100755 --- a/elements/opensuse/root.d/10-opensuse-cloud-image +++ b/elements/opensuse/root.d/10-opensuse-cloud-image @@ -18,11 +18,24 @@ if ! [ 'x86_64' = "$ARCH" ] ; then exit 1 fi -DIB_RELEASE=${DIB_RELEASE:-13.1} +# Set some image defaults +case ${DIB_RELEASE} in + # Old openSUSE releases + 13*) + OPENSUSE_IMAGE_BASEDIR=openSUSE + OPENSUSE_IMAGE_FILE=openSUSE-${DIB_RELEASE}-OS + ;; + # New Leap releases + 42*) + OPENSUSE_IMAGE_BASEDIR=Leap + OPENSUSE_IMAGE_FILE=openSUSE-Leap-${DIB_RELEASE}-OpenStack + ;; + # We handle unknown cases in environment.d/10-opensuse-distro-name.bash +esac # NOTE(toabctl): if something changes here on the buildservice side, please # first ask in #opensuse-cloud on freenode before you change the format here! -DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://download.opensuse.org/repositories/Cloud:/Images:/openSUSE_${DIB_RELEASE}/images/} -BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-openSUSE-${DIB_RELEASE}-OS-rootfs.${ARCH}.tbz} +DIB_CLOUD_IMAGES=${DIB_CLOUD_IMAGES:-http://download.opensuse.org/repositories/Cloud:/Images:/${OPENSUSE_IMAGE_BASEDIR}_${DIB_RELEASE}/images/} +BASE_IMAGE_FILE=${BASE_IMAGE_FILE:-${OPENSUSE_IMAGE_FILE}-rootfs.${ARCH}.tbz} SHA256SUMS_FILE=${SHA256SUMS_FILE:-${BASE_IMAGE_FILE}.sha256} CACHED_FILE=$DIB_IMAGE_CACHE/$BASE_IMAGE_FILE diff --git a/elements/redhat-common/bin/extract-image b/elements/redhat-common/bin/extract-image index 1dec0368c..ee01f689f 100755 --- a/elements/redhat-common/bin/extract-image +++ b/elements/redhat-common/bin/extract-image @@ -59,7 +59,7 @@ function extract_image() { qemu-img convert -f qcow2 -O raw $CACHED_IMAGE $RAW_FILE - ROOT_PARTITON=p$(sudo kpartx -l $RAW_FILE | awk "/loop[0-9]+p/"|wc -l) + ROOT_PARTITION=p$(sudo kpartx -l $RAW_FILE | awk "/loop[0-9]+p/"|wc -l) sudo udevadm settle # kpartx fails if no /dev/loop* exists, "losetup -f" prints first unused @@ -69,7 +69,7 @@ function extract_image() { # XXX: Parsing stdout is dangerous, would like a better way to discover # the device used for the image. ROOT_LOOPDEV=$(sudo kpartx -av $RAW_FILE | \ - awk "/loop[0-9]+$ROOT_PARTITON/ {print \$3}") + awk "/loop[0-9]+$ROOT_PARTITION/ {print \$3}") # If running inside Docker, make our nodes manually, because udev will not be working. if [ -f /.dockerenv ]; then sudo dmsetup --noudevsync mknodes diff --git a/elements/redhat-common/pre-install.d/15-remove-grub b/elements/redhat-common/pre-install.d/15-remove-grub index f116a8524..ba1d922a1 100755 --- a/elements/redhat-common/pre-install.d/15-remove-grub +++ b/elements/redhat-common/pre-install.d/15-remove-grub @@ -23,7 +23,7 @@ fi # XXX : it is not clear this is necessary for fedora/centos7 and it's # install hooks. Investigation is required. if rpm -q grub2; then - install-packages -e grub2 + install-packages -e grub-pc fi # now configure things to re-install grub at the end. We don't want @@ -43,5 +43,5 @@ fi # So we download the latest grub2 package and setup the install script # to just install the single-package, which will be called later by # vm/finalise.d/51-bootloader -install-packages -d /tmp/grub grub2 +install-packages -d /tmp/grub grub-pc echo "rpm -i /tmp/grub/*.rpm" > /tmp/grub/install diff --git a/elements/rhel-common/pre-install.d/00-rhel-registration b/elements/rhel-common/pre-install.d/00-rhel-registration index dcb69d8f8..60c449d85 100755 --- a/elements/rhel-common/pre-install.d/00-rhel-registration +++ b/elements/rhel-common/pre-install.d/00-rhel-registration @@ -97,6 +97,8 @@ case "${REG_METHOD:-}" in echo "Attaching with options: $attach_opts" subscription-manager attach $attach_opts fi + echo "Disabling all previous repos" + subscription-manager repos --disable=\* echo "Enabling repos: $repos" subscription-manager $repos ;; @@ -108,6 +110,8 @@ case "${REG_METHOD:-}" in rpm -Uvh "$REG_SAT_URL/pub/katello-ca-consumer-latest.noarch.rpm" || true echo "Registering with options: $sanitized_opts" subscription-manager register $opts + echo "Disabling all previous repos" + subscription-manager repos --disable=\* echo "Enabling repos: $user_repos" subscription-manager $repos echo "Disabling satellite repo because it is no longer needed" diff --git a/elements/runtime-ssh-host-keys/README.rst b/elements/runtime-ssh-host-keys/README.rst new file mode 100644 index 000000000..b00a2402e --- /dev/null +++ b/elements/runtime-ssh-host-keys/README.rst @@ -0,0 +1,10 @@ +===================== +runtime-ssh-host-keys +===================== +An element to generate SSH host keys on first boot. + +Since ssh key generation is not yet common to all operating systems, we need to +create a DIB element to manage this. We force the removal of the SSH host keys, +then add init scripts to generate them on first boot. + +This element currently supports Debian and Ubuntu (both systemd and upstart). diff --git a/elements/simple-init/cleanup.d/90-remove-ssh-host-keys b/elements/runtime-ssh-host-keys/cleanup.d/90-remove-ssh-host-keys similarity index 78% rename from elements/simple-init/cleanup.d/90-remove-ssh-host-keys rename to elements/runtime-ssh-host-keys/cleanup.d/90-remove-ssh-host-keys index c90626a8a..b14e03f1c 100755 --- a/elements/simple-init/cleanup.d/90-remove-ssh-host-keys +++ b/elements/runtime-ssh-host-keys/cleanup.d/90-remove-ssh-host-keys @@ -10,9 +10,6 @@ set -o pipefail # in so that they are regenerated on first boot and # are unique. -# TODO(greghaynes) This should be a thing we do for all images, not just -# simple-init. - if [ -d $TARGET_ROOT/etc/ssh ] ; then sudo find $TARGET_ROOT/etc/ssh -name 'ssh_host*' -type f -delete fi diff --git a/elements/runtime-ssh-host-keys/element-deps b/elements/runtime-ssh-host-keys/element-deps new file mode 100644 index 000000000..3a0277624 --- /dev/null +++ b/elements/runtime-ssh-host-keys/element-deps @@ -0,0 +1 @@ +dib-init-system diff --git a/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service b/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service new file mode 100644 index 000000000..90a831362 --- /dev/null +++ b/elements/runtime-ssh-host-keys/init-scripts/systemd/ssh-keygen.service @@ -0,0 +1,22 @@ +[Unit] +Description=OpenSSH Server Key Generation +Before=ssh.service + +ConditionPathExists=|!/etc/ssh/ssh_host_key +ConditionPathExists=|!/etc/ssh/ssh_host_key.pub +ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key +ConditionPathExists=|!/etc/ssh/ssh_host_rsa_key.pub +ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key +ConditionPathExists=|!/etc/ssh/ssh_host_dsa_key.pub +ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key +ConditionPathExists=|!/etc/ssh/ssh_host_ecdsa_key.pub +ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key +ConditionPathExists=|!/etc/ssh/ssh_host_ed25519_key.pub + +[Service] +ExecStart=/usr/bin/ssh-keygen -A +Type=oneshot +RemainAfterExit=yes + +[Install] +WantedBy=multi-user.target diff --git a/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf b/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf new file mode 100644 index 000000000..3fa2c0126 --- /dev/null +++ b/elements/runtime-ssh-host-keys/init-scripts/upstart/ssh-keygen.conf @@ -0,0 +1,8 @@ +description "OpenSSH Server Key Generation" + +start on starting ssh +console output + +task + +exec /usr/bin/ssh-keygen -A diff --git a/elements/runtime-ssh-host-keys/package-installs.yaml b/elements/runtime-ssh-host-keys/package-installs.yaml new file mode 100644 index 000000000..c5017af3f --- /dev/null +++ b/elements/runtime-ssh-host-keys/package-installs.yaml @@ -0,0 +1 @@ +openssh-client: diff --git a/elements/runtime-ssh-host-keys/pkg-map b/elements/runtime-ssh-host-keys/pkg-map new file mode 100644 index 000000000..ce9fd939e --- /dev/null +++ b/elements/runtime-ssh-host-keys/pkg-map @@ -0,0 +1,10 @@ +{ + "family": { + "redhat": { + "openssh-client": "openssh" + }, + "gentoo": { + "openssh-client": "" + } + } +} diff --git a/elements/runtime-ssh-host-keys/post-install.d/80-ssh-keygen b/elements/runtime-ssh-host-keys/post-install.d/80-ssh-keygen new file mode 100755 index 000000000..926a12d69 --- /dev/null +++ b/elements/runtime-ssh-host-keys/post-install.d/80-ssh-keygen @@ -0,0 +1,31 @@ +#!/bin/bash + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +case "$DIB_INIT_SYSTEM" in + upstart) + # nothing to do + exit 0 + ;; + systemd) + if [[ $DISTRO_NAME = "ubuntu" || $DISTRO_NAME = "debian" ]]; then + # NOTE(pabelanger): Only support ubuntu / debian today. + systemctl enable ssh-keygen.service + else + # Since we are not enabling it, delete it. + rm /usr/lib/systemd/system/ssh-keygen.service + fi + ;; + openrc) + # let dib-init-system's postinstall handle enabling init scripts + exit 0 + ;; + *) + echo "Unsupported init system" + exit 1 + ;; +esac diff --git a/elements/simple-init/README.rst b/elements/simple-init/README.rst index b5ad10221..dd131d9f8 100644 --- a/elements/simple-init/README.rst +++ b/elements/simple-init/README.rst @@ -33,3 +33,28 @@ not there. Finally, glean will handle ssh-keypair-injection from config drive if cloud-init is not installed. + +Chosing glean installation source +--------------------------------- + +By default glean is installed using pip using the latest release on pypi. +It is also possible to install glean from a specified git repository +location. This is useful for debugging and testing new glean changes +for example. To do this you need to set these variables:: + + DIB_INSTALLTYPE_simple_init=repo + DIB_REPOLOCATION_glean=/path/to/glean/repo + DIB_REPOREF_glean=name_of_git_ref + +For example to test glean change 364516 do:: + + git clone https://git.openstack.org/openstack-infra/glean /tmp/glean + cd /tmp/glean + git review -d 364516 + git checkout -b my-test-ref + +Then set your DIB env vars like this before running DIB:: + + DIB_INSTALLTYPE_simple_init=repo + DIB_REPOLOCATION_glean=/tmp/glean + DIB_REPOREF_glean=my-test-ref diff --git a/elements/simple-init/element-deps b/elements/simple-init/element-deps index d92bc778c..5c7f9bb38 100644 --- a/elements/simple-init/element-deps +++ b/elements/simple-init/element-deps @@ -1,5 +1,5 @@ cloud-init-datasources -dib-init-system install-types pip-and-virtualenv +runtime-ssh-host-keys source-repositories diff --git a/elements/source-repositories/pkg-map b/elements/source-repositories/pkg-map index d6bba0f29..7dfcd72ff 100644 --- a/elements/source-repositories/pkg-map +++ b/elements/source-repositories/pkg-map @@ -2,6 +2,9 @@ "family": { "gentoo": { "git": "dev-vcs/git" + }, + "suse": { + "git": "git-core" } }, "default": { diff --git a/elements/yum/bin/install-packages b/elements/yum/bin/install-packages index 2bef134a0..3dc72c846 100755 --- a/elements/yum/bin/install-packages +++ b/elements/yum/bin/install-packages @@ -14,7 +14,7 @@ # License for the specific language governing permissions and limitations # under the License. -if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then +if [ ${DIB_DEBUG_TRACE:-0} -gt 1 ]; then set -x fi set -eu diff --git a/elements/zypper-minimal/README.rst b/elements/zypper-minimal/README.rst new file mode 100644 index 000000000..d9c8fd323 --- /dev/null +++ b/elements/zypper-minimal/README.rst @@ -0,0 +1,19 @@ +============== +zypper-minimal +============== +Base element for creating minimal SUSE-based images + +This element is incomplete by itself so you probaby want to use it along +with the opensuse-minimal one. It requires 'zypper' to be installed on the +host. + +Repositories +------------ + +This element expects the `ZYPPER_REPOS` variable to be exported by the +operating system element. This variable contains repository mappings in +the following format: `${repo_name}==>${repo_url}`. For example:: + + ZYPPER_REPOS="update=>http://download.opensuse.org/update/leap/42.1/oss/ " + ZYPPER_REPOS+="oss=>http://download.opensuse.org/distribution/leap/42.1/repo/oss/" + export ZYPPER_REPOS diff --git a/elements/zypper-minimal/element-deps b/elements/zypper-minimal/element-deps new file mode 100644 index 000000000..846428bd8 --- /dev/null +++ b/elements/zypper-minimal/element-deps @@ -0,0 +1,3 @@ +dib-run-parts +package-installs +zypper diff --git a/elements/zypper-minimal/install.d/15-zypper-fstab b/elements/zypper-minimal/install.d/15-zypper-fstab new file mode 100755 index 000000000..cffbe1df5 --- /dev/null +++ b/elements/zypper-minimal/install.d/15-zypper-fstab @@ -0,0 +1,27 @@ +#!/bin/bash +# +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +cat << EOF > /etc/fstab +proc /proc proc nodev,noexec,nosuid 0 0 +LABEL=${DIB_ROOT_LABEL} / ${FS_TYPE} errors=remount-ro 0 1 +EOF diff --git a/elements/zypper-minimal/package-installs.yaml b/elements/zypper-minimal/package-installs.yaml new file mode 100644 index 000000000..d2ccc1605 --- /dev/null +++ b/elements/zypper-minimal/package-installs.yaml @@ -0,0 +1,10 @@ +# kernel +linux-image-generic: +# And a few useful tools. Some are pulled +# as dependencies but that may change so lets +# be explicit. +bash: +lsb-release: +openssl: +sed: +sudo: diff --git a/elements/zypper-minimal/root.d/08-zypper-chroot b/elements/zypper-minimal/root.d/08-zypper-chroot new file mode 100755 index 000000000..e6d55c30a --- /dev/null +++ b/elements/zypper-minimal/root.d/08-zypper-chroot @@ -0,0 +1,87 @@ +#!/bin/bash +# +# Copyright 2016 SUSE Linux Products Gmb +# Copyright 2015 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +# + +# dib-lint: disable=safe_sudo + +if [ "${DIB_DEBUG_TRACE:-0}" -gt 0 ]; then + set -x +fi +set -eu +set -o pipefail + +[ -n "${ZYPPER_REPOS}" ] + +function cleanup() { + sudo umount $TMP_MOUNT_PATH/var/cache/zypp +} + +trap cleanup EXIT + +ZYPPER_TARGET_OPTS="--non-interactive --gpg-auto-import-keys --root $TARGET_ROOT" +ZYPPER_INSTALL_OPTS="--no-confirm --no-recommends" + +for repo in ${ZYPPER_REPOS}; do + reponame=repo-${repo%%=>*} + repouri=${repo##*=>} + sudo zypper ${ZYPPER_TARGET_OPTS} addrepo --name ${reponame} --keep-packages ${repouri} ${reponame} +done + +# Refresh it +sudo zypper ${ZYPPER_TARGET_OPTS} refresh + +# It appears that zypper will clean up the repo's cache when it (re-)adds the +# repo so we need to add the cache now, once the repos are added. This is +# similar to what the zypper/50-zypper-cache script does +ZYPPER_CACHE_DIR=$DIB_IMAGE_CACHE/zypper +mkdir -p $ZYPPER_CACHE_DIR + +sudo mkdir -p $TMP_MOUNT_PATH/var/cache/zypp +sudo mount --bind $ZYPPER_CACHE_DIR $TMP_MOUNT_PATH/var/cache/zypp + +# Install filesystem, base and useful tools +sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} filesystem +# Install basic components in order +sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} -t pattern base +# Install a few useful tools +sudo zypper ${ZYPPER_TARGET_OPTS} install ${ZYPPER_INSTALL_OPTS} python zypper + +# Put in a dummy /etc/resolv.conf over the temporary one we used +# to bootstrap. systemd has a bug/feature [1] that it will assume +# you want systemd-networkd as the network manager and create a +# broken symlink to /run/... if the base image doesn't have one. +# This broken link confuses things like dhclient. +# [1] https://bugzilla.redhat.com/show_bug.cgi?id=1197204 +echo -e "# This file intentionally left blank\n" | \ + sudo tee $TARGET_ROOT/etc/resolv.conf + +# set the most reliable UTF-8 locale +echo -e 'LANG="en_US.UTF-8"' | \ +sudo tee $TARGET_ROOT/etc/locale.conf +# default to UTC +sudo -E chroot $TARGET_ROOT ln -sf /usr/share/zoneinfo/UTC \ + /etc/localtime + +# RPM doesn't know whether files have been changed since install +# At this point though, we know for certain that we have changed no +# config files, so anything marked .rpmnew is just a bug. +for newfile in $(sudo find $TARGET_ROOT -type f -name '*rpmnew') ; do + sudo mv $newfile $(echo $newfile | sed 's/.rpmnew$//') +done + +# Unmounting of the /var/cache/zypp is handled by the cleanup EXIT +# handler so there is nothing else to do here diff --git a/elements/opensuse/bin/install-packages b/elements/zypper/bin/install-packages similarity index 96% rename from elements/opensuse/bin/install-packages rename to elements/zypper/bin/install-packages index 0460ce21e..7728b2e46 100755 --- a/elements/opensuse/bin/install-packages +++ b/elements/zypper/bin/install-packages @@ -20,7 +20,7 @@ fi set -eu set -o pipefail -EXTRA_ARGS="" +EXTRA_ARGS="--no-recommends" MAP_ELEMENT="" ACTION=install @@ -47,7 +47,7 @@ eval set -- "$TEMP" while true ; do case "$1" in - -u) run_zypper dist-upgrade; exit 0;; + -u) run_zypper dist-upgrade --no-recommends; exit 0;; -e) ACTION="remove"; shift;; -d) EXTRA_ARGS="--download-only"; shift;; -m) MAP_ELEMENT=$2; shift 2;; diff --git a/elements/opensuse/bin/map-packages b/elements/zypper/bin/map-packages similarity index 100% rename from elements/opensuse/bin/map-packages rename to elements/zypper/bin/map-packages diff --git a/elements/zypper/bin/map-services b/elements/zypper/bin/map-services new file mode 100755 index 000000000..76162e0f3 --- /dev/null +++ b/elements/zypper/bin/map-services @@ -0,0 +1,66 @@ +#!/usr/bin/env python +# dib-lint: disable=indent +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# Copyright 2014 SUSE, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from __future__ import print_function +import os +import sys + +# Manually maintained for brevity; consider making this compiled from +# distromatch or other rich data sources. +# Debian name on the left, openSUSE on the right. +service_map = { + # openstack mapping + 'cinder-api': 'openstack-cinder-api', + 'cinder-backup': 'openstack-cinder-backup', + 'cinder-scheduler': 'openstack-cinder-scheduler', + 'cinder-volume': 'openstack-cinder-volume', + 'glance-api': 'openstack-glance-api', + 'glance-reg': 'openstack-glance-registry', + 'heat-api-cfn': 'openstack-heat-api-cfn', + 'heat-api-cloudwatch': 'openstack-heat-api-cloudwatch', + 'heat-api': 'openstack-heat-api', + 'heat-engine': 'openstack-heat-engine', + 'keystone': 'openstack-keystone', + 'libvirt-bin': 'libvirtd', + 'neutron-dhcp-agent': 'openstack-neutron-dhcp-agent', + 'neutron-openvswitch-agent': 'openstack-neutron-openvswitch-agent', + 'neutron-l3-agent': 'openstack-neutron-l3-agent', + 'neutron-metadata-agent': 'openstack-neutron-metadata-agent', + 'neutron-ovs-cleanup': 'openstack-neutron-ovs-cleanup', + 'neutron-server': 'openstack-neutron', + 'nova-api': 'openstack-nova-api', + 'nova-cert': 'openstack-nova-cert', + 'nova-compute': 'openstack-nova-compute', + 'nova-conductor': 'openstack-nova-conductor', + 'nova-consoleauth': 'openstack-nova-console', + 'nova-baremetal-deploy-helper': 'openstack-nova-baremetal-deploy-helper', + 'nova-novncproxy': 'openstack-nova-novncproxy', + 'nova-scheduler': 'openstack-nova-scheduler', +} + +print("WARNING: map-services has been deprecated. " + "Please use the svc-map element.", file=sys.stderr) + +for arg in sys.argv[1:]: + # We need to support the service name being different when installing from + # source vs. packages. So, if the requested service file already exists, + # just use that. + if os.path.exists('/usr/lib/systemd/system/%s.service' % arg): + print(arg) + else: + print(service_map.get(arg, arg)) +sys.exit(0) diff --git a/elements/zypper/element-deps b/elements/zypper/element-deps new file mode 100644 index 000000000..d888ae5c2 --- /dev/null +++ b/elements/zypper/element-deps @@ -0,0 +1 @@ +install-bin diff --git a/elements/opensuse/install.d/01-ccache-symlinks b/elements/zypper/install.d/01-ccache-symlinks similarity index 100% rename from elements/opensuse/install.d/01-ccache-symlinks rename to elements/zypper/install.d/01-ccache-symlinks diff --git a/elements/opensuse/install.d/01-login-defs b/elements/zypper/install.d/01-login-defs similarity index 100% rename from elements/opensuse/install.d/01-login-defs rename to elements/zypper/install.d/01-login-defs diff --git a/elements/opensuse/post-install.d/10-mkinitrd b/elements/zypper/post-install.d/10-mkinitrd similarity index 100% rename from elements/opensuse/post-install.d/10-mkinitrd rename to elements/zypper/post-install.d/10-mkinitrd diff --git a/lib/common-defaults b/lib/common-defaults index ee33eb470..e04718559 100644 --- a/lib/common-defaults +++ b/lib/common-defaults @@ -34,6 +34,7 @@ fi ARCH=${ARCH:-$_ARCH} export ARCH +export DIB_CHECKSUM=${DIB_CHECKSUM:-0} export DIB_NO_TMPFS=${DIB_NO_TMPFS:-0} export DIB_MIN_TMPFS=${DIB_MIN_TMPFS:-2} # Set via the CLI normally. diff --git a/lib/common-functions b/lib/common-functions index 42624667e..f0f007af2 100644 --- a/lib/common-functions +++ b/lib/common-functions @@ -51,9 +51,19 @@ function finish_image () { old_image="${1%.*}"-$(date +%Y.%m.%d-%H.%M.%S).${1##*.} echo "Old image found. Renaming it to $old_image" mv "$1" "$old_image" + if [ -f "$1.md5" ]; then + mv "$1.md5" "$old_image.md5" + fi + if [ -f "$1.sha256" ]; then + mv "$1.sha256" "$old_image.sha256" + fi fi mv $OUT_IMAGE_PATH $1 + if [ "$DIB_CHECKSUM" == "1" ]; then + md5sum $1 > $1.md5 + sha256sum $1 > $1.sha256 + fi echo "Image file $1 created..." } diff --git a/releasenotes/notes/opensuse-minimal-45267f5be1112c22.yaml b/releasenotes/notes/opensuse-minimal-45267f5be1112c22.yaml new file mode 100644 index 000000000..4480aeddf --- /dev/null +++ b/releasenotes/notes/opensuse-minimal-45267f5be1112c22.yaml @@ -0,0 +1,6 @@ +--- +features: + - New zypper-minimal and opensuse-minimal elements to create basic + openSUSE images. These two new elements are also making use of the + existing zypper element which has been extended to include the + functionality previously present in the opensuse element. diff --git a/releasenotes/notes/runtime-ssh-host-keys-7a2fc873cc90d33e.yaml b/releasenotes/notes/runtime-ssh-host-keys-7a2fc873cc90d33e.yaml new file mode 100644 index 000000000..3475ae7d0 --- /dev/null +++ b/releasenotes/notes/runtime-ssh-host-keys-7a2fc873cc90d33e.yaml @@ -0,0 +1,6 @@ +--- +features: + - New element (runtime-ssh-host-keys) to manage SSH host keys at boot. Since + SSH host key generation is not standard across operating systems, add + support for both Debian and Ubuntu to handle it. While this is a new + element, simple-init has been updated to depend on it. diff --git a/releasenotes/source/conf.py b/releasenotes/source/conf.py index 117ef54aa..ed01c8f27 100644 --- a/releasenotes/source/conf.py +++ b/releasenotes/source/conf.py @@ -235,3 +235,6 @@ texinfo_documents = [ # If true, do not generate a @detailmenu in the "Top" node's menu. #texinfo_no_detailmenu = False + +# -- Options for Internationalization output ------------------------------ +locale_dirs = ['locale/'] diff --git a/requirements.txt b/requirements.txt index 47f5abbbe..16b55619b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,7 +4,7 @@ Babel>=2.3.4 # BSD dib-utils # Apache-2.0 pbr>=1.6 # Apache-2.0 -PyYAML>=3.1.0 # MIT +PyYAML>=3.10.0 # MIT flake8<2.6.0,>=2.5.4 # MIT six>=1.9.0 # MIT -oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 +oslosphinx>=4.7.0 # Apache-2.0 diff --git a/test-requirements.txt b/test-requirements.txt index 04ab6d6b9..8671a4497 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,8 +7,8 @@ oslotest>=1.10.0 # Apache-2.0 testrepository>=0.0.18 # Apache-2.0/BSD # Doc requirements -sphinx!=1.3b1,<1.3,>=1.2.1 # BSD -oslosphinx!=3.4.0,>=2.5.0 # Apache-2.0 +sphinx!=1.3b1,<1.4,>=1.2.1 # BSD +oslosphinx>=4.7.0 # Apache-2.0 # releasenotes reno>=1.8.0 # Apache2 diff --git a/tests/run_functests.sh b/tests/run_functests.sh index 7a51f23b6..2bbd8c57f 100755 --- a/tests/run_functests.sh +++ b/tests/run_functests.sh @@ -25,21 +25,65 @@ DEFAULT_SKIP_TESTS=( centos-minimal/build-succeeds ) +function log_with_prefix { + local pr=$1 + + while read a; do + echo $(date +"%Y%m%d-%H%M%S.%N") "[$pr] $a" + done +} + +# Log job control messages +function log_jc { + local msg="$1" + printf "[JOB-CONTROL] %s %s\n" "$(date)" "${msg}" +} + +function job_cnt { + running_jobs=$(jobs -p) + echo ${running_jobs} | wc -w +} + +# This is needed, because the better 'wait -n' is +# available since bash 4.3 only. +function wait_minus_n { + if [ "${BASH_VERSINFO[0]}" -gt 4 \ + -o "${BASH_VERSINFO[0]}" = 4 \ + -a "${BASH_VERSINFO[1]}" -ge 3 ]; then + # Good way: wait on any job + wait -n + return $? + else + # Not that good way: wait on one specific job + # (others may be finished in the mean time) + local wait_for_pid=$(jobs -p | head -1) + wait ${wait_for_pid} + return $? + fi +} + # run_disk_element_test # Run a disk-image-build .tar build of ELEMENT including any elements # specified by TEST_ELEMENT function run_disk_element_test() { local test_element=$1 local element=$2 + local dont_use_tmp=$3 + local use_tmp_flag="" local dest_dir=$(mktemp -d) - trap "rm -rf $dest_dir /tmp/dib-test-should-fail" EXIT + trap "rm -rf $dest_dir" EXIT + + if [ "${dont_use_tmp}" = "yes" ]; then + use_tmp_flag="--no-tmpfs" + fi if break="after-error" break_outside_target=1 \ - break_cmd="cp \$TMP_MOUNT_PATH/tmp/dib-test-should-fail /tmp/ 2>&1 > /dev/null || true" \ + break_cmd="cp -v \$TMP_MOUNT_PATH/tmp/dib-test-should-fail ${dest_dir} || true" \ DIB_SHOW_IMAGE_USAGE=1 \ ELEMENTS_PATH=$DIB_ELEMENTS:$DIB_ELEMENTS/$element/test-elements \ - $DIB_CMD -x -t tar,qcow2 -o $dest_dir/image -n $element $test_element; then + $DIB_CMD -x -t tar,qcow2 ${use_tmp_flag} -o $dest_dir/image -n $element $test_element 2>&1 \ + | log_with_prefix "${element}/${test_element}"; then if ! [ -f "$dest_dir/image.qcow2" ]; then echo "Error: qcow2 build failed for element: $element, test-element: $test_element." @@ -60,7 +104,7 @@ function run_disk_element_test() { fi fi else - if [ -f "/tmp/dib-test-should-fail" ]; then + if [ -f "${dest_dir}/dib-test-should-fail" ]; then echo "PASS: Element $element, test-element: $test_element" else echo "Error: Build failed for element: $element, test-element: $test_element." @@ -81,7 +125,8 @@ function run_ramdisk_element_test() { local dest_dir=$(mktemp -d) if ELEMENTS_PATH=$DIB_ELEMENTS/$element/test-elements \ - $DIB_CMD -x -o $dest_dir/image $element $test_element; then + $DIB_CMD -x -o $dest_dir/image $element $test_element \ + | log_with_prefix "${element}/${test_element}"; then # TODO(dtantsur): test also kernel presence once we sort out its naming # problem (vmlinuz vs kernel) if ! [ -f "$dest_dir/image.initramfs" ]; then @@ -111,12 +156,15 @@ for e in $DIB_ELEMENTS/*/test-elements/*; do TESTS+=("$element/$test_element") done -while getopts ":hl" opt; do +JOB_MAX_CNT=1 + +while getopts ":hlpj:" opt; do case $opt in h) echo "run_functests.sh [-h] [-l] ..." echo " -h : show this help" echo " -l : list available tests" + echo " -p : run all tests in parallel" echo " : functional test to run" echo " Special test 'all' will run all tests" exit 0 @@ -130,6 +178,10 @@ while getopts ":hl" opt; do echo exit 0 ;; + j) + JOB_MAX_CNT=${OPTARG} + echo "Running parallel - using [${JOB_MAX_CNT}] jobs" + ;; \?) echo "Invalid option: -$OPTARG" exit 1 @@ -138,6 +190,15 @@ while getopts ":hl" opt; do done shift $((OPTIND-1)) +DONT_USE_TMP="no" +if [ "${JOB_MAX_CNT}" -gt 1 ]; then + # switch off using tmp dir for image building + # (The mem check using the tmp dir is currently done + # based on the available memory - and not on the free. + # See #1618124 for more details) + DONT_USE_TMP="yes" +fi + # cull the list of tests to run into TESTS_TO_RUN TESTS_TO_RUN=() title="" @@ -173,7 +234,36 @@ for test in "${TESTS_TO_RUN[@]}"; do done echo "------" +function wait_and_exit_on_failure { + local pid=$1 + + wait ${pid} + result=$? + + if [ "${result}" -ne 0 ]; then + exit ${result} + fi + return 0 +} + +EXIT_CODE=0 for test in "${TESTS_TO_RUN[@]}"; do + running_jobs_cnt=$(job_cnt) + log_jc "Number of running jobs [${running_jobs_cnt}] max jobs [${JOB_MAX_CNT}]" + if [ "${running_jobs_cnt}" -ge "${JOB_MAX_CNT}" ]; then + log_jc "Waiting for job to finish" + wait_minus_n + result=$? + + if [ "${result}" -ne 0 ]; then + EXIT_CODE=1 + # If a job fails, do not start any new ones. + break + fi + fi + + log_jc "Starting new job" + # from above; each array value is element/test_element. split it # back up element=${test%/*} @@ -188,7 +278,30 @@ for test in "${TESTS_TO_RUN[@]}"; do fi echo "Running $test ($element_type)" - run_${element_type}_element_test $test_element $element + run_${element_type}_element_test $test_element $element ${DONT_USE_TMP} & done -echo "Tests passed!" +# Wait for the rest of the jobs +while true; do + running_jobs_cnt=$(job_cnt) + log_jc "Number of running jobs left [${running_jobs_cnt}]" + + if [ "${running_jobs_cnt}" -eq 0 ]; then + break; + fi + + wait_minus_n + result=$? + + if [ "${result}" -ne 0 ]; then + EXIT_CODE=1 + fi +done + +if [ "${EXIT_CODE}" -eq 0 ]; then + echo "Tests passed!" + exit 0 +else + echo "At least one test failed" + exit 1 +fi