Release 1.24.0

-----BEGIN PGP SIGNATURE-----
 
 iQEcBAABAgAGBQJYP+r7AAoJEBty/58O8cX8ZcMIAI7Yy2BfGqY/jfyLYNryATSu
 g78McMxAeOiPYII+KKyVkeWsChKJUafLZr/ppO95pm7ZLXeHQeC4MHDI68RmghIX
 HVXn4hPYzwlZ+Yr4hqaEf11H+vw0TDxo2a5bxlQZw+FCx9+3lkR60QxWf8a6QCxv
 nLqKIhtpwGi9mIyvAya+MaOmHMHqbNYgIBHKP0yXhTdxoHZoJWZeXN1rkelwAF9N
 TS1+KwIk+uemn1ZlqzYDWj8euyFyAv8L0WlF0ztCKe7TAFvNeMo26fLykeKHVDEo
 GC5R8MshiOtihPC972rNIponIzd62AetTlDxfK+gcUZO21pNTrlgdLX5Jn04fo0=
 =BpQc
 -----END PGP SIGNATURE-----

Merge tag '1.24.0' into merge-branch

Release 1.24.0

Change-Id: I23cfa6f8dc66bb6a1d957a49a5b468cfd47c005b

diskimage_builder/elements/debian-minimal/package-installs.yaml

@@ -1,2 +1,6 @@
+linux-image-amd64:
+  arch: amd64
+linux-image-686:
+  arch: i386
 systemd:
 systemd-sysv:

diskimage_builder/elements/debian-minimal/root.d/75-debian-minimal-baseinstall

@@ -15,10 +15,6 @@
 #
 # See the License for the specific language governing permissions and
 # limitations under the License.
-#
-# The filename needs to be 02-...: because the install-package script
-# is installed in the dpkg/pre-install/01-dpkg and that has to be executed
-# first.
 
 if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
     set -x
@@ -34,20 +30,28 @@ set -o pipefail
 function apt_sources_write {
     local APT_SOURCES_CONF="$1"
 
-    mkdir -p /etc/apt/sources.list.d
+    sudo mkdir -p $TARGET_ROOT/etc/apt/sources.list.d
 
     echo "${APT_SOURCES_CONF}" \
         | while read line; do
             local name=$(echo ${line} | cut -d ":" -f 1)
             local value=$(echo ${line} | cut -d ":" -f 2-)
-            echo "$value" >>/etc/apt/sources.list.d/${name}.list
+            echo "$value" | sudo tee $TARGET_ROOT/etc/apt/sources.list.d/${name}.list
         done
 }
 
+sudo mount -t proc none $TARGET_ROOT/proc
+sudo mount -t sysfs none $TARGET_ROOT/sys
+trap "sudo umount $TARGET_ROOT/proc; sudo umount $TARGET_ROOT/sys" EXIT
+
+apt_get="sudo chroot $TARGET_ROOT /usr/bin/apt-get"
+
 apt_sources_write "${DIB_APT_SOURCES_CONF}"
 
 # Need to update to retrieve the signed Release file
-apt-get update
+$apt_get update
 
-apt-get clean
-install-packages -u
+$apt_get clean
+$apt_get dist-upgrade -y
+
+$apt_get install -y busybox python sudo

diskimage_builder/elements/debootstrap/root.d/08-debootstrap

@@ -28,23 +28,9 @@ fi
 [ -n "$DIB_RELEASE" ]
 [ -n "$DIB_DISTRIBUTION_MIRROR" ]
 
-if [ $DISTRO_NAME = 'ubuntu' ] ; then
-    KERNEL='generic'
-else
-    case $ARCH in
-        amd64) KERNEL='amd64' ;;
-        i386) KERNEL='686' ;;
-        arm7)
-            [ -n "$DIB_ARM_KERNEL" ]
-            KERNEL="$DIB_ARM_KERNEL"
-            ;;
-    esac
-fi
-KERNEL_PACKAGE="linux-image-$KERNEL"
-
 DIB_DEBIAN_COMPONENTS=${DIB_DEBIAN_COMPONENTS:-main}
 DIB_DEBOOTSTRAP_EXTRA_ARGS=${DIB_DEBOOTSTRAP_EXTRA_ARGS:-}
-DEBOOTSTRAP_TARBALL=$DIB_IMAGE_CACHE/debootstrap-${DISTRO_NAME}-${DIB_RELEASE}-${ARCH}-${KERNEL}.tar.gz
+DEBOOTSTRAP_TARBALL=$DIB_IMAGE_CACHE/debootstrap-${DISTRO_NAME}-${DIB_RELEASE}-${ARCH}.tar.gz
 http_proxy=${http_proxy:-}
 no_proxy=${no_proxy:-}
 
@@ -63,7 +49,6 @@ else
     # Have to --include=busybox because initramfs needs it
     sudo sh -c "http_proxy=$http_proxy no_proxy=$no_proxy debootstrap --verbose \
         --variant=minbase \
-        --include=python,sudo,busybox,$KERNEL_PACKAGE \
         --components=${DIB_DEBIAN_COMPONENTS} \
         --arch=${ARCH} \
         $KEYRING_OPT \
@@ -79,6 +64,7 @@ else
     sudo rm -fr ${TARGET_ROOT}/etc/apt/sources.list \
         ${TARGET_ROOT}/etc/apt/sources.list.d
 
+
     echo Caching debootstrap result in $DEBOOTSTRAP_TARBALL
     if [ "${DIB_DEBOOTSTRAP_CACHE:-0}" != "0" ]; then
         sudo tar --numeric-owner -C $TARGET_ROOT -zcf $DEBOOTSTRAP_TARBALL --exclude='./tmp/*' .

diskimage_builder/elements/element-manifest/README.rst

@@ -3,6 +3,6 @@ element-manifest
 ================
 
 Writes a manifest file that is the full list of elements that were used to
-build the image. The file path can be overriden by setting
+build the image. The file path can be overridden by setting
 $DIB\_ELEMENT\_MANIFEST\_PATH, and defaults to
 /etc/dib-manifests/element-manifest.

diskimage_builder/elements/package-installs/bin/package-installs-v2

@@ -20,20 +20,28 @@ import subprocess
 import sys
 
 
-def process_output(cmdline):
-    # Try to execute subprocess.check_output(), which is available
-    # in Python 2.7+, gracefully falling back to subprocess.Popen
-    # in older Python versions.
-    try:
-        return subprocess.check_output(cmdline).decode(encoding='utf-8')
-    except AttributeError:
-        proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
-        out = proc.communicate()[0]
-        if proc.returncode:
-            e = subprocess.CalledProcessError(proc.returncode, cmdline)
-            e.output = out
-            raise e
-        return out
+# run a command, return output
+#  if follow is set, output will be echoed to stdout
+def process_output(cmdline, follow=False):
+    proc = subprocess.Popen(cmdline, stdout=subprocess.PIPE)
+    if follow:
+        print("Running command: %s" % cmdline)
+        out = ""
+        with proc.stdout:
+            for line in iter(proc.stdout.readline, b''):
+                out += line.decode('utf-8')
+                print("> %s" % line.strip())
+        proc.wait()
+        print("> -- done")
+    else:
+        out = proc.communicate()[0].decode('utf-8')
+
+    if proc.returncode:
+        e = subprocess.CalledProcessError(proc.returncode, cmdline)
+        e.output = out
+        raise e
+
+    return out
 
 
 def main():
@@ -69,8 +77,7 @@ def main():
         pkg_map_args = ['pkg-map', '--missing-ok', '--element', element, pkg]
 
         try:
-            map_output = process_output(
-                pkg_map_args)
+            map_output = process_output(pkg_map_args)
             pkgs.extend(map_output.strip().split('\n'))
         except subprocess.CalledProcessError as e:
             if e.returncode == 1:
@@ -93,7 +100,7 @@ def main():
         print(" ".join(install_args))
     else:
         try:
-            process_output(install_args)
+            process_output(install_args, follow=True)
         except subprocess.CalledProcessError as e:
             print("install failed with error %s" % e.output)
             sys.exit(1)

diskimage_builder/elements/pip-and-virtualenv/package-installs.yaml

@@ -1,3 +1,7 @@
+# This package is only valid for SUSE and we need to pull it in
+# early enough because pip depends on it.
+python-xml:
+  phase: pre-install.d
 python-pip:
   installtype: package
 python-virtualenv:

diskimage_builder/elements/pip-and-virtualenv/pkg-map

@@ -3,6 +3,12 @@
     "gentoo": {
       "python-pip": "dev-python/pip",
       "python-virtualenv": "dev-python/virtualenv"
+    },
+    "suse" : {
+      "python-xml": "python-xml"
     }
+  },
+  "default": {
+    "python-xml": ""
   }
 }

diskimage_builder/elements/ubuntu-minimal/element-deps

@@ -1 +1,2 @@
 debootstrap
+package-installs

diskimage_builder/elements/ubuntu-minimal/package-installs.yaml

@@ -0,0 +1 @@
+linux-image-generic:

diskimage_builder/elements/ubuntu-minimal/root.d/75-ubuntu-minimal-baseinstall

@@ -0,0 +1,46 @@
+#!/bin/bash
+# Copyright (c) 2014 Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied.
+#
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+if [ ${DIB_DEBUG_TRACE:-0} -gt 0 ]; then
+    set -x
+fi
+set -eu
+set -o pipefail
+
+DIB_DISTRIBUTION_MIRROR=${DIB_DISTRIBUTION_MIRROR:-http://archive.ubuntu.com/ubuntu}
+
+# We should manage this in a betterer way
+sudo bash -c "cat << EOF >$TARGET_ROOT/etc/apt/sources.list
+deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE main restricted universe
+deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-updates main restricted universe
+deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-backports main restricted universe
+deb $DIB_DISTRIBUTION_MIRROR $DIB_RELEASE-security main restricted universe
+EOF"
+
+sudo mount -t proc none $TARGET_ROOT/proc
+sudo mount -t sysfs none $TARGET_ROOT/sys
+trap "sudo umount $TARGET_ROOT/proc; sudo umount $TARGET_ROOT/sys" EXIT
+
+apt_get="sudo chroot $TARGET_ROOT /usr/bin/apt-get" # dib-lint: safe_sudo
+
+# Need to update to retrieve the signed Release file
+$apt_get update
+
+$apt_get clean
+$apt_get dist-upgrade -y
+
+$apt_get install -y busybox python sudo

diskimage_builder/elements/zypper-minimal/package-installs.yaml

@@ -4,6 +4,7 @@ linux-image-generic:
 # as dependencies but that may change so lets
 # be explicit.
 bash:
+ca-certificates-mozilla:
 lsb-release:
 openssl:
 sed:

diskimage_builder/lib/common-functions

@@ -61,8 +61,9 @@ function finish_image () {
 
     mv $OUT_IMAGE_PATH $1
     if [ "$DIB_CHECKSUM" == "1" ]; then
-      md5sum $1 > $1.md5
-      sha256sum $1 > $1.sha256
+      # NOTE(pabelanger): Read image into memory once and generate both checksum
+      # files.
+      md5sum $1 > $1.md5 & sha256sum $1 > $1.sha256 & wait
     fi
     echo "Image file $1 created..."
 }

releasenotes/notes/package-outside-debootstrap-ac93e9ce991819f1.yaml

@@ -0,0 +1,5 @@
+---
+fixes:
+  - The `debian-minimal` and and `ubuntu-minimal` elements now install
+    directly from the updates repo, avoiding the need to
+    double-install packages during build.