diff --git a/diskimage_builder/dib_run_parts.py b/diskimage_builder/dib_run_parts.py new file mode 100644 index 000000000..77a74e5bb --- /dev/null +++ b/diskimage_builder/dib_run_parts.py @@ -0,0 +1,37 @@ +# Copyright 2016 Ian Wienand (iwienand@redhat.com) +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import os.path +import sys + +import diskimage_builder.paths + + +# simply wrap the dib-run-parts in lib +# +# Note to would-be modifiers : the same dib-run-parts script we are +# calling in "lib" here is actually copied into the chroot's +# /usr/local/bin by the dib-run-parts element, where it is run diretly +# by disk-image-create. Ergo, if you do something clever in here, it +# won't be reflected in the dib-run-parts that actually runs in the +# chroot. It may not always be like this, but it does reduce reliance +# on Python inside the chroot image. +def main(): + environ = os.environ + + script = "%s/%s" % (diskimage_builder.paths.get_path('lib'), + os.path.basename(sys.argv[0])) + + os.execve("/bin/bash", ['bash', script] + sys.argv[1:], environ) diff --git a/diskimage_builder/elements/dib-run-parts/root.d/90-base-dib-run-parts b/diskimage_builder/elements/dib-run-parts/root.d/90-base-dib-run-parts index c85dc82af..11586714b 100755 --- a/diskimage_builder/elements/dib-run-parts/root.d/90-base-dib-run-parts +++ b/diskimage_builder/elements/dib-run-parts/root.d/90-base-dib-run-parts @@ -8,7 +8,11 @@ set -o pipefail # Abort early if dib-run-parts is not found to prevent a meaningless # error message from the subsequent install command -DIB_RUN_PARTS=$(which dib-run-parts) +DIB_RUN_PARTS=${_LIB}/dib-run-parts + +if [ ! -f ${DIB_RUN_PARTS} ]; then + echo "Can't find dib-run-parts script!" +fi exec sudo install -m 0755 -o root -g root -D \ $DIB_RUN_PARTS \ diff --git a/diskimage_builder/lib/dib-run-parts b/diskimage_builder/lib/dib-run-parts new file mode 100644 index 000000000..0f81be195 --- /dev/null +++ b/diskimage_builder/lib/dib-run-parts @@ -0,0 +1,144 @@ +#!/bin/bash +# Inspired by Debian and RedHat run-parts but portable and specific to di-b. +# +# Copyright 2012 Hewlett-Packard Development Company, L.P. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +allowed_regex=${RUN_PARTS_REGEX:-"^[0-9A-Za-z_-]+$"} +show_list= + +set -ue +set -o pipefail + +name=$(basename $0) + +usage() { + echo "Usage: $name [OPTION] scripts_directory" + echo "Option:" + echo " --list print names of all valid files" + echo + echo "Examples:" + echo " dib-run-parts --list /opt/stack/os-config-refresh/configure.d/" + echo " dib-run-parts /opt/stack/os-config-refresh/configure.d/" + exit 1 +} >&2 + + +output_prefix() { + printf "%s %s " "${name}" "$(date)" >&2 +} + +output () { + output_prefix + echo $* >&2 +} + +output_printf () { + local FORMAT="$1" + shift + output_prefix + printf "${FORMAT}" $@ >&2 +} + +# source the environment files from environment.d +# arg : target_dir +source_environment() { + + local dir=$target_dir/../environment.d + local env_files + local xtrace + + if [ -d ${dir} ] ; then + env_files=$(find ${dir} -maxdepth 1 -xtype f | \ + grep -E "/[0-9A-Za-z_\.-]+$" | \ + LANG=C sort -n) + for env_file in $env_files ; do + output "Sourcing environment file ${env_file}" + # Set tracing as we import these environment files; it's + # nice to see the definitions in the logs + xtrace=$(set +o | grep xtrace) + set -o xtrace + source $env_file + $xtrace + done + fi +} + +if [ $# -lt 1 ] ; then + usage +fi + +if [ "$1" == "--list" ] ; then + show_list="1" + shift +fi + +target_dir="${1:-}" + +if ! [ -d "$target_dir" ] ; then + output "Scripts directory [$target_dir] must exist and be a directory" + usage +fi + +# We specifically only want to sort *by the numbers*. +# Lexical sorting is not guaranteed, and identical numbers may be +# parallelized later +# Note: -maxdepth 1 ensures only files in the target directory (but not +# subdirectories) are run, which is the way run-parts behaves. +targets=$(find $target_dir -maxdepth 1 -xtype f -executable -printf '%f\n' | grep -E "$allowed_regex" | LANG=C sort -n || echo "") + +if [ "$show_list" == "1" ] ; then + for target in $targets ; do + echo "${target_dir}/${target}" + done + exit 0 +fi + +PROFILE_DIR=$(mktemp -d --tmpdir profiledir.XXXXXX) + +# note, run this in a sub-shell so we don't pollute our +# own environment with source_environment +( + source_environment + + for target in $targets ; do + output "Running $target_dir/$target" + target_tag=${target//\//_} + date +%s.%N > $PROFILE_DIR/start_$target_tag + $target_dir/$target + target_tag=${target//\//_} + date +%s.%N > $PROFILE_DIR/stop_$target_tag + output "$target completed" + done +) + +output "----------------------- PROFILING -----------------------" +output "" +output "Target: $(basename $target_dir)" +output "" +output_printf "%-40s %9s\n" Script Seconds +output_printf "%-40s %9s\n" --------------------------------------- ---------- +output "" +pushd $PROFILE_DIR > /dev/null +for target in $(find . -name 'start_*' -printf '%f\n' | env LC_ALL=C sort -n) ; do + stop_file=stop_${target##start_} + start_seconds=$(cat $target) + stop_seconds=$(cat $stop_file) + duration=$(echo - | awk "{ print $stop_seconds - $start_seconds }") + LC_NUMERIC=C LC_ALL=C output_printf "%-40s %10.3f\n" ${target##start_} $duration +done +popd > /dev/null +rm -rf $PROFILE_DIR +output "" +output "--------------------- END PROFILING ---------------------" diff --git a/doc/source/developer/invocation.rst b/doc/source/developer/invocation.rst index 90c97ba25..ea3ad9450 100644 --- a/doc/source/developer/invocation.rst +++ b/doc/source/developer/invocation.rst @@ -1,12 +1,25 @@ -Installation -============ +Developer Installation +====================== -For general use, you can use distribution packages or install via -``pip`` in a ``virtualenv`` +Note that for non-development use you can use distribution packages or +install the latest release via ``pip`` in a ``virtualenv``. -For development purposes, you can use ``pip -e`` to install into a -local development/testing ``virtualenv``, or use ``tox -e venv -- -disk-image-create`` to run within a ``tox`` created environment. +For development purposes, you can use ``pip -e`` to install the latest +git tree checkout into a local development/testing ``virtualenv``, or +use ``tox -e venv -- disk-image-create`` to run within a ``tox`` +created environment. + +For example, to create a ``virtualenv`` and install + +:: + + $ mkdir dib + $ cd dib + $ virtualenv env + $ source env/bin/activate + $ git clone https://git.openstack.org/openstack/diskimage-builder + $ cd diskimage-builder + $ pip install -e . Invocation ========== diff --git a/doc/source/user_guide/installation.rst b/doc/source/user_guide/installation.rst index ee10e35a4..0391adf07 100644 --- a/doc/source/user_guide/installation.rst +++ b/doc/source/user_guide/installation.rst @@ -1,61 +1,40 @@ Installation ============ -Diskimage-builder can either be run directly out of the source repository or -installed via pip. If you plan on doing development on diskimage-builder or -the elements then we recommend you run the tool out of the source repository -as this installation requires minimal extra effort and does not require an -extra install step for your changes to take effect. +If your distribution does not proivde packages, you should install +``diskimage-builder`` via ``pip``, mostly likely in a ``virtualenv`` +to keep it separate. -Once installed, you will be able to :doc:`build images ` -using disk-image-create and the elements included in the main diskimage-builder -repository. +For example, to create a ``virtualenv`` and install from ``pip`` + +:: + + virtualenv ~/dib-virtualenv + . ~/dib-virtualenv/bin/activate + pip install diskimage-builder + + +Once installed, you will be able to :doc:`build images +` using ``disk-image-create`` and the elements +included in the main ``diskimage-builder`` repository. Requirements ------------ -Most image formats require the qemu-img tool which is provided by the -qemu-utils package on Ubuntu/Debian or the qemu package on +Most image formats require the ``qemu-img`` tool which is provided by +the ``qemu-utils`` package on Ubuntu/Debian or the ``qemu`` package on Fedora/RHEL/opensuse/Gentoo. -Some image formats, such as VHD, may require additional tools. Please see -the disk-image-create help output for more information. +Some image formats, such as ``VHD``, may require additional +tools. Please see the ``disk-image-create`` help output for more +information. Individual elements can also have additional dependencies for the build host. It is recommended you check the documentation for each element you are using to determine if there are any additional dependencies. Of particular note is the need for the `dev-python/pyyaml` package on Gentoo hosts. - -Source Installation -------------------- - -Clone the diskimage-builder and dib-utils repositories locally: - -:: - - git clone https://git.openstack.org/openstack/diskimage-builder - git clone https://git.openstack.org/openstack/dib-utils - - -Add the bin dirs to your path: - -:: - - export PATH=$PATH:$(pwd)/diskimage-builder/bin:$(pwd)/dib-utils/bin - - -Pip Installation ----------------- - -Installing via pip is as simple as: - -:: - - pip install diskimage-builder - - Package Installation -------------------- diff --git a/releasenotes/notes/remove-dib-utils-37f70dfad54900a0.yaml b/releasenotes/notes/remove-dib-utils-37f70dfad54900a0.yaml new file mode 100644 index 000000000..47d580564 --- /dev/null +++ b/releasenotes/notes/remove-dib-utils-37f70dfad54900a0.yaml @@ -0,0 +1,6 @@ +--- +deprecations: + - The `dib-utils` requirement has been removed as the + `dib-run-parts` script is now shipped from within + diskimage-builder. The `dib-utils` project is now considered + retired. diff --git a/requirements.txt b/requirements.txt index 16b55619b..c903697d6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,6 @@ # of appearance. Changing the order has an impact on the overall integration # process, which may cause wedges in the gate later. Babel>=2.3.4 # BSD -dib-utils # Apache-2.0 pbr>=1.6 # Apache-2.0 PyYAML>=3.10.0 # MIT flake8<2.6.0,>=2.5.4 # MIT diff --git a/setup.cfg b/setup.cfg index 4c5e9357f..4adae9d63 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,3 +56,4 @@ console_scripts = dib-block-device = diskimage_builder.block_device:main disk-image-create = diskimage_builder.disk_image_create:main ramdisk-image-create = diskimage_builder.disk_image_create:main + dib-run-parts = diskimage_builder.dib_run_parts:main