From 37a53354ec51a1d20c3ac7bfa70744fa858dcb88 Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Tue, 28 Jun 2016 15:57:55 +1000 Subject: [PATCH] Add IMAGE_ELEMENT_YAML and get_image_element_array These new variables are a list of elements chosen for the build along with their full paths. For Python elements, IMAGE_ELEMENT_YAML is a YAML formatted list that can be easily parsed. For bash elements, "get_image_element_array" will produce an associative-array of the same (working around lack of array export in Bash). This list is intended for consumption of elements who need to copy files from other elements, such as pkg-map and svc-map. As discussed in I2a29861c67de2d25c595cb35d850e92807d26ac6, this list has already been pruned and had overrides processed, so it is safe to simply walk over this list with no further processing. Since we're presenting the element list in a couple of different ways, we combine it all into the element-info script. It will output an eval-able string that declares the appropriate variables. I've added some inline documentation so they still appear in grep. The documentation is updated with examples, and moved to a more appropriate location as a sub-section of the element sytle guide. To test this out, use the associative-array in generate_hooks, where we can now find the element's directory without searching. Change-Id: Ibbd07d082ec827441def2d3f6240df3efdc6eae3 --- diskimage_builder/element_dependencies.py | 35 +++++++++++- doc/source/developer/developing_elements.rst | 35 ++++++++++-- lib/common-functions | 57 ++++++++++++++----- .../notes/element-vars-c6bf2e6795002f01.yaml | 6 ++ 4 files changed, 114 insertions(+), 19 deletions(-) create mode 100644 releasenotes/notes/element-vars-c6bf2e6795002f01.yaml diff --git a/diskimage_builder/element_dependencies.py b/diskimage_builder/element_dependencies.py index 6c2f60865..0a5ceeeec 100644 --- a/diskimage_builder/element_dependencies.py +++ b/diskimage_builder/element_dependencies.py @@ -19,6 +19,7 @@ import errno import logging import os import sys +import yaml import diskimage_builder.logging_config @@ -188,6 +189,10 @@ def main(argv): default=False, help=('(DEPRECATED) print expanded dependencies ' 'of all args')) + parser.add_argument('--env', '-e', action='store_true', + default=False, + help=('Output eval-able bash strings for ' + 'IMAGE_ELEMENT variables')) args = parser.parse_args(argv[1:]) @@ -197,5 +202,33 @@ def main(argv): logger.warning("expand-dependencies flag is deprecated, " "and is now on by default.", file=sys.stderr) - print(' '.join(expand_dependencies(args.elements, all_elements))) + elements = expand_dependencies(args.elements, all_elements) + + if args.env: + # first the "legacy" environment variable that just lists the + # elements + print("export IMAGE_ELEMENT='%s'" % ' '.join(elements)) + + # Then YAML + output = {} + for element in elements: + output[element] = all_elements[element].path + print("export IMAGE_ELEMENT_YAML='%s'" % yaml.safe_dump(output)) + + # Then bash array. Unfortunately, bash can't export array + # variables. So we take a compromise and produce an exported + # function that outputs the string to re-create the array. + # You can then simply do + # eval declare -A element_array=$(get_image_element_array) + # and you have it. + output = "" + for element in elements: + output += '[%s]=%s ' % (element, all_elements[element].path) + print("function get_image_element_array {\n" + " echo \"%s\"\n" + "};\n" + "export -f get_image_element_array;" % output) + else: + print(' '.join(elements)) + return 0 diff --git a/doc/source/developer/developing_elements.rst b/doc/source/developer/developing_elements.rst index 0cee5946e..7caf1a943 100644 --- a/doc/source/developer/developing_elements.rst +++ b/doc/source/developer/developing_elements.rst @@ -178,10 +178,6 @@ To set environment variables for other hooks, add a file to your element This directory contains bash script snippets that are sourced before running scripts in each phase. -DIB exposes an internal ``$IMAGE_ELEMENT`` variable which provides elements -access to the full set of elements that are included in the image build. This -can be used to process local in-element files across all the elements -(``pkg-map`` for example). Dependencies ^^^^^^^^^^^^ @@ -319,6 +315,37 @@ diskimage-builder, making it possible to use the same element to track more then one branch of a git repository or to get source for a local cache. See :doc:`../elements/source-repositories/README` for more information. +Finding other elements +---------------------- + +DIB exposes an internal ``$IMAGE_ELEMENT_YAML`` variable which +provides elements access to the full set of included elements and +their paths. This can be used to process local in-element files +across all the elements (``pkg-map`` for example). + +.. code-block:: python + + import os + import yaml + + elements = yaml.load(os.getenv('IMAGE_ELEMENT_YAML')) + for element, path in elements: + ... + +For elements written in Bash, there is a function +``get_image_element_array`` that can be used to instantiate an +associative-array of elements and paths (note arrays can not be +exported in bash). + +.. code-block:: bash + + # note eval to expand the result of the get function + eval declare -A image_elements=($(get_image_element_array)) + for i in ${!image_elements[$i]}; do + element=$i + path=${image_elements[$i]} + done + Debugging elements ------------------ diff --git a/lib/common-functions b/lib/common-functions index 152a48718..b919dedd4 100644 --- a/lib/common-functions +++ b/lib/common-functions @@ -75,19 +75,27 @@ function copy_hooks_not_overwrite () { } function generate_hooks () { - mkdir -p $TMP_HOOKS_PATH - for _ELEMENT in $IMAGE_ELEMENT ; do - for dir in ${ELEMENTS_PATH//:/ } ; do - [ -d $dir/$_ELEMENT ] || continue - for _DIR in $(find $dir/$_ELEMENT -follow -mindepth 1 -maxdepth 1 -type d -not -name tests); do - copy_hooks_not_overwrite $_DIR - done - for _FILE in $(find $dir/$_ELEMENT -follow -maxdepth 1 -type f); do - cp -t $TMP_HOOKS_PATH -a $_FILE - done - break + local dir + local file + + eval declare -A image_elements=($(get_image_element_array)) + + mkdir -p $TMP_HOOKS_PATH + + for i in "${!image_elements[@]}"; do + local element=$i + local element_dir=${image_elements[$i]} + + for dir in $(find $element_dir \ + -follow -mindepth 1 -maxdepth 1 \ + -type d -not -name tests); do + copy_hooks_not_overwrite $dir + done + for file in $(find $element_dir \ + -follow -maxdepth 1 -type f); do + cp -t $TMP_HOOKS_PATH -a $file + done done - done } # Call the supplied break-in routine if the named point is listed in the break @@ -196,8 +204,29 @@ function arg_to_elements() { fi echo "Building elements: $IMAGE_ELEMENT" - IMAGE_ELEMENT=$($SCRIPT_HOME/element-info $IMAGE_ELEMENT) - export IMAGE_ELEMENT + # element-info will output bash code to create + # * IMAGE_ELEMENT + # legacy list of elements + # + # * IMAGE_ELEMENT_YAML + # YAML dictionary with key=element, value=path + # + # import os + # import yaml + # yaml.load(os.getenv('IMAGE_ELEMENT_YAML') + # + # * function get_image_element_array + # Function to create Bash associative-array. Since bash can not + # export array variables, we provide a function to populate the + # variables. + # + # # we need the eval, it expands the string for the array create + # eval declare -A image_elements=($(get_image_element_array)) + # for i in ${!image_elements[@]}; do + # element=$i + # path=${image_elements[$i] + # done + eval "$($SCRIPT_HOME/element-info --env $IMAGE_ELEMENT)" echo "Expanded element dependencies to: $IMAGE_ELEMENT" } diff --git a/releasenotes/notes/element-vars-c6bf2e6795002f01.yaml b/releasenotes/notes/element-vars-c6bf2e6795002f01.yaml new file mode 100644 index 000000000..cf3d64ab2 --- /dev/null +++ b/releasenotes/notes/element-vars-c6bf2e6795002f01.yaml @@ -0,0 +1,6 @@ +--- +features: + + - Elements that need access to the other elements being used during + the build should use the new ``IMAGE_ELEMENT_YAML`` environment + variable and it's Bash equivalent ``get_image_element_array``. \ No newline at end of file