From 13df88774ac980f6456d00b367b90cbbe065abc0 Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Wed, 16 Dec 2015 17:06:41 +0000 Subject: [PATCH] Add helper tag to join arbitrary lists into strings Adds a yaml application tag '!join:' to support defining data as a list while having the resulting contents automatically joined together with the specified delimiter and passed as a string into the resulting object returned by yaml for JJB to process. This allows users to store long lists of data that is combined together for use with Jenkins plugins that expect a delimited string of arguments instead of needing each module in JJB to provide individual support. Change-Id: I745181ade3926d5c29708963189ae499a0378ece --- jenkins_jobs/local_yaml.py | 65 +++++++++++++++++ tests/localyaml/fixtures/joinlists.json | 8 +++ tests/localyaml/fixtures/joinlists.yaml | 13 ++++ .../fixtures/extended-choice-param-full.xml | 38 ++++++++++ .../fixtures/extended-choice-param-full.yaml | 25 +++++++ tests/yamlparser/fixtures/string_join.xml | 69 +++++++++++++++++++ tests/yamlparser/fixtures/string_join.yaml | 66 ++++++++++++++++++ 7 files changed, 284 insertions(+) create mode 100644 tests/localyaml/fixtures/joinlists.json create mode 100644 tests/localyaml/fixtures/joinlists.yaml create mode 100644 tests/yamlparser/fixtures/string_join.xml create mode 100644 tests/yamlparser/fixtures/string_join.yaml diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index c49393d65..493dac079 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -18,6 +18,52 @@ """Custom application specific yamls tags are supported to provide enhancements when reading yaml configuration. +Action Tags +^^^^^^^^^^^ + +These allow manipulation of data being stored in one layout in the source +yaml for convenience and/or clarity, to another format to be processed by +the targeted module instead of requiring all modules in JJB being capable +of supporting multiple input formats. + +The tag ``!join:`` will treat the first element of the following list as +the delimiter to use, when joining the remaining elements into a string +and returning a single string to be consumed by the specified module option. + +This allows users to maintain elements of data in a list structure for ease +of review/maintenance, and have the yaml parser convert it to a string for +consumption as any argument for modules. The main expected use case is to +allow for generic plugin data such as shell properties to be populated from +a list construct which the yaml parser converts to a single string, instead +of trying to support this within the module code which would require a +templating engine similar to Jinja. + +Generic Example: + + .. literalinclude:: /../../tests/localyaml/fixtures/joinlists.yaml + + +Environment Inject: + + .. literalinclude:: /../../tests/yamlparser/fixtures/string_join.yaml + + +While this mechanism can also be used items where delimiters are supported by +the module, that should be considered a bug that the existing code doesn't +handle being provided a list and delimiter to perform the correct conversion +for you. Should you discover a module that takes arguments with delimiters and +the existing JJB codebase does not handle accepting lists, then this can be +used as a temporary solution in place of using very long strings: + +Extended Params Example: + + .. literalinclude:: + /../../tests/parameters/fixtures/extended-choice-param-full.yaml + + +Inclusion Tags +^^^^^^^^^^^^^^ + These allow inclusion of arbitrary files as a method of having blocks of data managed separately to the yaml job configurations. A specific usage of this is inlining scripts contained in separate files, although such tags may also be @@ -327,6 +373,25 @@ class J2String(BaseYAMLObject): return Jinja2Loader(node.value, loader.search_path) +class YamlListJoin(BaseYAMLObject): + yaml_tag = u'!join:' + + @classmethod + def from_yaml(cls, loader, node): + if isinstance(node, yaml.SequenceNode): + delimiter = node.value[0].value + if not isinstance(node.value[1], yaml.SequenceNode): + raise yaml.constructor.ConstructorError( + None, None, "expected sequence node for join data, but " + "found %s" % node.value[1].id, node.start_mark) + + return delimiter.join((v.value for v in node.value[1].value)) + else: + raise yaml.constructor.ConstructorError( + None, None, "expected sequence node, but found %s" % node.id, + node.start_mark) + + class YamlInclude(BaseYAMLObject): yaml_tag = u'!include:' diff --git a/tests/localyaml/fixtures/joinlists.json b/tests/localyaml/fixtures/joinlists.json new file mode 100644 index 000000000..8702d17f7 --- /dev/null +++ b/tests/localyaml/fixtures/joinlists.json @@ -0,0 +1,8 @@ +[ + { + "string-with-comma": "item1,item2,item3" + }, + { + "string-with-space": "item1 item2 item3" + } +] diff --git a/tests/localyaml/fixtures/joinlists.yaml b/tests/localyaml/fixtures/joinlists.yaml new file mode 100644 index 000000000..c970df50d --- /dev/null +++ b/tests/localyaml/fixtures/joinlists.yaml @@ -0,0 +1,13 @@ +- string-with-comma: !join: + - ',' + - + - item1 + - item2 + - item3 + +- string-with-space: !join: + - ' ' + - + - item1 + - item2 + - item3 diff --git a/tests/parameters/fixtures/extended-choice-param-full.xml b/tests/parameters/fixtures/extended-choice-param-full.xml index f3716a90c..40560c999 100644 --- a/tests/parameters/fixtures/extended-choice-param-full.xml +++ b/tests/parameters/fixtures/extended-choice-param-full.xml @@ -22,6 +22,44 @@ + + OPTIONS_CHECKBOX + + OptionA,OptionB,OptionC + 2 + , + false + + + PT_CHECKBOX + + + + + + + + + + + MULTISELECTOPTIONS + Available options + foo|bar|select + 2 + | + true + foo + + PT_MULTI_SELECT + + key + + + + + + + diff --git a/tests/parameters/fixtures/extended-choice-param-full.yaml b/tests/parameters/fixtures/extended-choice-param-full.yaml index 9d2b0d37f..55a1f5ee6 100644 --- a/tests/parameters/fixtures/extended-choice-param-full.yaml +++ b/tests/parameters/fixtures/extended-choice-param-full.yaml @@ -12,3 +12,28 @@ parameters: default-value: foo default-property-file: /home/property.prop default-property-key: fookey + - extended-choice: + name: OPTIONS_CHECKBOX + type: checkbox + value: !join: + - ',' + - + - OptionA + - OptionB + - OptionC + visible-items: 2 + - extended-choice: + name: MULTISELECTOPTIONS + description: "Available options" + property-key: key + quote-value: true + type: multi-select + value: !join: + - '|' + - + - foo + - bar + - select + visible-items: 2 + multi-select-delimiter: '|' + default-value: foo diff --git a/tests/yamlparser/fixtures/string_join.xml b/tests/yamlparser/fixtures/string_join.xml new file mode 100644 index 000000000..2abfaa2d7 --- /dev/null +++ b/tests/yamlparser/fixtures/string_join.xml @@ -0,0 +1,69 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + FILE_LIST=/path/to/file1,/path/to/file2,/path/to/file3,/path/to/file4,/path/to/file5,/path/to/file6,/path/to/file7,/path/to/file8,/path/to/file9,/path/to/file10,/path/to/file11,/path/to/file12,/path/to/file13,/path/to/file14,/path/to/file15,/path/to/file16,/path/to/file17,/path/to/file18,/path/to/file19,/path/to/file20 + + false + + true + true + true + false + + + + + + echo "Template name: string-join-data-{name}" +echo "Data to be processed:" +echo "${INPUT_DATA}" + + + + + + + + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + FILE_LIST=/another/different/path/to/file1,/another/different/path/to/file2,/another/different/path/to/file3,/another/different/path/to/file4,/another/different/path/to/file5,/another/different/path/to/file6,/another/different/path/to/file7,/another/different/path/to/file8,/another/different/path/to/file9,/another/different/path/to/file10,/another/different/path/to/file11,/another/different/path/to/file12,/another/different/path/to/file13,/another/different/path/to/file14,/another/different/path/to/file15,/another/different/path/to/file16,/another/different/path/to/file17,/another/different/path/to/file18,/another/different/path/to/file19,/another/different/path/to/file20 + + false + + true + true + true + false + + + + + + echo "Template name: string-join-data-{name}" +echo "Data to be processed:" +echo "${INPUT_DATA}" + + + + + + diff --git a/tests/yamlparser/fixtures/string_join.yaml b/tests/yamlparser/fixtures/string_join.yaml new file mode 100644 index 000000000..0288d4b21 --- /dev/null +++ b/tests/yamlparser/fixtures/string_join.yaml @@ -0,0 +1,66 @@ +- project: + name: string_join_example + jobs: + - 'string-join-data-{name}': + name: set1 + files: !join: + - ',' + - + - /path/to/file1 + - /path/to/file2 + - /path/to/file3 + - /path/to/file4 + - /path/to/file5 + - /path/to/file6 + - /path/to/file7 + - /path/to/file8 + - /path/to/file9 + - /path/to/file10 + - /path/to/file11 + - /path/to/file12 + - /path/to/file13 + - /path/to/file14 + - /path/to/file15 + - /path/to/file16 + - /path/to/file17 + - /path/to/file18 + - /path/to/file19 + - /path/to/file20 + - 'string-join-data-{name}': + name: set2 + files: !join: + - ',' + - + - /another/different/path/to/file1 + - /another/different/path/to/file2 + - /another/different/path/to/file3 + - /another/different/path/to/file4 + - /another/different/path/to/file5 + - /another/different/path/to/file6 + - /another/different/path/to/file7 + - /another/different/path/to/file8 + - /another/different/path/to/file9 + - /another/different/path/to/file10 + - /another/different/path/to/file11 + - /another/different/path/to/file12 + - /another/different/path/to/file13 + - /another/different/path/to/file14 + - /another/different/path/to/file15 + - /another/different/path/to/file16 + - /another/different/path/to/file17 + - /another/different/path/to/file18 + - /another/different/path/to/file19 + - /another/different/path/to/file20 + +- job-template: + name: 'string-join-data-{name}' + properties: + - inject: + keep-system-variables: true + properties-content: | + FILE_LIST={files} + builders: + - shell: | + echo "Template name: {template-name}" + echo "Data to be processed:" + echo "${{INPUT_DATA}}"