Merge "Add helper tag to join arbitrary lists into strings"

jenkins_jobs/local_yaml.py

@@ -18,6 +18,52 @@
 """Custom application specific yamls tags are supported to provide
 enhancements when reading yaml configuration.
 
+Action Tags
+^^^^^^^^^^^
+
+These allow manipulation of data being stored in one layout in the source
+yaml for convenience and/or clarity, to another format to be processed by
+the targeted module instead of requiring all modules in JJB being capable
+of supporting multiple input formats.
+
+The tag ``!join:`` will treat the first element of the following list as
+the delimiter to use, when joining the remaining elements into a string
+and returning a single string to be consumed by the specified module option.
+
+This allows users to maintain elements of data in a list structure for ease
+of review/maintenance, and have the yaml parser convert it to a string for
+consumption as any argument for modules. The main expected use case is to
+allow for generic plugin data such as shell properties to be populated from
+a list construct which the yaml parser converts to a single string, instead
+of trying to support this within the module code which would require a
+templating engine similar to Jinja.
+
+Generic Example:
+
+    .. literalinclude:: /../../tests/localyaml/fixtures/joinlists.yaml
+
+
+Environment Inject:
+
+    .. literalinclude:: /../../tests/yamlparser/fixtures/string_join.yaml
+
+
+While this mechanism can also be used items where delimiters are supported by
+the module, that should be considered a bug that the existing code doesn't
+handle being provided a list and delimiter to perform the correct conversion
+for you. Should you discover a module that takes arguments with delimiters and
+the existing JJB codebase does not handle accepting lists, then this can be
+used as a temporary solution in place of using very long strings:
+
+Extended Params Example:
+
+    .. literalinclude::
+        /../../tests/parameters/fixtures/extended-choice-param-full.yaml
+
+
+Inclusion Tags
+^^^^^^^^^^^^^^
+
 These allow inclusion of arbitrary files as a method of having blocks of data
 managed separately to the yaml job configurations. A specific usage of this is
 inlining scripts contained in separate files, although such tags may also be
@@ -327,6 +373,25 @@ class J2String(BaseYAMLObject):
         return Jinja2Loader(node.value, loader.search_path)
 
 
+class YamlListJoin(BaseYAMLObject):
+    yaml_tag = u'!join:'
+
+    @classmethod
+    def from_yaml(cls, loader, node):
+        if isinstance(node, yaml.SequenceNode):
+            delimiter = node.value[0].value
+            if not isinstance(node.value[1], yaml.SequenceNode):
+                raise yaml.constructor.ConstructorError(
+                    None, None, "expected sequence node for join data, but "
+                                "found %s" % node.value[1].id, node.start_mark)
+
+            return delimiter.join((v.value for v in node.value[1].value))
+        else:
+            raise yaml.constructor.ConstructorError(
+                None, None, "expected sequence node, but found %s" % node.id,
+                node.start_mark)
+
+
 class YamlInclude(BaseYAMLObject):
     yaml_tag = u'!include:'
 

tests/localyaml/fixtures/joinlists.json

@@ -0,0 +1,8 @@
+[
+    {
+        "string-with-comma": "item1,item2,item3"
+    },
+    {
+        "string-with-space": "item1 item2 item3"
+    }
+]

tests/localyaml/fixtures/joinlists.yaml

@@ -0,0 +1,13 @@
+- string-with-comma: !join:
+    - ','
+    -
+        - item1
+        - item2
+        - item3
+
+- string-with-space: !join:
+    - ' '
+    -
+        - item1
+        - item2
+        - item3

tests/parameters/fixtures/extended-choice-param-full.xml

@@ -22,6 +22,44 @@
           <groovyScript/>
           <groovyClasspath/>
         </com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+        <com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+          <name>OPTIONS_CHECKBOX</name>
+          <description/>
+          <value>OptionA,OptionB,OptionC</value>
+          <visibleItemCount>2</visibleItemCount>
+          <multiSelectDelimiter>,</multiSelectDelimiter>
+          <quoteValue>false</quoteValue>
+          <defaultValue/>
+          <descriptionPropertyValue/>
+          <type>PT_CHECKBOX</type>
+          <propertyFile/>
+          <propertyKey/>
+          <defaultPropertyFile/>
+          <defaultPropertyKey/>
+          <descriptionPropertyFile/>
+          <descriptionPropertyKey/>
+          <groovyScript/>
+          <groovyClasspath/>
+        </com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+        <com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
+          <name>MULTISELECTOPTIONS</name>
+          <description>Available options</description>
+          <value>foo|bar|select</value>
+          <visibleItemCount>2</visibleItemCount>
+          <multiSelectDelimiter>|</multiSelectDelimiter>
+          <quoteValue>true</quoteValue>
+          <defaultValue>foo</defaultValue>
+          <descriptionPropertyValue/>
+          <type>PT_MULTI_SELECT</type>
+          <propertyFile/>
+          <propertyKey>key</propertyKey>
+          <defaultPropertyFile/>
+          <defaultPropertyKey/>
+          <descriptionPropertyFile/>
+          <descriptionPropertyKey/>
+          <groovyScript/>
+          <groovyClasspath/>
+        </com.cwctravel.hudson.plugins.extended__choice__parameter.ExtendedChoiceParameterDefinition>
       </parameterDefinitions>
     </hudson.model.ParametersDefinitionProperty>
   </properties>

tests/parameters/fixtures/extended-choice-param-full.yaml

@@ -12,3 +12,28 @@ parameters:
       default-value: foo
       default-property-file: /home/property.prop
       default-property-key: fookey
+  - extended-choice:
+      name: OPTIONS_CHECKBOX
+      type: checkbox
+      value: !join:
+        - ','
+        -
+            - OptionA
+            - OptionB
+            - OptionC
+      visible-items: 2
+  - extended-choice:
+      name: MULTISELECTOPTIONS
+      description: "Available options"
+      property-key: key
+      quote-value: true
+      type: multi-select
+      value: !join:
+        - '|'
+        -
+            - foo
+            - bar
+            - select
+      visible-items: 2
+      multi-select-delimiter: '|'
+      default-value: foo

tests/yamlparser/fixtures/string_join.xml

@@ -0,0 +1,69 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <propertiesContent>FILE_LIST=/path/to/file1,/path/to/file2,/path/to/file3,/path/to/file4,/path/to/file5,/path/to/file6,/path/to/file7,/path/to/file8,/path/to/file9,/path/to/file10,/path/to/file11,/path/to/file12,/path/to/file13,/path/to/file14,/path/to/file15,/path/to/file16,/path/to/file17,/path/to/file18,/path/to/file19,/path/to/file20
+</propertiesContent>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+      <overrideBuildParameters>false</overrideBuildParameters>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;Template name: string-join-data-{name}&quot;
+echo &quot;Data to be processed:&quot;
+echo &quot;${INPUT_DATA}&quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>
+<BLANKLINE>
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties>
+    <EnvInjectJobProperty>
+      <info>
+        <propertiesContent>FILE_LIST=/another/different/path/to/file1,/another/different/path/to/file2,/another/different/path/to/file3,/another/different/path/to/file4,/another/different/path/to/file5,/another/different/path/to/file6,/another/different/path/to/file7,/another/different/path/to/file8,/another/different/path/to/file9,/another/different/path/to/file10,/another/different/path/to/file11,/another/different/path/to/file12,/another/different/path/to/file13,/another/different/path/to/file14,/another/different/path/to/file15,/another/different/path/to/file16,/another/different/path/to/file17,/another/different/path/to/file18,/another/different/path/to/file19,/another/different/path/to/file20
+</propertiesContent>
+        <loadFilesFromMaster>false</loadFilesFromMaster>
+      </info>
+      <on>true</on>
+      <keepJenkinsSystemVariables>true</keepJenkinsSystemVariables>
+      <keepBuildVariables>true</keepBuildVariables>
+      <overrideBuildParameters>false</overrideBuildParameters>
+    </EnvInjectJobProperty>
+  </properties>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>echo &quot;Template name: string-join-data-{name}&quot;
+echo &quot;Data to be processed:&quot;
+echo &quot;${INPUT_DATA}&quot;
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/string_join.yaml

@@ -0,0 +1,66 @@
+- project:
+    name: string_join_example
+    jobs:
+        - 'string-join-data-{name}':
+            name: set1
+            files: !join:
+              - ','
+              -
+                - /path/to/file1
+                - /path/to/file2
+                - /path/to/file3
+                - /path/to/file4
+                - /path/to/file5
+                - /path/to/file6
+                - /path/to/file7
+                - /path/to/file8
+                - /path/to/file9
+                - /path/to/file10
+                - /path/to/file11
+                - /path/to/file12
+                - /path/to/file13
+                - /path/to/file14
+                - /path/to/file15
+                - /path/to/file16
+                - /path/to/file17
+                - /path/to/file18
+                - /path/to/file19
+                - /path/to/file20
+        - 'string-join-data-{name}':
+            name: set2
+            files: !join:
+              - ','
+              -
+                - /another/different/path/to/file1
+                - /another/different/path/to/file2
+                - /another/different/path/to/file3
+                - /another/different/path/to/file4
+                - /another/different/path/to/file5
+                - /another/different/path/to/file6
+                - /another/different/path/to/file7
+                - /another/different/path/to/file8
+                - /another/different/path/to/file9
+                - /another/different/path/to/file10
+                - /another/different/path/to/file11
+                - /another/different/path/to/file12
+                - /another/different/path/to/file13
+                - /another/different/path/to/file14
+                - /another/different/path/to/file15
+                - /another/different/path/to/file16
+                - /another/different/path/to/file17
+                - /another/different/path/to/file18
+                - /another/different/path/to/file19
+                - /another/different/path/to/file20
+
+- job-template:
+    name: 'string-join-data-{name}'
+    properties:
+      - inject:
+          keep-system-variables: true
+          properties-content: |
+            FILE_LIST={files}
+    builders:
+      - shell: |
+         echo "Template name: {template-name}"
+         echo "Data to be processed:"
+         echo "${{INPUT_DATA}}"