Merge "Add retain_anchors config option."

doc/source/execution.rst

@@ -57,6 +57,15 @@ job_builder section
   so user can be sure which instance was updated. User may click the link to
   go directly to that job. False by default.
 
+**retain_anchors**
+  (Optional) If set to True, YAML anchors will be retained across files,
+  allowing jobs to be composed from bits of YAML defined in separate files.
+  Note this means that the order of processing files matters - `jenkins-jobs`
+  loads files in alphabetical order (all files in a dir are loaded before any
+  files in subdirs). For example, if your anchors are in a file named `foo.yml`
+  they will be accessible in `qux.yml` but not in `bar.yml`. They will also be
+  accessible in `mydir/bar.yml` and `mydir/qux.yml`. False by default.
+
 
 jenkins section
 ^^^^^^^^^^^^^^^

jenkins_jobs/config.py

@@ -41,6 +41,7 @@ recursive=False
 exclude=.*
 allow_duplicates=False
 allow_empty_variables=False
+retain_anchors=False
 
 # other named sections could be used in addition to the implicit [jenkins]
 # if you have multiple jenkins servers.
@@ -305,6 +306,13 @@ class JJBConfig(object):
             config.has_option('job_builder', 'allow_empty_variables') and
             config.getboolean('job_builder', 'allow_empty_variables'))
 
+        # retain anchors across files?
+        retain_anchors = False
+        if config and config.has_option('job_builder', 'retain_anchors'):
+            retain_anchors = config.getboolean('job_builder',
+                                               'retain_anchors')
+        self.yamlparser['retain_anchors'] = retain_anchors
+
     def validate(self):
         # Inform the user as to what is likely to happen, as they may specify
         # a real jenkins instance in test mode to get the plugin info to check

jenkins_jobs/local_yaml.py

@@ -564,8 +564,9 @@ class LazyLoader(CustomLoader):
         return self._cls.from_yaml(self._loader, node)
 
 
-def load(stream, **kwargs):
-    LocalAnchorLoader.reset_anchors()
+def load(stream, retain_anchors=False, **kwargs):
+    if not retain_anchors:
+        LocalAnchorLoader.reset_anchors()
     return yaml.load(stream, functools.partial(LocalLoader, **kwargs))
 
 

jenkins_jobs/parser.py

@@ -98,7 +98,7 @@ class YamlParser(object):
         for path in fn:
             if not hasattr(path, 'read') and os.path.isdir(path):
                 files_to_process.extend([os.path.join(path, f)
-                                         for f in os.listdir(path)
+                                         for f in sorted(os.listdir(path))
                                          if (f.endswith('.yml')
                                              or f.endswith('.yaml'))])
             else:
@@ -134,7 +134,9 @@ class YamlParser(object):
 
     def _parse_fp(self, fp):
         # wrap provided file streams to ensure correct encoding used
-        data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path)
+        data = local_yaml.load(utils.wrap_stream(fp),
+                               self.jjb_config.yamlparser['retain_anchors'],
+                               search_path=self.path)
         if data:
             if not isinstance(data, list):
                 raise JenkinsJobsException(

jenkins_jobs/utils.py

@@ -50,6 +50,9 @@ def recurse_path(root, excludes=None):
     relative = [e for e in excludes if os.path.sep in e and
                 not os.path.isabs(e)]
     for root, dirs, files in os.walk(basepath, topdown=True):
+        # sort in-place to ensure dirnames are visited in alphabetical order
+        # a predictable order makes it easier to use the retain_anchors option
+        dirs.sort()
         dirs[:] = [
             d for d in dirs
             if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns])

tests/localyaml/fixtures/custom_retain_anchors.yaml

@@ -0,0 +1,8 @@
+- project:
+    name: retain_anchors
+    jobs:
+      - retain_anchors
+
+- job-template:
+    name: retain_anchors
+    <<: *retain_anchors_defaults

tests/localyaml/fixtures/custom_retain_anchors_include001.yaml

@@ -0,0 +1,10 @@
+- retain_anchors_wrapper_defaults: &retain_anchors_wrapper_defaults
+    name: 'retain_anchors_wrapper_defaults'
+    wrappers:
+      - timeout:
+          timeout: 180
+          fail: true
+
+- retain_anchors_defaults: &retain_anchors_defaults
+    name: 'retain_anchors_defaults'
+    <<: *retain_anchors_wrapper_defaults

tests/localyaml/test_localyaml.py

@@ -15,6 +15,7 @@
 # under the License.
 
 import os
+import yaml
 
 from testtools import ExpectedException
 from yaml.composer import ComposerError
@@ -80,3 +81,39 @@ class TestCaseLocalYamlIncludeAnchors(base.BaseTestCase):
         jjb_config.validate()
         j = YamlParser(jjb_config)
         j.load_files([os.path.join(self.fixtures_path, f) for f in files])
+
+
+class TestCaseLocalYamlRetainAnchors(base.BaseTestCase):
+
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+
+    def test_retain_anchors_default(self):
+        """
+        Verify that anchors are NOT retained across files by default.
+        """
+
+        files = ["custom_retain_anchors_include001.yaml",
+                 "custom_retain_anchors.yaml"]
+
+        jjb_config = JJBConfig()
+        # use the default value for retain_anchors
+        jjb_config.validate()
+        j = YamlParser(jjb_config)
+        with ExpectedException(yaml.composer.ComposerError,
+                               "found undefined alias.*"):
+            j.load_files([os.path.join(self.fixtures_path, f) for f in files])
+
+    def test_retain_anchors_enabled(self):
+        """
+        Verify that anchors are retained across files if retain_anchors is
+        enabled in the config.
+        """
+
+        files = ["custom_retain_anchors_include001.yaml",
+                 "custom_retain_anchors.yaml"]
+
+        jjb_config = JJBConfig()
+        jjb_config.yamlparser['retain_anchors'] = True
+        jjb_config.validate()
+        j = YamlParser(jjb_config)
+        j.load_files([os.path.join(self.fixtures_path, f) for f in files])