Add retain_anchors config option.

If set to True, YAML anchors can be referenced across files, allowing jobs to be
composed from bits of YAML defined in separate files. False by default.

Story: 2000338
Task: 2547
Change-Id: I034ce3bce0030093cb8d4266dabbdb06d96306d6
This commit is contained in:
Vicky Chijwani 2018-05-30 18:20:29 +05:30
parent 307c09cff3
commit 75d78b6540
8 changed files with 82 additions and 4 deletions

View File

@ -57,6 +57,15 @@ job_builder section
so user can be sure which instance was updated. User may click the link to
go directly to that job. False by default.
**retain_anchors**
(Optional) If set to True, YAML anchors will be retained across files,
allowing jobs to be composed from bits of YAML defined in separate files.
Note this means that the order of processing files matters - `jenkins-jobs`
loads files in alphabetical order (all files in a dir are loaded before any
files in subdirs). For example, if your anchors are in a file named `foo.yml`
they will be accessible in `qux.yml` but not in `bar.yml`. They will also be
accessible in `mydir/bar.yml` and `mydir/qux.yml`. False by default.
jenkins section
^^^^^^^^^^^^^^^

View File

@ -41,6 +41,7 @@ recursive=False
exclude=.*
allow_duplicates=False
allow_empty_variables=False
retain_anchors=False
# other named sections could be used in addition to the implicit [jenkins]
# if you have multiple jenkins servers.
@ -300,6 +301,13 @@ class JJBConfig(object):
config.has_option('job_builder', 'allow_empty_variables') and
config.getboolean('job_builder', 'allow_empty_variables'))
# retain anchors across files?
retain_anchors = False
if config and config.has_option('job_builder', 'retain_anchors'):
retain_anchors = config.getboolean('job_builder',
'retain_anchors')
self.yamlparser['retain_anchors'] = retain_anchors
def validate(self):
# Inform the user as to what is likely to happen, as they may specify
# a real jenkins instance in test mode to get the plugin info to check

View File

@ -564,8 +564,9 @@ class LazyLoader(CustomLoader):
return self._cls.from_yaml(self._loader, node)
def load(stream, **kwargs):
LocalAnchorLoader.reset_anchors()
def load(stream, retain_anchors=False, **kwargs):
if not retain_anchors:
LocalAnchorLoader.reset_anchors()
return yaml.load(stream, functools.partial(LocalLoader, **kwargs))

View File

@ -98,7 +98,7 @@ class YamlParser(object):
for path in fn:
if not hasattr(path, 'read') and os.path.isdir(path):
files_to_process.extend([os.path.join(path, f)
for f in os.listdir(path)
for f in sorted(os.listdir(path))
if (f.endswith('.yml')
or f.endswith('.yaml'))])
else:
@ -134,7 +134,9 @@ class YamlParser(object):
def _parse_fp(self, fp):
# wrap provided file streams to ensure correct encoding used
data = local_yaml.load(utils.wrap_stream(fp), search_path=self.path)
data = local_yaml.load(utils.wrap_stream(fp),
self.jjb_config.yamlparser['retain_anchors'],
search_path=self.path)
if data:
if not isinstance(data, list):
raise JenkinsJobsException(

View File

@ -50,6 +50,9 @@ def recurse_path(root, excludes=None):
relative = [e for e in excludes if os.path.sep in e and
not os.path.isabs(e)]
for root, dirs, files in os.walk(basepath, topdown=True):
# sort in-place to ensure dirnames are visited in alphabetical order
# a predictable order makes it easier to use the retain_anchors option
dirs.sort()
dirs[:] = [
d for d in dirs
if not any([fnmatch.fnmatch(d, pattern) for pattern in patterns])

View File

@ -0,0 +1,8 @@
- project:
name: retain_anchors
jobs:
- retain_anchors
- job-template:
name: retain_anchors
<<: *retain_anchors_defaults

View File

@ -0,0 +1,10 @@
- retain_anchors_wrapper_defaults: &retain_anchors_wrapper_defaults
name: 'retain_anchors_wrapper_defaults'
wrappers:
- timeout:
timeout: 180
fail: true
- retain_anchors_defaults: &retain_anchors_defaults
name: 'retain_anchors_defaults'
<<: *retain_anchors_wrapper_defaults

View File

@ -15,6 +15,7 @@
# under the License.
import os
import yaml
from testtools import ExpectedException
from yaml.composer import ComposerError
@ -80,3 +81,39 @@ class TestCaseLocalYamlIncludeAnchors(base.BaseTestCase):
jjb_config.validate()
j = YamlParser(jjb_config)
j.load_files([os.path.join(self.fixtures_path, f) for f in files])
class TestCaseLocalYamlRetainAnchors(base.BaseTestCase):
fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
def test_retain_anchors_default(self):
"""
Verify that anchors are NOT retained across files by default.
"""
files = ["custom_retain_anchors_include001.yaml",
"custom_retain_anchors.yaml"]
jjb_config = JJBConfig()
# use the default value for retain_anchors
jjb_config.validate()
j = YamlParser(jjb_config)
with ExpectedException(yaml.composer.ComposerError,
"found undefined alias.*"):
j.load_files([os.path.join(self.fixtures_path, f) for f in files])
def test_retain_anchors_enabled(self):
"""
Verify that anchors are retained across files if retain_anchors is
enabled in the config.
"""
files = ["custom_retain_anchors_include001.yaml",
"custom_retain_anchors.yaml"]
jjb_config = JJBConfig()
jjb_config.yamlparser['retain_anchors'] = True
jjb_config.validate()
j = YamlParser(jjb_config)
j.load_files([os.path.join(self.fixtures_path, f) for f in files])