From 64f9af07f3d66bee1a897ac2d507f5e6b4647620 Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Sat, 21 Dec 2013 20:21:49 +1100 Subject: [PATCH] Support lazy resolving of include yaml tags To allow filenames referenced by the application specific yaml tags to contain template job variables, use a lazy loading object that provides a format method that can be called by the deep_format function. Instead of processing the file, when a KeyError occurs on attempting to call format on the filename after the yaml tag, create a LazyLoader instance to wrap the data and provide a format method that can be called at a later stage. In order to call the correct method on the original Loader class, LazyLoader needs to be given the custom tag class, a reference to the loader and the node object. Using the tag class it can call the from_yaml() method with the loader and node object to return the file contents. Since the result from the LazyLoader instance is triggered by calling the format method, there is no need to escape the brackets used by pythons format method since the output will not be passed through it. In order to ensure this behaviour, nodes passed to the method handling the '!include-raw-escape:' tag class, which need to use the LazyLoader approach will convert to the '!include-raw:' tag class to the LazyLoader initialization instead. Due to a bug in sphinx with use of 'note' admonitions and manpage generation, need to update to a version >= 1.2.1. Change-Id: I187eb83ba54740c2c1b627bc99c2d9769687fbc7 Story: 2000522 --- jenkins_jobs/formatter.py | 22 +++-- jenkins_jobs/local_yaml.py | 99 +++++++++++++++++-- .../lazy-load-jobs-copy-files.yaml.inc | 8 ++ .../fixtures/lazy-load-jobs-multi001.xml | 44 +++++++++ .../fixtures/lazy-load-jobs-multi001.yaml | 20 ++++ .../lazy-load-jobs-pre-scm-shell-ant.yaml.inc | 15 +++ .../fixtures/lazy-load-jobs-timeout.yaml.inc | 7 ++ .../fixtures/lazy-load-jobs001.conf | 2 + .../yamlparser/fixtures/lazy-load-jobs001.xml | 41 ++++++++ .../fixtures/lazy-load-jobs001.yaml | 17 ++++ .../yamlparser/fixtures/lazy-load-jobs002.xml | 73 ++++++++++++++ .../fixtures/lazy-load-jobs002.yaml | 23 +++++ .../lazy-load-scripts/echo_vars_1.1.sh | 13 +++ .../lazy-load-scripts/echo_vars_1.2.sh | 13 +++ .../fixtures/lazy-load-wrappers-1.1.yaml.inc | 1 + .../fixtures/lazy-load-wrappers-1.2.yaml.inc | 3 + 16 files changed, 385 insertions(+), 16 deletions(-) create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs-copy-files.yaml.inc create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs-multi001.xml create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs-multi001.yaml create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs-pre-scm-shell-ant.yaml.inc create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs-timeout.yaml.inc create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs001.conf create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs001.xml create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs001.yaml create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs002.xml create mode 100644 tests/yamlparser/fixtures/lazy-load-jobs002.yaml create mode 100644 tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.1.sh create mode 100644 tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.2.sh create mode 100644 tests/yamlparser/fixtures/lazy-load-wrappers-1.1.yaml.inc create mode 100644 tests/yamlparser/fixtures/lazy-load-wrappers-1.2.yaml.inc diff --git a/jenkins_jobs/formatter.py b/jenkins_jobs/formatter.py index 224ad643f..0656bc060 100644 --- a/jenkins_jobs/formatter.py +++ b/jenkins_jobs/formatter.py @@ -35,15 +35,19 @@ def deep_format(obj, paramdict, allow_empty=False): if hasattr(obj, 'format'): try: result = re.match('^{obj:(?P\w+)}$', obj) - if result is not None: - ret = paramdict[result.group("key")] - else: - ret = CustomFormatter(allow_empty).format(obj, **paramdict) - except KeyError as exc: - missing_key = exc.args[0] - desc = "%s parameter missing to format %s\nGiven:\n%s" % ( - missing_key, obj, pformat(paramdict)) - raise JenkinsJobsException(desc) + except TypeError: + ret = obj.format(**paramdict) + else: + try: + if result is not None: + ret = paramdict[result.group("key")] + else: + ret = CustomFormatter(allow_empty).format(obj, **paramdict) + except KeyError as exc: + missing_key = exc.args[0] + desc = "%s parameter missing to format %s\nGiven:\n%s" % ( + missing_key, obj, pformat(paramdict)) + raise JenkinsJobsException(desc) elif isinstance(obj, list): ret = type(obj)() for item in obj: diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index aaeaead70..37d5c5a04 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -90,6 +90,34 @@ Examples: For all the multi file includes, the files are simply appended using a newline character. + +To allow for job templates to perform substitution on the path names, when a +filename containing a python format placeholder is encountered, lazy loading +support is enabled, where instead of returning the contents back during yaml +parsing, it is delayed until the variable substitution is performed. + +Example: + + .. literalinclude:: /../../tests/yamlparser/fixtures/lazy-load-jobs001.yaml + + using a list of files: + + .. literalinclude:: + /../../tests/yamlparser/fixtures/lazy-load-jobs-multi001.yaml + +.. note:: + + Because lazy-loading involves performing the substitution on the file + name, it means that jenkins-job-builder can not call the variable + substitution on the contents of the file. This means that the + ``!include-raw:`` tag will behave as though ``!include-raw-escape:`` tag + was used instead whenever name substitution on the filename is to be + performed. + + Given the behaviour described above, when substitution is to be performed + on any filename passed via ``!include-raw-escape:`` the tag will be + automatically converted to ``!include-raw:`` and no escaping will be + performed. """ import functools @@ -249,9 +277,14 @@ class YamlInclude(BaseYAMLObject): return filename @classmethod - def _open_file(cls, loader, scalar_node): - filename = cls._find_file(loader.construct_yaml_str(scalar_node), - loader.search_path) + def _open_file(cls, loader, node): + node_str = loader.construct_yaml_str(node) + try: + node_str.format() + except KeyError: + return cls._lazy_load(loader, cls.yaml_tag, node) + + filename = cls._find_file(node_str, loader.search_path) try: with io.open(filename, 'r', encoding='utf-8') as f: return f.read() @@ -262,18 +295,32 @@ class YamlInclude(BaseYAMLObject): @classmethod def _from_file(cls, loader, node): - data = yaml.load(cls._open_file(loader, node), + contents = cls._open_file(loader, node) + if isinstance(contents, LazyLoader): + return contents + + data = yaml.load(contents, functools.partial(cls.yaml_loader, search_path=loader.search_path)) return data + @classmethod + def _lazy_load(cls, loader, tag, node_str): + logger.info("Lazy loading of file template '{0}' enabled" + .format(node_str)) + return LazyLoader((cls, loader, node_str)) + @classmethod def from_yaml(cls, loader, node): if isinstance(node, yaml.ScalarNode): return cls._from_file(loader, node) elif isinstance(node, yaml.SequenceNode): - return u'\n'.join(cls._from_file(loader, scalar_node) - for scalar_node in node.value) + contents = [cls._from_file(loader, scalar_node) + for scalar_node in node.value] + if any(isinstance(s, LazyLoader) for s in contents): + return LazyLoaderCollection(contents) + + return u'\n'.join(contents) else: raise yaml.constructor.ConstructorError( None, None, "expected either a sequence or scalar node, but " @@ -293,7 +340,15 @@ class YamlIncludeRawEscape(YamlIncludeRaw): @classmethod def from_yaml(cls, loader, node): - return loader.escape_callback(YamlIncludeRaw.from_yaml(loader, node)) + data = YamlIncludeRaw.from_yaml(loader, node) + if isinstance(data, LazyLoader): + logger.warn("Replacing %s tag with %s since lazy loading means " + "file contents will not be deep formatted for " + "variable substitution.", cls.yaml_tag, + YamlIncludeRaw.yaml_tag) + return data + else: + return loader.escape_callback(data) class DeprecatedTag(BaseYAMLObject): @@ -320,6 +375,36 @@ class YamlIncludeRawEscapeDeprecated(DeprecatedTag): _new = YamlIncludeRawEscape +class LazyLoaderCollection(object): + """Helper class to format a collection of LazyLoader objects""" + def __init__(self, sequence): + self._data = sequence + + def format(self, *args, **kwargs): + return u'\n'.join(item.format(*args, **kwargs) for item in self._data) + + +class LazyLoader(object): + """Helper class to provide lazy loading of files included using !include* + tags where the path to the given file contains unresolved placeholders. + """ + + def __init__(self, data): + # str subclasses can only have one argument, so assume it is a tuple + # being passed and unpack as needed + self._cls, self._loader, self._node = data + + def __str__(self): + return "%s %s" % (self._cls.yaml_tag, self._node.value) + + def __repr__(self): + return "%s %s" % (self._cls.yaml_tag, self._node.value) + + def format(self, *args, **kwargs): + self._node.value = self._node.value.format(*args, **kwargs) + return self._cls.from_yaml(self._loader, self._node) + + def load(stream, **kwargs): LocalAnchorLoader.reset_anchors() return yaml.load(stream, functools.partial(LocalLoader, **kwargs)) diff --git a/tests/yamlparser/fixtures/lazy-load-jobs-copy-files.yaml.inc b/tests/yamlparser/fixtures/lazy-load-jobs-copy-files.yaml.inc new file mode 100644 index 000000000..499501c51 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs-copy-files.yaml.inc @@ -0,0 +1,8 @@ +name: copy-files +wrappers: + - copy-to-slave: + includes: + - file1 + - file2*.txt + excludes: + - file2bad.txt diff --git a/tests/yamlparser/fixtures/lazy-load-jobs-multi001.xml b/tests/yamlparser/fixtures/lazy-load-jobs-multi001.xml new file mode 100644 index 000000000..2df35f290 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs-multi001.xml @@ -0,0 +1,44 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + #!/bin/bash +# +# version 1.1 of the echo vars script + +MSG="hello world" +VERSION="1.1" + +[[ -n "${MSG}" ]] && { + # this next section is executed as one + echo "${MSG}" + echo "version: ${VERSION}" + exit 0 +} + +#!/bin/bash +echo "Doing somethiung cool" + + + + + + + 3 + true + false + 150 + 90 + elastic + + + diff --git a/tests/yamlparser/fixtures/lazy-load-jobs-multi001.yaml b/tests/yamlparser/fixtures/lazy-load-jobs-multi001.yaml new file mode 100644 index 000000000..f9ae90cf9 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs-multi001.yaml @@ -0,0 +1,20 @@ +- wrapper: + !include: lazy-load-jobs-timeout.yaml.inc + +- project: + name: test + num: "002" + version: + - 1.1 + jobs: + - 'build_myproject_{version}' + +- job-template: + name: 'build_myproject_{version}' + wrappers: + !include: lazy-load-wrappers-{version}.yaml.inc + builders: + - shell: + !include-raw: + - lazy-load-scripts/echo_vars_{version}.sh + - include-raw{num}-cool.sh diff --git a/tests/yamlparser/fixtures/lazy-load-jobs-pre-scm-shell-ant.yaml.inc b/tests/yamlparser/fixtures/lazy-load-jobs-pre-scm-shell-ant.yaml.inc new file mode 100644 index 000000000..56a0e1012 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs-pre-scm-shell-ant.yaml.inc @@ -0,0 +1,15 @@ +name: pre-scm-shell-ant +wrappers: + - pre-scm-buildstep: + - shell: | + #!/bin/bash + echo "Doing somethiung cool" + - shell: | + #!/bin/zsh + echo "Doing somethin cool with zsh" + - ant: + targets: "target1 target2" + ant-name: "Standard Ant" + - inject: + properties-file: example.prop + properties-content: EXAMPLE=foo-bar diff --git a/tests/yamlparser/fixtures/lazy-load-jobs-timeout.yaml.inc b/tests/yamlparser/fixtures/lazy-load-jobs-timeout.yaml.inc new file mode 100644 index 000000000..cdad0afdb --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs-timeout.yaml.inc @@ -0,0 +1,7 @@ +name: timeout-wrapper +wrappers: + - timeout: + fail: true + elastic-percentage: 150 + elastic-default-timeout: 90 + type: elastic diff --git a/tests/yamlparser/fixtures/lazy-load-jobs001.conf b/tests/yamlparser/fixtures/lazy-load-jobs001.conf new file mode 100644 index 000000000..e04ecff65 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs001.conf @@ -0,0 +1,2 @@ +[job_builder] +include_path=tests/yamlparser/fixtures/lazy-load-scripts diff --git a/tests/yamlparser/fixtures/lazy-load-jobs001.xml b/tests/yamlparser/fixtures/lazy-load-jobs001.xml new file mode 100644 index 000000000..62a32bfe9 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs001.xml @@ -0,0 +1,41 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + #!/bin/bash +# +# version 1.1 of the echo vars script + +MSG="hello world" +VERSION="1.1" + +[[ -n "${MSG}" ]] && { + # this next section is executed as one + echo "${MSG}" + echo "version: ${VERSION}" + exit 0 +} + + + + + + + 3 + true + false + 150 + 90 + elastic + + + diff --git a/tests/yamlparser/fixtures/lazy-load-jobs001.yaml b/tests/yamlparser/fixtures/lazy-load-jobs001.yaml new file mode 100644 index 000000000..2fcb1a56c --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs001.yaml @@ -0,0 +1,17 @@ +- wrapper: + !include: lazy-load-jobs-timeout.yaml.inc + +- project: + name: test + version: + - 1.1 + jobs: + - 'build_myproject_{version}' + +- job-template: + name: 'build_myproject_{version}' + wrappers: + !include: lazy-load-wrappers-{version}.yaml.inc + builders: + - shell: + !include-raw: echo_vars_{version}.sh diff --git a/tests/yamlparser/fixtures/lazy-load-jobs002.xml b/tests/yamlparser/fixtures/lazy-load-jobs002.xml new file mode 100644 index 000000000..446e1dc10 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs002.xml @@ -0,0 +1,73 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + #!/bin/bash +# +# version 1.2 of the echo vars script + +MSG="hello world" +VERSION="1.2" + +[[ -n "${MSG}" ]] && { + # this next section is executed as one + echo "${MSG}" + echo "version: ${VERSION}" + exit 0 +} + + + + + + + 3 + true + false + 150 + 90 + elastic + + + + + #!/bin/bash +echo "Doing somethiung cool" + + + + #!/bin/zsh +echo "Doing somethin cool with zsh" + + + + target1 target2 + Standard Ant + + + + example.prop + EXAMPLE=foo-bar + + + + + + file1,file2*.txt + file2bad.txt + false + false + userContent + false + + + diff --git a/tests/yamlparser/fixtures/lazy-load-jobs002.yaml b/tests/yamlparser/fixtures/lazy-load-jobs002.yaml new file mode 100644 index 000000000..2a5923414 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jobs002.yaml @@ -0,0 +1,23 @@ +- wrapper: + !include lazy-load-jobs-timeout.yaml.inc + +- wrapper: + !include lazy-load-jobs-pre-scm-shell-ant.yaml.inc + +- wrapper: + !include lazy-load-jobs-copy-files.yaml.inc + +- project: + name: test + version: + - 1.2 + jobs: + - 'build_myproject_{version}' + +- job-template: + name: 'build_myproject_{version}' + wrappers: + !include lazy-load-wrappers-{version}.yaml.inc + builders: + - shell: + !include-raw-escape lazy-load-scripts/echo_vars_{version}.sh diff --git a/tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.1.sh b/tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.1.sh new file mode 100644 index 000000000..23b75c8d3 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.1.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# version 1.1 of the echo vars script + +MSG="hello world" +VERSION="1.1" + +[[ -n "${MSG}" ]] && { + # this next section is executed as one + echo "${MSG}" + echo "version: ${VERSION}" + exit 0 +} diff --git a/tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.2.sh b/tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.2.sh new file mode 100644 index 000000000..afc9d96ff --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-scripts/echo_vars_1.2.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# version 1.2 of the echo vars script + +MSG="hello world" +VERSION="1.2" + +[[ -n "${MSG}" ]] && { + # this next section is executed as one + echo "${MSG}" + echo "version: ${VERSION}" + exit 0 +} diff --git a/tests/yamlparser/fixtures/lazy-load-wrappers-1.1.yaml.inc b/tests/yamlparser/fixtures/lazy-load-wrappers-1.1.yaml.inc new file mode 100644 index 000000000..e41c6be6c --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-wrappers-1.1.yaml.inc @@ -0,0 +1 @@ +- timeout-wrapper diff --git a/tests/yamlparser/fixtures/lazy-load-wrappers-1.2.yaml.inc b/tests/yamlparser/fixtures/lazy-load-wrappers-1.2.yaml.inc new file mode 100644 index 000000000..312284f4d --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-wrappers-1.2.yaml.inc @@ -0,0 +1,3 @@ +- timeout-wrapper +- pre-scm-shell-ant +- copy-files