From b95f194612029bfdb191da044e9a2b7e03ef3552 Mon Sep 17 00:00:00 2001 From: Daniel Watkins Date: Mon, 31 Jul 2017 17:46:57 -0400 Subject: [PATCH] Add !include-jinja2 for rendering templates with Jinja2 This template included using !include-jinja2: """ {{ my_var }} """ is rendered the same as an existing template that looks like this: """ {my_var} """ This also allows the use of Jinja2's richer syntax: """ {% for test_environment in configuration.get("envs", ["py35"]) %} tox -e {{ test_environment }} {% endfor %} """ Story: 2001135 Change-Id: Ia3ee21822d6e9237f5ea46796bc8810ecac61e2c --- jenkins_jobs/formatter.py | 5 ++ jenkins_jobs/local_yaml.py | 50 +++++++++++++++++-- requirements.txt | 1 + tests/yamlparser/fixtures/jinja01.xml | 23 +++++++++ tests/yamlparser/fixtures/jinja01.yaml | 15 ++++++ tests/yamlparser/fixtures/jinja01.yaml.inc | 4 ++ tests/yamlparser/fixtures/jinja02.xml | 19 +++++++ tests/yamlparser/fixtures/jinja02.yaml | 11 ++++ tests/yamlparser/fixtures/jinja02.yaml.inc | 1 + .../fixtures/lazy-load-jinja001.xml | 23 +++++++++ .../fixtures/lazy-load-jinja001.yaml | 16 ++++++ 11 files changed, 163 insertions(+), 5 deletions(-) create mode 100644 tests/yamlparser/fixtures/jinja01.xml create mode 100644 tests/yamlparser/fixtures/jinja01.yaml create mode 100644 tests/yamlparser/fixtures/jinja01.yaml.inc create mode 100644 tests/yamlparser/fixtures/jinja02.xml create mode 100644 tests/yamlparser/fixtures/jinja02.yaml create mode 100644 tests/yamlparser/fixtures/jinja02.yaml.inc create mode 100644 tests/yamlparser/fixtures/lazy-load-jinja001.xml create mode 100644 tests/yamlparser/fixtures/lazy-load-jinja001.yaml diff --git a/jenkins_jobs/formatter.py b/jenkins_jobs/formatter.py index 4d0468c15..008d66e74 100644 --- a/jenkins_jobs/formatter.py +++ b/jenkins_jobs/formatter.py @@ -21,6 +21,7 @@ import re from string import Formatter from jenkins_jobs.errors import JenkinsJobsException +from jenkins_jobs.local_yaml import CustomLoader logger = logging.getLogger(__name__) @@ -68,6 +69,10 @@ def deep_format(obj, paramdict, allow_empty=False): raise else: ret = obj + if isinstance(ret, CustomLoader): + # If we have a CustomLoader here, we've lazily-loaded a template; + # attempt to format it. + ret = deep_format(ret, paramdict, allow_empty=allow_empty) return ret diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py index bd396a08a..18b7802c0 100644 --- a/jenkins_jobs/local_yaml.py +++ b/jenkins_jobs/local_yaml.py @@ -127,6 +127,20 @@ Example: on any filename passed via ``!include-raw-escape:`` the tag will be automatically converted to ``!include-raw:`` and no escaping will be performed. + + +The tag ``!include-jinja2:`` will treat the given string or list of strings as +filenames to be opened as Jinja2 templates, which should be rendered to a +string and included in the calling YAML construct. (This is analogous to the +templating that will happen with ``!include-raw``.) + +Examples: + + .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml + + contents of jinja01.yaml.inc: + + .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml.inc """ import functools @@ -135,6 +149,7 @@ import logging import os import re +import jinja2 import yaml from yaml.constructor import BaseConstructor from yaml.representer import BaseRepresenter @@ -349,8 +364,8 @@ class YamlInclude(BaseYAMLObject): elif isinstance(node, yaml.SequenceNode): contents = [cls._from_file(loader, scalar_node) for scalar_node in node.value] - if any(isinstance(s, LazyLoader) for s in contents): - return LazyLoaderCollection(contents) + if any(isinstance(s, CustomLoader) for s in contents): + return CustomLoaderCollection(contents) return u'\n'.join(contents) else: @@ -383,6 +398,17 @@ class YamlIncludeRawEscape(YamlIncludeRaw): return loader.escape_callback(data) +class YamlIncludeJinja2(YamlIncludeRaw): + yaml_tag = u'!include-jinja2:' + + @classmethod + def _from_file(cls, loader, node): + contents = cls._open_file(loader, node) + if isinstance(contents, LazyLoader): + return contents + return Jinja2Loader(contents) + + class DeprecatedTag(BaseYAMLObject): @classmethod @@ -407,8 +433,22 @@ class YamlIncludeRawEscapeDeprecated(DeprecatedTag): _new = YamlIncludeRawEscape -class LazyLoaderCollection(object): - """Helper class to format a collection of LazyLoader objects""" +class CustomLoader(object): + """Parent class for non-standard loaders.""" + + +class Jinja2Loader(CustomLoader): + """A loader for Jinja2-templated files.""" + def __init__(self, contents): + self._template = jinja2.Template(contents) + self._template.environment.undefined = jinja2.StrictUndefined + + def format(self, **kwargs): + return self._template.render(kwargs) + + +class CustomLoaderCollection(object): + """Helper class to format a collection of CustomLoader objects""" def __init__(self, sequence): self._data = sequence @@ -416,7 +456,7 @@ class LazyLoaderCollection(object): return u'\n'.join(item.format(*args, **kwargs) for item in self._data) -class LazyLoader(object): +class LazyLoader(CustomLoader): """Helper class to provide lazy loading of files included using !include* tags where the path to the given file contains unresolved placeholders. """ diff --git a/requirements.txt b/requirements.txt index 07e73a21f..f0e7ab425 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,3 +7,4 @@ pbr>=1.8 # Apache-2.0 stevedore>=1.17.1 # Apache-2.0 python-jenkins>=0.4.8 fasteners +Jinja2 diff --git a/tests/yamlparser/fixtures/jinja01.xml b/tests/yamlparser/fixtures/jinja01.xml new file mode 100644 index 000000000..de5ed0184 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja01.xml @@ -0,0 +1,23 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + test variable +a +b +c + + + + + + diff --git a/tests/yamlparser/fixtures/jinja01.yaml b/tests/yamlparser/fixtures/jinja01.yaml new file mode 100644 index 000000000..6106b69a2 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja01.yaml @@ -0,0 +1,15 @@ +- builder: + name: test-builder + builders: + - shell: + !include-jinja2: jinja01.yaml.inc + +- job: + name: test-job + builders: + - test-builder: + var: "test variable" + test_list: + - a + - b + - c diff --git a/tests/yamlparser/fixtures/jinja01.yaml.inc b/tests/yamlparser/fixtures/jinja01.yaml.inc new file mode 100644 index 000000000..1f7c79266 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja01.yaml.inc @@ -0,0 +1,4 @@ +{{ var }} +{% for item in test_list -%} +{{ item }} +{% endfor %} diff --git a/tests/yamlparser/fixtures/jinja02.xml b/tests/yamlparser/fixtures/jinja02.xml new file mode 100644 index 000000000..d0e1c9095 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja02.xml @@ -0,0 +1,19 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + 1 + + + + + diff --git a/tests/yamlparser/fixtures/jinja02.yaml b/tests/yamlparser/fixtures/jinja02.yaml new file mode 100644 index 000000000..1e1d87476 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja02.yaml @@ -0,0 +1,11 @@ +- job-template: + name: 'test-job-{num}' + builders: + - shell: + !include-jinja2: jinja02.yaml.inc + +- project: + name: test-job-template-1 + num: 1 + jobs: + - 'test-job-{num}' diff --git a/tests/yamlparser/fixtures/jinja02.yaml.inc b/tests/yamlparser/fixtures/jinja02.yaml.inc new file mode 100644 index 000000000..a535c7426 --- /dev/null +++ b/tests/yamlparser/fixtures/jinja02.yaml.inc @@ -0,0 +1 @@ +{{ num }} diff --git a/tests/yamlparser/fixtures/lazy-load-jinja001.xml b/tests/yamlparser/fixtures/lazy-load-jinja001.xml new file mode 100644 index 000000000..de5ed0184 --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jinja001.xml @@ -0,0 +1,23 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + false + false + false + true + + + + + test variable +a +b +c + + + + + + diff --git a/tests/yamlparser/fixtures/lazy-load-jinja001.yaml b/tests/yamlparser/fixtures/lazy-load-jinja001.yaml new file mode 100644 index 000000000..b1f7a63fb --- /dev/null +++ b/tests/yamlparser/fixtures/lazy-load-jinja001.yaml @@ -0,0 +1,16 @@ +- builder: + name: test-builder + builders: + - shell: + !include-jinja2: jinja{include-number}.yaml.inc + +- job: + name: test-job + builders: + - test-builder: + var: "test variable" + test_list: + - a + - b + - c + include-number: "01"