From 18efe5066b8e0c2477eaea26a46d827494f1939e Mon Sep 17 00:00:00 2001 From: Vsevolod Fedorov Date: Mon, 13 Nov 2023 12:28:57 +0300 Subject: [PATCH] Unify variable and tags expansion inside jobs and macros Expand variables inside macros without parameters and jobs the same way as they are expanded inside macros with parameters and job templates. Make tags behave inside macros without parameters and jobs the same way as they are expanded inside macros with parameters and job templates. Update or fix affected tests. Story: 2010588 Story: 2010963 Story: 2010535 Task: 47394 Task: 49069 Task: 47151 Change-Id: Ie05ae6aa386c62ebbf68dd3e2c7001a4e444a47a --- doc/source/definition.rst | 44 +++---------- jenkins_jobs/defaults.py | 2 +- jenkins_jobs/dimensions.py | 7 ++- jenkins_jobs/expander.py | 49 +++++++++------ jenkins_jobs/job.py | 15 ++--- jenkins_jobs/loader.py | 5 +- jenkins_jobs/macro.py | 2 - jenkins_jobs/project.py | 2 +- jenkins_jobs/registry.py | 19 +++--- jenkins_jobs/root_base.py | 6 +- jenkins_jobs/view.py | 15 ++--- jenkins_jobs/yaml_objects.py | 63 ++----------------- .../fixtures/deprecated-include-raw001.yaml | 4 +- .../fixtures/include-raw-escaped001-job.json | 2 +- .../fixtures/include-raw-escaped001-job.xml | 8 +-- .../fixtures/include-raw-escaped001-job.yaml | 2 +- .../loader/fixtures/include-raw-multi001.yaml | 2 +- tests/loader/fixtures/include-raw001-job.yaml | 4 +- .../failure_formatting_params.yaml | 2 +- .../macro_with_null_params.error | 18 ++++++ .../macro_with_null_params.yaml | 4 +- .../error_fixtures/recursive_parameter.error | 6 +- .../error_fixtures/recursive_parameter.yaml | 4 +- .../job-and-macro-expansions.yaml | 8 +-- .../job_fixtures/job_honor_defaults.xml | 2 +- .../job_fixtures/job_honor_defaults.yaml | 2 +- .../job_fixtures/macro_with_null_params.xml | 19 ------ .../project_flow_template001.yaml | 4 +- .../project_pipeline_template001.yaml | 4 +- .../project_workflow_template001.yaml | 4 +- .../job_fixtures/regression-2006254.yaml | 2 +- .../job_fixtures/regression-2010535.xml | 2 +- .../job_fixtures/regression-2010535.yaml | 2 +- .../parameter-override-ordering-001.yaml | 4 +- 34 files changed, 133 insertions(+), 205 deletions(-) create mode 100644 tests/yamlparser/error_fixtures/macro_with_null_params.error rename tests/yamlparser/{job_fixtures => error_fixtures}/macro_with_null_params.yaml (68%) delete mode 100644 tests/yamlparser/job_fixtures/macro_with_null_params.xml diff --git a/doc/source/definition.rst b/doc/source/definition.rst index 828a4950d..ba41071f3 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -362,9 +362,6 @@ specific ones without having to duplicate code:: # Generic macro call with a parameter - add: number: "ZERO" - # Generic macro called without a parameter. Never do this! - # See below for the resulting wrong output :( - - add Then ```` section of the generated job show up as:: @@ -375,9 +372,6 @@ Then ```` section of the generated job show up as:: echo Adding ZERO - - echo Adding {number} - As you can see, the specialized macro ``addtwo`` reused the definition from @@ -386,31 +380,10 @@ the generic macro ``add``. Macro Notes ~~~~~~~~~~~ -If a macro is not passed any parameters it will not have any expansion -performed on it. Thus if you forget to provide `any` parameters to a -macro that expects some, the parameter-templates (``{foo}``) will be -left as is in the resulting output; this is almost certainly not what -you want. Note if you provide an invalid parameter, the expansion -will fail; the expansion will only be skipped if you provide `no` -parameters at all. - -Macros are expanded using Python string substitution rules. This can -especially cause confusion with shell snippets that use ``{`` as part -of their syntax. As described, if a macro has `no` parameters, no -expansion will be performed and thus it is correct to write the script -with no escaping, e.g.:: - - - builder: - name: a_builder - builders: - - shell: | - VARIABLE=${VARIABLE:-bar} - function foo { - echo "my shell function" - } - -However, if the macro `has` parameters, you must escape the ``{`` you -wish to make it through to the output, e.g.:: +Macros are expanded using Python string substitution rules. This can +especially cause confusion with shell snippets that use ``{`` and ``}`` as part +of their syntax. You must escape curly braces you wish to make it through +to the output, e.g.:: - builder: name: a_builder @@ -422,10 +395,11 @@ wish to make it through to the output, e.g.:: echo "my shell function" }} -Note that a ``job-template`` will have parameters by definition (at -least a ``name``). Thus embedded-shell within a ``job-template`` should -always use ``{{`` to achieve a literal ``{``. A generic builder will need -to consider the correct quoting based on its use of parameters. +The same apply for ``job`` and ``job-template``. Thus embedded-shell within +a ``job`` or ``job-template`` should always use ``{{`` to achieve a literal +``{``. + +You can also use ``!j2:`` tag instead - Jinja2 uses double quotes for variables. .. _folders: diff --git a/jenkins_jobs/defaults.py b/jenkins_jobs/defaults.py index 33ba1a482..913fbc0d7 100644 --- a/jenkins_jobs/defaults.py +++ b/jenkins_jobs/defaults.py @@ -176,7 +176,7 @@ class Defaults: contents: dict # Values that go to job contents. @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") contents, params = split_contents_params( diff --git a/jenkins_jobs/dimensions.py b/jenkins_jobs/dimensions.py index 45be299fa..08425addb 100644 --- a/jenkins_jobs/dimensions.py +++ b/jenkins_jobs/dimensions.py @@ -15,7 +15,7 @@ from collections import namedtuple from .errors import Context, JenkinsJobsException from .loc_loader import LocList, LocDict -from jenkins_jobs.expander import Expander +from jenkins_jobs.expander import YamlObjectsExpander Dimension = namedtuple("Dimension", "axis params") @@ -48,7 +48,7 @@ def _decode_axis_value(axis, value, key_pos, value_pos): def enum_dimensions_params(axes, params, defaults): - expander = Expander() + expander = YamlObjectsExpander() if not axes: # No axes - instantiate one job/view. yield {} @@ -60,9 +60,10 @@ def enum_dimensions_params(axes, params, defaults): except KeyError: try: value = defaults[axis] + key_pos = value_pos = None except KeyError: continue # May be, value would be received from an another axis values. - expanded_value = expander.expand(value, params) + expanded_value = expander.expand(value, params, key_pos, value_pos) value = [ Dimension(axis, params) for params in _decode_axis_value(axis, expanded_value, key_pos, value_pos) diff --git a/jenkins_jobs/expander.py b/jenkins_jobs/expander.py index f0420f3bc..a0143591e 100644 --- a/jenkins_jobs/expander.py +++ b/jenkins_jobs/expander.py @@ -55,9 +55,8 @@ def expand_tuple(expander, obj, params, key_pos, value_pos): class StrExpander: - def __init__(self, config): - allow_empty = config.yamlparser["allow_empty_variables"] - self._formatter = CustomFormatter(allow_empty) + def __init__(self, allow_empty_variables): + self._formatter = CustomFormatter(allow_empty_variables) def __call__(self, obj, params, key_pos, value_pos): try: @@ -79,14 +78,14 @@ def call_expand(expander, obj, params, key_pos, value_pos): return obj.expand(expander, params) -def call_subst(expander, obj, params, key_pos, value_pos): - return obj.subst(expander, params) - - def dont_expand(obj, params, key_pos, value_pos): return obj +def dont_expand_yaml_object(expander, obj, params, key_pos, value_pos): + return obj + + yaml_classes_list = [ J2String, J2Yaml, @@ -104,9 +103,13 @@ deprecated_yaml_tags = [ ] -# Does not expand string formats. Used in jobs and macros without parameters. +# Expand strings and yaml objects. class Expander: def __init__(self, config=None): + if config: + allow_empty_variables = config.yamlparser["allow_empty_variables"] + else: + allow_empty_variables = False _yaml_object_expanders = { cls: partial(call_expand, self) for cls in yaml_classes_list } @@ -116,8 +119,8 @@ class Expander: list: partial(expand_list, self), LocList: partial(expand_list, self), tuple: partial(expand_tuple, self), - str: dont_expand, - LocString: dont_expand, + str: StrExpander(allow_empty_variables), + LocString: StrExpander(allow_empty_variables), bool: dont_expand, int: dont_expand, float: dont_expand, @@ -136,20 +139,26 @@ class Expander: return expander(obj, params, key_pos, value_pos) -# Expands string formats also. Used in jobs templates and macros with parameters. -class ParamsExpander(Expander): +# Expand only yaml objects. +class YamlObjectsExpander(Expander): + def __init__(self): + super().__init__() + self.expanders.update( + { + str: dont_expand, + LocString: dont_expand, + } + ) + + +# Expand only string parameters. +class StringsOnlyExpander(Expander): def __init__(self, config): super().__init__(config) _yaml_object_expanders = { - cls: partial(call_subst, self) for cls in yaml_classes_list + cls: partial(dont_expand_yaml_object, self) for cls in yaml_classes_list } - self.expanders.update( - { - str: StrExpander(config), - LocString: StrExpander(config), - **_yaml_object_expanders, - } - ) + self.expanders.update(_yaml_object_expanders) def call_required_params(obj, pos): diff --git a/jenkins_jobs/job.py b/jenkins_jobs/job.py index 56dd0f434..903dc97fc 100644 --- a/jenkins_jobs/job.py +++ b/jenkins_jobs/job.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from .errors import JenkinsJobsException +from .expander import Expander from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .defaults import split_contents_params, job_contents_keys @@ -23,7 +24,7 @@ class JobBase(RootBase): folder: str @classmethod - def from_dict(cls, config, roots, expander, data, pos): + def from_dict(cls, config, roots, data, pos): keep_descriptions = config.yamlparser["keep_descriptions"] d = data.copy() name = d.pop_required_loc_string("name") @@ -35,7 +36,7 @@ class JobBase(RootBase): contents, params = split_contents_params(d, job_contents_keys) return cls( roots.defaults, - expander, + Expander(config), keep_descriptions, id, name, @@ -72,8 +73,8 @@ class JobBase(RootBase): class Job(JobBase, NonTemplateRootMixin): @classmethod - def add(cls, config, roots, expander, param_expander, data, pos): - job = cls.from_dict(config, roots, expander, data, pos) + def add(cls, config, roots, data, pos): + job = cls.from_dict(config, roots, data, pos) roots.assign(roots.jobs, job.id, job, "job") def __str__(self): @@ -82,8 +83,8 @@ class Job(JobBase, NonTemplateRootMixin): class JobTemplate(JobBase, TemplateRootMixin): @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): - template = cls.from_dict(config, roots, params_expander, data, pos) + def add(cls, config, roots, data, pos): + template = cls.from_dict(config, roots, data, pos) roots.assign(roots.job_templates, template.id, template, "job template") def __str__(self): @@ -96,7 +97,7 @@ class JobGroup(Group): _job_templates: dict @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") try: diff --git a/jenkins_jobs/loader.py b/jenkins_jobs/loader.py index c4ba948a3..6e9b6b1ce 100644 --- a/jenkins_jobs/loader.py +++ b/jenkins_jobs/loader.py @@ -17,7 +17,7 @@ from functools import partial from .errors import JenkinsJobsException from .loc_loader import LocLoader from .yaml_objects import BaseYamlObject -from .expander import Expander, ParamsExpander, deprecated_yaml_tags, yaml_classes_list +from .expander import Expander, deprecated_yaml_tags, yaml_classes_list from .roots import root_adders logger = logging.getLogger(__name__) @@ -115,7 +115,6 @@ def enum_expanded_paths(path_list): def load_files(config, roots, path_list): expander = Expander(config) - params_expander = ParamsExpander(config) loader = Loader.empty(config) for path in enum_expanded_paths(path_list): if is_stdin(path): @@ -155,4 +154,4 @@ def load_files(config, roots, path_list): f" known are: {','.join(root_adders)}.", pos=item.pos, ) - adder(config, roots, expander, params_expander, contents, item.pos) + adder(config, roots, contents, item.pos) diff --git a/jenkins_jobs/macro.py b/jenkins_jobs/macro.py index c6be59f92..e13d930bc 100644 --- a/jenkins_jobs/macro.py +++ b/jenkins_jobs/macro.py @@ -45,8 +45,6 @@ class Macro: elements_name, config, roots, - expander, - params_expander, data, pos, ): diff --git a/jenkins_jobs/project.py b/jenkins_jobs/project.py index c737eb2e5..165c4ef8d 100644 --- a/jenkins_jobs/project.py +++ b/jenkins_jobs/project.py @@ -33,7 +33,7 @@ class Project(GroupBase): params: dict @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") defaults = d.pop_loc_string("defaults", None) diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py index e9935e590..87194b055 100644 --- a/jenkins_jobs/registry.py +++ b/jenkins_jobs/registry.py @@ -27,7 +27,7 @@ from six import PY2 from jenkins.plugins import PluginVersion from jenkins_jobs.errors import JenkinsJobsException -from jenkins_jobs.expander import Expander, ParamsExpander +from jenkins_jobs.expander import Expander, StringsOnlyExpander from jenkins_jobs.yaml_objects import BaseYamlObject __all__ = ["ModuleRegistry"] @@ -49,7 +49,7 @@ class ModuleRegistry(object): self.masked_warned = {} self._macros = {} self._expander = Expander(jjb_config) - self._params_expander = ParamsExpander(jjb_config) + self._str_expander = StringsOnlyExpander(jjb_config) if plugins_list is None: self._plugin_version = {} @@ -290,20 +290,17 @@ class ModuleRegistry(object): "component type that is masking an inbuilt " "definition" % (name, component_type) ) - # Expand macro strings only if at least one macro parameter is provided. - if component_data: - expander = self._params_expander - else: - expander = self._expander - component_data = {} # It may be None. + if component_data is None: + component_data = {} expander_params = {**component_data, **(job_data or {})} elements = macro.elements if isinstance(elements, BaseYamlObject): - # Expand !j2-yaml element right below macro body. - elements = elements.expand(expander, expander_params) + # Expand !j2-yaml tag if it is right below macro body. + # But do not expand yaml tags inside it - they will be expanded later. + elements = elements.expand(self._str_expander, expander_params) for b in elements: try: - element = expander.expand(b, expander_params) + element = self._expander.expand(b, expander_params) except JenkinsJobsException as x: raise x.with_context( f"While expanding macro {name!r}", diff --git a/jenkins_jobs/root_base.py b/jenkins_jobs/root_base.py index 4ae8593b9..c5069232b 100644 --- a/jenkins_jobs/root_base.py +++ b/jenkins_jobs/root_base.py @@ -103,12 +103,16 @@ class NonTemplateRootMixin: def top_level_generate_items(self): try: defaults = self._pick_defaults(self.defaults_name, merge_global=False) + item_params = LocDict.merge( + defaults.params, + self.params, + ) contents = LocDict.merge( defaults.contents, self.contents, pos=self.pos, ) - expanded_contents = self._expand_contents(contents, self.params) + expanded_contents = self._expand_contents(contents, item_params) context = [Context(f"In {self}", self.pos)] yield JobViewData(expanded_contents, context) except JenkinsJobsException as x: diff --git a/jenkins_jobs/view.py b/jenkins_jobs/view.py index 4ba9303aa..8964b0852 100644 --- a/jenkins_jobs/view.py +++ b/jenkins_jobs/view.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from .errors import JenkinsJobsException from .loc_loader import LocDict +from .expander import Expander from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .defaults import split_contents_params, view_contents_keys @@ -23,7 +24,7 @@ class ViewBase(RootBase): view_type: str @classmethod - def from_dict(cls, config, roots, expander, data, pos): + def from_dict(cls, config, roots, data, pos): keep_descriptions = config.yamlparser["keep_descriptions"] d = data.copy() name = d.pop_required_loc_string("name") @@ -34,7 +35,7 @@ class ViewBase(RootBase): contents, params = split_contents_params(d, view_contents_keys) return cls( roots.defaults, - expander, + Expander(config), keep_descriptions, id, name, @@ -59,8 +60,8 @@ class ViewBase(RootBase): class View(ViewBase, NonTemplateRootMixin): @classmethod - def add(cls, config, roots, expander, param_expander, data, pos): - view = cls.from_dict(config, roots, expander, data, pos) + def add(cls, config, roots, data, pos): + view = cls.from_dict(config, roots, data, pos) roots.assign(roots.views, view.id, view, "view") def __str__(self): @@ -69,8 +70,8 @@ class View(ViewBase, NonTemplateRootMixin): class ViewTemplate(ViewBase, TemplateRootMixin): @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): - template = cls.from_dict(config, roots, params_expander, data, pos) + def add(cls, config, roots, data, pos): + template = cls.from_dict(config, roots, data, pos) roots.assign(roots.view_templates, template.id, template, "view template") def __str__(self): @@ -83,7 +84,7 @@ class ViewGroup(Group): _view_templates: dict @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") try: diff --git a/jenkins_jobs/yaml_objects.py b/jenkins_jobs/yaml_objects.py index 04bd1363f..883461a62 100644 --- a/jenkins_jobs/yaml_objects.py +++ b/jenkins_jobs/yaml_objects.py @@ -83,40 +83,9 @@ filenames to be opened as one or more data blob, which should be read into the calling yaml construct without any further parsing. Any data in a file included through this tag, will be treated as string data. -Examples: - - .. literalinclude:: /../../tests/loader/fixtures/include-raw001-job.yaml - - contents of include-raw001-hello-world.sh: - - .. literalinclude:: - /../../tests/loader/fixtures/include-raw001-hello-world.sh - - contents of include-raw001-vars.sh: - - .. literalinclude:: - /../../tests/loader/fixtures/include-raw001-vars.sh - - using a list of files: - - .. literalinclude:: - /../../tests/loader/fixtures/include-raw-multi001.yaml - -The tag ``!include-raw-escape:`` treats the given string or list of strings as -filenames to be opened as one or more data blobs, which should be escaped -before being read in as string data. This allows job-templates to use this tag -to include scripts from files without needing to escape braces in the original -file. - -.. warning:: - - When used as a macro ``!include-raw-escape:`` should only be used if - parameters are passed into the escaped file and you would like to escape - those parameters. If the file does not have any jjb parameters passed into - it then ``!include-raw:`` should be used instead otherwise you will run - into an interesting issue where ``include-raw-escape:`` actually adds - additional curly braces around existing curly braces. For example - ${PROJECT} becomes ${{PROJECT}} which may break bash scripts. +It will expand variables inside the file. If your file contains curly braces, +you should double them. Or, you can use tag ``!include-raw-escape``, which +does not substitute variables. Examples: @@ -259,12 +228,8 @@ class BaseYamlObject(metaclass=abc.ABCMeta): @abc.abstractmethod def expand(self, expander, params): - """Expand object but do not substitute template parameters""" - pass - - def subst(self, expander, params): """Expand object and substitute template parameters""" - return self.expand(expander, params) + pass def _find_file(self, rel_path, pos): search_path = self._search_path @@ -286,10 +251,6 @@ class BaseYamlObject(metaclass=abc.ABCMeta): for idx, path in enumerate(path_list): yield self._expand_path(path, path_list.value_pos[idx], *args) - def _subst_path_list(self, path_list, *args): - for idx, path in enumerate(path_list): - yield self._subst_path(path, path_list.value_pos[idx], *args) - class J2BaseYamlObject(BaseYamlObject): def __init__(self, jjb_config, loader, pos): @@ -438,19 +399,11 @@ class IncludeRawBase(IncludeBaseObject): def expand(self, expander, params): return "\n".join(self._expand_path_list(self._path_list, params)) - def subst(self, expander, params): - return "\n".join(self._subst_path_list(self._path_list, params)) - class IncludeRaw(IncludeRawBase): yaml_tag = "!include-raw:" def _expand_path(self, rel_path_template, pos, params): - rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path, pos) - return full_path.read_text() - - def _subst_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) full_path = self._find_file(rel_path, pos) template = full_path.read_text() @@ -464,14 +417,6 @@ class IncludeRawEscape(IncludeRawBase): yaml_tag = "!include-raw-escape:" def _expand_path(self, rel_path_template, pos, params): - rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path, pos) - text = full_path.read_text() - # Backward compatibility: - # if used inside job or macro without parameters, curly braces are duplicated. - return text.replace("{", "{{").replace("}", "}}") - - def _subst_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) full_path = self._find_file(rel_path, pos) return full_path.read_text() diff --git a/tests/loader/fixtures/deprecated-include-raw001.yaml b/tests/loader/fixtures/deprecated-include-raw001.yaml index ba2f8ef0c..972816c2e 100644 --- a/tests/loader/fixtures/deprecated-include-raw001.yaml +++ b/tests/loader/fixtures/deprecated-include-raw001.yaml @@ -2,6 +2,6 @@ name: test-job-include-raw-1 builders: - shell: - !include-raw include-raw001-hello-world.sh + !include-raw-escape include-raw001-hello-world.sh - shell: - !include-raw include-raw001-vars.sh + !include-raw-escape include-raw001-vars.sh diff --git a/tests/loader/fixtures/include-raw-escaped001-job.json b/tests/loader/fixtures/include-raw-escaped001-job.json index 21af3e732..6a210d138 100644 --- a/tests/loader/fixtures/include-raw-escaped001-job.json +++ b/tests/loader/fixtures/include-raw-escaped001-job.json @@ -5,7 +5,7 @@ "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n" }, { - "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n # this next section is executed as one\n echo \"${{VAR3}}\"\n exit 0\n}}\n" + "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n # this next section is executed as one\n echo \"${VAR3}\"\n exit 0\n}\n" } ], "description": "", diff --git a/tests/loader/fixtures/include-raw-escaped001-job.xml b/tests/loader/fixtures/include-raw-escaped001-job.xml index 552942ff0..9a48dc3d9 100644 --- a/tests/loader/fixtures/include-raw-escaped001-job.xml +++ b/tests/loader/fixtures/include-raw-escaped001-job.xml @@ -30,13 +30,13 @@ exit 0 VAR1="hello" VAR2="world" -VAR3="${{VAR1}} ${{VAR2}}" +VAR3="${VAR1} ${VAR2}" -[[ -n "${{VAR3}}" ]] && {{ +[[ -n "${VAR3}" ]] && { # this next section is executed as one - echo "${{VAR3}}" + echo "${VAR3}" exit 0 -}} +} diff --git a/tests/loader/fixtures/include-raw-escaped001-job.yaml b/tests/loader/fixtures/include-raw-escaped001-job.yaml index b9d51c020..4cfcc9c7d 100644 --- a/tests/loader/fixtures/include-raw-escaped001-job.yaml +++ b/tests/loader/fixtures/include-raw-escaped001-job.yaml @@ -1,4 +1,4 @@ -# Using include-raw-excape inside job cause double braces in included file, like: {{VAR1}}. +# Using include-raw-escape inside job works the same way as inside job template. - job: name: test-job-include-raw builders: diff --git a/tests/loader/fixtures/include-raw-multi001.yaml b/tests/loader/fixtures/include-raw-multi001.yaml index 2f0f2af55..bf0927c57 100644 --- a/tests/loader/fixtures/include-raw-multi001.yaml +++ b/tests/loader/fixtures/include-raw-multi001.yaml @@ -2,6 +2,6 @@ name: test-job-include-raw-1 builders: - shell: - !include-raw: + !include-raw-escape: - include-raw001-hello-world.sh - include-raw001-vars.sh diff --git a/tests/loader/fixtures/include-raw001-job.yaml b/tests/loader/fixtures/include-raw001-job.yaml index dfa5fb71f..ab718448f 100644 --- a/tests/loader/fixtures/include-raw001-job.yaml +++ b/tests/loader/fixtures/include-raw001-job.yaml @@ -2,6 +2,6 @@ name: test-job-include-raw-1 builders: - shell: - !include-raw: include-raw001-hello-world.sh + !include-raw-escape: include-raw001-hello-world.sh - shell: - !include-raw: include-raw001-vars.sh + !include-raw-escape: include-raw001-vars.sh diff --git a/tests/yamlparser/error_fixtures/failure_formatting_params.yaml b/tests/yamlparser/error_fixtures/failure_formatting_params.yaml index ef387ac06..aa8f072d5 100644 --- a/tests/yamlparser/error_fixtures/failure_formatting_params.yaml +++ b/tests/yamlparser/error_fixtures/failure_formatting_params.yaml @@ -11,7 +11,7 @@ - xenial-{bdate} jobs: - 'template-requiring-param-{os}': - os: 'ubuntu-{flavour}' + os: 'ubuntu-{flavor}' - job-template: name: 'template-requiring-param-{os}' diff --git a/tests/yamlparser/error_fixtures/macro_with_null_params.error b/tests/yamlparser/error_fixtures/macro_with_null_params.error new file mode 100644 index 000000000..a93473c93 --- /dev/null +++ b/tests/yamlparser/error_fixtures/macro_with_null_params.error @@ -0,0 +1,18 @@ +macro_with_null_params.yaml:14:3: In project 'sample-project' + - project: + ^ +macro_with_null_params.yaml:17:9: Defined here + - sample-job + ^ +macro_with_null_params.yaml:8:3: In job template 'sample-job' + - job-template: + ^ +macro_with_null_params.yaml:12:9: While expanding builder macro call 'sample-macro' + - sample-macro: + ^ +macro_with_null_params.yaml:1:3: While expanding macro 'sample-macro' + - builder: + ^ +macro_with_null_params.yaml:6:17: While formatting string 'echo {hello}': Missing parameter: 'hello' + - shell: 'echo {hello}' + ^ diff --git a/tests/yamlparser/job_fixtures/macro_with_null_params.yaml b/tests/yamlparser/error_fixtures/macro_with_null_params.yaml similarity index 68% rename from tests/yamlparser/job_fixtures/macro_with_null_params.yaml rename to tests/yamlparser/error_fixtures/macro_with_null_params.yaml index 9988313a6..e97e028d3 100644 --- a/tests/yamlparser/job_fixtures/macro_with_null_params.yaml +++ b/tests/yamlparser/error_fixtures/macro_with_null_params.yaml @@ -2,13 +2,13 @@ name: sample-macro builders: # Add parameter to check if macro behaves the same way as if no params were provided. - # That is, it should not be expanded. + # That is, it should try to expand the parameter. - shell: 'echo {hello}' - job-template: name: sample-job builders: - # Place colon but define no parameters. + # Place colon but define no parameters (parameters are null instead of dict). - sample-macro: - project: diff --git a/tests/yamlparser/error_fixtures/recursive_parameter.error b/tests/yamlparser/error_fixtures/recursive_parameter.error index 9a931f709..7514a6221 100644 --- a/tests/yamlparser/error_fixtures/recursive_parameter.error +++ b/tests/yamlparser/error_fixtures/recursive_parameter.error @@ -5,13 +5,13 @@ recursive_parameter.yaml:7:3: In job template 'sample-job-{param_1}' - job-template: ^ recursive_parameter.yaml:5:9: Used by param_1 - - '{param_2}-at-project' + - '{param_2}-at-globals' ^ recursive_parameter.yaml:9:14: Used by param_2 param_2: '{param_3}-at-template' ^ recursive_parameter.yaml:13:14: Used by param_3 - param_3: '{param_4}-at-globals' + param_3: '{param_4}-at-project' ^ recursive_parameter.yaml:16:20: Used by param_4 param_4: '{param_1}-at-job-spec' @@ -20,5 +20,5 @@ recursive_parameter.yaml:3:5: While expanding 'param_1' param_1: ^ recursive_parameter.yaml:5:9: Recursive parameters usage: param_1 <- param_2 <- param_3 <- param_4 - - '{param_2}-at-project' + - '{param_2}-at-globals' ^ diff --git a/tests/yamlparser/error_fixtures/recursive_parameter.yaml b/tests/yamlparser/error_fixtures/recursive_parameter.yaml index 6596aadce..c32f7bdff 100644 --- a/tests/yamlparser/error_fixtures/recursive_parameter.yaml +++ b/tests/yamlparser/error_fixtures/recursive_parameter.yaml @@ -2,7 +2,7 @@ name: global param_1: - param_1_value_1 - - '{param_2}-at-project' + - '{param_2}-at-globals' - job-template: name: 'sample-job-{param_1}' @@ -10,7 +10,7 @@ - project: name: sample-project - param_3: '{param_4}-at-globals' + param_3: '{param_4}-at-project' jobs: - 'sample-job-{param_1}': param_4: '{param_1}-at-job-spec' diff --git a/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml b/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml index d52c8b2eb..fa68acc12 100644 --- a/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml +++ b/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml @@ -9,8 +9,8 @@ name: builder-without-params builders: - shell: | - echo Should not be expanded: {param} - - shell: !include-raw: job-and-macro-expansions.yaml.no-expand.inc + echo Should not be expanded: {{param}} + - shell: !include-raw-escape: job-and-macro-expansions.yaml.no-expand.inc - builder: name: builder-with-params @@ -27,8 +27,8 @@ display-name: sample-job builders: - shell: | - echo Should not be expanded: {param} - - shell: !include-raw: job-and-macro-expansions.yaml.no-expand.inc + echo Should not be expanded: {{param}} + - shell: !include-raw-escape: job-and-macro-expansions.yaml.no-expand.inc - job-template: name: sample-job-template diff --git a/tests/yamlparser/job_fixtures/job_honor_defaults.xml b/tests/yamlparser/job_fixtures/job_honor_defaults.xml index b9639469e..0e0b40b9c 100644 --- a/tests/yamlparser/job_fixtures/job_honor_defaults.xml +++ b/tests/yamlparser/job_fixtures/job_honor_defaults.xml @@ -11,7 +11,7 @@ - echo Build arch {arch}. + echo Build arch i386. diff --git a/tests/yamlparser/job_fixtures/job_honor_defaults.yaml b/tests/yamlparser/job_fixtures/job_honor_defaults.yaml index 70fe442b9..a414fc4e3 100644 --- a/tests/yamlparser/job_fixtures/job_honor_defaults.yaml +++ b/tests/yamlparser/job_fixtures/job_honor_defaults.yaml @@ -1,4 +1,4 @@ -# Variables in jobs are not expanded. +# Variables in jobs are expanded using defaults. - defaults: name: global diff --git a/tests/yamlparser/job_fixtures/macro_with_null_params.xml b/tests/yamlparser/job_fixtures/macro_with_null_params.xml deleted file mode 100644 index b6a46c944..000000000 --- a/tests/yamlparser/job_fixtures/macro_with_null_params.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - <!-- Managed by Jenkins Job Builder --> - false - false - false - false - true - - - - - echo {hello} - - - - - diff --git a/tests/yamlparser/job_fixtures/project_flow_template001.yaml b/tests/yamlparser/job_fixtures/project_flow_template001.yaml index 65316ec3b..7183de393 100644 --- a/tests/yamlparser/job_fixtures/project_flow_template001.yaml +++ b/tests/yamlparser/job_fixtures/project_flow_template001.yaml @@ -4,6 +4,6 @@ dsl: | build("job1") parallel ( - { build("job2a") }, - { build("job2b") } + {{ build("job2a") }}, + {{ build("job2b") }} ) diff --git a/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml b/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml index 2044da3bf..39f641728 100644 --- a/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml +++ b/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml @@ -5,7 +5,7 @@ build job: "job1" parallel [ 2a: build job: "job2a", - 2b: node "dummynode" { + 2b: node "dummynode" {{ sh "echo I'm alive!" - } + }} ] diff --git a/tests/yamlparser/job_fixtures/project_workflow_template001.yaml b/tests/yamlparser/job_fixtures/project_workflow_template001.yaml index c0cc57089..f0a937a9d 100644 --- a/tests/yamlparser/job_fixtures/project_workflow_template001.yaml +++ b/tests/yamlparser/job_fixtures/project_workflow_template001.yaml @@ -5,7 +5,7 @@ build job: "job1" parallel [ 2a: build job: "job2a", - 2b: node "dummynode" { + 2b: node "dummynode" {{ sh "echo I'm alive!" - } + }} ] diff --git a/tests/yamlparser/job_fixtures/regression-2006254.yaml b/tests/yamlparser/job_fixtures/regression-2006254.yaml index ea928a67b..2bab04613 100644 --- a/tests/yamlparser/job_fixtures/regression-2006254.yaml +++ b/tests/yamlparser/job_fixtures/regression-2006254.yaml @@ -25,7 +25,7 @@ - timed: "H 14 * * *" builders: - - shell: !include-raw: regression-2006254.inc + - shell: !include-raw-escape: regression-2006254.inc parameters: - bool: diff --git a/tests/yamlparser/job_fixtures/regression-2010535.xml b/tests/yamlparser/job_fixtures/regression-2010535.xml index 9bee35216..69c4d3673 100644 --- a/tests/yamlparser/job_fixtures/regression-2010535.xml +++ b/tests/yamlparser/job_fixtures/regression-2010535.xml @@ -13,7 +13,7 @@ PARAM_1 - {default|my_default} + my_default false diff --git a/tests/yamlparser/job_fixtures/regression-2010535.yaml b/tests/yamlparser/job_fixtures/regression-2010535.yaml index 3694578b1..ebb65884b 100644 --- a/tests/yamlparser/job_fixtures/regression-2010535.yaml +++ b/tests/yamlparser/job_fixtures/regression-2010535.yaml @@ -1,5 +1,5 @@ # https://storyboard.openstack.org/#!/story/2010535 -# Bug: JJB doesn't expand macro in case of usage without arguments +# Fixed: Bug: JJB doesn't expand macro in case of usage without arguments # String templates in macro calls without parameters are NOT expanded. # Jinja2 templates in macro calls without parameters ARE expanded. diff --git a/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml b/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml index 423ad258a..497702d23 100644 --- a/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml +++ b/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml @@ -12,7 +12,7 @@ publishers: - trigger-parameterized-builds: - project: first_job - predefined-parameters: BUILD_NUM=${BUILD_NUMBER} + predefined-parameters: BUILD_NUM=${{BUILD_NUMBER}} property-file: default_version.prop current-parameters: true - project: second_job @@ -25,7 +25,7 @@ publishers: - trigger-parameterized-builds: - project: 1.2_first_job - predefined-parameters: BUILD_NUM=${BUILD_NUMBER} + predefined-parameters: BUILD_NUM=${{BUILD_NUMBER}} current-parameters: true property-file: version.prop - project: 1.2_second_job