diff --git a/doc/source/definition.rst b/doc/source/definition.rst index 828a4950d..ba41071f3 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -362,9 +362,6 @@ specific ones without having to duplicate code:: # Generic macro call with a parameter - add: number: "ZERO" - # Generic macro called without a parameter. Never do this! - # See below for the resulting wrong output :( - - add Then ```` section of the generated job show up as:: @@ -375,9 +372,6 @@ Then ```` section of the generated job show up as:: echo Adding ZERO - - echo Adding {number} - As you can see, the specialized macro ``addtwo`` reused the definition from @@ -386,31 +380,10 @@ the generic macro ``add``. Macro Notes ~~~~~~~~~~~ -If a macro is not passed any parameters it will not have any expansion -performed on it. Thus if you forget to provide `any` parameters to a -macro that expects some, the parameter-templates (``{foo}``) will be -left as is in the resulting output; this is almost certainly not what -you want. Note if you provide an invalid parameter, the expansion -will fail; the expansion will only be skipped if you provide `no` -parameters at all. - -Macros are expanded using Python string substitution rules. This can -especially cause confusion with shell snippets that use ``{`` as part -of their syntax. As described, if a macro has `no` parameters, no -expansion will be performed and thus it is correct to write the script -with no escaping, e.g.:: - - - builder: - name: a_builder - builders: - - shell: | - VARIABLE=${VARIABLE:-bar} - function foo { - echo "my shell function" - } - -However, if the macro `has` parameters, you must escape the ``{`` you -wish to make it through to the output, e.g.:: +Macros are expanded using Python string substitution rules. This can +especially cause confusion with shell snippets that use ``{`` and ``}`` as part +of their syntax. You must escape curly braces you wish to make it through +to the output, e.g.:: - builder: name: a_builder @@ -422,10 +395,11 @@ wish to make it through to the output, e.g.:: echo "my shell function" }} -Note that a ``job-template`` will have parameters by definition (at -least a ``name``). Thus embedded-shell within a ``job-template`` should -always use ``{{`` to achieve a literal ``{``. A generic builder will need -to consider the correct quoting based on its use of parameters. +The same apply for ``job`` and ``job-template``. Thus embedded-shell within +a ``job`` or ``job-template`` should always use ``{{`` to achieve a literal +``{``. + +You can also use ``!j2:`` tag instead - Jinja2 uses double quotes for variables. .. _folders: diff --git a/jenkins_jobs/defaults.py b/jenkins_jobs/defaults.py index 33ba1a482..913fbc0d7 100644 --- a/jenkins_jobs/defaults.py +++ b/jenkins_jobs/defaults.py @@ -176,7 +176,7 @@ class Defaults: contents: dict # Values that go to job contents. @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") contents, params = split_contents_params( diff --git a/jenkins_jobs/dimensions.py b/jenkins_jobs/dimensions.py index 45be299fa..08425addb 100644 --- a/jenkins_jobs/dimensions.py +++ b/jenkins_jobs/dimensions.py @@ -15,7 +15,7 @@ from collections import namedtuple from .errors import Context, JenkinsJobsException from .loc_loader import LocList, LocDict -from jenkins_jobs.expander import Expander +from jenkins_jobs.expander import YamlObjectsExpander Dimension = namedtuple("Dimension", "axis params") @@ -48,7 +48,7 @@ def _decode_axis_value(axis, value, key_pos, value_pos): def enum_dimensions_params(axes, params, defaults): - expander = Expander() + expander = YamlObjectsExpander() if not axes: # No axes - instantiate one job/view. yield {} @@ -60,9 +60,10 @@ def enum_dimensions_params(axes, params, defaults): except KeyError: try: value = defaults[axis] + key_pos = value_pos = None except KeyError: continue # May be, value would be received from an another axis values. - expanded_value = expander.expand(value, params) + expanded_value = expander.expand(value, params, key_pos, value_pos) value = [ Dimension(axis, params) for params in _decode_axis_value(axis, expanded_value, key_pos, value_pos) diff --git a/jenkins_jobs/expander.py b/jenkins_jobs/expander.py index f0420f3bc..a0143591e 100644 --- a/jenkins_jobs/expander.py +++ b/jenkins_jobs/expander.py @@ -55,9 +55,8 @@ def expand_tuple(expander, obj, params, key_pos, value_pos): class StrExpander: - def __init__(self, config): - allow_empty = config.yamlparser["allow_empty_variables"] - self._formatter = CustomFormatter(allow_empty) + def __init__(self, allow_empty_variables): + self._formatter = CustomFormatter(allow_empty_variables) def __call__(self, obj, params, key_pos, value_pos): try: @@ -79,14 +78,14 @@ def call_expand(expander, obj, params, key_pos, value_pos): return obj.expand(expander, params) -def call_subst(expander, obj, params, key_pos, value_pos): - return obj.subst(expander, params) - - def dont_expand(obj, params, key_pos, value_pos): return obj +def dont_expand_yaml_object(expander, obj, params, key_pos, value_pos): + return obj + + yaml_classes_list = [ J2String, J2Yaml, @@ -104,9 +103,13 @@ deprecated_yaml_tags = [ ] -# Does not expand string formats. Used in jobs and macros without parameters. +# Expand strings and yaml objects. class Expander: def __init__(self, config=None): + if config: + allow_empty_variables = config.yamlparser["allow_empty_variables"] + else: + allow_empty_variables = False _yaml_object_expanders = { cls: partial(call_expand, self) for cls in yaml_classes_list } @@ -116,8 +119,8 @@ class Expander: list: partial(expand_list, self), LocList: partial(expand_list, self), tuple: partial(expand_tuple, self), - str: dont_expand, - LocString: dont_expand, + str: StrExpander(allow_empty_variables), + LocString: StrExpander(allow_empty_variables), bool: dont_expand, int: dont_expand, float: dont_expand, @@ -136,20 +139,26 @@ class Expander: return expander(obj, params, key_pos, value_pos) -# Expands string formats also. Used in jobs templates and macros with parameters. -class ParamsExpander(Expander): +# Expand only yaml objects. +class YamlObjectsExpander(Expander): + def __init__(self): + super().__init__() + self.expanders.update( + { + str: dont_expand, + LocString: dont_expand, + } + ) + + +# Expand only string parameters. +class StringsOnlyExpander(Expander): def __init__(self, config): super().__init__(config) _yaml_object_expanders = { - cls: partial(call_subst, self) for cls in yaml_classes_list + cls: partial(dont_expand_yaml_object, self) for cls in yaml_classes_list } - self.expanders.update( - { - str: StrExpander(config), - LocString: StrExpander(config), - **_yaml_object_expanders, - } - ) + self.expanders.update(_yaml_object_expanders) def call_required_params(obj, pos): diff --git a/jenkins_jobs/job.py b/jenkins_jobs/job.py index 56dd0f434..903dc97fc 100644 --- a/jenkins_jobs/job.py +++ b/jenkins_jobs/job.py @@ -13,6 +13,7 @@ from dataclasses import dataclass from .errors import JenkinsJobsException +from .expander import Expander from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .defaults import split_contents_params, job_contents_keys @@ -23,7 +24,7 @@ class JobBase(RootBase): folder: str @classmethod - def from_dict(cls, config, roots, expander, data, pos): + def from_dict(cls, config, roots, data, pos): keep_descriptions = config.yamlparser["keep_descriptions"] d = data.copy() name = d.pop_required_loc_string("name") @@ -35,7 +36,7 @@ class JobBase(RootBase): contents, params = split_contents_params(d, job_contents_keys) return cls( roots.defaults, - expander, + Expander(config), keep_descriptions, id, name, @@ -72,8 +73,8 @@ class JobBase(RootBase): class Job(JobBase, NonTemplateRootMixin): @classmethod - def add(cls, config, roots, expander, param_expander, data, pos): - job = cls.from_dict(config, roots, expander, data, pos) + def add(cls, config, roots, data, pos): + job = cls.from_dict(config, roots, data, pos) roots.assign(roots.jobs, job.id, job, "job") def __str__(self): @@ -82,8 +83,8 @@ class Job(JobBase, NonTemplateRootMixin): class JobTemplate(JobBase, TemplateRootMixin): @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): - template = cls.from_dict(config, roots, params_expander, data, pos) + def add(cls, config, roots, data, pos): + template = cls.from_dict(config, roots, data, pos) roots.assign(roots.job_templates, template.id, template, "job template") def __str__(self): @@ -96,7 +97,7 @@ class JobGroup(Group): _job_templates: dict @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") try: diff --git a/jenkins_jobs/loader.py b/jenkins_jobs/loader.py index c4ba948a3..6e9b6b1ce 100644 --- a/jenkins_jobs/loader.py +++ b/jenkins_jobs/loader.py @@ -17,7 +17,7 @@ from functools import partial from .errors import JenkinsJobsException from .loc_loader import LocLoader from .yaml_objects import BaseYamlObject -from .expander import Expander, ParamsExpander, deprecated_yaml_tags, yaml_classes_list +from .expander import Expander, deprecated_yaml_tags, yaml_classes_list from .roots import root_adders logger = logging.getLogger(__name__) @@ -115,7 +115,6 @@ def enum_expanded_paths(path_list): def load_files(config, roots, path_list): expander = Expander(config) - params_expander = ParamsExpander(config) loader = Loader.empty(config) for path in enum_expanded_paths(path_list): if is_stdin(path): @@ -155,4 +154,4 @@ def load_files(config, roots, path_list): f" known are: {','.join(root_adders)}.", pos=item.pos, ) - adder(config, roots, expander, params_expander, contents, item.pos) + adder(config, roots, contents, item.pos) diff --git a/jenkins_jobs/macro.py b/jenkins_jobs/macro.py index c6be59f92..e13d930bc 100644 --- a/jenkins_jobs/macro.py +++ b/jenkins_jobs/macro.py @@ -45,8 +45,6 @@ class Macro: elements_name, config, roots, - expander, - params_expander, data, pos, ): diff --git a/jenkins_jobs/project.py b/jenkins_jobs/project.py index c737eb2e5..165c4ef8d 100644 --- a/jenkins_jobs/project.py +++ b/jenkins_jobs/project.py @@ -33,7 +33,7 @@ class Project(GroupBase): params: dict @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") defaults = d.pop_loc_string("defaults", None) diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py index e9935e590..87194b055 100644 --- a/jenkins_jobs/registry.py +++ b/jenkins_jobs/registry.py @@ -27,7 +27,7 @@ from six import PY2 from jenkins.plugins import PluginVersion from jenkins_jobs.errors import JenkinsJobsException -from jenkins_jobs.expander import Expander, ParamsExpander +from jenkins_jobs.expander import Expander, StringsOnlyExpander from jenkins_jobs.yaml_objects import BaseYamlObject __all__ = ["ModuleRegistry"] @@ -49,7 +49,7 @@ class ModuleRegistry(object): self.masked_warned = {} self._macros = {} self._expander = Expander(jjb_config) - self._params_expander = ParamsExpander(jjb_config) + self._str_expander = StringsOnlyExpander(jjb_config) if plugins_list is None: self._plugin_version = {} @@ -290,20 +290,17 @@ class ModuleRegistry(object): "component type that is masking an inbuilt " "definition" % (name, component_type) ) - # Expand macro strings only if at least one macro parameter is provided. - if component_data: - expander = self._params_expander - else: - expander = self._expander - component_data = {} # It may be None. + if component_data is None: + component_data = {} expander_params = {**component_data, **(job_data or {})} elements = macro.elements if isinstance(elements, BaseYamlObject): - # Expand !j2-yaml element right below macro body. - elements = elements.expand(expander, expander_params) + # Expand !j2-yaml tag if it is right below macro body. + # But do not expand yaml tags inside it - they will be expanded later. + elements = elements.expand(self._str_expander, expander_params) for b in elements: try: - element = expander.expand(b, expander_params) + element = self._expander.expand(b, expander_params) except JenkinsJobsException as x: raise x.with_context( f"While expanding macro {name!r}", diff --git a/jenkins_jobs/root_base.py b/jenkins_jobs/root_base.py index 4ae8593b9..c5069232b 100644 --- a/jenkins_jobs/root_base.py +++ b/jenkins_jobs/root_base.py @@ -103,12 +103,16 @@ class NonTemplateRootMixin: def top_level_generate_items(self): try: defaults = self._pick_defaults(self.defaults_name, merge_global=False) + item_params = LocDict.merge( + defaults.params, + self.params, + ) contents = LocDict.merge( defaults.contents, self.contents, pos=self.pos, ) - expanded_contents = self._expand_contents(contents, self.params) + expanded_contents = self._expand_contents(contents, item_params) context = [Context(f"In {self}", self.pos)] yield JobViewData(expanded_contents, context) except JenkinsJobsException as x: diff --git a/jenkins_jobs/view.py b/jenkins_jobs/view.py index 4ba9303aa..8964b0852 100644 --- a/jenkins_jobs/view.py +++ b/jenkins_jobs/view.py @@ -14,6 +14,7 @@ from dataclasses import dataclass from .errors import JenkinsJobsException from .loc_loader import LocDict +from .expander import Expander from .root_base import RootBase, NonTemplateRootMixin, TemplateRootMixin, Group from .defaults import split_contents_params, view_contents_keys @@ -23,7 +24,7 @@ class ViewBase(RootBase): view_type: str @classmethod - def from_dict(cls, config, roots, expander, data, pos): + def from_dict(cls, config, roots, data, pos): keep_descriptions = config.yamlparser["keep_descriptions"] d = data.copy() name = d.pop_required_loc_string("name") @@ -34,7 +35,7 @@ class ViewBase(RootBase): contents, params = split_contents_params(d, view_contents_keys) return cls( roots.defaults, - expander, + Expander(config), keep_descriptions, id, name, @@ -59,8 +60,8 @@ class ViewBase(RootBase): class View(ViewBase, NonTemplateRootMixin): @classmethod - def add(cls, config, roots, expander, param_expander, data, pos): - view = cls.from_dict(config, roots, expander, data, pos) + def add(cls, config, roots, data, pos): + view = cls.from_dict(config, roots, data, pos) roots.assign(roots.views, view.id, view, "view") def __str__(self): @@ -69,8 +70,8 @@ class View(ViewBase, NonTemplateRootMixin): class ViewTemplate(ViewBase, TemplateRootMixin): @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): - template = cls.from_dict(config, roots, params_expander, data, pos) + def add(cls, config, roots, data, pos): + template = cls.from_dict(config, roots, data, pos) roots.assign(roots.view_templates, template.id, template, "view template") def __str__(self): @@ -83,7 +84,7 @@ class ViewGroup(Group): _view_templates: dict @classmethod - def add(cls, config, roots, expander, params_expander, data, pos): + def add(cls, config, roots, data, pos): d = data.copy() name = d.pop_required_loc_string("name") try: diff --git a/jenkins_jobs/yaml_objects.py b/jenkins_jobs/yaml_objects.py index 04bd1363f..883461a62 100644 --- a/jenkins_jobs/yaml_objects.py +++ b/jenkins_jobs/yaml_objects.py @@ -83,40 +83,9 @@ filenames to be opened as one or more data blob, which should be read into the calling yaml construct without any further parsing. Any data in a file included through this tag, will be treated as string data. -Examples: - - .. literalinclude:: /../../tests/loader/fixtures/include-raw001-job.yaml - - contents of include-raw001-hello-world.sh: - - .. literalinclude:: - /../../tests/loader/fixtures/include-raw001-hello-world.sh - - contents of include-raw001-vars.sh: - - .. literalinclude:: - /../../tests/loader/fixtures/include-raw001-vars.sh - - using a list of files: - - .. literalinclude:: - /../../tests/loader/fixtures/include-raw-multi001.yaml - -The tag ``!include-raw-escape:`` treats the given string or list of strings as -filenames to be opened as one or more data blobs, which should be escaped -before being read in as string data. This allows job-templates to use this tag -to include scripts from files without needing to escape braces in the original -file. - -.. warning:: - - When used as a macro ``!include-raw-escape:`` should only be used if - parameters are passed into the escaped file and you would like to escape - those parameters. If the file does not have any jjb parameters passed into - it then ``!include-raw:`` should be used instead otherwise you will run - into an interesting issue where ``include-raw-escape:`` actually adds - additional curly braces around existing curly braces. For example - ${PROJECT} becomes ${{PROJECT}} which may break bash scripts. +It will expand variables inside the file. If your file contains curly braces, +you should double them. Or, you can use tag ``!include-raw-escape``, which +does not substitute variables. Examples: @@ -259,12 +228,8 @@ class BaseYamlObject(metaclass=abc.ABCMeta): @abc.abstractmethod def expand(self, expander, params): - """Expand object but do not substitute template parameters""" - pass - - def subst(self, expander, params): """Expand object and substitute template parameters""" - return self.expand(expander, params) + pass def _find_file(self, rel_path, pos): search_path = self._search_path @@ -286,10 +251,6 @@ class BaseYamlObject(metaclass=abc.ABCMeta): for idx, path in enumerate(path_list): yield self._expand_path(path, path_list.value_pos[idx], *args) - def _subst_path_list(self, path_list, *args): - for idx, path in enumerate(path_list): - yield self._subst_path(path, path_list.value_pos[idx], *args) - class J2BaseYamlObject(BaseYamlObject): def __init__(self, jjb_config, loader, pos): @@ -438,19 +399,11 @@ class IncludeRawBase(IncludeBaseObject): def expand(self, expander, params): return "\n".join(self._expand_path_list(self._path_list, params)) - def subst(self, expander, params): - return "\n".join(self._subst_path_list(self._path_list, params)) - class IncludeRaw(IncludeRawBase): yaml_tag = "!include-raw:" def _expand_path(self, rel_path_template, pos, params): - rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path, pos) - return full_path.read_text() - - def _subst_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) full_path = self._find_file(rel_path, pos) template = full_path.read_text() @@ -464,14 +417,6 @@ class IncludeRawEscape(IncludeRawBase): yaml_tag = "!include-raw-escape:" def _expand_path(self, rel_path_template, pos, params): - rel_path = self._formatter.format(rel_path_template, **params) - full_path = self._find_file(rel_path, pos) - text = full_path.read_text() - # Backward compatibility: - # if used inside job or macro without parameters, curly braces are duplicated. - return text.replace("{", "{{").replace("}", "}}") - - def _subst_path(self, rel_path_template, pos, params): rel_path = self._formatter.format(rel_path_template, **params) full_path = self._find_file(rel_path, pos) return full_path.read_text() diff --git a/tests/loader/fixtures/deprecated-include-raw001.yaml b/tests/loader/fixtures/deprecated-include-raw001.yaml index ba2f8ef0c..972816c2e 100644 --- a/tests/loader/fixtures/deprecated-include-raw001.yaml +++ b/tests/loader/fixtures/deprecated-include-raw001.yaml @@ -2,6 +2,6 @@ name: test-job-include-raw-1 builders: - shell: - !include-raw include-raw001-hello-world.sh + !include-raw-escape include-raw001-hello-world.sh - shell: - !include-raw include-raw001-vars.sh + !include-raw-escape include-raw001-vars.sh diff --git a/tests/loader/fixtures/include-raw-escaped001-job.json b/tests/loader/fixtures/include-raw-escaped001-job.json index 21af3e732..6a210d138 100644 --- a/tests/loader/fixtures/include-raw-escaped001-job.json +++ b/tests/loader/fixtures/include-raw-escaped001-job.json @@ -5,7 +5,7 @@ "shell": "#!/bin/bash\n#\n# Sample script showing how the yaml include-raw tag can be used\n# to inline scripts that are maintained outside of the jenkins\n# job yaml configuration.\n\necho \"hello world\"\n\nexit 0\n" }, { - "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${{VAR1}} ${{VAR2}}\"\n\n[[ -n \"${{VAR3}}\" ]] && {{\n # this next section is executed as one\n echo \"${{VAR3}}\"\n exit 0\n}}\n" + "shell": "#!/bin/bash\n#\n# sample script to check that brackets aren't escaped\n# when using the include-raw application yaml tag\n\nVAR1=\"hello\"\nVAR2=\"world\"\nVAR3=\"${VAR1} ${VAR2}\"\n\n[[ -n \"${VAR3}\" ]] && {\n # this next section is executed as one\n echo \"${VAR3}\"\n exit 0\n}\n" } ], "description": "", diff --git a/tests/loader/fixtures/include-raw-escaped001-job.xml b/tests/loader/fixtures/include-raw-escaped001-job.xml index 552942ff0..9a48dc3d9 100644 --- a/tests/loader/fixtures/include-raw-escaped001-job.xml +++ b/tests/loader/fixtures/include-raw-escaped001-job.xml @@ -30,13 +30,13 @@ exit 0 VAR1="hello" VAR2="world" -VAR3="${{VAR1}} ${{VAR2}}" +VAR3="${VAR1} ${VAR2}" -[[ -n "${{VAR3}}" ]] && {{ +[[ -n "${VAR3}" ]] && { # this next section is executed as one - echo "${{VAR3}}" + echo "${VAR3}" exit 0 -}} +} diff --git a/tests/loader/fixtures/include-raw-escaped001-job.yaml b/tests/loader/fixtures/include-raw-escaped001-job.yaml index b9d51c020..4cfcc9c7d 100644 --- a/tests/loader/fixtures/include-raw-escaped001-job.yaml +++ b/tests/loader/fixtures/include-raw-escaped001-job.yaml @@ -1,4 +1,4 @@ -# Using include-raw-excape inside job cause double braces in included file, like: {{VAR1}}. +# Using include-raw-escape inside job works the same way as inside job template. - job: name: test-job-include-raw builders: diff --git a/tests/loader/fixtures/include-raw-multi001.yaml b/tests/loader/fixtures/include-raw-multi001.yaml index 2f0f2af55..bf0927c57 100644 --- a/tests/loader/fixtures/include-raw-multi001.yaml +++ b/tests/loader/fixtures/include-raw-multi001.yaml @@ -2,6 +2,6 @@ name: test-job-include-raw-1 builders: - shell: - !include-raw: + !include-raw-escape: - include-raw001-hello-world.sh - include-raw001-vars.sh diff --git a/tests/loader/fixtures/include-raw001-job.yaml b/tests/loader/fixtures/include-raw001-job.yaml index dfa5fb71f..ab718448f 100644 --- a/tests/loader/fixtures/include-raw001-job.yaml +++ b/tests/loader/fixtures/include-raw001-job.yaml @@ -2,6 +2,6 @@ name: test-job-include-raw-1 builders: - shell: - !include-raw: include-raw001-hello-world.sh + !include-raw-escape: include-raw001-hello-world.sh - shell: - !include-raw: include-raw001-vars.sh + !include-raw-escape: include-raw001-vars.sh diff --git a/tests/yamlparser/error_fixtures/failure_formatting_params.yaml b/tests/yamlparser/error_fixtures/failure_formatting_params.yaml index ef387ac06..aa8f072d5 100644 --- a/tests/yamlparser/error_fixtures/failure_formatting_params.yaml +++ b/tests/yamlparser/error_fixtures/failure_formatting_params.yaml @@ -11,7 +11,7 @@ - xenial-{bdate} jobs: - 'template-requiring-param-{os}': - os: 'ubuntu-{flavour}' + os: 'ubuntu-{flavor}' - job-template: name: 'template-requiring-param-{os}' diff --git a/tests/yamlparser/error_fixtures/macro_with_null_params.error b/tests/yamlparser/error_fixtures/macro_with_null_params.error new file mode 100644 index 000000000..a93473c93 --- /dev/null +++ b/tests/yamlparser/error_fixtures/macro_with_null_params.error @@ -0,0 +1,18 @@ +macro_with_null_params.yaml:14:3: In project 'sample-project' + - project: + ^ +macro_with_null_params.yaml:17:9: Defined here + - sample-job + ^ +macro_with_null_params.yaml:8:3: In job template 'sample-job' + - job-template: + ^ +macro_with_null_params.yaml:12:9: While expanding builder macro call 'sample-macro' + - sample-macro: + ^ +macro_with_null_params.yaml:1:3: While expanding macro 'sample-macro' + - builder: + ^ +macro_with_null_params.yaml:6:17: While formatting string 'echo {hello}': Missing parameter: 'hello' + - shell: 'echo {hello}' + ^ diff --git a/tests/yamlparser/job_fixtures/macro_with_null_params.yaml b/tests/yamlparser/error_fixtures/macro_with_null_params.yaml similarity index 68% rename from tests/yamlparser/job_fixtures/macro_with_null_params.yaml rename to tests/yamlparser/error_fixtures/macro_with_null_params.yaml index 9988313a6..e97e028d3 100644 --- a/tests/yamlparser/job_fixtures/macro_with_null_params.yaml +++ b/tests/yamlparser/error_fixtures/macro_with_null_params.yaml @@ -2,13 +2,13 @@ name: sample-macro builders: # Add parameter to check if macro behaves the same way as if no params were provided. - # That is, it should not be expanded. + # That is, it should try to expand the parameter. - shell: 'echo {hello}' - job-template: name: sample-job builders: - # Place colon but define no parameters. + # Place colon but define no parameters (parameters are null instead of dict). - sample-macro: - project: diff --git a/tests/yamlparser/error_fixtures/recursive_parameter.error b/tests/yamlparser/error_fixtures/recursive_parameter.error index 9a931f709..7514a6221 100644 --- a/tests/yamlparser/error_fixtures/recursive_parameter.error +++ b/tests/yamlparser/error_fixtures/recursive_parameter.error @@ -5,13 +5,13 @@ recursive_parameter.yaml:7:3: In job template 'sample-job-{param_1}' - job-template: ^ recursive_parameter.yaml:5:9: Used by param_1 - - '{param_2}-at-project' + - '{param_2}-at-globals' ^ recursive_parameter.yaml:9:14: Used by param_2 param_2: '{param_3}-at-template' ^ recursive_parameter.yaml:13:14: Used by param_3 - param_3: '{param_4}-at-globals' + param_3: '{param_4}-at-project' ^ recursive_parameter.yaml:16:20: Used by param_4 param_4: '{param_1}-at-job-spec' @@ -20,5 +20,5 @@ recursive_parameter.yaml:3:5: While expanding 'param_1' param_1: ^ recursive_parameter.yaml:5:9: Recursive parameters usage: param_1 <- param_2 <- param_3 <- param_4 - - '{param_2}-at-project' + - '{param_2}-at-globals' ^ diff --git a/tests/yamlparser/error_fixtures/recursive_parameter.yaml b/tests/yamlparser/error_fixtures/recursive_parameter.yaml index 6596aadce..c32f7bdff 100644 --- a/tests/yamlparser/error_fixtures/recursive_parameter.yaml +++ b/tests/yamlparser/error_fixtures/recursive_parameter.yaml @@ -2,7 +2,7 @@ name: global param_1: - param_1_value_1 - - '{param_2}-at-project' + - '{param_2}-at-globals' - job-template: name: 'sample-job-{param_1}' @@ -10,7 +10,7 @@ - project: name: sample-project - param_3: '{param_4}-at-globals' + param_3: '{param_4}-at-project' jobs: - 'sample-job-{param_1}': param_4: '{param_1}-at-job-spec' diff --git a/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml b/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml index d52c8b2eb..fa68acc12 100644 --- a/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml +++ b/tests/yamlparser/job_fixtures/job-and-macro-expansions.yaml @@ -9,8 +9,8 @@ name: builder-without-params builders: - shell: | - echo Should not be expanded: {param} - - shell: !include-raw: job-and-macro-expansions.yaml.no-expand.inc + echo Should not be expanded: {{param}} + - shell: !include-raw-escape: job-and-macro-expansions.yaml.no-expand.inc - builder: name: builder-with-params @@ -27,8 +27,8 @@ display-name: sample-job builders: - shell: | - echo Should not be expanded: {param} - - shell: !include-raw: job-and-macro-expansions.yaml.no-expand.inc + echo Should not be expanded: {{param}} + - shell: !include-raw-escape: job-and-macro-expansions.yaml.no-expand.inc - job-template: name: sample-job-template diff --git a/tests/yamlparser/job_fixtures/job_honor_defaults.xml b/tests/yamlparser/job_fixtures/job_honor_defaults.xml index b9639469e..0e0b40b9c 100644 --- a/tests/yamlparser/job_fixtures/job_honor_defaults.xml +++ b/tests/yamlparser/job_fixtures/job_honor_defaults.xml @@ -11,7 +11,7 @@ - echo Build arch {arch}. + echo Build arch i386. diff --git a/tests/yamlparser/job_fixtures/job_honor_defaults.yaml b/tests/yamlparser/job_fixtures/job_honor_defaults.yaml index 70fe442b9..a414fc4e3 100644 --- a/tests/yamlparser/job_fixtures/job_honor_defaults.yaml +++ b/tests/yamlparser/job_fixtures/job_honor_defaults.yaml @@ -1,4 +1,4 @@ -# Variables in jobs are not expanded. +# Variables in jobs are expanded using defaults. - defaults: name: global diff --git a/tests/yamlparser/job_fixtures/macro_with_null_params.xml b/tests/yamlparser/job_fixtures/macro_with_null_params.xml deleted file mode 100644 index b6a46c944..000000000 --- a/tests/yamlparser/job_fixtures/macro_with_null_params.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - <!-- Managed by Jenkins Job Builder --> - false - false - false - false - true - - - - - echo {hello} - - - - - diff --git a/tests/yamlparser/job_fixtures/project_flow_template001.yaml b/tests/yamlparser/job_fixtures/project_flow_template001.yaml index 65316ec3b..7183de393 100644 --- a/tests/yamlparser/job_fixtures/project_flow_template001.yaml +++ b/tests/yamlparser/job_fixtures/project_flow_template001.yaml @@ -4,6 +4,6 @@ dsl: | build("job1") parallel ( - { build("job2a") }, - { build("job2b") } + {{ build("job2a") }}, + {{ build("job2b") }} ) diff --git a/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml b/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml index 2044da3bf..39f641728 100644 --- a/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml +++ b/tests/yamlparser/job_fixtures/project_pipeline_template001.yaml @@ -5,7 +5,7 @@ build job: "job1" parallel [ 2a: build job: "job2a", - 2b: node "dummynode" { + 2b: node "dummynode" {{ sh "echo I'm alive!" - } + }} ] diff --git a/tests/yamlparser/job_fixtures/project_workflow_template001.yaml b/tests/yamlparser/job_fixtures/project_workflow_template001.yaml index c0cc57089..f0a937a9d 100644 --- a/tests/yamlparser/job_fixtures/project_workflow_template001.yaml +++ b/tests/yamlparser/job_fixtures/project_workflow_template001.yaml @@ -5,7 +5,7 @@ build job: "job1" parallel [ 2a: build job: "job2a", - 2b: node "dummynode" { + 2b: node "dummynode" {{ sh "echo I'm alive!" - } + }} ] diff --git a/tests/yamlparser/job_fixtures/regression-2006254.yaml b/tests/yamlparser/job_fixtures/regression-2006254.yaml index ea928a67b..2bab04613 100644 --- a/tests/yamlparser/job_fixtures/regression-2006254.yaml +++ b/tests/yamlparser/job_fixtures/regression-2006254.yaml @@ -25,7 +25,7 @@ - timed: "H 14 * * *" builders: - - shell: !include-raw: regression-2006254.inc + - shell: !include-raw-escape: regression-2006254.inc parameters: - bool: diff --git a/tests/yamlparser/job_fixtures/regression-2010535.xml b/tests/yamlparser/job_fixtures/regression-2010535.xml index 9bee35216..69c4d3673 100644 --- a/tests/yamlparser/job_fixtures/regression-2010535.xml +++ b/tests/yamlparser/job_fixtures/regression-2010535.xml @@ -13,7 +13,7 @@ PARAM_1 - {default|my_default} + my_default false diff --git a/tests/yamlparser/job_fixtures/regression-2010535.yaml b/tests/yamlparser/job_fixtures/regression-2010535.yaml index 3694578b1..ebb65884b 100644 --- a/tests/yamlparser/job_fixtures/regression-2010535.yaml +++ b/tests/yamlparser/job_fixtures/regression-2010535.yaml @@ -1,5 +1,5 @@ # https://storyboard.openstack.org/#!/story/2010535 -# Bug: JJB doesn't expand macro in case of usage without arguments +# Fixed: Bug: JJB doesn't expand macro in case of usage without arguments # String templates in macro calls without parameters are NOT expanded. # Jinja2 templates in macro calls without parameters ARE expanded. diff --git a/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml b/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml index 423ad258a..497702d23 100644 --- a/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml +++ b/tests/yamlparser/job_fixtures/trigger_parameterized_builds/parameter-override-ordering-001.yaml @@ -12,7 +12,7 @@ publishers: - trigger-parameterized-builds: - project: first_job - predefined-parameters: BUILD_NUM=${BUILD_NUMBER} + predefined-parameters: BUILD_NUM=${{BUILD_NUMBER}} property-file: default_version.prop current-parameters: true - project: second_job @@ -25,7 +25,7 @@ publishers: - trigger-parameterized-builds: - project: 1.2_first_job - predefined-parameters: BUILD_NUM=${BUILD_NUMBER} + predefined-parameters: BUILD_NUM=${{BUILD_NUMBER}} current-parameters: true property-file: version.prop - project: 1.2_second_job