diff --git a/doc/source/changelog.rst b/doc/source/changelog.rst index 7305b28b5..400fd1ae8 100644 --- a/doc/source/changelog.rst +++ b/doc/source/changelog.rst @@ -9,6 +9,11 @@ Features added * Update TAP publisher plugin (by Kienan Stewart). * Add tags: ``!include-raw-verbatim:`` and ``!include-raw-expand:``. Tags ``!include-raw:`` and ``!include-raw-escape:`` are now deprecated. +* Macros can now use parameters specified in defaults, the same way as job-templates. + See this examples: + + * `macro-uses-global-defaults.yaml `_ + * `macro-uses-custom-defaults.yaml `_ .. note:: After moving to 6.1.0 release, to remove deprecation warnings make these adjustments to your JJB sources: diff --git a/jenkins_jobs/macro.py b/jenkins_jobs/macro.py index e13d930bc..aab1d1111 100644 --- a/jenkins_jobs/macro.py +++ b/jenkins_jobs/macro.py @@ -13,6 +13,10 @@ from dataclasses import dataclass from functools import partial +from .root_base import ElementBase +from .expander import Expander, StringsOnlyExpander +from .yaml_objects import BaseYamlObject +from .loc_loader import LocDict from .errors import JenkinsJobsException from .position import Pos @@ -33,8 +37,12 @@ macro_specs = [ @dataclass -class Macro: +class Macro(ElementBase): + _expander: Expander + _str_expander: StringsOnlyExpander + _type_name: str name: str + defaults_name: str pos: Pos elements: list @@ -50,16 +58,61 @@ class Macro: ): d = data.copy() name = d.pop_required_loc_string("name") + defaults = d.pop_loc_string("defaults", "global") elements = d.pop_required_element(elements_name) + expander = Expander(config) + str_expander = StringsOnlyExpander(config) if d: example_key = next(iter(d.keys())) raise JenkinsJobsException( f"In {type_name} macro {name!r}: unexpected elements: {','.join(d.keys())}", pos=data.key_pos.get(example_key), ) - macro = cls(name, pos, elements or []) + macro = cls( + roots.defaults, + expander, + str_expander, + type_name, + name, + defaults, + pos, + elements or [], + ) roots.assign(roots.macros[type_name], name, macro, "macro") + def __str__(self): + return f"macro {self.name!r}" + + def dispatch_elements(self, registry, xml_parent, component_data, job_data, params): + defaults = self._pick_defaults(self.defaults_name) + full_params = LocDict.merge( + defaults.params, + params, + ) + element_list = self.elements + if isinstance(element_list, BaseYamlObject): + # Expand !j2-yaml tag if it is right below macro body. + # But do not expand yaml tags inside it - they will be expanded later. + element_list = element_list.expand(self._str_expander, full_params) + for element in element_list: + try: + expanded_element = self._expander.expand(element, full_params) + except JenkinsJobsException as x: + raise x.with_context( + f"While expanding {self}", + pos=self.pos, + ) + # Pass component_data in as template data to this function + # so that if the macro is invoked with arguments, + # the arguments are interpolated into the real defn. + registry.dispatch( + self._type_name, + xml_parent, + expanded_element, + component_data, + job_data=job_data, + ) + macro_adders = { macro_type: partial(Macro.add, macro_type, elements_name) diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py index 2d4eaf363..963fa9332 100644 --- a/jenkins_jobs/registry.py +++ b/jenkins_jobs/registry.py @@ -27,8 +27,6 @@ from six import PY2 from jenkins.plugins import PluginVersion from jenkins_jobs.errors import JenkinsJobsException -from jenkins_jobs.expander import Expander, StringsOnlyExpander -from jenkins_jobs.yaml_objects import BaseYamlObject __all__ = ["ModuleRegistry"] @@ -48,8 +46,6 @@ class ModuleRegistry(object): self.jjb_config = jjb_config self.masked_warned = {} self._macros = {} - self._expander = Expander(jjb_config) - self._str_expander = StringsOnlyExpander(jjb_config) if plugins_list is None: self._plugin_version = {} @@ -290,30 +286,8 @@ class ModuleRegistry(object): ) if component_data is None: component_data = {} - expander_params = {**component_data, **(job_data or {})} - elements = macro.elements - if isinstance(elements, BaseYamlObject): - # Expand !j2-yaml tag if it is right below macro body. - # But do not expand yaml tags inside it - they will be expanded later. - elements = elements.expand(self._str_expander, expander_params) - for b in elements: - try: - element = self._expander.expand(b, expander_params) - except JenkinsJobsException as x: - raise x.with_context( - f"While expanding macro {name!r}", - pos=macro.pos, - ) - # Pass component_data in as template data to this function - # so that if the macro is invoked with arguments, - # the arguments are interpolated into the real defn. - self.dispatch( - component_type, - xml_parent, - element, - component_data, - job_data=job_data, - ) + params = {**component_data, **(job_data or {})} + macro.dispatch_elements(self, xml_parent, component_data, job_data, params) def _load_eps(self, component_list_type, component_type, entry_point, name): logging.debug("Caching entrypoints for %s" % component_list_type) diff --git a/jenkins_jobs/root_base.py b/jenkins_jobs/root_base.py index b7a897d1b..7ce005756 100644 --- a/jenkins_jobs/root_base.py +++ b/jenkins_jobs/root_base.py @@ -40,10 +40,34 @@ class JobViewData: @dataclass -class RootBase: - """Base class for YAML root elements - job, view or template""" +class ElementBase: + """Base class for YAML elements - job, view, template, or macro""" _defaults: dict + + @property + def title(self): + return str(self).capitalize() + + def _pick_defaults(self, name): + try: + defaults = self._defaults[name] + except KeyError: + if name == "global": + return Defaults.empty() + raise JenkinsJobsException( + f"{self.title} wants defaults {name!r}, but it was never defined", + pos=name.pos, + ) + if name == "global": + return defaults + return defaults.merged_with_global(self._pick_defaults("global")) + + +@dataclass +class RootBase(ElementBase): + """Base class for YAML root elements - job, view or template""" + _expander: Expander _keep_descriptions: bool _id: str @@ -61,10 +85,6 @@ class RootBase: else: return self.name - @property - def title(self): - return str(self).capitalize() - @property def contents(self): contents = self._contents.copy() @@ -80,21 +100,6 @@ class RootBase: expanded_contents["description"] = amended_description return expanded_contents - def _pick_defaults(self, name): - try: - defaults = self._defaults[name] - except KeyError: - if name == "global": - return Defaults.empty() - raise JenkinsJobsException( - f"{self.title} wants defaults {self.defaults_name!r}" - " but it was never defined", - pos=name.pos, - ) - if name == "global": - return defaults - return defaults.merged_with_global(self._pick_defaults("global")) - class NonTemplateRootMixin: def top_level_generate_items(self): diff --git a/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.error b/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.error deleted file mode 100644 index 4d13a5177..000000000 --- a/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.error +++ /dev/null @@ -1,18 +0,0 @@ -macro-uses-custom-defaults.yaml:34:3: In project 'sample-project' - - project: - ^ -macro-uses-custom-defaults.yaml:37:7: Defined here - - sample-job-template - ^ -macro-uses-custom-defaults.yaml:26:3: In job template 'sample-job-template' - - job-template: - ^ -macro-uses-custom-defaults.yaml:30:7: While expanding builder macro call 'builder-without-params' - - builder-without-params - ^ -macro-uses-custom-defaults.yaml:7:3: While expanding macro 'builder-without-params' - - builder: - ^ -macro-uses-custom-defaults.yaml:12:9: While formatting string 'echo "builder-without-params: Fails when trying to expand: {global_param}"\n...': Missing parameter: 'global_param' - echo "builder-without-params: Fails when tryi ... - ^ diff --git a/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml.inc b/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml.inc deleted file mode 100644 index 7c187717d..000000000 --- a/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml.inc +++ /dev/null @@ -1 +0,0 @@ -echo "include-raw-expand: Fails when trying to expand: {global_param}" diff --git a/tests/yamlparser/error_fixtures/macro-uses-global-defaults.error b/tests/yamlparser/error_fixtures/macro-uses-global-defaults.error deleted file mode 100644 index 771fa729e..000000000 --- a/tests/yamlparser/error_fixtures/macro-uses-global-defaults.error +++ /dev/null @@ -1,18 +0,0 @@ -macro-uses-global-defaults.yaml:32:3: In project 'sample-project' - - project: - ^ -macro-uses-global-defaults.yaml:35:7: Defined here - - sample-job-template - ^ -macro-uses-global-defaults.yaml:24:3: In job template 'sample-job-template' - - job-template: - ^ -macro-uses-global-defaults.yaml:28:7: While expanding builder macro call 'builder-without-params' - - builder-without-params - ^ -macro-uses-global-defaults.yaml:7:3: While expanding macro 'builder-without-params' - - builder: - ^ -macro-uses-global-defaults.yaml:11:9: While formatting string 'echo "builder-without-params: Fails when trying to expand: {global_param}"\n...': Missing parameter: 'global_param' - echo "builder-without-params: Fails when tryi ... - ^ diff --git a/tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml.inc b/tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml.inc deleted file mode 100644 index 7c187717d..000000000 --- a/tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml.inc +++ /dev/null @@ -1 +0,0 @@ -echo "include-raw-expand: Fails when trying to expand: {global_param}" diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_job.error b/tests/yamlparser/error_fixtures/missing_defaults_at_job.error index cd5436ab2..55d519abd 100644 --- a/tests/yamlparser/error_fixtures/missing_defaults_at_job.error +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_job.error @@ -1,6 +1,6 @@ missing_defaults_at_job.yaml:1:3: In job 'sample-job' - job: ^ -missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults' but it was never defined +missing_defaults_at_job.yaml:3:15: Job 'sample-job' wants defaults 'missing-defaults', but it was never defined defaults: missing-defaults ^ diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error b/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error index 2b56f7646..f639de4f1 100644 --- a/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_job_template.error @@ -4,6 +4,6 @@ missing_defaults_at_job_template.yaml:5:3: In project 'sample-project' missing_defaults_at_job_template.yaml:1:3: In job template 'sample-job' - job-template: ^ -missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults' but it was never defined +missing_defaults_at_job_template.yaml:3:15: Job template 'sample-job' wants defaults 'missing-defaults', but it was never defined defaults: missing-defaults ^ diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_macro.error b/tests/yamlparser/error_fixtures/missing_defaults_at_macro.error new file mode 100644 index 000000000..aa9fd9d0f --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_macro.error @@ -0,0 +1,15 @@ +missing_defaults_at_macro.yaml:12:3: In project 'sample-project' + - project: + ^ +missing_defaults_at_macro.yaml:15:7: Defined here + - sample-job + ^ +missing_defaults_at_macro.yaml:7:3: In job template 'sample-job' + - job-template: + ^ +missing_defaults_at_macro.yaml:10:7: While expanding builder macro call 'sample-builder' + - sample-builder + ^ +missing_defaults_at_macro.yaml:3:15: Macro 'sample-builder' wants defaults 'missing-defaults', but it was never defined + defaults: missing-defaults + ^ diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_macro.yaml b/tests/yamlparser/error_fixtures/missing_defaults_at_macro.yaml new file mode 100644 index 000000000..480137db5 --- /dev/null +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_macro.yaml @@ -0,0 +1,15 @@ +- builder: + name: sample-builder + defaults: missing-defaults + builders: + - shell: hello + +- job-template: + name: sample-job + builders: + - sample-builder + +- project: + name: sample-project + jobs: + - sample-job diff --git a/tests/yamlparser/error_fixtures/missing_defaults_at_project.error b/tests/yamlparser/error_fixtures/missing_defaults_at_project.error index 1d58bad72..f738edf56 100644 --- a/tests/yamlparser/error_fixtures/missing_defaults_at_project.error +++ b/tests/yamlparser/error_fixtures/missing_defaults_at_project.error @@ -4,6 +4,6 @@ missing_defaults_at_project.yaml:4:3: In project 'sample-project' missing_defaults_at_project.yaml:1:3: In job template 'sample-job' - job-template: ^ -missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'global' but it was never defined +missing_defaults_at_project.yaml:6:15: Job template 'sample-job' wants defaults 'missing-defaults', but it was never defined defaults: missing-defaults ^ diff --git a/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.xml b/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.xml new file mode 100644 index 000000000..c92c9391d --- /dev/null +++ b/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.xml @@ -0,0 +1,37 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + sample-job-template + false + false + false + true + + + + + echo "builder-without-params: Should be expanded: sample global param value" + + + + echo "include-raw-expand: Should be expanded: sample global param value" + + + + echo "builder-with-params: Should be expanded: sample macro param value" + + + + echo "builder-with-params: Should be expanded: sample global param value" + + + + echo "include-raw-expand: Should be expanded: sample global param value" + + + + + + diff --git a/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml b/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml similarity index 71% rename from tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml rename to tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml index 9b5a5b669..4db1a877d 100644 --- a/tests/yamlparser/error_fixtures/macro-uses-custom-defaults.yaml +++ b/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml @@ -1,4 +1,4 @@ -# Macros do not use custom defaults. +# Macros use custom defaults. - defaults: name: custom @@ -6,21 +6,20 @@ - builder: name: builder-without-params - # defaults: custom + defaults: custom builders: - shell: | - echo "builder-without-params: Fails when trying to expand: {global_param}" - # Also fails when trying to expand: + echo "builder-without-params: Should be expanded: {global_param}" - shell: !include-raw-expand: macro-uses-custom-defaults.yaml.inc - builder: name: builder-with-params - # defaults: custom + defaults: custom builders: - shell: | echo "builder-with-params: Should be expanded: {param}" - shell: | - echo "builder-with-params: Fails when trying to expand: {global_param}" + echo "builder-with-params: Should be expanded: {global_param}" - shell: !include-raw-expand: macro-uses-custom-defaults.yaml.inc - job-template: diff --git a/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml.inc b/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml.inc new file mode 100644 index 000000000..b16f88c4b --- /dev/null +++ b/tests/yamlparser/job_fixtures/macro-uses-custom-defaults.yaml.inc @@ -0,0 +1 @@ +echo "include-raw-expand: Should be expanded: {global_param}" diff --git a/tests/yamlparser/job_fixtures/macro-uses-global-defaults.xml b/tests/yamlparser/job_fixtures/macro-uses-global-defaults.xml new file mode 100644 index 000000000..c92c9391d --- /dev/null +++ b/tests/yamlparser/job_fixtures/macro-uses-global-defaults.xml @@ -0,0 +1,37 @@ + + + + <!-- Managed by Jenkins Job Builder --> + false + sample-job-template + false + false + false + true + + + + + echo "builder-without-params: Should be expanded: sample global param value" + + + + echo "include-raw-expand: Should be expanded: sample global param value" + + + + echo "builder-with-params: Should be expanded: sample macro param value" + + + + echo "builder-with-params: Should be expanded: sample global param value" + + + + echo "include-raw-expand: Should be expanded: sample global param value" + + + + + + diff --git a/tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml b/tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml similarity index 74% rename from tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml rename to tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml index e7d3fc5f8..a3b58fc11 100644 --- a/tests/yamlparser/error_fixtures/macro-uses-global-defaults.yaml +++ b/tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml @@ -1,4 +1,4 @@ -# Macros do not use global defaults. +# Macros use global defaults. - defaults: name: global @@ -8,8 +8,7 @@ name: builder-without-params builders: - shell: | - echo "builder-without-params: Fails when trying to expand: {global_param}" - # Also fails when trying to expand: + echo "builder-without-params: Should be expanded: {global_param}" - shell: !include-raw-expand: macro-uses-global-defaults.yaml.inc - builder: @@ -18,7 +17,7 @@ - shell: | echo "builder-with-params: Should be expanded: {param}" - shell: | - echo "builder-with-params: Fails when trying to expand: {global_param}" + echo "builder-with-params: Should be expanded: {global_param}" - shell: !include-raw-expand: macro-uses-global-defaults.yaml.inc - job-template: diff --git a/tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml.inc b/tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml.inc new file mode 100644 index 000000000..b16f88c4b --- /dev/null +++ b/tests/yamlparser/job_fixtures/macro-uses-global-defaults.yaml.inc @@ -0,0 +1 @@ +echo "include-raw-expand: Should be expanded: {global_param}"