summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorThanh Ha <thanh.ha@linuxfoundation.org>2017-10-25 10:38:29 -0400
committerWayne Warren <wayne.warren.s@gmail.com>2018-01-01 10:54:10 -0600
commiteddb40babdfd1acc16b923f2542cfe191d74bed7 (patch)
tree26798638bf65d787b77d26a7f883a8deb04fe134
parente3e8d161f31647dcb6099f9937bda13600fab55b (diff)
Revert "Move macro expansion into YamlParser."
Notes
Notes (review): Code-Review+2: Thanh Ha <zxiiro@gmail.com> Code-Review+2: Darragh Bailey <daragh.bailey@gmail.com> Workflow+1: Darragh Bailey <daragh.bailey@gmail.com> Verified+2: Zuul Submitted-by: Zuul Submitted-at: Fri, 05 Jan 2018 15:48:57 +0000 Reviewed-on: https://review.openstack.org/530419 Project: openstack-infra/jenkins-job-builder Branch: refs/heads/master
-rw-r--r--jenkins_jobs/local_yaml.py34
-rw-r--r--jenkins_jobs/parser.py14
-rw-r--r--jenkins_jobs/registry.py259
-rw-r--r--setup.cfg13
-rw-r--r--tests/xml_config/test_xml_config.py5
5 files changed, 40 insertions, 285 deletions
diff --git a/jenkins_jobs/local_yaml.py b/jenkins_jobs/local_yaml.py
index 49faf65..18b7802 100644
--- a/jenkins_jobs/local_yaml.py
+++ b/jenkins_jobs/local_yaml.py
@@ -143,7 +143,6 @@ Examples:
143 .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml.inc 143 .. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml.inc
144""" 144"""
145 145
146import copy
147import functools 146import functools
148import io 147import io
149import logging 148import logging
@@ -291,32 +290,6 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
291 def _escape(self, data): 290 def _escape(self, data):
292 return re.sub(r'({|})', r'\1\1', data) 291 return re.sub(r'({|})', r'\1\1', data)
293 292
294 def __deepcopy__(self, memo):
295 """
296 Make a deep copy of a LocalLoader excluding the uncopyable self.stream.
297
298 This is achieved by performing a shallow copy of self, setting the
299 stream attribute to None and then performing a deep copy of the shallow
300 copy.
301
302 (As this method will be called again on that deep copy, we also set a
303 sentinel attribute on the shallow copy to ensure that we don't recurse
304 infinitely.)
305 """
306 assert self.done, 'Unsafe to copy an in-progress loader'
307 if getattr(self, '_copy', False):
308 # This is a shallow copy for an in-progress deep copy, remove the
309 # _copy marker and return self
310 del self._copy
311 return self
312 # Make a shallow copy
313 shallow = copy.copy(self)
314 shallow.stream = None
315 shallow._copy = True
316 deep = copy.deepcopy(shallow, memo)
317 memo[id(self)] = deep
318 return deep
319
320 293
321class LocalDumper(OrderedRepresenter, yaml.Dumper): 294class LocalDumper(OrderedRepresenter, yaml.Dumper):
322 def __init__(self, *args, **kwargs): 295 def __init__(self, *args, **kwargs):
@@ -467,12 +440,11 @@ class CustomLoader(object):
467class Jinja2Loader(CustomLoader): 440class Jinja2Loader(CustomLoader):
468 """A loader for Jinja2-templated files.""" 441 """A loader for Jinja2-templated files."""
469 def __init__(self, contents): 442 def __init__(self, contents):
470 self._contents = contents 443 self._template = jinja2.Template(contents)
444 self._template.environment.undefined = jinja2.StrictUndefined
471 445
472 def format(self, **kwargs): 446 def format(self, **kwargs):
473 _template = jinja2.Template(self._contents) 447 return self._template.render(kwargs)
474 _template.environment.undefined = jinja2.StrictUndefined
475 return _template.render(kwargs)
476 448
477 449
478class CustomLoaderCollection(object): 450class CustomLoaderCollection(object):
diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py
index 1033b9f..b470466 100644
--- a/jenkins_jobs/parser.py
+++ b/jenkins_jobs/parser.py
@@ -25,7 +25,6 @@ import os
25from jenkins_jobs.constants import MAGIC_MANAGE_STRING 25from jenkins_jobs.constants import MAGIC_MANAGE_STRING
26from jenkins_jobs.errors import JenkinsJobsException 26from jenkins_jobs.errors import JenkinsJobsException
27from jenkins_jobs.formatter import deep_format 27from jenkins_jobs.formatter import deep_format
28from jenkins_jobs.registry import MacroRegistry
29import jenkins_jobs.local_yaml as local_yaml 28import jenkins_jobs.local_yaml as local_yaml
30from jenkins_jobs import utils 29from jenkins_jobs import utils
31 30
@@ -82,8 +81,6 @@ class YamlParser(object):
82 self.keep_desc = jjb_config.yamlparser['keep_descriptions'] 81 self.keep_desc = jjb_config.yamlparser['keep_descriptions']
83 self.path = jjb_config.yamlparser['include_path'] 82 self.path = jjb_config.yamlparser['include_path']
84 83
85 self._macro_registry = MacroRegistry()
86
87 def load_files(self, fn): 84 def load_files(self, fn):
88 85
89 # handle deprecated behavior, and check that it's not a file like 86 # handle deprecated behavior, and check that it's not a file like
@@ -221,11 +218,6 @@ class YamlParser(object):
221 job["description"] = description + \ 218 job["description"] = description + \
222 self._get_managed_string().lstrip() 219 self._get_managed_string().lstrip()
223 220
224 def _register_macros(self):
225 for component_type in self._macro_registry.component_types:
226 for macro in self.data.get(component_type, {}).values():
227 self._macro_registry.register(component_type, macro)
228
229 def _getfullname(self, data): 221 def _getfullname(self, data):
230 if 'folder' in data: 222 if 'folder' in data:
231 return "%s/%s" % (data['folder'], data['name']) 223 return "%s/%s" % (data['folder'], data['name'])
@@ -241,13 +233,10 @@ class YamlParser(object):
241 if module.handle_data(self.data): 233 if module.handle_data(self.data):
242 changed = True 234 changed = True
243 235
244 self._register_macros()
245 for default in self.data.get('defaults', {}).values():
246 self._macro_registry.expand_macros(default)
247 for job in self.data.get('job', {}).values(): 236 for job in self.data.get('job', {}).values():
248 self._macro_registry.expand_macros(job)
249 job = self._applyDefaults(job) 237 job = self._applyDefaults(job)
250 job['name'] = self._getfullname(job) 238 job['name'] = self._getfullname(job)
239
251 if jobs_glob and not matches(job['name'], jobs_glob): 240 if jobs_glob and not matches(job['name'], jobs_glob):
252 logger.debug("Ignoring job {0}".format(job['name'])) 241 logger.debug("Ignoring job {0}".format(job['name']))
253 continue 242 continue
@@ -410,7 +399,6 @@ class YamlParser(object):
410 raise 399 raise
411 expanded['name'] = self._getfullname(expanded) 400 expanded['name'] = self._getfullname(expanded)
412 401
413 self._macro_registry.expand_macros(expanded, params)
414 job_name = expanded.get('name') 402 job_name = expanded.get('name')
415 if jobs_glob and not matches(job_name, jobs_glob): 403 if jobs_glob and not matches(job_name, jobs_glob):
416 continue 404 continue
diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py
index 9321ad1..a107078 100644
--- a/jenkins_jobs/registry.py
+++ b/jenkins_jobs/registry.py
@@ -15,7 +15,6 @@
15 15
16# Manage Jenkins plugin module registry. 16# Manage Jenkins plugin module registry.
17 17
18import copy
19import logging 18import logging
20import operator 19import operator
21import pkg_resources 20import pkg_resources
@@ -32,223 +31,6 @@ __all__ = [
32logger = logging.getLogger(__name__) 31logger = logging.getLogger(__name__)
33 32
34 33
35class MacroRegistry(object):
36
37 _component_to_component_list_mapping = {}
38 _component_list_to_component_mapping = {}
39 _macros_by_component_type = {}
40 _macros_by_component_list_type = {}
41
42 def __init__(self):
43
44 for entrypoint in pkg_resources.iter_entry_points(
45 group='jenkins_jobs.macros'):
46 Mod = entrypoint.load()
47 self._component_list_to_component_mapping[
48 Mod.component_list_type] = Mod.component_type
49 self._component_to_component_list_mapping[
50 Mod.component_type] = Mod.component_list_type
51 self._macros_by_component_type[
52 Mod.component_type] = {}
53 self._macros_by_component_list_type[
54 Mod.component_list_type] = {}
55
56 self._mask_warned = {}
57
58 @property
59 def _nonempty_component_list_types(self):
60 return [clt for clt in self._macros_by_component_list_type
61 if len(self._macros_by_component_list_type[clt]) != 0]
62
63 @property
64 def component_types(self):
65 return self._macros_by_component_type.keys()
66
67 def _is_macro(self, component_name, component_list_type):
68 return (component_name in
69 self._macros_by_component_list_type[component_list_type])
70
71 def register(self, component_type, macro):
72 macro_name = macro["name"]
73 clt = self._component_to_component_list_mapping[component_type]
74 self._macros_by_component_type[component_type][macro_name] = macro
75 self._macros_by_component_list_type[clt][macro_name] = macro
76
77 def expand_macros(self, jobish, template_data=None):
78 """Create a copy of the given job-like thing, expand macros in place on
79 the copy, and return that object to calling context.
80
81 :arg dict jobish: A job-like JJB data structure. Could be anything that
82 might provide JJB "components" that get expanded to XML configuration.
83 This includes "job", "job-template", and "default" DSL items. This
84 argument is not modified in place, but rather copied so that the copy
85 may be returned to calling context.
86
87 :arg dict template_data: If jobish is a job-template, use the same
88 template data used to fill in job-template variables to fill in macro
89 variables.
90 """
91 for component_list_type in self._nonempty_component_list_types:
92 self._expand_macros_for_component_list_type(
93 jobish, component_list_type, template_data)
94
95 def _expand_macros_for_component_list_type(self,
96 jobish,
97 component_list_type,
98 template_data=None):
99 """In-place expansion of macros on jobish.
100
101 :arg dict jobish: A job-like JJB data structure. Could be anything that
102 might provide JJB "components" that get expanded to XML configuration.
103 This includes "job", "job-template", and "default" DSL items. This
104 argument is not modified in place, but rather copied so that the copy
105 may be returned to calling context.
106
107 :arg str component_list_type: A string value indicating which type of
108 component we are expanding macros for.
109
110 :arg dict template_data: If jobish is a job-template, use the same
111 template data used to fill in job-template variables to fill in macro
112 variables.
113 """
114 if (jobish.get("project-type", None) == "pipeline"
115 and component_list_type == "scm"):
116 # Pipeline projects have an atypical scm type, eg:
117 #
118 # - job:
119 # name: whatever
120 # project-type: pipeline
121 # pipeline-scm:
122 # script-path: nonstandard-scriptpath.groovy
123 # scm:
124 # - macro_name
125 #
126 # as opposed to the more typical:
127 #
128 # - job:
129 # name: whatever2
130 # scm:
131 # - macro_name
132 #
133 # So we treat that case specially here.
134 component_list = jobish.get("pipeline-scm", {}).get("scm", [])
135 else:
136 component_list = jobish.get(component_list_type, [])
137
138 component_substitutions = []
139 for component in component_list:
140 macro_component_list = self._maybe_expand_macro(
141 component, component_list_type, template_data)
142
143 if macro_component_list is not None:
144 # Since macros can contain other macros, we need to recurse
145 # into the newly-expanded macro component list to expand any
146 # macros that might be hiding in there. In order to do this we
147 # have to make the macro component list look like a job by
148 # embedding it in a dictionary like so.
149 self._expand_macros_for_component_list_type(
150 {component_list_type: macro_component_list},
151 component_list_type,
152 template_data)
153
154 component_substitutions.append(
155 (component, macro_component_list))
156
157 for component, macro_component_list in component_substitutions:
158 component_index = component_list.index(component)
159 component_list.remove(component)
160 i = 0
161 for macro_component in macro_component_list:
162 component_list.insert(component_index + i, macro_component)
163 i += 1
164
165 def _maybe_expand_macro(self,
166 component,
167 component_list_type,
168 template_data=None):
169 """For a given component, if it refers to a macro, return the
170 components defined for that macro with template variables (if any)
171 interpolated in.
172
173 :arg str component_list_type: A string value indicating which type of
174 component we are expanding macros for.
175
176 :arg dict template_data: If component is a macro and contains template
177 variables, use the same template data used to fill in job-template
178 variables to fill in macro variables.
179 """
180 component_copy = copy.deepcopy(component)
181
182 if isinstance(component, dict):
183 # The component is a singleton dictionary of name:
184 # dict(args)
185 component_name, component_data = next(iter(component_copy.items()))
186 else:
187 # The component is a simple string name, eg "run-tests".
188 component_name, component_data = component_copy, None
189
190 if template_data:
191 # Address the case where a macro name contains a variable to be
192 # interpolated by template variables.
193 component_name = deep_format(component_name, template_data, True)
194
195 # Check that the component under consideration actually is a
196 # macro.
197 if not self._is_macro(component_name, component_list_type):
198 return None
199
200 # Warn if the macro shadows an actual module type name for this
201 # component list type.
202 if ModuleRegistry.is_module_name(component_name, component_list_type):
203 self._mask_warned[component_name] = True
204 logger.warning(
205 "You have a macro ('%s') defined for '%s' "
206 "component list type that is masking an inbuilt "
207 "definition" % (component_name, component_list_type))
208
209 macro_component_list = self._get_macro_components(component_name,
210 component_list_type)
211
212 # If macro instance contains component_data, interpolate that
213 # into macro components.
214 if component_data:
215
216 # Also use template_data, but prefer data obtained directly from
217 # the macro instance.
218 if template_data:
219 template_data = copy.deepcopy(template_data)
220 template_data.update(component_data)
221
222 macro_component_list = deep_format(
223 macro_component_list, template_data, False)
224 else:
225 macro_component_list = deep_format(
226 macro_component_list, component_data, False)
227
228 return macro_component_list
229
230 def _get_macro_components(self, macro_name, component_list_type):
231 """Return the list of components that a macro expands into. For example:
232
233 - wrapper:
234 name: timeout-wrapper
235 wrappers:
236 - timeout:
237 fail: true
238 elastic-percentage: 150
239 elastic-default-timeout: 90
240 type: elastic
241
242 Provides a single "wrapper" type (corresponding to the "wrappers" list
243 type) component named "timeout" with the values shown above.
244
245 The macro_name argument in this case would be "timeout-wrapper".
246 """
247 macro_component_list = self._macros_by_component_list_type[
248 component_list_type][macro_name][component_list_type]
249 return copy.deepcopy(macro_component_list)
250
251
252class ModuleRegistry(object): 34class ModuleRegistry(object):
253 _entry_points_cache = {} 35 _entry_points_cache = {}
254 36
@@ -347,7 +129,8 @@ class ModuleRegistry(object):
347 def set_parser_data(self, parser_data): 129 def set_parser_data(self, parser_data):
348 self.__parser_data = parser_data 130 self.__parser_data = parser_data
349 131
350 def dispatch(self, component_type, xml_parent, component): 132 def dispatch(self, component_type, xml_parent,
133 component, template_data={}):
351 """This is a method that you can call from your implementation of 134 """This is a method that you can call from your implementation of
352 Base.gen_xml or component. It allows modules to define a type 135 Base.gen_xml or component. It allows modules to define a type
353 of component, and benefit from extensibility via Python 136 of component, and benefit from extensibility via Python
@@ -357,6 +140,8 @@ class ModuleRegistry(object):
357 (e.g., `builder`) 140 (e.g., `builder`)
358 :arg YAMLParser parser: the global YAML Parser 141 :arg YAMLParser parser: the global YAML Parser
359 :arg Element xml_parent: the parent XML element 142 :arg Element xml_parent: the parent XML element
143 :arg dict template_data: values that should be interpolated into
144 the component definition
360 145
361 See :py:class:`jenkins_jobs.modules.base.Base` for how to register 146 See :py:class:`jenkins_jobs.modules.base.Base` for how to register
362 components of a module. 147 components of a module.
@@ -375,6 +160,18 @@ class ModuleRegistry(object):
375 if isinstance(component, dict): 160 if isinstance(component, dict):
376 # The component is a singleton dictionary of name: dict(args) 161 # The component is a singleton dictionary of name: dict(args)
377 name, component_data = next(iter(component.items())) 162 name, component_data = next(iter(component.items()))
163 if template_data:
164 # Template data contains values that should be interpolated
165 # into the component definition
166 try:
167 component_data = deep_format(
168 component_data, template_data,
169 self.jjb_config.yamlparser['allow_empty_variables'])
170 except Exception:
171 logging.error(
172 "Failure formatting component ('%s') data '%s'",
173 name, component_data)
174 raise
378 else: 175 else:
379 # The component is a simple string name, eg "run-tests" 176 # The component is a simple string name, eg "run-tests"
380 name = component 177 name = component
@@ -437,17 +234,25 @@ class ModuleRegistry(object):
437 logger.debug("Cached entry point group %s = %s", 234 logger.debug("Cached entry point group %s = %s",
438 component_list_type, eps) 235 component_list_type, eps)
439 236
440 if name in eps: 237 # check for macro first
238 component = self.parser_data.get(component_type, {}).get(name)
239 if component:
240 if name in eps and name not in self.masked_warned:
241 self.masked_warned[name] = True
242 logger.warning(
243 "You have a macro ('%s') defined for '%s' "
244 "component type that is masking an inbuilt "
245 "definition" % (name, component_type))
246
247 for b in component[component_list_type]:
248 # Pass component_data in as template data to this function
249 # so that if the macro is invoked with arguments,
250 # the arguments are interpolated into the real defn.
251 self.dispatch(component_type, xml_parent, b, component_data)
252 elif name in eps:
441 func = eps[name].load() 253 func = eps[name].load()
442 func(self, xml_parent, component_data) 254 func(self, xml_parent, component_data)
443 else: 255 else:
444 raise JenkinsJobsException("Unknown entry point or macro '{0}' " 256 raise JenkinsJobsException("Unknown entry point or macro '{0}' "
445 "for component type: '{1}'.". 257 "for component type: '{1}'.".
446 format(name, component_type)) 258 format(name, component_type))
447
448 @classmethod
449 def is_module_name(self, name, component_list_type):
450 eps = self._entry_points_cache.get(component_list_type)
451 if not eps:
452 return False
453 return (name in eps)
diff --git a/setup.cfg b/setup.cfg
index 77edb7e..cfc34e5 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -87,16 +87,3 @@ jenkins_jobs.modules =
87 triggers=jenkins_jobs.modules.triggers:Triggers 87 triggers=jenkins_jobs.modules.triggers:Triggers
88 wrappers=jenkins_jobs.modules.wrappers:Wrappers 88 wrappers=jenkins_jobs.modules.wrappers:Wrappers
89 zuul=jenkins_jobs.modules.zuul:Zuul 89 zuul=jenkins_jobs.modules.zuul:Zuul
90jenkins_jobs.macros =
91 builder=jenkins_jobs.modules.builders:Builders
92 general=jenkins_jobs.modules.general:General
93 hipchat=jenkins_jobs.modules.hipchat_notif:HipChat
94 metadata=jenkins_jobs.modules.metadata:Metadata
95 notification=jenkins_jobs.modules.notifications:Notifications
96 parameter=jenkins_jobs.modules.parameters:Parameters
97 property=jenkins_jobs.modules.properties:Properties
98 publisher=jenkins_jobs.modules.publishers:Publishers
99 reporter=jenkins_jobs.modules.reporters:Reporters
100 scm=jenkins_jobs.modules.scm:SCM
101 trigger=jenkins_jobs.modules.triggers:Triggers
102 wrapper=jenkins_jobs.modules.wrappers:Wrappers
diff --git a/tests/xml_config/test_xml_config.py b/tests/xml_config/test_xml_config.py
index e50fc95..d64890d 100644
--- a/tests/xml_config/test_xml_config.py
+++ b/tests/xml_config/test_xml_config.py
@@ -68,6 +68,9 @@ class TestXmlJobGeneratorExceptions(base.BaseTestCase):
68 68
69 reg = registry.ModuleRegistry(config) 69 reg = registry.ModuleRegistry(config)
70 reg.set_parser_data(yp.data) 70 reg.set_parser_data(yp.data)
71 job_data_list, view_data_list = yp.expandYaml(reg)
71 72
72 self.assertRaises(errors.JenkinsJobsException, yp.expandYaml, reg) 73 xml_generator = xml_config.XmlJobGenerator(reg)
74 self.assertRaises(Exception, xml_generator.generateXML, job_data_list)
75 self.assertIn("Failure formatting component", self.logger.output)
73 self.assertIn("Problem formatting with args", self.logger.output) 76 self.assertIn("Problem formatting with args", self.logger.output)