Merge "Move macro expansion into YamlParser."

This commit is contained in:
Jenkins 2017-09-08 15:41:08 +00:00 committed by Gerrit Code Review
commit b58f6b17de
5 changed files with 288 additions and 42 deletions

View File

@ -143,6 +143,7 @@ Examples:
.. literalinclude:: /../../tests/yamlparser/fixtures/jinja01.yaml.inc
"""
import copy
import functools
import io
import logging
@ -290,6 +291,32 @@ class LocalLoader(OrderedConstructor, LocalAnchorLoader):
def _escape(self, data):
return re.sub(r'({|})', r'\1\1', data)
def __deepcopy__(self, memo):
"""
Make a deep copy of a LocalLoader excluding the uncopyable self.stream.
This is achieved by performing a shallow copy of self, setting the
stream attribute to None and then performing a deep copy of the shallow
copy.
(As this method will be called again on that deep copy, we also set a
sentinel attribute on the shallow copy to ensure that we don't recurse
infinitely.)
"""
assert self.done, 'Unsafe to copy an in-progress loader'
if getattr(self, '_copy', False):
# This is a shallow copy for an in-progress deep copy, remove the
# _copy marker and return self
del self._copy
return self
# Make a shallow copy
shallow = copy.copy(self)
shallow.stream = None
shallow._copy = True
deep = copy.deepcopy(shallow, memo)
memo[id(self)] = deep
return deep
class LocalDumper(OrderedRepresenter, yaml.Dumper):
def __init__(self, *args, **kwargs):
@ -440,11 +467,12 @@ class CustomLoader(object):
class Jinja2Loader(CustomLoader):
"""A loader for Jinja2-templated files."""
def __init__(self, contents):
self._template = jinja2.Template(contents)
self._template.environment.undefined = jinja2.StrictUndefined
self._contents = contents
def format(self, **kwargs):
return self._template.render(kwargs)
_template = jinja2.Template(self._contents)
_template.environment.undefined = jinja2.StrictUndefined
return _template.render(kwargs)
class CustomLoaderCollection(object):

View File

@ -25,6 +25,7 @@ import os
from jenkins_jobs.constants import MAGIC_MANAGE_STRING
from jenkins_jobs.errors import JenkinsJobsException
from jenkins_jobs.formatter import deep_format
from jenkins_jobs.registry import MacroRegistry
import jenkins_jobs.local_yaml as local_yaml
from jenkins_jobs import utils
@ -81,6 +82,8 @@ class YamlParser(object):
self.keep_desc = jjb_config.yamlparser['keep_descriptions']
self.path = jjb_config.yamlparser['include_path']
self._macro_registry = MacroRegistry()
def load_files(self, fn):
# handle deprecated behavior, and check that it's not a file like
@ -218,6 +221,11 @@ class YamlParser(object):
job["description"] = description + \
self._get_managed_string().lstrip()
def _register_macros(self):
for component_type in self._macro_registry.component_types:
for macro in self.data.get(component_type, {}).values():
self._macro_registry.register(component_type, macro)
def expandYaml(self, registry, jobs_glob=None):
changed = True
while changed:
@ -227,7 +235,11 @@ class YamlParser(object):
if module.handle_data(self.data):
changed = True
self._register_macros()
for default in self.data.get('defaults', {}).values():
self._macro_registry.expand_macros(default)
for job in self.data.get('job', {}).values():
self._macro_registry.expand_macros(job)
if jobs_glob and not matches(job['name'], jobs_glob):
logger.debug("Ignoring job {0}".format(job['name']))
continue
@ -389,6 +401,7 @@ class YamlParser(object):
"params '%s'", template_name, template, params)
raise
self._macro_registry.expand_macros(expanded, params)
job_name = expanded.get('name')
if jobs_glob and not matches(job_name, jobs_glob):
continue

View File

@ -15,6 +15,7 @@
# Manage Jenkins plugin module registry.
import copy
import logging
import operator
import pkg_resources
@ -31,8 +32,225 @@ __all__ = [
logger = logging.getLogger(__name__)
class MacroRegistry(object):
_component_to_component_list_mapping = {}
_component_list_to_component_mapping = {}
_macros_by_component_type = {}
_macros_by_component_list_type = {}
def __init__(self):
for entrypoint in pkg_resources.iter_entry_points(
group='jenkins_jobs.macros'):
Mod = entrypoint.load()
self._component_list_to_component_mapping[
Mod.component_list_type] = Mod.component_type
self._component_to_component_list_mapping[
Mod.component_type] = Mod.component_list_type
self._macros_by_component_type[
Mod.component_type] = {}
self._macros_by_component_list_type[
Mod.component_list_type] = {}
self._mask_warned = {}
@property
def _nonempty_component_list_types(self):
return [clt for clt in self._macros_by_component_list_type
if len(self._macros_by_component_list_type[clt]) != 0]
@property
def component_types(self):
return self._macros_by_component_type.keys()
def _is_macro(self, component_name, component_list_type):
return (component_name in
self._macros_by_component_list_type[component_list_type])
def register(self, component_type, macro):
macro_name = macro["name"]
clt = self._component_to_component_list_mapping[component_type]
self._macros_by_component_type[component_type][macro_name] = macro
self._macros_by_component_list_type[clt][macro_name] = macro
def expand_macros(self, jobish, template_data=None):
"""Create a copy of the given job-like thing, expand macros in place on
the copy, and return that object to calling context.
:arg dict jobish: A job-like JJB data structure. Could be anything that
might provide JJB "components" that get expanded to XML configuration.
This includes "job", "job-template", and "default" DSL items. This
argument is not modified in place, but rather copied so that the copy
may be returned to calling context.
:arg dict template_data: If jobish is a job-template, use the same
template data used to fill in job-template variables to fill in macro
variables.
"""
for component_list_type in self._nonempty_component_list_types:
self._expand_macros_for_component_list_type(
jobish, component_list_type, template_data)
def _expand_macros_for_component_list_type(self,
jobish,
component_list_type,
template_data=None):
"""In-place expansion of macros on jobish.
:arg dict jobish: A job-like JJB data structure. Could be anything that
might provide JJB "components" that get expanded to XML configuration.
This includes "job", "job-template", and "default" DSL items. This
argument is not modified in place, but rather copied so that the copy
may be returned to calling context.
:arg str component_list_type: A string value indicating which type of
component we are expanding macros for.
:arg dict template_data: If jobish is a job-template, use the same
template data used to fill in job-template variables to fill in macro
variables.
"""
if (jobish.get("project-type", None) == "pipeline"
and component_list_type == "scm"):
# Pipeline projects have an atypical scm type, eg:
#
# - job:
# name: whatever
# project-type: pipeline
# pipeline-scm:
# script-path: nonstandard-scriptpath.groovy
# scm:
# - macro_name
#
# as opposed to the more typical:
#
# - job:
# name: whatever2
# scm:
# - macro_name
#
# So we treat that case specially here.
component_list = jobish.get("pipeline-scm", {}).get("scm", [])
else:
component_list = jobish.get(component_list_type, [])
component_substitutions = []
for component in component_list:
macro_component_list = self._maybe_expand_macro(
component, component_list_type, template_data)
if macro_component_list is not None:
# Since macros can contain other macros, we need to recurse
# into the newly-expanded macro component list to expand any
# macros that might be hiding in there. In order to do this we
# have to make the macro component list look like a job by
# embedding it in a dictionary like so.
self._expand_macros_for_component_list_type(
{component_list_type: macro_component_list},
component_list_type,
template_data)
component_substitutions.append(
(component, macro_component_list))
for component, macro_component_list in component_substitutions:
component_index = component_list.index(component)
component_list.remove(component)
i = 0
for macro_component in macro_component_list:
component_list.insert(component_index + i, macro_component)
i += 1
def _maybe_expand_macro(self,
component,
component_list_type,
template_data=None):
"""For a given component, if it refers to a macro, return the
components defined for that macro with template variables (if any)
interpolated in.
:arg str component_list_type: A string value indicating which type of
component we are expanding macros for.
:arg dict template_data: If component is a macro and contains template
variables, use the same template data used to fill in job-template
variables to fill in macro variables.
"""
component_copy = copy.deepcopy(component)
if isinstance(component, dict):
# The component is a singleton dictionary of name:
# dict(args)
component_name, component_data = next(iter(component_copy.items()))
else:
# The component is a simple string name, eg "run-tests".
component_name, component_data = component_copy, None
if template_data:
# Address the case where a macro name contains a variable to be
# interpolated by template variables.
component_name = deep_format(component_name, template_data, True)
# Check that the component under consideration actually is a
# macro.
if not self._is_macro(component_name, component_list_type):
return None
# Warn if the macro shadows an actual module type name for this
# component list type.
if ModuleRegistry.is_module_name(component_name, component_list_type):
self._mask_warned[component_name] = True
logger.warning(
"You have a macro ('%s') defined for '%s' "
"component list type that is masking an inbuilt "
"definition" % (component_name, component_list_type))
macro_component_list = self._get_macro_components(component_name,
component_list_type)
# If macro instance contains component_data, interpolate that
# into macro components.
if component_data:
# Also use template_data, but prefer data obtained directly from
# the macro instance.
if template_data:
template_data = copy.deepcopy(template_data)
template_data.update(component_data)
macro_component_list = deep_format(
macro_component_list, template_data, False)
else:
macro_component_list = deep_format(
macro_component_list, component_data, False)
return macro_component_list
def _get_macro_components(self, macro_name, component_list_type):
"""Return the list of components that a macro expands into. For example:
- wrapper:
name: timeout-wrapper
wrappers:
- timeout:
fail: true
elastic-percentage: 150
elastic-default-timeout: 90
type: elastic
Provides a single "wrapper" type (corresponding to the "wrappers" list
type) component named "timeout" with the values shown above.
The macro_name argument in this case would be "timeout-wrapper".
"""
macro_component_list = self._macros_by_component_list_type[
component_list_type][macro_name][component_list_type]
return copy.deepcopy(macro_component_list)
class ModuleRegistry(object):
entry_points_cache = {}
_entry_points_cache = {}
def __init__(self, jjb_config, plugins_list=None):
self.modules = []
@ -129,8 +347,7 @@ class ModuleRegistry(object):
def set_parser_data(self, parser_data):
self.__parser_data = parser_data
def dispatch(self, component_type, xml_parent,
component, template_data={}):
def dispatch(self, component_type, xml_parent, component):
"""This is a method that you can call from your implementation of
Base.gen_xml or component. It allows modules to define a type
of component, and benefit from extensibility via Python
@ -140,8 +357,6 @@ class ModuleRegistry(object):
(e.g., `builder`)
:arg YAMLParser parser: the global YAML Parser
:arg Element xml_parent: the parent XML element
:arg dict template_data: values that should be interpolated into
the component definition
See :py:class:`jenkins_jobs.modules.base.Base` for how to register
components of a module.
@ -160,25 +375,13 @@ class ModuleRegistry(object):
if isinstance(component, dict):
# The component is a singleton dictionary of name: dict(args)
name, component_data = next(iter(component.items()))
if template_data:
# Template data contains values that should be interpolated
# into the component definition
try:
component_data = deep_format(
component_data, template_data,
self.jjb_config.yamlparser['allow_empty_variables'])
except Exception:
logging.error(
"Failure formatting component ('%s') data '%s'",
name, component_data)
raise
else:
# The component is a simple string name, eg "run-tests"
name = component
component_data = {}
# Look for a component function defined in an entry point
eps = ModuleRegistry.entry_points_cache.get(component_list_type)
eps = self._entry_points_cache.get(component_list_type)
if eps is None:
module_eps = []
# auto build entry points by inferring from base component_types
@ -230,29 +433,21 @@ class ModuleRegistry(object):
eps[module_ep.name] = module_ep
# cache both sets of entry points
ModuleRegistry.entry_points_cache[component_list_type] = eps
self._entry_points_cache[component_list_type] = eps
logger.debug("Cached entry point group %s = %s",
component_list_type, eps)
# check for macro first
component = self.parser_data.get(component_type, {}).get(name)
if component:
if name in eps and name not in self.masked_warned:
self.masked_warned[name] = True
logger.warning(
"You have a macro ('%s') defined for '%s' "
"component type that is masking an inbuilt "
"definition" % (name, component_type))
for b in component[component_list_type]:
# Pass component_data in as template data to this function
# so that if the macro is invoked with arguments,
# the arguments are interpolated into the real defn.
self.dispatch(component_type, xml_parent, b, component_data)
elif name in eps:
if name in eps:
func = eps[name].load()
func(self, xml_parent, component_data)
else:
raise JenkinsJobsException("Unknown entry point or macro '{0}' "
"for component type: '{1}'.".
format(name, component_type))
@classmethod
def is_module_name(self, name, component_list_type):
eps = self._entry_points_cache.get(component_list_type)
if not eps:
return False
return (name in eps)

View File

@ -85,3 +85,16 @@ jenkins_jobs.modules =
triggers=jenkins_jobs.modules.triggers:Triggers
wrappers=jenkins_jobs.modules.wrappers:Wrappers
zuul=jenkins_jobs.modules.zuul:Zuul
jenkins_jobs.macros =
builder=jenkins_jobs.modules.builders:Builders
general=jenkins_jobs.modules.general:General
hipchat=jenkins_jobs.modules.hipchat_notif:HipChat
metadata=jenkins_jobs.modules.metadata:Metadata
notification=jenkins_jobs.modules.notifications:Notifications
parameter=jenkins_jobs.modules.parameters:Parameters
property=jenkins_jobs.modules.properties:Properties
publisher=jenkins_jobs.modules.publishers:Publishers
reporter=jenkins_jobs.modules.reporters:Reporters
scm=jenkins_jobs.modules.scm:SCM
trigger=jenkins_jobs.modules.triggers:Triggers
wrapper=jenkins_jobs.modules.wrappers:Wrappers

View File

@ -52,9 +52,6 @@ class TestXmlJobGeneratorExceptions(base.BaseTestCase):
reg = registry.ModuleRegistry(config)
reg.set_parser_data(yp.data)
job_data_list, view_data_list = yp.expandYaml(reg)
xml_generator = xml_config.XmlJobGenerator(reg)
self.assertRaises(Exception, xml_generator.generateXML, job_data_list)
self.assertIn("Failure formatting component", self.logger.output)
self.assertRaises(errors.JenkinsJobsException, yp.expandYaml, reg)
self.assertIn("Problem formatting with args", self.logger.output)