From 4a8b93b8c2d517c8dc27d91437454890cc49a640 Mon Sep 17 00:00:00 2001 From: Wayne Date: Wed, 22 Apr 2015 09:18:45 -0700 Subject: [PATCH] Remove YamlParser from jenkins_jobs.builder The goal of this patch is simply to move some classes out of jenkins_jobs.builder into more appropriately-named modules. This started with simply moving YamlParser into jenkins_jobs.parser but led to other moves in order to avoid cyclic imports since YamlParser uses other classes previously defined in jenkins_jobs.builder. That said, this patch doesn't intend to address all of the clutter in jenkins_jobs.builder, mostly just what is necessary to get started working on YamlParser independent of other classes in that module. Change-Id: Ie88bf683e495033eb0b670fe29c256a70282735f --- doc/source/extending.rst | 2 +- jenkins_jobs/builder.py | 553 +------------------- jenkins_jobs/constants.py | 18 + jenkins_jobs/formatter.py | 86 +++ jenkins_jobs/parser.py | 322 ++++++++++++ jenkins_jobs/registry.py | 204 ++++++++ jenkins_jobs/xml_config.py | 33 ++ tests/base.py | 3 +- tests/cmd/subcommands/test_test.py | 4 +- tests/moduleregistry/test_moduleregistry.py | 2 +- 10 files changed, 672 insertions(+), 555 deletions(-) create mode 100644 jenkins_jobs/constants.py create mode 100644 jenkins_jobs/formatter.py create mode 100644 jenkins_jobs/parser.py create mode 100644 jenkins_jobs/registry.py create mode 100644 jenkins_jobs/xml_config.py diff --git a/doc/source/extending.rst b/doc/source/extending.rst index 5d3018ba0..c2def6b6f 100644 --- a/doc/source/extending.rst +++ b/doc/source/extending.rst @@ -83,5 +83,5 @@ All modules and their associated components are registered in the module registry. It can be accessed either from modules via the registry field, or via the parser parameter of components. -.. autoclass:: jenkins_jobs.builder.ModuleRegistry +.. autoclass:: jenkins_jobs.registry.ModuleRegistry :members: diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index 4a08a435d..a07a97958 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -26,40 +26,13 @@ import xml from xml.dom import minidom import jenkins import re -import pkg_resources from pprint import pformat import logging -import copy -import itertools -import fnmatch -from string import Formatter -from jenkins_jobs.errors import JenkinsJobsException -import jenkins_jobs.local_yaml as local_yaml + +from jenkins_jobs.constants import MAGIC_MANAGE_STRING +from jenkins_jobs.parser import YamlParser logger = logging.getLogger(__name__) -MAGIC_MANAGE_STRING = "" - - -class CustomFormatter(Formatter): - """ - Custom formatter to allow non-existing key references when formatting a - string - """ - def __init__(self, allow_empty=False): - super(CustomFormatter, self).__init__() - self.allow_empty = allow_empty - - def get_value(self, key, args, kwargs): - try: - return Formatter.get_value(self, key, args, kwargs) - except KeyError: - if self.allow_empty: - logger.debug( - 'Found uninitialized key %s, replaced with empty string', - key - ) - return '' - raise # Python 2.6's minidom toprettyxml produces broken output by adding extraneous @@ -99,526 +72,6 @@ if sys.version_info[:3] < (2, 7, 3) or xml.__name__ != 'xml': minidom.Element.writexml = writexml -def deep_format(obj, paramdict, allow_empty=False): - """Apply the paramdict via str.format() to all string objects found within - the supplied obj. Lists and dicts are traversed recursively.""" - # YAML serialisation was originally used to achieve this, but that places - # limitations on the values in paramdict - the post-format result must - # still be valid YAML (so substituting-in a string containing quotes, for - # example, is problematic). - if hasattr(obj, 'format'): - try: - result = re.match('^{obj:(?P\w+)}$', obj) - if result is not None: - ret = paramdict[result.group("key")] - else: - ret = CustomFormatter(allow_empty).format(obj, **paramdict) - except KeyError as exc: - missing_key = exc.message - desc = "%s parameter missing to format %s\nGiven:\n%s" % ( - missing_key, obj, pformat(paramdict)) - raise JenkinsJobsException(desc) - elif isinstance(obj, list): - ret = [] - for item in obj: - ret.append(deep_format(item, paramdict, allow_empty)) - elif isinstance(obj, dict): - ret = {} - for item in obj: - try: - ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \ - deep_format(obj[item], paramdict, allow_empty) - except KeyError as exc: - missing_key = exc.message - desc = "%s parameter missing to format %s\nGiven:\n%s" % ( - missing_key, obj, pformat(paramdict)) - raise JenkinsJobsException(desc) - else: - ret = obj - return ret - - -def matches(what, glob_patterns): - """ - Checks if the given string, ``what``, matches any of the glob patterns in - the iterable, ``glob_patterns`` - - :arg str what: String that we want to test if it matches a pattern - :arg iterable glob_patterns: glob patterns to match (list, tuple, set, - etc.) - """ - return any(fnmatch.fnmatch(what, glob_pattern) - for glob_pattern in glob_patterns) - - -class YamlParser(object): - def __init__(self, config=None, plugins_info=None): - self.data = {} - self.jobs = [] - self.xml_jobs = [] - self.config = config - self.registry = ModuleRegistry(self.config, plugins_info) - self.path = ["."] - if self.config: - if config.has_section('job_builder') and \ - config.has_option('job_builder', 'include_path'): - self.path = config.get('job_builder', - 'include_path').split(':') - self.keep_desc = self.get_keep_desc() - - def get_keep_desc(self): - keep_desc = False - if self.config and self.config.has_section('job_builder') and \ - self.config.has_option('job_builder', 'keep_descriptions'): - keep_desc = self.config.getboolean('job_builder', - 'keep_descriptions') - return keep_desc - - def parse_fp(self, fp): - data = local_yaml.load(fp, search_path=self.path) - if data: - if not isinstance(data, list): - raise JenkinsJobsException( - "The topmost collection in file '{fname}' must be a list," - " not a {cls}".format(fname=getattr(fp, 'name', fp), - cls=type(data))) - for item in data: - cls, dfn = next(iter(item.items())) - group = self.data.get(cls, {}) - if len(item.items()) > 1: - n = None - for k, v in item.items(): - if k == "name": - n = v - break - # Syntax error - raise JenkinsJobsException("Syntax error, for item " - "named '{0}'. Missing indent?" - .format(n)) - name = dfn['name'] - if name in group: - self._handle_dups("Duplicate entry found in '{0}: '{1}' " - "already defined".format(fp.name, name)) - group[name] = dfn - self.data[cls] = group - - def parse(self, fn): - with open(fn) as fp: - self.parse_fp(fp) - - def _handle_dups(self, message): - - if not (self.config and self.config.has_section('job_builder') and - self.config.getboolean('job_builder', 'allow_duplicates')): - logger.error(message) - raise JenkinsJobsException(message) - else: - logger.warn(message) - - def getJob(self, name): - job = self.data.get('job', {}).get(name, None) - if not job: - return job - return self.applyDefaults(job) - - def getJobGroup(self, name): - return self.data.get('job-group', {}).get(name, None) - - def getJobTemplate(self, name): - job = self.data.get('job-template', {}).get(name, None) - if not job: - return job - return self.applyDefaults(job) - - def applyDefaults(self, data, override_dict=None): - if override_dict is None: - override_dict = {} - - whichdefaults = data.get('defaults', 'global') - defaults = copy.deepcopy(self.data.get('defaults', - {}).get(whichdefaults, {})) - if defaults == {} and whichdefaults != 'global': - raise JenkinsJobsException("Unknown defaults set: '{0}'" - .format(whichdefaults)) - - for key in override_dict.keys(): - if key in defaults.keys(): - defaults[key] = override_dict[key] - - newdata = {} - newdata.update(defaults) - newdata.update(data) - return newdata - - def formatDescription(self, job): - if self.keep_desc: - description = job.get("description", None) - else: - description = job.get("description", '') - if description is not None: - job["description"] = description + \ - self.get_managed_string().lstrip() - - def expandYaml(self, jobs_glob=None): - changed = True - while changed: - changed = False - for module in self.registry.modules: - if hasattr(module, 'handle_data'): - if module.handle_data(self): - changed = True - - for job in self.data.get('job', {}).values(): - if jobs_glob and not matches(job['name'], jobs_glob): - logger.debug("Ignoring job {0}".format(job['name'])) - continue - logger.debug("Expanding job '{0}'".format(job['name'])) - job = self.applyDefaults(job) - self.formatDescription(job) - self.jobs.append(job) - for project in self.data.get('project', {}).values(): - logger.debug("Expanding project '{0}'".format(project['name'])) - # use a set to check for duplicate job references in projects - seen = set() - for jobspec in project.get('jobs', []): - if isinstance(jobspec, dict): - # Singleton dict containing dict of job-specific params - jobname, jobparams = next(iter(jobspec.items())) - if not isinstance(jobparams, dict): - jobparams = {} - else: - jobname = jobspec - jobparams = {} - job = self.getJob(jobname) - if job: - # Just naming an existing defined job - if jobname in seen: - self._handle_dups("Duplicate job '{0}' specified " - "for project '{1}'".format( - jobname, project['name'])) - seen.add(jobname) - continue - # see if it's a job group - group = self.getJobGroup(jobname) - if group: - for group_jobspec in group['jobs']: - if isinstance(group_jobspec, dict): - group_jobname, group_jobparams = \ - next(iter(group_jobspec.items())) - if not isinstance(group_jobparams, dict): - group_jobparams = {} - else: - group_jobname = group_jobspec - group_jobparams = {} - job = self.getJob(group_jobname) - if job: - if group_jobname in seen: - self._handle_dups( - "Duplicate job '{0}' specified for " - "project '{1}'".format(group_jobname, - project['name'])) - seen.add(group_jobname) - continue - template = self.getJobTemplate(group_jobname) - # Allow a group to override parameters set by a project - d = {} - d.update(project) - d.update(jobparams) - d.update(group) - d.update(group_jobparams) - # Except name, since the group's name is not useful - d['name'] = project['name'] - if template: - self.expandYamlForTemplateJob(d, template, - jobs_glob) - continue - # see if it's a template - template = self.getJobTemplate(jobname) - if template: - d = {} - d.update(project) - d.update(jobparams) - self.expandYamlForTemplateJob(d, template, jobs_glob) - else: - raise JenkinsJobsException("Failed to find suitable " - "template named '{0}'" - .format(jobname)) - # check for duplicate generated jobs - seen = set() - # walk the list in reverse so that last definition wins - for job in self.jobs[::-1]: - if job['name'] in seen: - self._handle_dups("Duplicate definitions for job '{0}' " - "specified".format(job['name'])) - self.jobs.remove(job) - seen.add(job['name']) - - def expandYamlForTemplateJob(self, project, template, jobs_glob=None): - dimensions = [] - template_name = template['name'] - # reject keys that are not useful during yaml expansion - for k in ['jobs']: - project.pop(k) - for (k, v) in project.items(): - tmpk = '{{{0}}}'.format(k) - if tmpk not in template_name: - logger.debug("Variable %s not in name %s, rejecting from job" - " matrix expansion.", tmpk, template_name) - continue - if type(v) == list: - dimensions.append(zip([k] * len(v), v)) - # XXX somewhat hackish to ensure we actually have a single - # pass through the loop - if len(dimensions) == 0: - dimensions = [(("", ""),)] - - for values in itertools.product(*dimensions): - params = copy.deepcopy(project) - params = self.applyDefaults(params, template) - - expanded_values = {} - for (k, v) in values: - if isinstance(v, dict): - inner_key = next(iter(v)) - expanded_values[k] = inner_key - expanded_values.update(v[inner_key]) - else: - expanded_values[k] = v - - params.update(expanded_values) - params = deep_format(params, params) - allow_empty_variables = self.config \ - and self.config.has_section('job_builder') \ - and self.config.has_option( - 'job_builder', 'allow_empty_variables') \ - and self.config.getboolean( - 'job_builder', 'allow_empty_variables') - expanded = deep_format(template, params, allow_empty_variables) - - job_name = expanded.get('name') - if jobs_glob and not matches(job_name, jobs_glob): - continue - - self.formatDescription(expanded) - self.jobs.append(expanded) - - def get_managed_string(self): - # The \n\n is not hard coded, because they get stripped if the - # project does not otherwise have a description. - return "\n\n" + MAGIC_MANAGE_STRING - - def generateXML(self): - for job in self.jobs: - self.xml_jobs.append(self.getXMLForJob(job)) - - def getXMLForJob(self, data): - kind = data.get('project-type', 'freestyle') - - for ep in pkg_resources.iter_entry_points( - group='jenkins_jobs.projects', name=kind): - Mod = ep.load() - mod = Mod(self.registry) - xml = mod.root_xml(data) - self.gen_xml(xml, data) - job = XmlJob(xml, data['name']) - return job - - def gen_xml(self, xml, data): - for module in self.registry.modules: - if hasattr(module, 'gen_xml'): - module.gen_xml(self, xml, data) - - -class ModuleRegistry(object): - entry_points_cache = {} - - def __init__(self, config, plugins_list=None): - self.modules = [] - self.modules_by_component_type = {} - self.handlers = {} - self.global_config = config - - if plugins_list is None: - self.plugins_dict = {} - else: - self.plugins_dict = self._get_plugins_info_dict(plugins_list) - - for entrypoint in pkg_resources.iter_entry_points( - group='jenkins_jobs.modules'): - Mod = entrypoint.load() - mod = Mod(self) - self.modules.append(mod) - self.modules.sort(key=operator.attrgetter('sequence')) - if mod.component_type is not None: - self.modules_by_component_type[mod.component_type] = mod - - @staticmethod - def _get_plugins_info_dict(plugins_list): - def mutate_plugin_info(plugin_info): - """ - We perform mutations on a single member of plugin_info here, then - return a dictionary with the longName and shortName of the plugin - mapped to its plugin info dictionary. - """ - version = plugin_info.get('version', '0') - plugin_info['version'] = re.sub(r'(.*)-(?:SNAPSHOT|BETA)', - r'\g<1>.preview', version) - - aliases = [] - for key in ['longName', 'shortName']: - value = plugin_info.get(key, None) - if value is not None: - aliases.append(value) - - plugin_info_dict = {} - for name in aliases: - plugin_info_dict[name] = plugin_info - - return plugin_info_dict - - list_of_dicts = [mutate_plugin_info(v) for v in plugins_list] - - plugins_info_dict = {} - for d in list_of_dicts: - plugins_info_dict.update(d) - - return plugins_info_dict - - def get_plugin_info(self, plugin_name): - """ This method is intended to provide information about plugins within - a given module's implementation of Base.gen_xml. The return value is a - dictionary with data obtained directly from a running Jenkins instance. - This allows module authors to differentiate generated XML output based - on information such as specific plugin versions. - - :arg string plugin_name: Either the shortName or longName of a plugin - as see in a query that looks like: - ``http:///pluginManager/api/json?pretty&depth=2`` - - During a 'test' run, it is possible to override JJB's query to a live - Jenkins instance by passing it a path to a file containing a YAML list - of dictionaries that mimics the plugin properties you want your test - output to reflect:: - - jenkins-jobs test -p /path/to/plugins-info.yaml - - Below is example YAML that might be included in - /path/to/plugins-info.yaml. - - .. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml - - """ - return self.plugins_dict.get(plugin_name, {}) - - def registerHandler(self, category, name, method): - cat_dict = self.handlers.get(category, {}) - if not cat_dict: - self.handlers[category] = cat_dict - cat_dict[name] = method - - def getHandler(self, category, name): - return self.handlers[category][name] - - def dispatch(self, component_type, - parser, xml_parent, - component, template_data={}): - """This is a method that you can call from your implementation of - Base.gen_xml or component. It allows modules to define a type - of component, and benefit from extensibility via Python - entry points and Jenkins Job Builder :ref:`Macros `. - - :arg string component_type: the name of the component - (e.g., `builder`) - :arg YAMLParser parser: the global YAML Parser - :arg Element xml_parent: the parent XML element - :arg dict template_data: values that should be interpolated into - the component definition - - See :py:class:`jenkins_jobs.modules.base.Base` for how to register - components of a module. - - See the Publishers module for a simple example of how to use - this method. - """ - - if component_type not in self.modules_by_component_type: - raise JenkinsJobsException("Unknown component type: " - "'{0}'.".format(component_type)) - - component_list_type = self.modules_by_component_type[component_type] \ - .component_list_type - - if isinstance(component, dict): - # The component is a singleton dictionary of name: dict(args) - name, component_data = next(iter(component.items())) - if template_data: - # Template data contains values that should be interpolated - # into the component definition - s = yaml.dump(component_data, default_flow_style=False) - allow_empty_variables = self.global_config \ - and self.global_config.has_section('job_builder') \ - and self.global_config.has_option( - 'job_builder', 'allow_empty_variables') \ - and self.global_config.getboolean( - 'job_builder', 'allow_empty_variables') - s = CustomFormatter( - allow_empty_variables).format(s, **template_data) - component_data = yaml.load(s) - else: - # The component is a simple string name, eg "run-tests" - name = component - component_data = {} - - # Look for a component function defined in an entry point - eps = ModuleRegistry.entry_points_cache.get(component_list_type) - if eps is None: - module_eps = list(pkg_resources.iter_entry_points( - group='jenkins_jobs.{0}'.format(component_list_type))) - eps = {} - for module_ep in module_eps: - if module_ep.name in eps: - raise JenkinsJobsException( - "Duplicate entry point found for component type: " - "'{0}', '{0}'," - "name: '{1}'".format(component_type, name)) - eps[module_ep.name] = module_ep - - ModuleRegistry.entry_points_cache[component_list_type] = eps - logger.debug("Cached entry point group %s = %s", - component_list_type, eps) - - if name in eps: - func = eps[name].load() - func(parser, xml_parent, component_data) - else: - # Otherwise, see if it's defined as a macro - component = parser.data.get(component_type, {}).get(name) - if component: - for b in component[component_list_type]: - # Pass component_data in as template data to this function - # so that if the macro is invoked with arguments, - # the arguments are interpolated into the real defn. - self.dispatch(component_type, - parser, xml_parent, b, component_data) - else: - raise JenkinsJobsException("Unknown entry point or macro '{0}'" - " for component type: '{1}'.". - format(name, component_type)) - - -class XmlJob(object): - def __init__(self, xml, name): - self.xml = xml - self.name = name - - def md5(self): - return hashlib.md5(self.output()).hexdigest() - - def output(self): - out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8')) - return out.toprettyxml(indent=' ', encoding='utf-8') - - class CacheStorage(object): # ensure each instance of the class has a reference to the required # modules so that they are available to be used when the destructor diff --git a/jenkins_jobs/constants.py b/jenkins_jobs/constants.py new file mode 100644 index 000000000..a7dd30c4d --- /dev/null +++ b/jenkins_jobs/constants.py @@ -0,0 +1,18 @@ +#!/usr/bin/env python +# Copyright (C) 2015 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Manage JJB Constants + +MAGIC_MANAGE_STRING = "" diff --git a/jenkins_jobs/formatter.py b/jenkins_jobs/formatter.py new file mode 100644 index 000000000..9080847a4 --- /dev/null +++ b/jenkins_jobs/formatter.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python +# Copyright (C) 2015 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Manage interpolation of JJB variables into template strings. + +import logging +from pprint import pformat +import re +from string import Formatter + +from jenkins_jobs.errors import JenkinsJobsException + +logger = logging.getLogger(__name__) + + +def deep_format(obj, paramdict, allow_empty=False): + """Apply the paramdict via str.format() to all string objects found within + the supplied obj. Lists and dicts are traversed recursively.""" + # YAML serialisation was originally used to achieve this, but that places + # limitations on the values in paramdict - the post-format result must + # still be valid YAML (so substituting-in a string containing quotes, for + # example, is problematic). + if hasattr(obj, 'format'): + try: + result = re.match('^{obj:(?P\w+)}$', obj) + if result is not None: + ret = paramdict[result.group("key")] + else: + ret = CustomFormatter(allow_empty).format(obj, **paramdict) + except KeyError as exc: + missing_key = exc.message + desc = "%s parameter missing to format %s\nGiven:\n%s" % ( + missing_key, obj, pformat(paramdict)) + raise JenkinsJobsException(desc) + elif isinstance(obj, list): + ret = [] + for item in obj: + ret.append(deep_format(item, paramdict, allow_empty)) + elif isinstance(obj, dict): + ret = {} + for item in obj: + try: + ret[CustomFormatter(allow_empty).format(item, **paramdict)] = \ + deep_format(obj[item], paramdict, allow_empty) + except KeyError as exc: + missing_key = exc.message + desc = "%s parameter missing to format %s\nGiven:\n%s" % ( + missing_key, obj, pformat(paramdict)) + raise JenkinsJobsException(desc) + else: + ret = obj + return ret + + +class CustomFormatter(Formatter): + """ + Custom formatter to allow non-existing key references when formatting a + string + """ + def __init__(self, allow_empty=False): + super(CustomFormatter, self).__init__() + self.allow_empty = allow_empty + + def get_value(self, key, args, kwargs): + try: + return Formatter.get_value(self, key, args, kwargs) + except KeyError: + if self.allow_empty: + logger.debug( + 'Found uninitialized key %s, replaced with empty string', + key + ) + return '' + raise diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py new file mode 100644 index 000000000..ed351671c --- /dev/null +++ b/jenkins_jobs/parser.py @@ -0,0 +1,322 @@ +#!/usr/bin/env python +# Copyright (C) 2015 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Manage JJB yaml feature implementation + +import copy +import fnmatch +import itertools +import logging +import pkg_resources + +import jenkins_jobs.local_yaml as local_yaml +from jenkins_jobs.constants import MAGIC_MANAGE_STRING +from jenkins_jobs.errors import JenkinsJobsException +from jenkins_jobs.registry import ModuleRegistry +from jenkins_jobs.formatter import deep_format +from jenkins_jobs.xml_config import XmlJob + +logger = logging.getLogger(__name__) + + +def matches(what, glob_patterns): + """ + Checks if the given string, ``what``, matches any of the glob patterns in + the iterable, ``glob_patterns`` + + :arg str what: String that we want to test if it matches a pattern + :arg iterable glob_patterns: glob patterns to match (list, tuple, set, + etc.) + """ + return any(fnmatch.fnmatch(what, glob_pattern) + for glob_pattern in glob_patterns) + + +class YamlParser(object): + def __init__(self, config=None, plugins_info=None): + self.data = {} + self.jobs = [] + self.xml_jobs = [] + self.config = config + self.registry = ModuleRegistry(self.config, plugins_info) + self.path = ["."] + if self.config: + if config.has_section('job_builder') and \ + config.has_option('job_builder', 'include_path'): + self.path = config.get('job_builder', + 'include_path').split(':') + self.keep_desc = self.get_keep_desc() + + def get_keep_desc(self): + keep_desc = False + if self.config and self.config.has_section('job_builder') and \ + self.config.has_option('job_builder', 'keep_descriptions'): + keep_desc = self.config.getboolean('job_builder', + 'keep_descriptions') + return keep_desc + + def parse_fp(self, fp): + data = local_yaml.load(fp, search_path=self.path) + if data: + if not isinstance(data, list): + raise JenkinsJobsException( + "The topmost collection in file '{fname}' must be a list," + " not a {cls}".format(fname=getattr(fp, 'name', fp), + cls=type(data))) + for item in data: + cls, dfn = next(iter(item.items())) + group = self.data.get(cls, {}) + if len(item.items()) > 1: + n = None + for k, v in item.items(): + if k == "name": + n = v + break + # Syntax error + raise JenkinsJobsException("Syntax error, for item " + "named '{0}'. Missing indent?" + .format(n)) + name = dfn['name'] + if name in group: + self._handle_dups("Duplicate entry found in '{0}: '{1}' " + "already defined".format(fp.name, name)) + group[name] = dfn + self.data[cls] = group + + def parse(self, fn): + with open(fn) as fp: + self.parse_fp(fp) + + def _handle_dups(self, message): + + if not (self.config and self.config.has_section('job_builder') and + self.config.getboolean('job_builder', 'allow_duplicates')): + logger.error(message) + raise JenkinsJobsException(message) + else: + logger.warn(message) + + def getJob(self, name): + job = self.data.get('job', {}).get(name, None) + if not job: + return job + return self.applyDefaults(job) + + def getJobGroup(self, name): + return self.data.get('job-group', {}).get(name, None) + + def getJobTemplate(self, name): + job = self.data.get('job-template', {}).get(name, None) + if not job: + return job + return self.applyDefaults(job) + + def applyDefaults(self, data, override_dict=None): + if override_dict is None: + override_dict = {} + + whichdefaults = data.get('defaults', 'global') + defaults = copy.deepcopy(self.data.get('defaults', + {}).get(whichdefaults, {})) + if defaults == {} and whichdefaults != 'global': + raise JenkinsJobsException("Unknown defaults set: '{0}'" + .format(whichdefaults)) + + for key in override_dict.keys(): + if key in defaults.keys(): + defaults[key] = override_dict[key] + + newdata = {} + newdata.update(defaults) + newdata.update(data) + return newdata + + def formatDescription(self, job): + if self.keep_desc: + description = job.get("description", None) + else: + description = job.get("description", '') + if description is not None: + job["description"] = description + \ + self.get_managed_string().lstrip() + + def expandYaml(self, jobs_glob=None): + changed = True + while changed: + changed = False + for module in self.registry.modules: + if hasattr(module, 'handle_data'): + if module.handle_data(self): + changed = True + + for job in self.data.get('job', {}).values(): + if jobs_glob and not matches(job['name'], jobs_glob): + logger.debug("Ignoring job {0}".format(job['name'])) + continue + logger.debug("Expanding job '{0}'".format(job['name'])) + job = self.applyDefaults(job) + self.formatDescription(job) + self.jobs.append(job) + for project in self.data.get('project', {}).values(): + logger.debug("Expanding project '{0}'".format(project['name'])) + # use a set to check for duplicate job references in projects + seen = set() + for jobspec in project.get('jobs', []): + if isinstance(jobspec, dict): + # Singleton dict containing dict of job-specific params + jobname, jobparams = next(iter(jobspec.items())) + if not isinstance(jobparams, dict): + jobparams = {} + else: + jobname = jobspec + jobparams = {} + job = self.getJob(jobname) + if job: + # Just naming an existing defined job + if jobname in seen: + self._handle_dups("Duplicate job '{0}' specified " + "for project '{1}'".format( + jobname, project['name'])) + seen.add(jobname) + continue + # see if it's a job group + group = self.getJobGroup(jobname) + if group: + for group_jobspec in group['jobs']: + if isinstance(group_jobspec, dict): + group_jobname, group_jobparams = \ + next(iter(group_jobspec.items())) + if not isinstance(group_jobparams, dict): + group_jobparams = {} + else: + group_jobname = group_jobspec + group_jobparams = {} + job = self.getJob(group_jobname) + if job: + if group_jobname in seen: + self._handle_dups( + "Duplicate job '{0}' specified for " + "project '{1}'".format(group_jobname, + project['name'])) + seen.add(group_jobname) + continue + template = self.getJobTemplate(group_jobname) + # Allow a group to override parameters set by a project + d = {} + d.update(project) + d.update(jobparams) + d.update(group) + d.update(group_jobparams) + # Except name, since the group's name is not useful + d['name'] = project['name'] + if template: + self.expandYamlForTemplateJob(d, template, + jobs_glob) + continue + # see if it's a template + template = self.getJobTemplate(jobname) + if template: + d = {} + d.update(project) + d.update(jobparams) + self.expandYamlForTemplateJob(d, template, jobs_glob) + else: + raise JenkinsJobsException("Failed to find suitable " + "template named '{0}'" + .format(jobname)) + # check for duplicate generated jobs + seen = set() + # walk the list in reverse so that last definition wins + for job in self.jobs[::-1]: + if job['name'] in seen: + self._handle_dups("Duplicate definitions for job '{0}' " + "specified".format(job['name'])) + self.jobs.remove(job) + seen.add(job['name']) + + def expandYamlForTemplateJob(self, project, template, jobs_glob=None): + dimensions = [] + template_name = template['name'] + # reject keys that are not useful during yaml expansion + for k in ['jobs']: + project.pop(k) + for (k, v) in project.items(): + tmpk = '{{{0}}}'.format(k) + if tmpk not in template_name: + logger.debug("Variable %s not in name %s, rejecting from job" + " matrix expansion.", tmpk, template_name) + continue + if type(v) == list: + dimensions.append(zip([k] * len(v), v)) + # XXX somewhat hackish to ensure we actually have a single + # pass through the loop + if len(dimensions) == 0: + dimensions = [(("", ""),)] + + for values in itertools.product(*dimensions): + params = copy.deepcopy(project) + params = self.applyDefaults(params, template) + + expanded_values = {} + for (k, v) in values: + if isinstance(v, dict): + inner_key = next(iter(v)) + expanded_values[k] = inner_key + expanded_values.update(v[inner_key]) + else: + expanded_values[k] = v + + params.update(expanded_values) + params = deep_format(params, params) + allow_empty_variables = self.config \ + and self.config.has_section('job_builder') \ + and self.config.has_option( + 'job_builder', 'allow_empty_variables') \ + and self.config.getboolean( + 'job_builder', 'allow_empty_variables') + expanded = deep_format(template, params, allow_empty_variables) + + job_name = expanded.get('name') + if jobs_glob and not matches(job_name, jobs_glob): + continue + + self.formatDescription(expanded) + self.jobs.append(expanded) + + def get_managed_string(self): + # The \n\n is not hard coded, because they get stripped if the + # project does not otherwise have a description. + return "\n\n" + MAGIC_MANAGE_STRING + + def generateXML(self): + for job in self.jobs: + self.xml_jobs.append(self.getXMLForJob(job)) + + def getXMLForJob(self, data): + kind = data.get('project-type', 'freestyle') + + for ep in pkg_resources.iter_entry_points( + group='jenkins_jobs.projects', name=kind): + Mod = ep.load() + mod = Mod(self.registry) + xml = mod.root_xml(data) + self.gen_xml(xml, data) + job = XmlJob(xml, data['name']) + return job + + def gen_xml(self, xml, data): + for module in self.registry.modules: + if hasattr(module, 'gen_xml'): + module.gen_xml(self, xml, data) diff --git a/jenkins_jobs/registry.py b/jenkins_jobs/registry.py new file mode 100644 index 000000000..087bc744a --- /dev/null +++ b/jenkins_jobs/registry.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python +# Copyright (C) 2015 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Manage Jenkins plugin module registry. + +import logging +import operator +import pkg_resources +import re +import yaml + +from jenkins_jobs.errors import JenkinsJobsException +from jenkins_jobs.formatter import CustomFormatter + +logger = logging.getLogger(__name__) + + +class ModuleRegistry(object): + entry_points_cache = {} + + def __init__(self, config, plugins_list=None): + self.modules = [] + self.modules_by_component_type = {} + self.handlers = {} + self.global_config = config + + if plugins_list is None: + self.plugins_dict = {} + else: + self.plugins_dict = self._get_plugins_info_dict(plugins_list) + + for entrypoint in pkg_resources.iter_entry_points( + group='jenkins_jobs.modules'): + Mod = entrypoint.load() + mod = Mod(self) + self.modules.append(mod) + self.modules.sort(key=operator.attrgetter('sequence')) + if mod.component_type is not None: + self.modules_by_component_type[mod.component_type] = mod + + @staticmethod + def _get_plugins_info_dict(plugins_list): + def mutate_plugin_info(plugin_info): + """ + We perform mutations on a single member of plugin_info here, then + return a dictionary with the longName and shortName of the plugin + mapped to its plugin info dictionary. + """ + version = plugin_info.get('version', '0') + plugin_info['version'] = re.sub(r'(.*)-(?:SNAPSHOT|BETA)', + r'\g<1>.preview', version) + + aliases = [] + for key in ['longName', 'shortName']: + value = plugin_info.get(key, None) + if value is not None: + aliases.append(value) + + plugin_info_dict = {} + for name in aliases: + plugin_info_dict[name] = plugin_info + + return plugin_info_dict + + list_of_dicts = [mutate_plugin_info(v) for v in plugins_list] + + plugins_info_dict = {} + for d in list_of_dicts: + plugins_info_dict.update(d) + + return plugins_info_dict + + def get_plugin_info(self, plugin_name): + """ This method is intended to provide information about plugins within + a given module's implementation of Base.gen_xml. The return value is a + dictionary with data obtained directly from a running Jenkins instance. + This allows module authors to differentiate generated XML output based + on information such as specific plugin versions. + + :arg string plugin_name: Either the shortName or longName of a plugin + as see in a query that looks like: + ``http:///pluginManager/api/json?pretty&depth=2`` + + During a 'test' run, it is possible to override JJB's query to a live + Jenkins instance by passing it a path to a file containing a YAML list + of dictionaries that mimics the plugin properties you want your test + output to reflect:: + + jenkins-jobs test -p /path/to/plugins-info.yaml + + Below is example YAML that might be included in + /path/to/plugins-info.yaml. + + .. literalinclude:: /../../tests/cmd/fixtures/plugins-info.yaml + + """ + return self.plugins_dict.get(plugin_name, {}) + + def registerHandler(self, category, name, method): + cat_dict = self.handlers.get(category, {}) + if not cat_dict: + self.handlers[category] = cat_dict + cat_dict[name] = method + + def getHandler(self, category, name): + return self.handlers[category][name] + + def dispatch(self, component_type, + parser, xml_parent, + component, template_data={}): + """This is a method that you can call from your implementation of + Base.gen_xml or component. It allows modules to define a type + of component, and benefit from extensibility via Python + entry points and Jenkins Job Builder :ref:`Macros `. + + :arg string component_type: the name of the component + (e.g., `builder`) + :arg YAMLParser parser: the global YAML Parser + :arg Element xml_parent: the parent XML element + :arg dict template_data: values that should be interpolated into + the component definition + + See :py:class:`jenkins_jobs.modules.base.Base` for how to register + components of a module. + + See the Publishers module for a simple example of how to use + this method. + """ + + if component_type not in self.modules_by_component_type: + raise JenkinsJobsException("Unknown component type: " + "'{0}'.".format(component_type)) + + component_list_type = self.modules_by_component_type[component_type] \ + .component_list_type + + if isinstance(component, dict): + # The component is a singleton dictionary of name: dict(args) + name, component_data = next(iter(component.items())) + if template_data: + # Template data contains values that should be interpolated + # into the component definition + s = yaml.dump(component_data, default_flow_style=False) + allow_empty_variables = self.global_config \ + and self.global_config.has_section('job_builder') \ + and self.global_config.has_option( + 'job_builder', 'allow_empty_variables') \ + and self.global_config.getboolean( + 'job_builder', 'allow_empty_variables') + s = CustomFormatter( + allow_empty_variables).format(s, **template_data) + component_data = yaml.load(s) + else: + # The component is a simple string name, eg "run-tests" + name = component + component_data = {} + + # Look for a component function defined in an entry point + eps = ModuleRegistry.entry_points_cache.get(component_list_type) + if eps is None: + module_eps = list(pkg_resources.iter_entry_points( + group='jenkins_jobs.{0}'.format(component_list_type))) + eps = {} + for module_ep in module_eps: + if module_ep.name in eps: + raise JenkinsJobsException( + "Duplicate entry point found for component type: " + "'{0}', '{0}'," + "name: '{1}'".format(component_type, name)) + eps[module_ep.name] = module_ep + + ModuleRegistry.entry_points_cache[component_list_type] = eps + logger.debug("Cached entry point group %s = %s", + component_list_type, eps) + + if name in eps: + func = eps[name].load() + func(parser, xml_parent, component_data) + else: + # Otherwise, see if it's defined as a macro + component = parser.data.get(component_type, {}).get(name) + if component: + for b in component[component_list_type]: + # Pass component_data in as template data to this function + # so that if the macro is invoked with arguments, + # the arguments are interpolated into the real defn. + self.dispatch(component_type, + parser, xml_parent, b, component_data) + else: + raise JenkinsJobsException("Unknown entry point or macro '{0}'" + " for component type: '{1}'.". + format(name, component_type)) diff --git a/jenkins_jobs/xml_config.py b/jenkins_jobs/xml_config.py new file mode 100644 index 000000000..28483026e --- /dev/null +++ b/jenkins_jobs/xml_config.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# Copyright (C) 2015 OpenStack, LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Manage Jenkins XML config file output. + +import hashlib +from xml.dom import minidom +import xml.etree.ElementTree as XML + + +class XmlJob(object): + def __init__(self, xml, name): + self.xml = xml + self.name = name + + def md5(self): + return hashlib.md5(self.output()).hexdigest() + + def output(self): + out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8')) + return out.toprettyxml(indent=' ', encoding='utf-8') diff --git a/tests/base.py b/tests/base.py index 10476d3fe..7d47e672f 100644 --- a/tests/base.py +++ b/tests/base.py @@ -37,7 +37,8 @@ try: except ImportError: import mock # noqa import jenkins_jobs.local_yaml as yaml -from jenkins_jobs.builder import XmlJob, YamlParser +from jenkins_jobs.parser import YamlParser +from jenkins_jobs.xml_config import XmlJob from jenkins_jobs.modules import (project_flow, project_matrix, project_maven, diff --git a/tests/cmd/subcommands/test_test.py b/tests/cmd/subcommands/test_test.py index 037c29539..5c10ee985 100644 --- a/tests/cmd/subcommands/test_test.py +++ b/tests/cmd/subcommands/test_test.py @@ -190,7 +190,7 @@ class TestTests(CmdTestsBase): "http://test-jenkins.with.non.default.url:8080/") @mock.patch('jenkins_jobs.builder.YamlParser.generateXML') - @mock.patch('jenkins_jobs.builder.ModuleRegistry') + @mock.patch('jenkins_jobs.parser.ModuleRegistry') def test_plugins_info_stub_option(self, registry_mock, generateXML_mock): """ Test handling of plugins_info stub option. @@ -214,7 +214,7 @@ class TestTests(CmdTestsBase): registry_mock.assert_called_with(self.config, plugins_info_list) @mock.patch('jenkins_jobs.builder.YamlParser.generateXML') - @mock.patch('jenkins_jobs.builder.ModuleRegistry') + @mock.patch('jenkins_jobs.parser.ModuleRegistry') def test_bogus_plugins_info_stub_option(self, registry_mock, generateXML_mock): """ diff --git a/tests/moduleregistry/test_moduleregistry.py b/tests/moduleregistry/test_moduleregistry.py index bf34a26ba..fbe969c7f 100644 --- a/tests/moduleregistry/test_moduleregistry.py +++ b/tests/moduleregistry/test_moduleregistry.py @@ -5,7 +5,7 @@ from testscenarios.testcase import TestWithScenarios from six.moves import configparser, StringIO from jenkins_jobs import cmd -from jenkins_jobs.builder import ModuleRegistry +from jenkins_jobs.registry import ModuleRegistry class ModuleRegistryPluginInfoTestsWithScenarios(TestWithScenarios,