#!/usr/bin/env python # Copyright (C) 2015 OpenStack, LLC. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. # Manage JJB yaml feature implementation import copy import fnmatch import io import itertools import logging import re import os from jenkins_jobs.constants import MAGIC_MANAGE_STRING from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.formatter import deep_format import jenkins_jobs.local_yaml as local_yaml from jenkins_jobs import utils __all__ = [ "YamlParser" ] logger = logging.getLogger(__name__) def matches(what, glob_patterns): """ Checks if the given string, ``what``, matches any of the glob patterns in the iterable, ``glob_patterns`` :arg str what: String that we want to test if it matches a pattern :arg iterable glob_patterns: glob patterns to match (list, tuple, set, etc.) """ return any(fnmatch.fnmatch(what, glob_pattern) for glob_pattern in glob_patterns) def combination_matches(combination, match_combinations): """ Checks if the given combination is matches for any of the given combination globs, being those a set of combinations where if a key is missing, it's considered matching (key1=2, key2=3) would match the combination match: (key2=3) but not: (key1=2, key2=2) """ for cmatch in match_combinations: for key, val in combination.items(): if cmatch.get(key, val) != val: break else: return True return False class YamlParser(object): def __init__(self, jjb_config=None): self.data = {} self.jobs = [] self.views = [] self.jjb_config = jjb_config self.keep_desc = jjb_config.yamlparser['keep_descriptions'] self.path = jjb_config.yamlparser['include_path'] def load_files(self, fn): # handle deprecated behavior, and check that it's not a file like # object as these may implement the '__iter__' attribute. if not hasattr(fn, '__iter__') or hasattr(fn, 'read'): logger.warning( 'Passing single elements for the `fn` argument in ' 'Builder.load_files is deprecated. Please update your code ' 'to use a list as support for automatic conversion will be ' 'removed in a future version.') fn = [fn] files_to_process = [] for path in fn: if not hasattr(path, 'read') and os.path.isdir(path): files_to_process.extend([os.path.join(path, f) for f in sorted(os.listdir(path)) if (f.endswith('.yml') or f.endswith('.yaml'))]) else: files_to_process.append(path) # symlinks used to allow loading of sub-dirs can result in duplicate # definitions of macros and templates when loading all from top-level unique_files = [] for f in files_to_process: if hasattr(f, 'read'): unique_files.append(f) continue rpf = os.path.realpath(f) if rpf not in unique_files: unique_files.append(rpf) else: logger.warning("File '%s' already added as '%s', ignoring " "reference to avoid duplicating yaml " "definitions." % (f, rpf)) for in_file in unique_files: # use of ask-for-permissions instead of ask-for-forgiveness # performs better when low use cases. if hasattr(in_file, 'name'): fname = in_file.name else: fname = in_file logger.debug("Parsing YAML file {0}".format(fname)) if hasattr(in_file, 'read'): self._parse_fp(in_file) else: self.parse(in_file) def _parse_fp(self, fp): # wrap provided file streams to ensure correct encoding used data = local_yaml.load(utils.wrap_stream(fp), self.jjb_config.yamlparser['retain_anchors'], search_path=self.path) if data: if not isinstance(data, list): raise JenkinsJobsException( "The topmost collection in file '{fname}' must be a list," " not a {cls}".format(fname=getattr(fp, 'name', fp), cls=type(data))) for item in data: cls, dfn = next(iter(item.items())) group = self.data.get(cls, {}) if len(item.items()) > 1: n = None for k, v in item.items(): if k == "name": n = v break # Syntax error raise JenkinsJobsException("Syntax error, for item " "named '{0}'. Missing indent?" .format(n)) # allow any entry to specify an id that can also be used _id = dfn.get('id', dfn['name']) if _id in group: self._handle_dups( "Duplicate entry found in '{0}: '{1}' already " "defined".format(fp.name, _id)) group[_id] = dfn self.data[cls] = group def parse(self, fn): with io.open(fn, 'r', encoding='utf-8') as fp: self._parse_fp(fp) def _handle_dups(self, message): if not self.jjb_config.yamlparser['allow_duplicates']: logger.error(message) raise JenkinsJobsException(message) else: logger.warning(message) def _getJob(self, name): job = self.data.get('job', {}).get(name, None) if not job: return job return self._applyDefaults(job) def _getJobGroup(self, name): return self.data.get('job-group', {}).get(name, None) def _getJobTemplate(self, name): job = self.data.get('job-template', {}).get(name, None) if not job: return job return self._applyDefaults(job) def _applyDefaults(self, data, override_dict=None): if override_dict is None: override_dict = {} whichdefaults = data.get('defaults', 'global') defaults = copy.deepcopy(self.data.get('defaults', {}).get(whichdefaults, {})) if defaults == {} and whichdefaults != 'global': raise JenkinsJobsException("Unknown defaults set: '{0}'" .format(whichdefaults)) for key in override_dict.keys(): if key in defaults.keys(): defaults[key] = override_dict[key] newdata = {} newdata.update(defaults) newdata.update(data) return newdata def _formatDescription(self, job): if self.keep_desc: description = job.get("description", None) else: description = job.get("description", '') if description is not None: job["description"] = description + \ self._get_managed_string().lstrip() def _getfullname(self, data): if 'folder' in data: return "%s/%s" % (data['folder'], data['name']) return data['name'] def expandYaml(self, registry, jobs_glob=None): changed = True while changed: changed = False for module in registry.modules: if hasattr(module, 'handle_data'): if module.handle_data(self.data): changed = True for job in self.data.get('job', {}).values(): job = self._applyDefaults(job) job['name'] = self._getfullname(job) if jobs_glob and not matches(job['name'], jobs_glob): logger.debug("Ignoring job {0}".format(job['name'])) continue logger.debug("Expanding job '{0}'".format(job['name'])) self._formatDescription(job) self.jobs.append(job) for view in self.data.get('view', {}).values(): view['name'] = self._getfullname(view) logger.debug("Expanding view '{0}'".format(view['name'])) self._formatDescription(view) self.views.append(view) for project in self.data.get('project', {}).values(): logger.debug("Expanding project '{0}'".format(project['name'])) # use a set to check for duplicate job references in projects seen = set() for jobspec in project.get('jobs', []): if isinstance(jobspec, dict): # Singleton dict containing dict of job-specific params jobname, jobparams = next(iter(jobspec.items())) if not isinstance(jobparams, dict): jobparams = {} else: jobname = jobspec jobparams = {} job = self._getJob(jobname) if job: # Just naming an existing defined job if jobname in seen: self._handle_dups("Duplicate job '{0}' specified " "for project '{1}'" .format(jobname, project['name'])) seen.add(jobname) continue # see if it's a job group group = self._getJobGroup(jobname) if group: for group_jobspec in group['jobs']: if isinstance(group_jobspec, dict): group_jobname, group_jobparams = \ next(iter(group_jobspec.items())) if not isinstance(group_jobparams, dict): group_jobparams = {} else: group_jobname = group_jobspec group_jobparams = {} job = self._getJob(group_jobname) if job: if group_jobname in seen: self._handle_dups( "Duplicate job '{0}' specified for " "project '{1}'".format(group_jobname, project['name'])) seen.add(group_jobname) continue template = self._getJobTemplate(group_jobname) # Allow a group to override parameters set by a project d = type(project)(project) d.update(jobparams) d.update(group) d.update(group_jobparams) # Except name, since the group's name is not useful d['name'] = project['name'] if template: self._expandYamlForTemplateJob(d, template, jobs_glob) continue # see if it's a template template = self._getJobTemplate(jobname) if template: d = type(project)(project) d.update(jobparams) self._expandYamlForTemplateJob(d, template, jobs_glob) else: raise JenkinsJobsException("Failed to find suitable " "template named '{0}'" .format(jobname)) for viewspec in project.get('views', []): if isinstance(viewspec, dict): # Singleton dict containing dict of view-specific params viewname, viewparams = next(iter(viewspec.items())) if not isinstance(viewparams, dict): viewparams = {} else: viewname = viewspec viewparams = {} view = self._getView(viewname) if view: # Just naming an existing defined view if viewname in seen: self._handle_dups("Duplicate view '{0}' specified " "for project '{1}'" .format(viewname, project['name'])) seen.add(viewname) continue # see if it's a view group group = self._getViewGroup(viewname) if group: for group_viewspec in group['views']: if isinstance(group_viewspec, dict): group_viewname, group_viewparams = \ next(iter(group_viewspec.items())) if not isinstance(group_viewparams, dict): group_viewparams = {} else: group_viewname = group_viewspec group_viewparams = {} view = self._getView(group_viewname) if view: if group_viewname in seen: self._handle_dups( "Duplicate view '{0}' specified for " "project '{1}'".format(group_viewname, project['name'])) seen.add(group_viewname) continue template = self._getViewTemplate(group_viewname) # Allow a group to override parameters set by a project d = type(project)(project) d.update(viewparams) d.update(group) d.update(group_viewparams) # Except name, since the group's name is not useful d['name'] = project['name'] if template: self._expandYamlForTemplateView( d, template, jobs_glob) continue # see if it's a template template = self._getViewTemplate(viewname) if template: d = type(project)(project) d.update(viewparams) self._expandYamlForTemplateView(d, template, jobs_glob) else: raise JenkinsJobsException("Failed to find suitable " "template named '{0}'" .format(viewname)) # check for duplicate generated jobs seen = set() # walk the list in reverse so that last definition wins for job in self.jobs[::-1]: if job['name'] in seen: self._handle_dups("Duplicate definitions for job '{0}' " "specified".format(job['name'])) self.jobs.remove(job) seen.add(job['name']) # check for duplicate generated views seen_views = set() # walk the list in reverse so that last definition wins for view in self.views[::-1]: if view['name'] in seen_views: self._handle_dups("Duplicate definitions for view '{0}' " "specified".format(view['name'])) self.views.remove(view) seen_views.add(view['name']) return self.jobs, self.views def _expandYamlForTemplateJob(self, project, template, jobs_glob=None): dimensions = [] template_name = template['name'] # reject keys that are not useful during yaml expansion for k in ['jobs']: project.pop(k) excludes = project.pop('exclude', []) for (k, v) in project.items(): tmpk = '{{{0}}}'.format(k) if tmpk not in template_name: continue if type(v) == list: dimensions.append(zip([k] * len(v), v)) # XXX somewhat hackish to ensure we actually have a single # pass through the loop if len(dimensions) == 0: dimensions = [(("", ""),)] for values in itertools.product(*dimensions): params = copy.deepcopy(project) params = self._applyDefaults(params, template) params['template-name'] = re.sub(r'({|})', r'\1\1', template_name) try: expanded_values = {} for (k, v) in values: if isinstance(v, dict): inner_key = next(iter(v)) expanded_values[k] = inner_key expanded_values.update(v[inner_key]) else: expanded_values[k] = v except TypeError: project_name = project.pop('name') logger.error( "Exception thrown while expanding template '%s' for " "project '%s', with expansion arguments of:\n%s\n" "Original project input variables for template:\n%s\n" "Most likely the inputs have items indented incorrectly " "to describe how they should be applied.\n\nNote yaml " "'null' is mapped to python's 'None'", template_name, project_name, "".join(local_yaml.dump({k: v}, default_flow_style=False) for (k, v) in values), local_yaml.dump(project, default_flow_style=False)) raise params.update(expanded_values) try: params = deep_format(params, params) except Exception: logging.error( "Failure formatting params '%s' with itself", params) raise if combination_matches(params, excludes): logger.debug('Excluding combination %s', str(params)) continue for key in template.keys(): if key not in params: params[key] = template[key] try: expanded = deep_format( template, params, self.jjb_config.yamlparser['allow_empty_variables']) except Exception: logging.error( "Failure formatting template '%s', containing '%s' with " "params '%s'", template_name, template, params) raise expanded['name'] = self._getfullname(expanded) job_name = expanded.get('name') if jobs_glob and not matches(job_name, jobs_glob): continue self._formatDescription(expanded) self.jobs.append(expanded) def _get_managed_string(self): # The \n\n is not hard coded, because they get stripped if the # project does not otherwise have a description. return "\n\n" + MAGIC_MANAGE_STRING # Views related def _getView(self, name): view = self.data.get('view', {}).get(name, None) if not view: return view return self._applyDefaults(view) def _getViewGroup(self, name): return self.data.get('view-group', {}).get(name, None) def _getViewTemplate(self, name): view = self.data.get('view-template', {}).get(name, None) if not view: return view return self._applyDefaults(view) def _expandYamlForTemplateView(self, project, template, views_glob=None): dimensions = [] template_name = template['name'] # reject keys that are not useful during yaml expansion for k in ['views']: project.pop(k) excludes = project.pop('exclude', []) for (k, v) in project.items(): tmpk = '{{{0}}}'.format(k) if tmpk not in template_name: continue if type(v) == list: dimensions.append(zip([k] * len(v), v)) # XXX somewhat hackish to ensure we actually have a single # pass through the loop if len(dimensions) == 0: dimensions = [(("", ""),)] for values in itertools.product(*dimensions): params = copy.deepcopy(project) params = self._applyDefaults(params, template) expanded_values = {} for (k, v) in values: if isinstance(v, dict): inner_key = next(iter(v)) expanded_values[k] = inner_key expanded_values.update(v[inner_key]) else: expanded_values[k] = v params.update(expanded_values) params = deep_format(params, params) if combination_matches(params, excludes): logger.debug('Excluding combination %s', str(params)) continue for key in template.keys(): if key not in params: params[key] = template[key] params['template-name'] = template_name expanded = deep_format( template, params, self.jjb_config.yamlparser['allow_empty_variables']) view_name = expanded.get('name') if views_glob and not matches(view_name, views_glob): continue self._formatDescription(expanded) self.views.append(expanded)