From aaae83c6235a80ff706c3d5330c9f952dbbc80a2 Mon Sep 17 00:00:00 2001 From: Wayne Warren Date: Sun, 3 Jan 2016 12:19:44 -0800 Subject: [PATCH] Factor XmlJobGenerator out of YamlParser. Separate XML generation from Yaml parsing/interpreting. The goal here is to allow different sources to provide data for XML generation, including external API users writing job definitions in pure Python or JJB developers who would like to work on alternative Yaml parsing code since the current YamlParser has very likely reached the limits of what can be reasonably done with one giant expandYaml procedure. Change-Id: I9da848acac7e944c0e07286b7399b2e1956a58a5 --- jenkins_jobs/cli/subcommand/update.py | 9 +++++--- jenkins_jobs/parser.py | 25 +------------------- jenkins_jobs/xml_config.py | 33 +++++++++++++++++++++++++++ tests/base.py | 13 +++++++---- tests/cmd/subcommands/test_test.py | 6 +++-- 5 files changed, 52 insertions(+), 34 deletions(-) diff --git a/jenkins_jobs/cli/subcommand/update.py b/jenkins_jobs/cli/subcommand/update.py index 6ac1115a7..bf3e76362 100644 --- a/jenkins_jobs/cli/subcommand/update.py +++ b/jenkins_jobs/cli/subcommand/update.py @@ -20,6 +20,7 @@ import time from jenkins_jobs.builder import Builder from jenkins_jobs.parser import YamlParser from jenkins_jobs.registry import ModuleRegistry +from jenkins_jobs.xml_config import XmlJobGenerator from jenkins_jobs.errors import JenkinsJobsException import jenkins_jobs.cli.subcommand.base as base @@ -74,19 +75,21 @@ class UpdateSubCommand(base.BaseSubCommand): # Generate XML parser = YamlParser(jjb_config) registry = ModuleRegistry(jjb_config, builder.plugins_list) + xml_generator = XmlJobGenerator(registry) parser.load_files(options.path) registry.set_parser_data(parser.data) - parser.expandYaml(registry, options.names) - parser.generateXML(registry) + job_data_list = parser.expandYaml(registry, options.names) + + xml_jobs = xml_generator.generateXML(job_data_list) jobs = parser.jobs step = time.time() logging.debug('%d XML files generated in %ss', len(jobs), str(step - orig)) - return builder, parser.xml_jobs + return builder, xml_jobs def execute(self, options, jjb_config): diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index d9194fd1e..5faad2253 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -21,14 +21,12 @@ import io import itertools import logging import os -import pkg_resources from jenkins_jobs.constants import MAGIC_MANAGE_STRING from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.formatter import deep_format import jenkins_jobs.local_yaml as local_yaml from jenkins_jobs import utils -from jenkins_jobs.xml_config import XmlJob logger = logging.getLogger(__name__) @@ -73,7 +71,6 @@ class YamlParser(object): def __init__(self, jjb_config=None): self.data = {} self.jobs = [] - self.xml_jobs = [] self.jjb_config = jjb_config self.keep_desc = jjb_config.yamlparser['keep_descriptions'] @@ -309,6 +306,7 @@ class YamlParser(object): "specified".format(job['name'])) self.jobs.remove(job) seen.add(job['name']) + return self.jobs def _expandYamlForTemplateJob(self, project, template, jobs_glob=None): dimensions = [] @@ -367,24 +365,3 @@ class YamlParser(object): # The \n\n is not hard coded, because they get stripped if the # project does not otherwise have a description. return "\n\n" + MAGIC_MANAGE_STRING - - def generateXML(self, registry): - for job in self.jobs: - self.xml_jobs.append(self.getXMLForJob(job, registry)) - - def getXMLForJob(self, data, registry): - kind = data.get('project-type', 'freestyle') - - for ep in pkg_resources.iter_entry_points( - group='jenkins_jobs.projects', name=kind): - Mod = ep.load() - mod = Mod(registry) - xml = mod.root_xml(data) - self.gen_xml(xml, data, registry) - job = XmlJob(xml, data['name']) - return job - - def gen_xml(self, xml, data, registry): - for module in registry.modules: - if hasattr(module, 'gen_xml'): - module.gen_xml(xml, data) diff --git a/jenkins_jobs/xml_config.py b/jenkins_jobs/xml_config.py index 2e74f5343..3aa33564d 100644 --- a/jenkins_jobs/xml_config.py +++ b/jenkins_jobs/xml_config.py @@ -16,6 +16,7 @@ # Manage Jenkins XML config file output. import hashlib +import pkg_resources from xml.dom import minidom import xml.etree.ElementTree as XML @@ -53,3 +54,35 @@ class XmlJob(object): def output(self): out = minidom.parseString(XML.tostring(self.xml, encoding='UTF-8')) return out.toprettyxml(indent=' ', encoding='utf-8') + + +class XmlJobGenerator(object): + """ This class is responsible for generating Jenkins Configuration XML from + a compatible intermediate representation of Jenkins Jobs. + """ + + def __init__(self, registry): + self.registry = registry + + def generateXML(self, jobdict_list): + xml_jobs = [] + for job in jobdict_list: + xml_jobs.append(self.__getXMLForJob(job)) + return xml_jobs + + def __getXMLForJob(self, data): + kind = data.get('project-type', 'freestyle') + + for ep in pkg_resources.iter_entry_points( + group='jenkins_jobs.projects', name=kind): + Mod = ep.load() + mod = Mod(self.registry) + xml = mod.root_xml(data) + self.__gen_xml(xml, data) + job = XmlJob(xml, data['name']) + return job + + def __gen_xml(self, xml, data): + for module in self.registry.modules: + if hasattr(module, 'gen_xml'): + module.gen_xml(xml, data) diff --git a/tests/base.py b/tests/base.py index 661656950..6cca3a5ab 100644 --- a/tests/base.py +++ b/tests/base.py @@ -42,6 +42,7 @@ from jenkins_jobs.modules import project_multijob from jenkins_jobs.parser import YamlParser from jenkins_jobs.registry import ModuleRegistry from jenkins_jobs.xml_config import XmlJob +from jenkins_jobs.xml_config import XmlJobGenerator # This dance deals with the fact that we want unittest.mock if # we're on Python 3.4 and later, and non-stdlib mock otherwise. @@ -202,15 +203,17 @@ class SingleJobTestCase(BaseTestCase): registry = ModuleRegistry(config) registry.set_parser_data(parser.data) - # Generate the XML tree - parser.expandYaml(registry) - parser.generateXML(registry) + job_data_list = parser.expandYaml(registry) - parser.xml_jobs.sort(key=operator.attrgetter('name')) + # Generate the XML tree + xml_generator = XmlJobGenerator(registry) + xml_jobs = xml_generator.generateXML(job_data_list) + + xml_jobs.sort(key=operator.attrgetter('name')) # Prettify generated XML pretty_xml = u"\n".join(job.output().decode('utf-8') - for job in parser.xml_jobs) + for job in xml_jobs) self.assertThat( pretty_xml, diff --git a/tests/cmd/subcommands/test_test.py b/tests/cmd/subcommands/test_test.py index 64aa21167..4122a90dd 100644 --- a/tests/cmd/subcommands/test_test.py +++ b/tests/cmd/subcommands/test_test.py @@ -130,7 +130,8 @@ class TestTests(CmdTestsBase): e = self.assertRaises(UnicodeError, jenkins_jobs.execute) self.assertIn("'ascii' codec can't encode character", str(e)) - @mock.patch('jenkins_jobs.cli.subcommand.update.YamlParser.generateXML') + @mock.patch( + 'jenkins_jobs.cli.subcommand.update.XmlJobGenerator.generateXML') @mock.patch('jenkins_jobs.cli.subcommand.update.ModuleRegistry') def test_plugins_info_stub_option(self, registry_mock, generateXML_mock): """ @@ -154,7 +155,8 @@ class TestTests(CmdTestsBase): registry_mock.assert_called_with(mock.ANY, plugins_info_list) - @mock.patch('jenkins_jobs.cli.subcommand.update.YamlParser.generateXML') + @mock.patch( + 'jenkins_jobs.cli.subcommand.update.XmlJobGenerator.generateXML') @mock.patch('jenkins_jobs.cli.subcommand.update.ModuleRegistry') def test_bogus_plugins_info_stub_option(self, registry_mock, generateXML_mock):