From 1deb3aff4c1ff57b92d967c67411b316ef5b8952 Mon Sep 17 00:00:00 2001 From: Thanh Ha Date: Wed, 14 Sep 2016 16:44:50 -0400 Subject: [PATCH] Add view management functionality - Adds the ability for JJB to work with views - Views can be created, updated, and deleted. - New modules for List view and Build Pipeline view are added - New tests for testing the deletion of views Example View configuration: - view: name: MyView view-type: list Change-Id: Idb29a4407bcc14593e10a4d951036cb04e8e6c27 Co-Authored-By: Brandon Leonard Co-Authored-By: Joao Vale Co-Authored-By: Lucas Dutra Nunes Signed-off-by: Thanh Ha --- doc/source/definition.rst | 15 ++- doc/source/execution.rst | 47 ++++--- doc/source/view_list.rst | 7 + jenkins_jobs/builder.py | 143 ++++++++++++++++++++ jenkins_jobs/cli/subcommand/delete.py | 27 +++- jenkins_jobs/cli/subcommand/delete_all.py | 38 +++++- jenkins_jobs/cli/subcommand/test.py | 4 +- jenkins_jobs/cli/subcommand/update.py | 19 ++- jenkins_jobs/modules/view_list.py | 102 +++++++++++++++ jenkins_jobs/modules/view_pipeline.py | 145 +++++++++++++++++++++ jenkins_jobs/parser.py | 9 +- jenkins_jobs/xml_config.py | 32 +++++ setup.cfg | 3 + tests/base.py | 14 +- tests/cmd/subcommands/test_delete.py | 8 +- tests/cmd/test_config.py | 2 + tests/views/__init__.py | 0 tests/views/fixtures/view_list001.xml | 27 ++++ tests/views/fixtures/view_list001.yaml | 20 +++ tests/views/fixtures/view_list002.xml | 21 +++ tests/views/fixtures/view_list002.yaml | 10 ++ tests/views/fixtures/view_pipeline001.xml | 22 ++++ tests/views/fixtures/view_pipeline001.yaml | 17 +++ tests/views/fixtures/view_pipeline002.xml | 21 +++ tests/views/fixtures/view_pipeline002.yaml | 3 + tests/views/test_views.py | 30 +++++ 26 files changed, 755 insertions(+), 31 deletions(-) create mode 100644 doc/source/view_list.rst create mode 100644 jenkins_jobs/modules/view_list.py create mode 100644 jenkins_jobs/modules/view_pipeline.py create mode 100644 tests/views/__init__.py create mode 100644 tests/views/fixtures/view_list001.xml create mode 100644 tests/views/fixtures/view_list001.yaml create mode 100644 tests/views/fixtures/view_list002.xml create mode 100644 tests/views/fixtures/view_list002.yaml create mode 100644 tests/views/fixtures/view_pipeline001.xml create mode 100644 tests/views/fixtures/view_pipeline001.yaml create mode 100644 tests/views/fixtures/view_pipeline002.xml create mode 100644 tests/views/fixtures/view_pipeline002.yaml create mode 100644 tests/views/test_views.py diff --git a/doc/source/definition.rst b/doc/source/definition.rst index bc279d024..41e07076d 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -178,6 +178,20 @@ the Job Templates in the Job Group will be realized. For example: Would cause the jobs `project-name-unit-tests` and `project-name-perf-tests` to be created in Jenkins. +.. _views: + +Views +^^^^^ + +A view is a particular way of displaying a specific set of jobs. To +create a view, you must define a view in a YAML file and have a variable called view-type with a valid value. It looks like this:: + + - view: + name: view-name + view-type: list + +Views are processed differently than Jobs and therefore will not work within a `Project`_ or a `Job Template`_. + .. _macro: Macro @@ -494,4 +508,3 @@ Generally the sequence is: #. builders (maven, freestyle, matrix, etc..) #. postbuilders (maven only, configured like :ref:`builders`) #. publishers/reporters/notifications - diff --git a/doc/source/execution.rst b/doc/source/execution.rst index 4323ea726..7cf0ec379 100644 --- a/doc/source/execution.rst +++ b/doc/source/execution.rst @@ -161,16 +161,17 @@ When you're satisfied with the generated XML from the test, you can run:: jenkins-jobs update /path/to/defs -which will upload the job definitions to Jenkins if needed. Jenkins Job -Builder maintains, for each host, a cache [#f1]_ of previously configured jobs, -so that you can run that command as often as you like, and it will only -update the jobs configurations in Jenkins if the defined definitions has -changed since the last time it was run. Note: if you modify a job +which will upload the job and view definitions to Jenkins if needed. Jenkins +Job Builder maintains, for each host, a cache [#f1]_ of previously configured +jobs and views, so that you can run that command as often as you like, and it +will only update the jobs configurations in Jenkins if the defined definitions +has changed since the last time it was run. Note: if you modify a job directly in Jenkins, jenkins-jobs will not know about it and will not update it. -To update a specific list of jobs, simply pass the job names as additional -arguments after the job definition path. To update Foo1 and Foo2 run:: +To update a specific list of jobs/views, simply pass the job/view names as +additional arguments after the job definition path. To update Foo1 and Foo2 +run:: jenkins-jobs update /path/to/defs Foo1 Foo2 @@ -248,19 +249,25 @@ are denoted by starting from the root, relative by containing the path separator, and patterns by having neither. Patterns use simple shell globing to match directories. -Deleting Jobs -^^^^^^^^^^^^^ -Jenkins Job Builder supports deleting jobs from Jenkins. +Deleting Jobs/Views +^^^^^^^^^^^^^^^^^^^ +Jenkins Job Builder supports deleting jobs and views from Jenkins. To delete a specific job:: jenkins-jobs delete Foo1 -To delete a list of jobs, simply pass them as additional +To delete a list of jobs or views, simply pass them as additional arguments after the command:: jenkins-jobs delete Foo1 Foo2 +To delete only views or only jobs, simply add the argument +--views-only or --jobs-only after the command:: + + jenkins-jobs delete --views-only Foo1 + jenkins-jobs delete --jobs-only Foo1 + The ``update`` command includes a ``delete-old`` option to remove obsolete jobs:: @@ -270,21 +277,31 @@ Obsolete jobs are jobs once managed by JJB (as distinguished by a special comment that JJB appends to their description), that were not generated in this JJB run. -There is also a command to delete **all** jobs. -**WARNING**: Use with caution:: +There is also a command to delete **all** jobs and/or views. +**WARNING**: Use with caution. + +To delete **all** jobs and views:: jenkins-jobs delete-all +TO delete **all** jobs:: + + jenkins-jobs delete-all --jobs-only + +To delete **all** views:: + + jenkins-jobs delete-all --views-only + Globbed Parameters ^^^^^^^^^^^^^^^^^^ Jenkins job builder supports globbed parameters to identify jobs from a set of definition files. This feature only supports JJB managed jobs. -To update jobs that only have 'foo' in their name:: +To update jobs/views that only have 'foo' in their name:: jenkins-jobs update ./myjobs \*foo\* -To delete jobs that only have 'foo' in their name:: +To delete jobs/views that only have 'foo' in their name:: jenkins-jobs delete --path ./myjobs \*foo\* diff --git a/doc/source/view_list.rst b/doc/source/view_list.rst new file mode 100644 index 000000000..3142bcf37 --- /dev/null +++ b/doc/source/view_list.rst @@ -0,0 +1,7 @@ +.. view_list: + +List View +========= + +.. automodule:: view_list + :members: diff --git a/jenkins_jobs/builder.py b/jenkins_jobs/builder.py index cf59d5704..91881e082 100644 --- a/jenkins_jobs/builder.py +++ b/jenkins_jobs/builder.py @@ -61,6 +61,8 @@ class JenkinsManager(object): self._plugins_list = jjb_config.builder['plugins_info'] self._jobs = None self._job_list = None + self._views = None + self._view_list = None self._jjb_config = jjb_config @property @@ -274,3 +276,144 @@ class JenkinsManager(object): def parallel_update_job(self, job): self.update_job(job.name, job.output().decode('utf-8')) return (job.name, job.md5()) + + ################ + # View related # + ################ + + @property + def views(self): + if self._views is None: + # populate views + self._views = self.jenkins.get_views() + return self._views + + @property + def view_list(self): + if self._view_list is None: + self._view_list = set(view['name'] for view in self.views) + return self._view_list + + def get_views(self, cache=True): + if not cache: + self._views = None + self._view_list = None + return self.views + + def is_view(self, view_name): + # first use cache + if view_name in self.view_list: + return True + + # if not exists, use jenkins + return self.jenkins.view_exists(view_name) + + def delete_view(self, view_name): + if self.is_view(view_name): + logger.info("Deleting jenkins view {}".format(view_name)) + self.jenkins.delete_view(view_name) + + def delete_views(self, views): + if views is not None: + logger.info("Removing jenkins view(s): %s" % ", ".join(views)) + for view in views: + self.delete_view(view) + if self.cache.is_cached(view): + self.cache.set(view, '') + self.cache.save() + + def delete_all_views(self): + views = self.get_views() + # Jenkins requires at least one view present. Don't remove the first + # view as it is likely the default view. + views.pop(0) + logger.info("Number of views to delete: %d", len(views)) + for view in views: + self.delete_view(view['name']) + # Need to clear the JJB cache after deletion + self.cache.clear() + + def update_view(self, view_name, xml): + if self.is_view(view_name): + logger.info("Reconfiguring jenkins view {0}".format(view_name)) + self.jenkins.reconfig_view(view_name, xml) + else: + logger.info("Creating jenkins view {0}".format(view_name)) + self.jenkins.create_view(view_name, xml) + + def update_views(self, xml_views, output=None, n_workers=None): + orig = time.time() + + logger.info("Number of views generated: %d", len(xml_views)) + xml_views.sort(key=operator.attrgetter('name')) + + if output: + # ensure only wrapped once + if hasattr(output, 'write'): + output = utils.wrap_stream(output) + + for view in xml_views: + if hasattr(output, 'write'): + # `output` is a file-like object + logger.info("View name: %s", view.name) + logger.debug("Writing XML to '{0}'".format(output)) + try: + output.write(view.output()) + except IOError as exc: + if exc.errno == errno.EPIPE: + # EPIPE could happen if piping output to something + # that doesn't read the whole input (e.g.: the UNIX + # `head` command) + return + raise + continue + + output_fn = os.path.join(output, view.name) + logger.debug("Writing XML to '{0}'".format(output_fn)) + with io.open(output_fn, 'w', encoding='utf-8') as f: + f.write(view.output().decode('utf-8')) + return xml_views, len(xml_views) + + # Filter out the views that did not change + logging.debug('Filtering %d views for changed views', + len(xml_views)) + step = time.time() + views = [view for view in xml_views + if self.changed(view)] + logging.debug("Filtered for changed views in %ss", + (time.time() - step)) + + if not views: + return [], 0 + + # Update the views + logging.debug('Updating views') + step = time.time() + p_params = [{'view': view} for view in views] + results = self.parallel_update_view( + n_workers=n_workers, + concurrent=p_params) + logging.debug("Parsing results") + # generalize the result parsing, as a concurrent view always returns a + # list + if len(p_params) in (1, 0): + results = [results] + for result in results: + if isinstance(result, Exception): + raise result + else: + # update in-memory cache + v_name, v_md5 = result + self.cache.set(v_name, v_md5) + # write cache to disk + self.cache.save() + logging.debug("Updated %d views in %ss", + len(views), + time.time() - step) + logging.debug("Total run took %ss", (time.time() - orig)) + return views, len(views) + + @concurrent + def parallel_update_view(self, view): + self.update_view(view.name, view.output().decode('utf-8')) + return (view.name, view.md5()) diff --git a/jenkins_jobs/cli/subcommand/delete.py b/jenkins_jobs/cli/subcommand/delete.py index 95100cf13..b589d88c3 100644 --- a/jenkins_jobs/cli/subcommand/delete.py +++ b/jenkins_jobs/cli/subcommand/delete.py @@ -15,6 +15,7 @@ from jenkins_jobs.builder import JenkinsManager +from jenkins_jobs.errors import JenkinsJobsException from jenkins_jobs.parser import YamlParser from jenkins_jobs.registry import ModuleRegistry import jenkins_jobs.cli.subcommand.base as base @@ -36,10 +37,26 @@ class DeleteSubCommand(base.BaseSubCommand): default=None, help="colon-separated list of paths to YAML files " "or directories") + delete.add_argument( + '-j', '--jobs-only', + action='store_true', dest='del_jobs', + default=False, + help='delete only jobs' + ) + delete.add_argument( + '-v', '--views-only', + action='store_true', dest='del_views', + default=False, + help='delete only views' + ) def execute(self, options, jjb_config): builder = JenkinsManager(jjb_config) + if options.del_jobs and options.del_views: + raise JenkinsJobsException( + '"--views-only" and "--jobs-only" cannot be used together.') + fn = options.path registry = ModuleRegistry(jjb_config, builder.plugins_list) parser = YamlParser(jjb_config) @@ -48,7 +65,15 @@ class DeleteSubCommand(base.BaseSubCommand): parser.load_files(fn) parser.expandYaml(registry, options.name) jobs = [j['name'] for j in parser.jobs] + views = [v['name'] for v in parser.views] else: jobs = options.name + views = options.name - builder.delete_jobs(jobs) + if options.del_jobs: + builder.delete_jobs(jobs) + elif options.del_views: + builder.delete_views(views) + else: + builder.delete_jobs(jobs) + builder.delete_views(views) diff --git a/jenkins_jobs/cli/subcommand/delete_all.py b/jenkins_jobs/cli/subcommand/delete_all.py index 14f34881b..8886d26ac 100644 --- a/jenkins_jobs/cli/subcommand/delete_all.py +++ b/jenkins_jobs/cli/subcommand/delete_all.py @@ -19,6 +19,7 @@ import sys from jenkins_jobs import utils from jenkins_jobs.builder import JenkinsManager +from jenkins_jobs.errors import JenkinsJobsException import jenkins_jobs.cli.subcommand.base as base @@ -35,14 +36,43 @@ class DeleteAllSubCommand(base.BaseSubCommand): self.parse_option_recursive_exclude(delete_all) + delete_all.add_argument( + '-j', '--jobs-only', + action='store_true', dest='del_jobs', + default=False, + help='delete only jobs' + ) + delete_all.add_argument( + '-v', '--views-only', + action='store_true', dest='del_views', + default=False, + help='delete only views' + ) + def execute(self, options, jjb_config): builder = JenkinsManager(jjb_config) + reach = set() + if options.del_jobs and options.del_views: + raise JenkinsJobsException( + '"--views-only" and "--jobs-only" cannot be used together.') + elif options.del_jobs and not options.del_views: + reach.add('jobs') + elif options.del_views and not options.del_jobs: + reach.add('views') + else: + reach.update(('jobs', 'views')) + if not utils.confirm( - 'Sure you want to delete *ALL* jobs from Jenkins ' + 'Sure you want to delete *ALL* {} from Jenkins ' 'server?\n(including those not managed by Jenkins ' - 'Job Builder)'): + 'Job Builder)'.format(" AND ".join(reach))): sys.exit('Aborted') - logger.info("Deleting all jobs") - builder.delete_all_jobs() + if options.del_jobs: + logger.info("Deleting all jobs") + builder.delete_all_jobs() + + if options.del_views: + logger.info("Deleting all views") + builder.delete_all_views() diff --git a/jenkins_jobs/cli/subcommand/test.py b/jenkins_jobs/cli/subcommand/test.py index e68f725d0..b10754fa0 100644 --- a/jenkins_jobs/cli/subcommand/test.py +++ b/jenkins_jobs/cli/subcommand/test.py @@ -45,6 +45,8 @@ class TestSubCommand(update.UpdateSubCommand): def execute(self, options, jjb_config): - builder, xml_jobs = self._generate_xmljobs(options, jjb_config) + builder, xml_jobs, xml_views = self._generate_xmljobs( + options, jjb_config) builder.update_jobs(xml_jobs, output=options.output_dir, n_workers=1) + builder.update_views(xml_views, output=options.output_dir, n_workers=1) diff --git a/jenkins_jobs/cli/subcommand/update.py b/jenkins_jobs/cli/subcommand/update.py index bb14ddee8..03901c2f4 100644 --- a/jenkins_jobs/cli/subcommand/update.py +++ b/jenkins_jobs/cli/subcommand/update.py @@ -21,6 +21,7 @@ from jenkins_jobs.builder import JenkinsManager from jenkins_jobs.parser import YamlParser from jenkins_jobs.registry import ModuleRegistry from jenkins_jobs.xml_config import XmlJobGenerator +from jenkins_jobs.xml_config import XmlViewGenerator from jenkins_jobs.errors import JenkinsJobsException import jenkins_jobs.cli.subcommand.base as base @@ -75,21 +76,24 @@ class UpdateSubCommand(base.BaseSubCommand): # Generate XML parser = YamlParser(jjb_config) registry = ModuleRegistry(jjb_config, builder.plugins_list) - xml_generator = XmlJobGenerator(registry) + xml_job_generator = XmlJobGenerator(registry) + xml_view_generator = XmlViewGenerator(registry) parser.load_files(options.path) registry.set_parser_data(parser.data) - job_data_list = parser.expandYaml(registry, options.names) + job_data_list, view_data_list = parser.expandYaml( + registry, options.names) - xml_jobs = xml_generator.generateXML(job_data_list) + xml_jobs = xml_job_generator.generateXML(job_data_list) + xml_views = xml_view_generator.generateXML(view_data_list) jobs = parser.jobs step = time.time() logging.debug('%d XML files generated in %ss', len(jobs), str(step - orig)) - return builder, xml_jobs + return builder, xml_jobs, xml_views def execute(self, options, jjb_config): @@ -97,12 +101,17 @@ class UpdateSubCommand(base.BaseSubCommand): raise JenkinsJobsException( 'Number of workers must be equal or greater than 0') - builder, xml_jobs = self._generate_xmljobs(options, jjb_config) + builder, xml_jobs, xml_views = self._generate_xmljobs( + options, jjb_config) jobs, num_updated_jobs = builder.update_jobs( xml_jobs, n_workers=options.n_workers) logger.info("Number of jobs updated: %d", num_updated_jobs) + views, num_updated_views = builder.update_views( + xml_views, n_workers=options.n_workers) + logger.info("Number of views updated: %d", num_updated_views) + keep_jobs = [job.name for job in xml_jobs] if options.delete_old: n = builder.delete_old_managed(keep=keep_jobs) diff --git a/jenkins_jobs/modules/view_list.py b/jenkins_jobs/modules/view_list.py new file mode 100644 index 000000000..b4741d53e --- /dev/null +++ b/jenkins_jobs/modules/view_list.py @@ -0,0 +1,102 @@ +# Copyright 2015 Openstack Foundation + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import xml.etree.ElementTree as XML +import jenkins_jobs.modules.base + +""" +The view list module handles creating Jenkins List views. + +To create a list view specify ``list`` in the ``view-type`` attribute +to the :ref:`View-list` definition. + +:View Parameters: + * **name** (`str`): The name of the view. + * **view-type** (`str`): The type of view. + * **description** (`str`): A description of the view. (optional) + * **filter-executors** (`bool`): Show only executors that can + execute the included views. (default false) + * **filter-queue** (`bool`): Show only included jobs in builder + queue. (default false) + * **job-name** (`list`): List of jobs to be included. + * **columns** (`list`): List of columns to be shown in view. + * **regex** (`str`): . Regular expression for selecting jobs + (optional) + * **recurse** (`bool`): Recurse in subfolders.(default false) + * **status-filter** (`bool`): Filter job list by enabled/disabled + status. (optional) +""" + +COLUMN_DICT = { + 'status': 'hudson.views.StatusColumn', + 'weather': 'hudson.views.WeatherColumn', + 'job': 'hudson.views.JobColumn', + 'last-success': 'hudson.views.LastSuccessColumn', + 'last-failure': 'hudson.views.LastFailureColumn', + 'last-duration': 'hudson.views.LastDurationColumn', + 'build-button': 'hudson.views.BuildButtonColumn', + 'last-stable': 'hudson.views.LastStableColumn', +} + + +class List(jenkins_jobs.modules.base.Base): + sequence = 0 + + def root_xml(self, data): + root = XML.Element('hudson.model.ListView') + XML.SubElement(root, 'name').text = data['name'] + desc_text = data.get('description', None) + if desc_text is not None: + XML.SubElement(root, 'description').text = desc_text + + filterExecutors = data.get('filter-executors', False) + FE_element = XML.SubElement(root, 'filterExecutors') + FE_element.text = 'true' if filterExecutors else 'false' + + filterQueue = data.get('filter-queue', False) + FQ_element = XML.SubElement(root, 'filterQueue') + FQ_element.text = 'true' if filterQueue else 'false' + + XML.SubElement(root, 'properties', + {'class': 'hudson.model.View$PropertyList'}) + + jn_xml = XML.SubElement(root, 'jobNames') + jobnames = data.get('job-name', None) + XML.SubElement(jn_xml, 'comparator', {'class': + 'hudson.util.CaseInsensitiveComparator'}) + if jobnames is not None: + for jobname in jobnames: + XML.SubElement(jn_xml, 'string').text = str(jobname) + XML.SubElement(root, 'jobFilters') + + c_xml = XML.SubElement(root, 'columns') + columns = data.get('columns', []) + for column in columns: + if column in COLUMN_DICT: + XML.SubElement(c_xml, COLUMN_DICT[column]) + + regex = data.get('regex', None) + if regex is not None: + XML.SubElement(root, 'includeRegex').text = regex + + recurse = data.get('recurse', False) + R_element = XML.SubElement(root, 'recurse') + R_element.text = 'true' if recurse else 'false' + + statusfilter = data.get('status-filter', None) + if statusfilter is not None: + SF_element = XML.SubElement(root, 'statusFilter') + SF_element.text = 'true' if statusfilter else 'false' + + return root diff --git a/jenkins_jobs/modules/view_pipeline.py b/jenkins_jobs/modules/view_pipeline.py new file mode 100644 index 000000000..d956a0564 --- /dev/null +++ b/jenkins_jobs/modules/view_pipeline.py @@ -0,0 +1,145 @@ +# Copyright 2015 Openstack Foundation + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import xml.etree.ElementTree as XML +import jenkins_jobs.modules.base + +""" +The view pipeline module handles creating Jenkins Build Pipeline views. +To create a list view specify ``list`` in the ``view-type`` attribute +to the :ref:`View-pipeline` definition. +Requires the Jenkins +:jenkins-wiki:`Build Pipeline Plugin `. + +:View Parameters: + * **name** (`str`): The name of the view. + * **view-type** (`str`): The type of view. + * **description** (`str`): A description of the view. (optional) + * **filter-executors** (`bool`): Show only executors that can + execute the included views. (default false) + * **filter-queue** (`bool`): Show only included jobs in builder + queue. (default false) + * **first-job** (`str`): Parent Job in the view. + * **no-of-displayed-builds** (`str`): Number of builds to display. + (default 1) + * **title** (`str`): Build view title. (optional) + * **linkStyle** (`str`): Console output link style. Can be + 'Lightbox', 'New Window', or 'This Window'. (default Lightbox) + * **css-Url** (`str`): Url for Custom CSS files (optional) + * **latest-job-only** (`bool`) Trigger only latest job. + (default false) + * **manual-trigger** (`bool`) Always allow manual trigger. + (default false) + * **show-parameters** (`bool`) Show pipeline parameters. + (default false) + * **parameters-in-headers** (`bool`) Show pipeline parameters in + headers. (default false) + * **starts-with-parameters** (`bool`) Use Starts with parameters. + (default false) + * **refresh-frequency** (`str`) Frequency to refresh in seconds. + (default '3') + * **definition-header** (`bool`) Show pipeline definition header. + (default false) + +Example: + + .. literalinclude:: + /../../tests/views/fixtures/pipeline_view001.yaml + +Example: + + .. literalinclude:: + /../../tests/views/fixtures/pipeline_view002.yaml +""" + + +class Pipeline(jenkins_jobs.modules.base.Base): + sequence = 0 + + def root_xml(self, data): + linktypes = ['Lightbox', 'New Window'] + root = XML.Element('au.com.centrumsystems.hudson.' + 'plugin.buildpipeline.BuildPipelineView', + {'plugin': 'build-pipeline-plugin'}) + XML.SubElement(root, 'name').text = data['name'] + desc_text = data.get('description', None) + if desc_text is not None: + XML.SubElement(root, 'description').text = desc_text + + filterExecutors = data.get('filter-executors', False) + FE_element = XML.SubElement(root, 'filterExecutors') + FE_element.text = 'true' if filterExecutors else 'false' + + filterQueue = data.get('filter-queue', False) + FQ_element = XML.SubElement(root, 'filterQueue') + FQ_element.text = 'true' if filterQueue else 'false' + + XML.SubElement(root, 'properties', + {'class': 'hudson.model.View$PropertyList'}) + + GBurl = ('au.com.centrumsystems.hudson.plugin.buildpipeline.' + 'DownstreamProjectGridBuilder') + gridBuilder = XML.SubElement(root, 'gridBuilder', {'class': GBurl}) + + jobname = data.get('first-job', '') + XML.SubElement(gridBuilder, 'firstJob').text = jobname + + builds = str(data.get('no-of-displayed-builds', 1)) + XML.SubElement(root, 'noOfDisplayedBuilds').text = builds + + title = data.get('title', None) + BVT_element = XML.SubElement(root, 'buildViewTitle') + if title is not None: + BVT_element.text = title + + linkStyle = data.get('link-style', 'Lightbox') + LS_element = XML.SubElement(root, 'consoleOutputLinkStyle') + if linkStyle in linktypes: + LS_element.text = linkStyle + else: + LS_element.text = 'Lightbox' + + cssUrl = data.get('css-Url', None) + CU_element = XML.SubElement(root, 'cssUrl') + if cssUrl is not None: + CU_element.text = cssUrl + + latest_job_only = data.get('latest-job-only', False) + OLJ_element = XML.SubElement(root, 'triggerOnlyLatestJob') + OLJ_element.text = 'true' if latest_job_only else 'false' + + manual_trigger = data.get('manual-trigger', False) + AMT_element = XML.SubElement(root, 'alwaysAllowManualTrigger') + AMT_element.text = 'true' if manual_trigger else 'false' + + show_parameters = data.get('show-parameters', False) + PP_element = XML.SubElement(root, 'showPipelineParameters') + PP_element.text = 'true' if show_parameters else 'false' + + parameters_in_headers = data.get('parameters-in-headers', False) + PIH_element = XML.SubElement(root, 'showPipelineParametersInHeaders') + PIH_element.text = 'true' if parameters_in_headers else 'false' + + start_with_parameters = data.get('start-with-parameters', False) + SWP_element = XML.SubElement(root, 'startsWithParameters') + SWP_element.text = 'true' if start_with_parameters else 'false' + + refresh_frequency = str(data.get('refresh-frequency', 3)) + XML.SubElement(root, 'refreshFrequency').text = refresh_frequency + + headers = data.get('definition-header', False) + DH_element = XML.SubElement(root, 'showPipelineDefinitionHeader') + DH_element.text = 'true' if headers else 'false' + + return root diff --git a/jenkins_jobs/parser.py b/jenkins_jobs/parser.py index f39d08eff..4c269bc79 100644 --- a/jenkins_jobs/parser.py +++ b/jenkins_jobs/parser.py @@ -75,6 +75,7 @@ class YamlParser(object): def __init__(self, jjb_config=None): self.data = {} self.jobs = [] + self.views = [] self.jjb_config = jjb_config self.keep_desc = jjb_config.yamlparser['keep_descriptions'] @@ -234,6 +235,12 @@ class YamlParser(object): job = self._applyDefaults(job) self._formatDescription(job) self.jobs.append(job) + + for view in self.data.get('view', {}).values(): + logger.debug("Expanding view '{0}'".format(view['name'])) + self._formatDescription(view) + self.views.append(view) + for project in self.data.get('project', {}).values(): logger.debug("Expanding project '{0}'".format(project['name'])) # use a set to check for duplicate job references in projects @@ -310,7 +317,7 @@ class YamlParser(object): "specified".format(job['name'])) self.jobs.remove(job) seen.add(job['name']) - return self.jobs + return self.jobs, self.views def _expandYamlForTemplateJob(self, project, template, jobs_glob=None): dimensions = [] diff --git a/jenkins_jobs/xml_config.py b/jenkins_jobs/xml_config.py index acf84c41c..185246490 100644 --- a/jenkins_jobs/xml_config.py +++ b/jenkins_jobs/xml_config.py @@ -96,3 +96,35 @@ class XmlJobGenerator(object): for module in self.registry.modules: if hasattr(module, 'gen_xml'): module.gen_xml(xml, data) + + +class XmlViewGenerator(object): + """ This class is responsible for generating Jenkins Configuration XML from + a compatible intermediate representation of Jenkins Views. + """ + + def __init__(self, registry): + self.registry = registry + + def generateXML(self, viewdict_list): + xml_views = [] + for view in viewdict_list: + xml_views.append(self.__getXMLForView(view)) + return xml_views + + def __getXMLForView(self, data): + kind = data.get('view-type', 'list') + + for ep in pkg_resources.iter_entry_points( + group='jenkins_jobs.views', name=kind): + Mod = ep.load() + mod = Mod(self.registry) + xml = mod.root_xml(data) + self.__gen_xml(xml, data) + view = XmlJob(xml, data['name']) + return view + + def __gen_xml(self, xml, data): + for module in self.registry.modules: + if hasattr(module, 'gen_xml'): + module.gen_xml(xml, data) diff --git a/setup.cfg b/setup.cfg index d506c19db..fc240abcc 100644 --- a/setup.cfg +++ b/setup.cfg @@ -48,6 +48,9 @@ jenkins_jobs.projects = maven=jenkins_jobs.modules.project_maven:Maven multijob=jenkins_jobs.modules.project_multijob:MultiJob workflow=jenkins_jobs.modules.project_workflow:Workflow +jenkins_jobs.views = + list=jenkins_jobs.modules.view_list:List + pipeline=jenkins_jobs.modules.view_pipeline:Pipeline jenkins_jobs.builders = raw=jenkins_jobs.modules.general:raw jenkins_jobs.reporters = diff --git a/tests/base.py b/tests/base.py index 3647ee526..bc6d159d2 100644 --- a/tests/base.py +++ b/tests/base.py @@ -34,12 +34,15 @@ import testscenarios from yaml import safe_dump from jenkins_jobs.config import JJBConfig +from jenkins_jobs.errors import InvalidAttributeError import jenkins_jobs.local_yaml as yaml from jenkins_jobs.modules import project_externaljob from jenkins_jobs.modules import project_flow from jenkins_jobs.modules import project_matrix from jenkins_jobs.modules import project_maven from jenkins_jobs.modules import project_multijob +from jenkins_jobs.modules import view_list +from jenkins_jobs.modules import view_pipeline from jenkins_jobs.parser import YamlParser from jenkins_jobs.registry import ModuleRegistry from jenkins_jobs.xml_config import XmlJob @@ -175,6 +178,15 @@ class BaseScenariosTestCase(testscenarios.TestWithScenarios, BaseTestCase): elif (yaml_content['project-type'] == "externaljob"): project = project_externaljob.ExternalJob(registry) + if 'view-type' in yaml_content: + if yaml_content['view-type'] == "list": + project = view_list.List(None) + elif yaml_content['view-type'] == "pipeline": + project = view_pipeline.Pipeline(None) + else: + raise InvalidAttributeError( + 'view-type', yaml_content['view-type']) + if project: xml_project = project.root_xml(yaml_content) else: @@ -206,7 +218,7 @@ class SingleJobTestCase(BaseScenariosTestCase): registry = ModuleRegistry(config) registry.set_parser_data(parser.data) - job_data_list = parser.expandYaml(registry) + job_data_list, view_data_list = parser.expandYaml(registry) # Generate the XML tree xml_generator = XmlJobGenerator(registry) diff --git a/tests/cmd/subcommands/test_delete.py b/tests/cmd/subcommands/test_delete.py index 13b60927d..4e577fd66 100644 --- a/tests/cmd/subcommands/test_delete.py +++ b/tests/cmd/subcommands/test_delete.py @@ -30,7 +30,9 @@ class DeleteTests(CmdTestsBase): @mock.patch('jenkins_jobs.cli.subcommand.update.' 'JenkinsManager.delete_jobs') - def test_delete_single_job(self, delete_job_mock): + @mock.patch('jenkins_jobs.cli.subcommand.update.' + 'JenkinsManager.delete_views') + def test_delete_single_job(self, delete_job_mock, delete_view_mock): """ Test handling the deletion of a single Jenkins job. """ @@ -40,7 +42,9 @@ class DeleteTests(CmdTestsBase): @mock.patch('jenkins_jobs.cli.subcommand.update.' 'JenkinsManager.delete_jobs') - def test_delete_multiple_jobs(self, delete_job_mock): + @mock.patch('jenkins_jobs.cli.subcommand.update.' + 'JenkinsManager.delete_views') + def test_delete_multiple_jobs(self, delete_job_mock, delete_view_mock): """ Test handling the deletion of multiple Jenkins jobs. """ diff --git a/tests/cmd/test_config.py b/tests/cmd/test_config.py index 5236cdd47..de8257f4a 100644 --- a/tests/cmd/test_config.py +++ b/tests/cmd/test_config.py @@ -123,6 +123,7 @@ class TestConfigs(CmdTestsBase): args = ['--conf', self.default_config_file, 'update', path] jenkins_mock.return_value.update_jobs.return_value = ([], 0) + jenkins_mock.return_value.update_views.return_value = ([], 0) self.execute_jenkins_jobs_with_args(args) # validate that the JJBConfig used to initialize builder.Jenkins @@ -146,6 +147,7 @@ class TestConfigs(CmdTestsBase): args = ['--conf', config_file, 'update', path] jenkins_mock.return_value.update_jobs.return_value = ([], 0) + jenkins_mock.return_value.update_views.return_value = ([], 0) self.execute_jenkins_jobs_with_args(args) # validate that the JJBConfig used to initialize builder.Jenkins diff --git a/tests/views/__init__.py b/tests/views/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/views/fixtures/view_list001.xml b/tests/views/fixtures/view_list001.xml new file mode 100644 index 000000000..02a345657 --- /dev/null +++ b/tests/views/fixtures/view_list001.xml @@ -0,0 +1,27 @@ + + + list-view-name01 + Sample description + true + true + + + + job-name-1 + job-name-2 + job-name-3 + + + + + + + + + + + + + true + false + diff --git a/tests/views/fixtures/view_list001.yaml b/tests/views/fixtures/view_list001.yaml new file mode 100644 index 000000000..29b48c27e --- /dev/null +++ b/tests/views/fixtures/view_list001.yaml @@ -0,0 +1,20 @@ +name: list-view-name01 +view-type: list +description: 'Sample description' +filter-executors: true +filter-queue: true +job-name: + - job-name-1 + - job-name-2 + - job-name-3 +columns: + - status + - weather + - job + - last-success + - last-failure + - last-duration + - build-button + - last-stable +recurse: true +status-filter: false diff --git a/tests/views/fixtures/view_list002.xml b/tests/views/fixtures/view_list002.xml new file mode 100644 index 000000000..57bbfd500 --- /dev/null +++ b/tests/views/fixtures/view_list002.xml @@ -0,0 +1,21 @@ + + + regex-example + false + false + + + + + + + + + + + + + + (?!test.*).* + false + diff --git a/tests/views/fixtures/view_list002.yaml b/tests/views/fixtures/view_list002.yaml new file mode 100644 index 000000000..dc468564f --- /dev/null +++ b/tests/views/fixtures/view_list002.yaml @@ -0,0 +1,10 @@ +name: regex-example +view-type: list +columns: + - status + - weather + - job + - last-success + - last-failure + - last-duration +regex: (?!test.*).* diff --git a/tests/views/fixtures/view_pipeline001.xml b/tests/views/fixtures/view_pipeline001.xml new file mode 100644 index 000000000..0e545e6fd --- /dev/null +++ b/tests/views/fixtures/view_pipeline001.xml @@ -0,0 +1,22 @@ + + + testBPview + This is a description + false + false + + + job-one + + 5 + Title + New Window + fake.urlfor.css + true + true + true + true + true + 3 + true + diff --git a/tests/views/fixtures/view_pipeline001.yaml b/tests/views/fixtures/view_pipeline001.yaml new file mode 100644 index 000000000..e6562cecf --- /dev/null +++ b/tests/views/fixtures/view_pipeline001.yaml @@ -0,0 +1,17 @@ +name: testBPview +view-type: pipeline +description: 'This is a description' +filter-executors: false +filter-queue: false +first-job: job-one +no-of-displayed-builds: 5 +title: Title +link-style: New Window +css-Url: fake.urlfor.css +latest-job-only: true +manual-trigger: true +show-parameters: true +parameters-in-headers: true +start-with-parameters: true +refresh-frequency: 3 +definition-header: true diff --git a/tests/views/fixtures/view_pipeline002.xml b/tests/views/fixtures/view_pipeline002.xml new file mode 100644 index 000000000..a26ac623b --- /dev/null +++ b/tests/views/fixtures/view_pipeline002.xml @@ -0,0 +1,21 @@ + + + testBPview + false + false + + + job-one + + 1 + + Lightbox + + false + false + false + false + false + 3 + false + diff --git a/tests/views/fixtures/view_pipeline002.yaml b/tests/views/fixtures/view_pipeline002.yaml new file mode 100644 index 000000000..78ffff7c0 --- /dev/null +++ b/tests/views/fixtures/view_pipeline002.yaml @@ -0,0 +1,3 @@ +name: testBPview +view-type: pipeline +first-job: job-one diff --git a/tests/views/test_views.py b/tests/views/test_views.py new file mode 100644 index 000000000..1f9924fc2 --- /dev/null +++ b/tests/views/test_views.py @@ -0,0 +1,30 @@ +# Copyright 2015 Openstack Foundation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License.import os + +import os +from jenkins_jobs.modules import view_list +from jenkins_jobs.modules import view_pipeline +from tests import base + + +class TestCaseModuleViewList(base.BaseScenariosTestCase): + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + scenarios = base.get_scenarios(fixtures_path) + klass = view_list.List + + +class TestCaseModuleViewPipeline(base.BaseScenariosTestCase): + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + scenarios = base.get_scenarios(fixtures_path) + klass = view_pipeline.Pipeline