Added config options to not overwrite jobs desc

When you do not specify any description for a job, right now it creates a
new one overwriting whatever you had put there manually. Now when
setting the configuration option 'keep_descriptions' to True, it will
only overwrite the description if you specified one explicitly in the
yaml.
That way you can avoid specifying any description and it will not
overwrite the ones you put there manually through the jenkins ui.

Added a couple of tests for it also, and extended the tests to allow custom
configuration by adding a *.conf file for it

Change-Id: I408f5ee06a6939a8cb8f4f2e6c6e0d060462259c
Signed-off-by: David Caro <dcaroest@redhat.com>

doc/source/installation.rst

@@ -46,6 +46,21 @@ the following format:
 .. literalinclude:: ../../etc/jenkins_jobs.ini-sample
    :language: ini
 
+job_builder section
+^^^^^^^^^^^^^^^^^^^
+
+**ignore_cache**
+  (Optional) If set to True, Jenkins Job Builder won't use any cache.
+
+**keep_descriptions**
+  By default `jenkins-jobs` will overwrite the jobs descriptions even if no
+  description has been defined explicitly.
+  When this option is set to True, that behavior changes and it will only
+  overwrite the description if you specified it in the yaml. False by default.
+
+jenkins section
+^^^^^^^^^^^^^^^
+
 **user**
   This should be the name of a user previously defined in Jenkins.
   Appropriate user permissions must be set under the Jenkins security
@@ -61,9 +76,6 @@ the following format:
 **url**
   The base URL for your Jenkins installation.
 
-**ignore_cache**
-  (Optional) If set to True, Jenkins Job Builder won't use any cache.
-
 
 Running
 -------

etc/jenkins_jobs.ini-sample

@@ -1,5 +1,10 @@
+[job_builder]
+ignore_cache=True
+keep_descriptions=False
+
 [jenkins]
 user=jenkins
 password=1234567890abcdef1234567890abcdef
 url=https://jenkins.example.com
-ignore_cache=True
+##### This is deprecated, use job_builder section instead
+#ignore_cache=True

jenkins_jobs/builder.py

@@ -117,9 +117,10 @@ def matches(what, where):
 
 class YamlParser(object):
     def __init__(self, config=None):
-        self.registry = ModuleRegistry(config)
         self.data = {}
         self.jobs = []
+        self.config = config
+        self.registry = ModuleRegistry(self.config)
 
     def parse(self, fn):
         data = yaml.load(open(fn))
@@ -283,8 +284,18 @@ class YamlParser(object):
 
     def getXMLForJob(self, data):
         kind = data.get('project-type', 'freestyle')
-        data["description"] = (data.get("description", "") +
-                               self.get_managed_string()).lstrip()
+        if self.config:
+            keep_desc = self.config.getboolean('job_builder',
+                                               'keep_descriptions')
+        else:
+            keep_desc = False
+        if keep_desc:
+            description = data.get("description", None)
+        else:
+            description = data.get("description", '')
+        if description is not None:
+            data["description"] = description + \
+                self.get_managed_string().lstrip()
         for ep in pkg_resources.iter_entry_points(
                 group='jenkins_jobs.projects', name=kind):
             Mod = ep.load()

jenkins_jobs/cmd.py

@@ -18,6 +18,19 @@ import ConfigParser
 import logging
 import os
 import sys
+import cStringIO
+
+
+DEFAULT_CONF = """
+[job_builder]
+keep_descriptions=False
+ignore_cache=False
+
+[jenkins]
+url=http://localhost:8080/
+user=
+password=
+"""
 
 
 def confirm(question):
@@ -78,20 +91,15 @@ def main():
                                  'jenkins_jobs.ini')
         if os.path.isfile(localconf):
             conf = localconf
-
     config = ConfigParser.ConfigParser()
-    if os.path.isfile(conf):
+    ## Load default config always
+    config.readfp(cStringIO.StringIO(DEFAULT_CONF))
+    if options.command == 'test':
+        logger.debug("Not reading config for test output generation")
+    elif os.path.isfile(conf):
         logger.debug("Reading config from {0}".format(conf))
         conffp = open(conf, 'r')
         config.readfp(conffp)
-    elif options.command == 'test':
-        ## to avoid the 'no section' and 'no option' errors when testing
-        config.add_section("jenkins")
-        config.set("jenkins", "url", "http://localhost:8080")
-        config.set("jenkins", "user", None)
-        config.set("jenkins", "password", None)
-        config.set("jenkins", "ignore_cache", False)
-        logger.debug("Not reading config for test output generation")
     else:
         raise jenkins_jobs.errors.JenkinsJobsException(
             "A valid configuration file is required when not run as a test"
@@ -105,7 +113,12 @@ def main():
     if options.ignore_cache:
         ignore_cache = options.ignore_cache
     elif config.has_option('jenkins', 'ignore_cache'):
+        logging.warn('ignore_cache option should be moved to the [job_builder]'
+                     ' section in the config file, the one specified in the '
+                     '[jenkins] section will be ignored in the future')
         ignore_cache = config.get('jenkins', 'ignore_cache')
+    elif config.has_option('job_builder', 'ignore_cache'):
+        ignore_cache = config.get('job_builder', 'ignore_cache')
 
     # workaround for python 2.6 interpolation error
     # https://bugs.launchpad.net/openstack-ci/+bug/1259631

tests/base.py

@@ -47,9 +47,15 @@ def get_scenarios(fixtures_path):
             raise Exception(
                 "No XML file named '%s' to match "
                 "YAML file '%s'" % (xml_candidate, yaml_filename))
+        conf_candidate = re.sub(r'\.yaml$', '.conf', yaml_filename)
+        # If present, add the configuration file
+        if conf_candidate not in files:
+            conf_candidate = None
 
         scenarios.append((yaml_filename, {
-            'yaml_filename': yaml_filename, 'xml_filename': xml_candidate
+            'yaml_filename': yaml_filename,
+            'xml_filename': xml_candidate,
+            'conf_filename': conf_candidate,
         }))
 
     return scenarios

tests/yamlparser/fixtures/add_managed_to_desc.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <description>My nice description&lt;!-- Managed by Jenkins Job Builder --&gt;</description>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/usr/bin/env python
+#
+print(&quot;Doing something cool with python&quot;)
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/add_managed_to_desc.yaml

@@ -0,0 +1,13 @@
+- project:
+    name: add_managed_to_desc
+    jobs:
+        - 'add_managed_to_desc'
+
+- job-template:
+    name: 'add_managed_to_desc'
+    description: 'My nice description'
+    builders:
+      - shell: |
+         #!/usr/bin/env python
+         #
+         print("Doing something cool with python")

tests/yamlparser/fixtures/complete001.yaml

@@ -66,14 +66,14 @@
          choosing-strategy: gerrit
 
 - project:
-    name: test
+    name: complete001
     version:
         - 1.2
     jobs:
-        - 'build_myproject_{version}'
+        - 'complete001_{version}'
 
 - job-template:
-    name: 'build_myproject_{version}'
+    name: 'complete001_{version}'
     scm:
       - gerrit-scm:
          project_pattern: openstack-infra/jenkins-job-builder

tests/yamlparser/fixtures/dont_overwrite_desc.conf

@@ -0,0 +1,2 @@
+[job_builder]
+keep_descriptions = True

tests/yamlparser/fixtures/dont_overwrite_desc.xml

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<project>
+  <actions/>
+  <keepDependencies>false</keepDependencies>
+  <blockBuildWhenDownstreamBuilding>false</blockBuildWhenDownstreamBuilding>
+  <blockBuildWhenUpstreamBuilding>false</blockBuildWhenUpstreamBuilding>
+  <concurrentBuild>false</concurrentBuild>
+  <canRoam>true</canRoam>
+  <properties/>
+  <scm class="hudson.scm.NullSCM"/>
+  <builders>
+    <hudson.tasks.Shell>
+      <command>#!/usr/bin/env python
+#
+print(&quot;Doing something cool with python&quot;)
+</command>
+    </hudson.tasks.Shell>
+  </builders>
+  <publishers/>
+  <buildWrappers/>
+</project>

tests/yamlparser/fixtures/dont_overwrite_desc.yaml

@@ -0,0 +1,12 @@
+- project:
+    name: dont_overwrite_desc
+    jobs:
+        - 'dont_overwrite_desc'
+
+- job-template:
+    name: 'dont_overwrite_desc'
+    builders:
+      - shell: |
+         #!/usr/bin/env python
+         #
+         print("Doing something cool with python")

tests/yamlparser/test_yamlparser.py

@@ -16,6 +16,7 @@
 # under the License.
 
 import os
+from ConfigParser import ConfigParser
 from testtools import TestCase
 from testscenarios.testcase import TestWithScenarios
 from tests.base import get_scenarios, BaseTestCase
@@ -37,7 +38,14 @@ class TestCaseModuleYamlInclude(TestWithScenarios, TestCase, BaseTestCase):
 
         yaml_filepath = os.path.join(self.fixtures_path, self.yaml_filename)
 
-        parser = YamlParser()
+        if self.conf_filename:
+            config = ConfigParser()
+            conf_filepath = os.path.join(self.fixtures_path,
+                                         self.conf_filename)
+            config.readfp(open(conf_filepath))
+        else:
+            config = None
+        parser = YamlParser(config)
         parser.parse(yaml_filepath)
 
         # Generate the XML tree