Tests for publishers

I am trying to build up a very basic testing framework so we can tests
our publishers. The idea is to provide a YAML input file and expected
XML output, run the parser on the YAML file and compares its output with
the fixture XML.

That test suite should speed up development of new publishers.

TestCaseModulePublisher is declared inside a function so that unittest
discovery does not consider it a usable test directly. load_test is
responsible for instantiating the test suite.

Change-Id: If3260113eb1337ac47c3883b11c600e5a595dae3
[fabre.arnaud@gmail.com: fixed some runtime errors]
Signed-off-by: Arnaud Fabre <fabre.arnaud@gmail.com>

.gitignore

@@ -3,6 +3,7 @@
 *.egg-info
 *.pyc
 .test
+.testrepository
 .tox
 AUTHORS
 build/*

.testr.conf

@@ -0,0 +1,4 @@
+[DEFAULT]
+test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} ${PYTHON:-python} -m subunit.run discover -t ./ tests $LISTOPT $IDOPTION
+test_id_option=--load-list $IDFILE
+test_list_option=--list

tests/publishers/fixtures/README

@@ -0,0 +1,14 @@
+This directory contains fixtures to test the publishers.
+
+The filename should start with a publisher name (example: xunit) and you must
+provide two files:
+ - .yaml : yaml snippet representing the publisher as it would be written
+           in jenkins job builder configuration file.
+ - .xml  : xml Jenkins snippet that should be rendered by the publisher
+
+Each yaml file MUST have a corresponding xml file.
+
+Once the YAML file has been parsed, it is prettify using python minidom
+which also means that:
+- your XML file must start with: <?xml version="1.0" ?>
+- self closing elements do not contains space eg: <element/>

tests/publishers/fixtures/scp001.xml

@@ -0,0 +1,17 @@
+<?xml version="1.0" ?>
+<project>
+  <publishers>
+    <be.certipost.hudson.plugin.SCPRepositoryPublisher>
+      <siteName>example.com</siteName>
+      <entries>
+        <be.certipost.hudson.plugin.Entry>
+          <filePath>dest/dir</filePath>
+          <sourceFile>base/source/dir/**</sourceFile>
+          <keepHierarchy>true</keepHierarchy>
+          <copyConsoleLog>false</copyConsoleLog>
+          <copyAfterFailure>true</copyAfterFailure>
+        </be.certipost.hudson.plugin.Entry>
+      </entries>
+    </be.certipost.hudson.plugin.SCPRepositoryPublisher>
+  </publishers>
+</project>

tests/publishers/fixtures/scp001.yaml

@@ -0,0 +1,9 @@
+# vim: sw=4 ts=4 et
+publishers:
+  - scp:
+      site: 'example.com'
+      files:
+        - target: 'dest/dir'
+          source: 'base/source/dir/**'
+          keep-hierarchy: true
+          copy-after-failure: true

tests/publishers/fixtures/xunit001.xml

@@ -0,0 +1,36 @@
+<?xml version="1.0" ?>
+<project>
+  <publishers>
+    <xunit>
+      <types>
+        <PHPUnitJunitHudsonTestType>
+          <pattern>junit.log</pattern>
+          <failIfNotNew>true</failIfNotNew>
+          <deleteOutputFiles>true</deleteOutputFiles>
+          <stopProcessingIfError>true</stopProcessingIfError>
+        </PHPUnitJunitHudsonTestType>
+        <CppUnitJunitHudsonTestType>
+          <pattern>cppunit.log</pattern>
+          <failIfNotNew>true</failIfNotNew>
+          <deleteOutputFiles>true</deleteOutputFiles>
+          <stopProcessingIfError>true</stopProcessingIfError>
+        </CppUnitJunitHudsonTestType>
+      </types>
+      <thresholds>
+        <org.jenkinsci.plugins.xunit.threshold.FailedThreshold>
+          <failureThreshold/>
+          <unstableThreshold/>
+          <unstableNewThreshold/>
+          <failureNewThreshold/>
+        </org.jenkinsci.plugins.xunit.threshold.FailedThreshold>
+        <org.jenkinsci.plugins.xunit.threshold.SkippedThreshold>
+          <failureThreshold/>
+          <unstableThreshold/>
+          <unstableNewThreshold/>
+          <failureNewThreshold/>
+        </org.jenkinsci.plugins.xunit.threshold.SkippedThreshold>
+      </thresholds>
+      <thresholdMode>2</thresholdMode>
+    </xunit>
+  </publishers>
+</project>

tests/publishers/fixtures/xunit001.yaml

@@ -0,0 +1,21 @@
+# vim: sw=4 ts=4 et
+publishers:
+  - xunit:
+      thresholdmode: 'percent'
+      thresholds:
+          - failed:
+                unstable: 0
+                unstablenew: 0
+                failure: 0
+                failurenew: 0
+          - skipped:
+                unstable: 0
+                unstablenew: 0
+                failure: 0
+                failurenew: 0
+      types:
+          - phpunit:
+              pattern: "junit.log"
+              stoponerror: true
+          - cppunit:
+              pattern: "cppunit.log"

tests/publishers/test_publishers.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python
+#
+# Joint copyright:
+#  - Copyright 2012,2013 Wikimedia Foundation
+#  - Copyright 2012,2013 Antoine "hashar" Musso
+#  - Copyright 2013 Arnaud Fabre
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+import testtools
+import unittest
+import xml.etree.ElementTree as XML
+import yaml
+
+from jenkins_jobs.builder import XmlJob, YamlParser, ModuleRegistry
+from jenkins_jobs.modules import publishers
+
+FIXTURES_PATH = os.path.join(
+    os.path.dirname(__file__), 'fixtures')
+
+
+def load_tests(loader, tests, pattern):
+    return unittest.TestSuite(
+        build_test_case(xml, yamldef, files)
+        for xml, yamldef, files in get_fixtures()
+    )
+
+
+def get_fixtures():
+    """Returns a list of tuples containing, in order:
+        - content of the fixture .xml file (aka expected)
+        - content of the fixture .yaml file
+        - list of the filenames
+    """
+    fixtures = []
+    files = os.listdir(FIXTURES_PATH)
+    yaml_files = [f for f in files if re.match(r'.*\.yaml$', f)]
+
+    for yaml_filename in yaml_files:
+        xml_candidate = re.sub(r'\.yaml$', '.xml', yaml_filename)
+        # Make sure the yaml file has a xml counterpart
+        if xml_candidate not in files:
+            raise Exception(
+                "No XML file named '%s' to match " +
+                "YAML file '%s'" % (xml_candidate, yaml_filename))
+
+        # Read XML content, assuming it is unicode encoded
+        xml_filename = os.path.join(FIXTURES_PATH, xml_candidate)
+        xml_content = u"%s" % open(xml_filename, 'r').read()
+
+        yaml_file = file(os.path.join(FIXTURES_PATH, yaml_filename), 'r')
+        yaml_content = yaml.load(yaml_file)
+
+        fixtures.append((
+            xml_content,
+            yaml_content,
+            [xml_filename, yaml_filename],
+        ))
+
+    return fixtures
+
+
+# The class is wrapped in a def to prevent it from being discovered by
+# python-discover, it would try to load the class passing unexpected parameters
+# which breaks everything.
+def build_test_case(expected_xml, yaml, files):
+    class TestCaseModulePublisher(testtools.TestCase):
+
+        # testtools.TestCase settings:
+        maxDiff = None      # always dump text difference
+        longMessage = True  # keep normal error message when providing our
+
+        def __init__(self, expected_xml, yaml, files):
+            testtools.TestCase.__init__(self, 'test_yaml_snippet')
+            self.xml = expected_xml
+            self.yaml = yaml
+            self.files = files
+
+        def test_yaml_snippet(self):
+            xml_project = XML.Element('project')  # root element
+            parser = YamlParser()
+            pub = publishers.Publishers(ModuleRegistry({}))
+
+            # Generate the XML tree directly with modules/publishers/*
+            pub.gen_xml(parser, xml_project, self.yaml)
+
+            # Prettify generated XML
+            pretty_xml = XmlJob(xml_project, 'fixturejob').output()
+
+            self.assertMultiLineEqual(
+                self.xml, pretty_xml,
+                'Test inputs: %s' % ', '.join(self.files)
+            )
+    return TestCaseModulePublisher(expected_xml, yaml, files)
+
+if __name__ == "__main__":
+    unittest.main()

tools/test-requires

@@ -1,2 +1,7 @@
+discover
+fixtures
+python-subunit
 sphinx
 setuptools_git>=0.4
+testtools
+testrepository

tox.ini

@@ -1,20 +1,24 @@
 [tox]
-envlist = pep8, pyflakes
+envlist = pep8, pyflakes, py27
 
 [tox:jenkins]
 downloadcache = ~/cache/pip
 
 [testenv]
+setenv VIRTUAL_ENV={envdir}
+       SUBUNIT_FORMATTER=tee testr_subunit_log
+       OS_STDOUT_NOCAPTURE=False
 deps = -r{toxinidir}/tools/pip-requires
        -r{toxinidir}/tools/test-requires
+commands = python setup.py testr --slowest --testr-args='{posargs}'
 
 [testenv:pep8]
 deps = pep8==1.3.3
-commands = pep8 --repeat --show-source --ignore=E125 --exclude=.venv,.tox,dist,doc,build,*egg .
+commands = pep8 --repeat --show-source --ignore=E125 --exclude=.venv,.tox,dist,doc,build,*egg . {posargs}
 
 [testenv:pyflakes]
 deps = pyflakes
-commands = pyflakes jenkins_jobs setup.py
+commands = pyflakes jenkins_jobs tests setup.py
 
 [testenv:compare-xml-old]
 commands = jenkins-jobs test -o .test/old/out/ .test/old/config/