From b829874916287022fe9dc8423e458c51e1b6a797 Mon Sep 17 00:00:00 2001 From: Antoine Musso Date: Sun, 14 Jul 2013 19:29:33 +0200 Subject: [PATCH] Tests for publishers I am trying to build up a very basic testing framework so we can tests our publishers. The idea is to provide a YAML input file and expected XML output, run the parser on the YAML file and compares its output with the fixture XML. That test suite should speed up development of new publishers. TestCaseModulePublisher is declared inside a function so that unittest discovery does not consider it a usable test directly. load_test is responsible for instantiating the test suite. Change-Id: If3260113eb1337ac47c3883b11c600e5a595dae3 [fabre.arnaud@gmail.com: fixed some runtime errors] Signed-off-by: Arnaud Fabre --- .gitignore | 1 + .testr.conf | 4 + tests/__init__.py | 0 tests/publishers/__init__.py | 0 tests/publishers/fixtures/README | 14 +++ tests/publishers/fixtures/scp001.xml | 17 ++++ tests/publishers/fixtures/scp001.yaml | 9 ++ tests/publishers/fixtures/xunit001.xml | 36 ++++++++ tests/publishers/fixtures/xunit001.yaml | 21 +++++ tests/publishers/test_publishers.py | 109 ++++++++++++++++++++++++ tools/test-requires | 5 ++ tox.ini | 10 ++- 12 files changed, 223 insertions(+), 3 deletions(-) create mode 100644 .testr.conf create mode 100644 tests/__init__.py create mode 100644 tests/publishers/__init__.py create mode 100644 tests/publishers/fixtures/README create mode 100644 tests/publishers/fixtures/scp001.xml create mode 100644 tests/publishers/fixtures/scp001.yaml create mode 100644 tests/publishers/fixtures/xunit001.xml create mode 100644 tests/publishers/fixtures/xunit001.yaml create mode 100644 tests/publishers/test_publishers.py diff --git a/.gitignore b/.gitignore index 91b4f08ac..4de625721 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,7 @@ *.egg-info *.pyc .test +.testrepository .tox AUTHORS build/* diff --git a/.testr.conf b/.testr.conf new file mode 100644 index 000000000..5433c070e --- /dev/null +++ b/.testr.conf @@ -0,0 +1,4 @@ +[DEFAULT] +test_command=OS_STDOUT_CAPTURE=${OS_STDOUT_CAPTURE:-1} OS_STDERR_CAPTURE=${OS_STDERR_CAPTURE:-1} OS_LOG_CAPTURE=${OS_LOG_CAPTURE:-1} ${PYTHON:-python} -m subunit.run discover -t ./ tests $LISTOPT $IDOPTION +test_id_option=--load-list $IDFILE +test_list_option=--list diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/publishers/__init__.py b/tests/publishers/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/publishers/fixtures/README b/tests/publishers/fixtures/README new file mode 100644 index 000000000..dc505aba6 --- /dev/null +++ b/tests/publishers/fixtures/README @@ -0,0 +1,14 @@ +This directory contains fixtures to test the publishers. + +The filename should start with a publisher name (example: xunit) and you must +provide two files: + - .yaml : yaml snippet representing the publisher as it would be written + in jenkins job builder configuration file. + - .xml : xml Jenkins snippet that should be rendered by the publisher + +Each yaml file MUST have a corresponding xml file. + +Once the YAML file has been parsed, it is prettify using python minidom +which also means that: +- your XML file must start with: +- self closing elements do not contains space eg: diff --git a/tests/publishers/fixtures/scp001.xml b/tests/publishers/fixtures/scp001.xml new file mode 100644 index 000000000..78af67769 --- /dev/null +++ b/tests/publishers/fixtures/scp001.xml @@ -0,0 +1,17 @@ + + + + + example.com + + + dest/dir + base/source/dir/** + true + false + true + + + + + diff --git a/tests/publishers/fixtures/scp001.yaml b/tests/publishers/fixtures/scp001.yaml new file mode 100644 index 000000000..d705043bf --- /dev/null +++ b/tests/publishers/fixtures/scp001.yaml @@ -0,0 +1,9 @@ +# vim: sw=4 ts=4 et +publishers: + - scp: + site: 'example.com' + files: + - target: 'dest/dir' + source: 'base/source/dir/**' + keep-hierarchy: true + copy-after-failure: true diff --git a/tests/publishers/fixtures/xunit001.xml b/tests/publishers/fixtures/xunit001.xml new file mode 100644 index 000000000..2535b6249 --- /dev/null +++ b/tests/publishers/fixtures/xunit001.xml @@ -0,0 +1,36 @@ + + + + + + + junit.log + true + true + true + + + cppunit.log + true + true + true + + + + + + + + + + + + + + + + + 2 + + + diff --git a/tests/publishers/fixtures/xunit001.yaml b/tests/publishers/fixtures/xunit001.yaml new file mode 100644 index 000000000..44cb1741d --- /dev/null +++ b/tests/publishers/fixtures/xunit001.yaml @@ -0,0 +1,21 @@ +# vim: sw=4 ts=4 et +publishers: + - xunit: + thresholdmode: 'percent' + thresholds: + - failed: + unstable: 0 + unstablenew: 0 + failure: 0 + failurenew: 0 + - skipped: + unstable: 0 + unstablenew: 0 + failure: 0 + failurenew: 0 + types: + - phpunit: + pattern: "junit.log" + stoponerror: true + - cppunit: + pattern: "cppunit.log" diff --git a/tests/publishers/test_publishers.py b/tests/publishers/test_publishers.py new file mode 100644 index 000000000..4c173c557 --- /dev/null +++ b/tests/publishers/test_publishers.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# +# Joint copyright: +# - Copyright 2012,2013 Wikimedia Foundation +# - Copyright 2012,2013 Antoine "hashar" Musso +# - Copyright 2013 Arnaud Fabre +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import os +import re +import testtools +import unittest +import xml.etree.ElementTree as XML +import yaml + +from jenkins_jobs.builder import XmlJob, YamlParser, ModuleRegistry +from jenkins_jobs.modules import publishers + +FIXTURES_PATH = os.path.join( + os.path.dirname(__file__), 'fixtures') + + +def load_tests(loader, tests, pattern): + return unittest.TestSuite( + build_test_case(xml, yamldef, files) + for xml, yamldef, files in get_fixtures() + ) + + +def get_fixtures(): + """Returns a list of tuples containing, in order: + - content of the fixture .xml file (aka expected) + - content of the fixture .yaml file + - list of the filenames + """ + fixtures = [] + files = os.listdir(FIXTURES_PATH) + yaml_files = [f for f in files if re.match(r'.*\.yaml$', f)] + + for yaml_filename in yaml_files: + xml_candidate = re.sub(r'\.yaml$', '.xml', yaml_filename) + # Make sure the yaml file has a xml counterpart + if xml_candidate not in files: + raise Exception( + "No XML file named '%s' to match " + + "YAML file '%s'" % (xml_candidate, yaml_filename)) + + # Read XML content, assuming it is unicode encoded + xml_filename = os.path.join(FIXTURES_PATH, xml_candidate) + xml_content = u"%s" % open(xml_filename, 'r').read() + + yaml_file = file(os.path.join(FIXTURES_PATH, yaml_filename), 'r') + yaml_content = yaml.load(yaml_file) + + fixtures.append(( + xml_content, + yaml_content, + [xml_filename, yaml_filename], + )) + + return fixtures + + +# The class is wrapped in a def to prevent it from being discovered by +# python-discover, it would try to load the class passing unexpected parameters +# which breaks everything. +def build_test_case(expected_xml, yaml, files): + class TestCaseModulePublisher(testtools.TestCase): + + # testtools.TestCase settings: + maxDiff = None # always dump text difference + longMessage = True # keep normal error message when providing our + + def __init__(self, expected_xml, yaml, files): + testtools.TestCase.__init__(self, 'test_yaml_snippet') + self.xml = expected_xml + self.yaml = yaml + self.files = files + + def test_yaml_snippet(self): + xml_project = XML.Element('project') # root element + parser = YamlParser() + pub = publishers.Publishers(ModuleRegistry({})) + + # Generate the XML tree directly with modules/publishers/* + pub.gen_xml(parser, xml_project, self.yaml) + + # Prettify generated XML + pretty_xml = XmlJob(xml_project, 'fixturejob').output() + + self.assertMultiLineEqual( + self.xml, pretty_xml, + 'Test inputs: %s' % ', '.join(self.files) + ) + return TestCaseModulePublisher(expected_xml, yaml, files) + +if __name__ == "__main__": + unittest.main() diff --git a/tools/test-requires b/tools/test-requires index 5a74ecc72..cca00c0d9 100644 --- a/tools/test-requires +++ b/tools/test-requires @@ -1,2 +1,7 @@ +discover +fixtures +python-subunit sphinx setuptools_git>=0.4 +testtools +testrepository diff --git a/tox.ini b/tox.ini index 61ac8baac..63848b125 100644 --- a/tox.ini +++ b/tox.ini @@ -1,20 +1,24 @@ [tox] -envlist = pep8, pyflakes +envlist = pep8, pyflakes, py27 [tox:jenkins] downloadcache = ~/cache/pip [testenv] +setenv VIRTUAL_ENV={envdir} + SUBUNIT_FORMATTER=tee testr_subunit_log + OS_STDOUT_NOCAPTURE=False deps = -r{toxinidir}/tools/pip-requires -r{toxinidir}/tools/test-requires +commands = python setup.py testr --slowest --testr-args='{posargs}' [testenv:pep8] deps = pep8==1.3.3 -commands = pep8 --repeat --show-source --ignore=E125 --exclude=.venv,.tox,dist,doc,build,*egg . +commands = pep8 --repeat --show-source --ignore=E125 --exclude=.venv,.tox,dist,doc,build,*egg . {posargs} [testenv:pyflakes] deps = pyflakes -commands = pyflakes jenkins_jobs setup.py +commands = pyflakes jenkins_jobs tests setup.py [testenv:compare-xml-old] commands = jenkins-jobs test -o .test/old/out/ .test/old/config/