From 1e70128d309ce781dff9445c791cc7681c79ce26 Mon Sep 17 00:00:00 2001 From: Darragh Bailey Date: Mon, 23 Feb 2015 15:52:57 +0000 Subject: [PATCH] Add tests for yaml anchor behaviour Adds two tests to ensure correct behaviour with referencing yaml anchors and a third test to verify the expansion internally of yaml anchors and aliases. * test that anchors are not carried across subsequent top level invocations of yaml.load(). This will be used subsequently to ensure that where anchors are allowed across included files they are correctly reset on each top level call. * test that where anchors are defined in a top level file and subsequently included files, duplicate anchors raise exceptions as though they were defined within the same file. * test that data returned from yaml loading contains the additional data specified by the alias. Uses json to force conversion so that the outputted yaml contains the results of the anchors and aliases instead of them. Update documentation to contain details of the use of anchors and aliases including a refernce to a simple generic example from the specification as well as a JJB specific example. Change-Id: I0f2b55e1e2f2bad09f65b1b981baa0159372ee10 --- doc/source/definition.rst | 41 +++++++++++---- tests/base.py | 30 ++++++++++- .../localyaml/fixtures/anchors_aliases.iyaml | 15 ++++++ .../localyaml/fixtures/anchors_aliases.oyaml | 23 +++++++++ .../custom_same_anchor-001-part1.yaml | 11 ++++ .../custom_same_anchor-001-part2.yaml | 11 ++++ .../fixtures/exception_include001.json | 0 .../fixtures/exception_include001.yaml | 15 ++++++ .../fixtures/exception_include001.yaml.inc | 13 +++++ tests/localyaml/test_localyaml.py | 51 ++++++++++++++++++- 10 files changed, 198 insertions(+), 12 deletions(-) create mode 100644 tests/localyaml/fixtures/anchors_aliases.iyaml create mode 100644 tests/localyaml/fixtures/anchors_aliases.oyaml create mode 100644 tests/localyaml/fixtures/custom_same_anchor-001-part1.yaml create mode 100644 tests/localyaml/fixtures/custom_same_anchor-001-part2.yaml create mode 100644 tests/localyaml/fixtures/exception_include001.json create mode 100644 tests/localyaml/fixtures/exception_include001.yaml create mode 100644 tests/localyaml/fixtures/exception_include001.yaml.inc diff --git a/doc/source/definition.rst b/doc/source/definition.rst index dc04ab680..4bc400ed1 100644 --- a/doc/source/definition.rst +++ b/doc/source/definition.rst @@ -252,15 +252,6 @@ For example: .. literalinclude:: /../../tests/yamlparser/fixtures/custom_distri.yaml -The yaml specification supports `anchors and aliases`__ which means -that JJB definitions allow references to variables in templates. - -__ http://yaml.org/spec/1.2/spec.html#id2765878 - -For example: - -.. literalinclude:: /../../tests/yamlparser/fixtures/yaml_anchor.yaml - JJB also supports interpolation of parameters within parameters. This allows a little more flexibility when ordering template jobs as components in different projects and job groups. @@ -269,6 +260,38 @@ For example: .. literalinclude:: /../../tests/yamlparser/fixtures/second_order_parameter_interpolation002.yaml + +Yaml Anchors & Aliases +^^^^^^^^^^^^^^^^^^^^^^ + +The yaml specification supports `anchors and aliases`_ which means +that JJB definitions allow references to variables in templates. + +For example: + +.. literalinclude:: /../../tests/yamlparser/fixtures/yaml_anchor.yaml + + +The `anchors and aliases`_ are expanded internally within JJB's yaml loading +calls, and are limited to individual documents. That means you use the same +anchor name in separate files without collisions, but also means that you must +define the anchor in the same file that you intend to reference it. + +A simple example can be seen in the specs `full length example`_ with the +following being more representative of usage within JJB: + +.. literalinclude:: /../../tests/localyaml/fixtures/anchors_aliases.iyaml + + +Which will be expanded to the following yaml before being processed: + +.. literalinclude:: /../../tests/localyaml/fixtures/anchors_aliases.oyaml + + +.. _full length example: http://www.yaml.org/spec/1.2/spec.html#id2761803 +.. _anchors and aliases: http://www.yaml.org/spec/1.2/spec.html#id2765878 + + Custom Yaml Tags ---------------- diff --git a/tests/base.py b/tests/base.py index 9d09cd5f4..10476d3fe 100644 --- a/tests/base.py +++ b/tests/base.py @@ -28,6 +28,8 @@ import testtools from testtools.content import text_content import xml.etree.ElementTree as XML from six.moves import configparser +from six.moves import StringIO +from yaml import safe_dump # This dance deals with the fact that we want unittest.mock if # we're on Python 3.4 and later, and non-stdlib mock otherwise. try: @@ -43,7 +45,8 @@ from jenkins_jobs.modules import (project_flow, def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml', - plugins_info_ext='plugins_info.yaml'): + plugins_info_ext='plugins_info.yaml', + filter_func=None): """Returns a list of scenarios, each scenario being described by two parameters (yaml and xml filenames by default). - content of the fixture output file (aka expected) @@ -59,6 +62,9 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml', if input_filename.endswith(plugins_info_ext): continue + if callable(filter_func) and filter_func(input_filename): + continue + output_candidate = re.sub(r'\.{0}$'.format(in_ext), '.{0}'.format(out_ext), input_filename) # Make sure the input file has a output counterpart @@ -212,3 +218,25 @@ class JsonTestCase(BaseTestCase): doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF) ) + + +class YamlTestCase(BaseTestCase): + + def test_yaml_snippet(self): + expected_yaml = self._read_utf8_content() + yaml_content = self._read_yaml_content(self.in_filename) + + # using json forces expansion of yaml anchors and aliases in the + # outputted yaml, otherwise it would simply appear exactly as + # entered which doesn't show that the net effect of the yaml + data = StringIO(json.dumps(yaml_content)) + + pretty_yaml = safe_dump(json.load(data), default_flow_style=False) + + self.assertThat( + pretty_yaml, + testtools.matchers.DocTestMatches(expected_yaml, + doctest.ELLIPSIS | + doctest.NORMALIZE_WHITESPACE | + doctest.REPORT_NDIFF) + ) diff --git a/tests/localyaml/fixtures/anchors_aliases.iyaml b/tests/localyaml/fixtures/anchors_aliases.iyaml new file mode 100644 index 000000000..89d581df6 --- /dev/null +++ b/tests/localyaml/fixtures/anchors_aliases.iyaml @@ -0,0 +1,15 @@ +- wrapper_defaults: &wrapper_defaults + name: 'wrapper_defaults' + wrappers: + - timeout: + timeout: 180 + fail: true + - timestamps + +- job_defaults: &job_defaults + name: 'defaults' + <<: *wrapper_defaults + +- job-template: + name: 'myjob' + <<: *job_defaults diff --git a/tests/localyaml/fixtures/anchors_aliases.oyaml b/tests/localyaml/fixtures/anchors_aliases.oyaml new file mode 100644 index 000000000..77fee59b7 --- /dev/null +++ b/tests/localyaml/fixtures/anchors_aliases.oyaml @@ -0,0 +1,23 @@ +- wrapper_defaults: + name: wrapper_defaults + wrappers: + - timeout: + fail: true + timeout: 180 + - timestamps + +- job_defaults: + name: defaults + wrappers: + - timeout: + fail: true + timeout: 180 + - timestamps + +- job-template: + name: myjob + wrappers: + - timeout: + fail: true + timeout: 180 + - timestamps diff --git a/tests/localyaml/fixtures/custom_same_anchor-001-part1.yaml b/tests/localyaml/fixtures/custom_same_anchor-001-part1.yaml new file mode 100644 index 000000000..7dfb26fb2 --- /dev/null +++ b/tests/localyaml/fixtures/custom_same_anchor-001-part1.yaml @@ -0,0 +1,11 @@ +- builders: + name: custom-copytarball1 + builders: + - copyartifact: &custom-copytarball + project: foo + filter: "*.tar.gz" + target: /home/foo + which-build: last-successful + optional: true + flatten: true + parameter-filters: PUBLISH=true diff --git a/tests/localyaml/fixtures/custom_same_anchor-001-part2.yaml b/tests/localyaml/fixtures/custom_same_anchor-001-part2.yaml new file mode 100644 index 000000000..641582f8f --- /dev/null +++ b/tests/localyaml/fixtures/custom_same_anchor-001-part2.yaml @@ -0,0 +1,11 @@ +- builders: + name: custom-copytarball2 + builders: + - copyartifact: &custom-copytarball + project: foo + filter: "*.tar.gz" + target: /home/foo + which-build: last-successful + optional: true + flatten: true + parameter-filters: PUBLISH=true diff --git a/tests/localyaml/fixtures/exception_include001.json b/tests/localyaml/fixtures/exception_include001.json new file mode 100644 index 000000000..e69de29bb diff --git a/tests/localyaml/fixtures/exception_include001.yaml b/tests/localyaml/fixtures/exception_include001.yaml new file mode 100644 index 000000000..828b541b0 --- /dev/null +++ b/tests/localyaml/fixtures/exception_include001.yaml @@ -0,0 +1,15 @@ + +- builders: + - copyartifact: ©tarball + project: foo + filter: "*.tar.gz" + target: /home/foo + which-build: last-successful + optional: true + flatten: true + parameter-filters: PUBLISH=true + +- job: + name: test-job-1 + builders: + !include exception_include001.yaml.inc diff --git a/tests/localyaml/fixtures/exception_include001.yaml.inc b/tests/localyaml/fixtures/exception_include001.yaml.inc new file mode 100644 index 000000000..34e91b8fd --- /dev/null +++ b/tests/localyaml/fixtures/exception_include001.yaml.inc @@ -0,0 +1,13 @@ +- copyartifact: ©tarball + project: foo + filter: "*.tar.gz" + target: /home/foo + which-build: last-successful + optional: true + flatten: true + parameter-filters: PUBLISH=true +- copyartifact: + <<: *copytarball + project: bar + which-build: specific-build + build-number: 123 diff --git a/tests/localyaml/test_localyaml.py b/tests/localyaml/test_localyaml.py index 2f4eb51d6..d1827037b 100644 --- a/tests/localyaml/test_localyaml.py +++ b/tests/localyaml/test_localyaml.py @@ -15,9 +15,17 @@ # under the License. import os +from testtools import ExpectedException +from testtools.matchers import MismatchError from testtools import TestCase from testscenarios.testcase import TestWithScenarios -from tests.base import get_scenarios, JsonTestCase + +from jenkins_jobs import builder +from tests.base import get_scenarios, JsonTestCase, YamlTestCase + + +def _exclude_scenarios(input_filename): + return os.path.basename(input_filename).startswith("custom_") class TestCaseLocalYamlInclude(TestWithScenarios, TestCase, JsonTestCase): @@ -26,4 +34,43 @@ class TestCaseLocalYamlInclude(TestWithScenarios, TestCase, JsonTestCase): modules XML parsing behaviour """ fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') - scenarios = get_scenarios(fixtures_path, 'yaml', 'json') + scenarios = get_scenarios(fixtures_path, 'yaml', 'json', + filter_func=_exclude_scenarios) + + def test_yaml_snippet(self): + + if os.path.basename(self.in_filename).startswith("exception_"): + with ExpectedException(MismatchError): + super(TestCaseLocalYamlInclude, self).test_yaml_snippet() + else: + super(TestCaseLocalYamlInclude, self).test_yaml_snippet() + + +class TestCaseLocalYamlAnchorAlias(TestWithScenarios, TestCase, YamlTestCase): + """ + Verify yaml input is expanded to the expected yaml output when using yaml + anchors and aliases. + """ + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + scenarios = get_scenarios(fixtures_path, 'iyaml', 'oyaml') + + +class TestCaseLocalYamlIncludeAnchors(TestCase): + + fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures') + + def test_multiple_same_anchor_in_multiple_toplevel_yaml(self): + """ + Verify that anchors/aliases only span use of '!include' tag + + To ensure that any yaml loaded by the include tag is in the same + space as the top level file, but individual top level yaml definitions + are treated by the yaml loader as independent. + """ + + files = ["custom_same_anchor-001-part1.yaml", + "custom_same_anchor-001-part2.yaml"] + + b = builder.Builder("http://example.com", "jenkins", None, + plugins_list=[]) + b.load_files([os.path.join(self.fixtures_path, f) for f in files])