Add tests for yaml anchor behaviour

Adds two tests to ensure correct behaviour with referencing yaml
anchors and a third test to verify the expansion internally of yaml
anchors and aliases.

* test that anchors are not carried across subsequent top level
  invocations of yaml.load(). This will be used subsequently to ensure
  that where anchors are allowed across included files they are
  correctly reset on each top level call.
* test that where anchors are defined in a top level file and
  subsequently included files, duplicate anchors raise exceptions as
  though they were defined within the same file.
* test that data returned from yaml loading contains the additional data
  specified by the alias. Uses json to force conversion so that the
  outputted yaml contains the results of the anchors and aliases instead
  of them.

Update documentation to contain details of the use of anchors and
aliases including a refernce to a simple generic example from the
specification as well as a JJB specific example.

Change-Id: I0f2b55e1e2f2bad09f65b1b981baa0159372ee10

doc/source/definition.rst

@@ -252,15 +252,6 @@ For example:
 .. literalinclude::  /../../tests/yamlparser/fixtures/custom_distri.yaml
 
 
-The yaml specification supports `anchors and aliases`__ which means
-that JJB definitions allow references to variables in templates.
-
-__ http://yaml.org/spec/1.2/spec.html#id2765878
-
-For example:
-
-.. literalinclude::  /../../tests/yamlparser/fixtures/yaml_anchor.yaml
-
 JJB also supports interpolation of parameters within parameters. This allows a
 little more flexibility when ordering template jobs as components in different
 projects and job groups.
@@ -269,6 +260,38 @@ For example:
 
 .. literalinclude:: /../../tests/yamlparser/fixtures/second_order_parameter_interpolation002.yaml
 
+
+Yaml Anchors & Aliases
+^^^^^^^^^^^^^^^^^^^^^^
+
+The yaml specification supports `anchors and aliases`_ which means
+that JJB definitions allow references to variables in templates.
+
+For example:
+
+.. literalinclude::  /../../tests/yamlparser/fixtures/yaml_anchor.yaml
+
+
+The `anchors and aliases`_ are expanded internally within JJB's yaml loading
+calls, and are limited to individual documents. That means you use the same
+anchor name in separate files without collisions, but also means that you must
+define the anchor in the same file that you intend to reference it.
+
+A simple example can be seen in the specs `full length example`_ with the
+following being more representative of usage within JJB:
+
+.. literalinclude:: /../../tests/localyaml/fixtures/anchors_aliases.iyaml
+
+
+Which will be expanded to the following yaml before being processed:
+
+.. literalinclude:: /../../tests/localyaml/fixtures/anchors_aliases.oyaml
+
+
+.. _full length example: http://www.yaml.org/spec/1.2/spec.html#id2761803
+.. _anchors and aliases: http://www.yaml.org/spec/1.2/spec.html#id2765878
+
+
 Custom Yaml Tags
 ----------------
 

tests/base.py

@@ -28,6 +28,8 @@ import testtools
 from testtools.content import text_content
 import xml.etree.ElementTree as XML
 from six.moves import configparser
+from six.moves import StringIO
+from yaml import safe_dump
 # This dance deals with the fact that we want unittest.mock if
 # we're on Python 3.4 and later, and non-stdlib mock otherwise.
 try:
@@ -43,7 +45,8 @@ from jenkins_jobs.modules import (project_flow,
 
 
 def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
-                  plugins_info_ext='plugins_info.yaml'):
+                  plugins_info_ext='plugins_info.yaml',
+                  filter_func=None):
     """Returns a list of scenarios, each scenario being described
     by two parameters (yaml and xml filenames by default).
         - content of the fixture output file (aka expected)
@@ -59,6 +62,9 @@ def get_scenarios(fixtures_path, in_ext='yaml', out_ext='xml',
         if input_filename.endswith(plugins_info_ext):
             continue
 
+        if callable(filter_func) and filter_func(input_filename):
+            continue
+
         output_candidate = re.sub(r'\.{0}$'.format(in_ext),
                                   '.{0}'.format(out_ext), input_filename)
         # Make sure the input file has a output counterpart
@@ -212,3 +218,25 @@ class JsonTestCase(BaseTestCase):
                                               doctest.NORMALIZE_WHITESPACE |
                                               doctest.REPORT_NDIFF)
         )
+
+
+class YamlTestCase(BaseTestCase):
+
+    def test_yaml_snippet(self):
+        expected_yaml = self._read_utf8_content()
+        yaml_content = self._read_yaml_content(self.in_filename)
+
+        # using json forces expansion of yaml anchors and aliases in the
+        # outputted yaml, otherwise it would simply appear exactly as
+        # entered which doesn't show that the net effect of the yaml
+        data = StringIO(json.dumps(yaml_content))
+
+        pretty_yaml = safe_dump(json.load(data), default_flow_style=False)
+
+        self.assertThat(
+            pretty_yaml,
+            testtools.matchers.DocTestMatches(expected_yaml,
+                                              doctest.ELLIPSIS |
+                                              doctest.NORMALIZE_WHITESPACE |
+                                              doctest.REPORT_NDIFF)
+        )

tests/localyaml/fixtures/anchors_aliases.iyaml

@@ -0,0 +1,15 @@
+- wrapper_defaults: &wrapper_defaults
+    name: 'wrapper_defaults'
+    wrappers:
+      - timeout:
+          timeout: 180
+          fail: true
+      - timestamps
+
+- job_defaults: &job_defaults
+    name: 'defaults'
+    <<: *wrapper_defaults
+
+- job-template:
+    name: 'myjob'
+    <<: *job_defaults

tests/localyaml/fixtures/anchors_aliases.oyaml

@@ -0,0 +1,23 @@
+- wrapper_defaults:
+    name: wrapper_defaults
+    wrappers:
+    - timeout:
+        fail: true
+        timeout: 180
+    - timestamps
+
+- job_defaults:
+    name: defaults
+    wrappers:
+    - timeout:
+        fail: true
+        timeout: 180
+    - timestamps
+
+- job-template:
+    name: myjob
+    wrappers:
+    - timeout:
+        fail: true
+        timeout: 180
+    - timestamps

tests/localyaml/fixtures/custom_same_anchor-001-part1.yaml

@@ -0,0 +1,11 @@
+- builders:
+    name: custom-copytarball1
+    builders:
+      - copyartifact: &custom-copytarball
+          project: foo
+          filter: "*.tar.gz"
+          target: /home/foo
+          which-build: last-successful
+          optional: true
+          flatten: true
+          parameter-filters: PUBLISH=true

tests/localyaml/fixtures/custom_same_anchor-001-part2.yaml

@@ -0,0 +1,11 @@
+- builders:
+    name: custom-copytarball2
+    builders:
+      - copyartifact: &custom-copytarball
+          project: foo
+          filter: "*.tar.gz"
+          target: /home/foo
+          which-build: last-successful
+          optional: true
+          flatten: true
+          parameter-filters: PUBLISH=true

tests/localyaml/fixtures/exception_include001.yaml

@@ -0,0 +1,15 @@
+
+- builders:
+    - copyartifact: &copytarball
+        project: foo
+        filter: "*.tar.gz"
+        target: /home/foo
+        which-build: last-successful
+        optional: true
+        flatten: true
+        parameter-filters: PUBLISH=true
+
+- job:
+    name: test-job-1
+    builders:
+      !include exception_include001.yaml.inc

tests/localyaml/fixtures/exception_include001.yaml.inc

@@ -0,0 +1,13 @@
+- copyartifact: &copytarball
+    project: foo
+    filter: "*.tar.gz"
+    target: /home/foo
+    which-build: last-successful
+    optional: true
+    flatten: true
+    parameter-filters: PUBLISH=true
+- copyartifact:
+    <<: *copytarball
+    project: bar
+    which-build: specific-build
+    build-number: 123

tests/localyaml/test_localyaml.py

@@ -15,9 +15,17 @@
 # under the License.
 
 import os
+from testtools import ExpectedException
+from testtools.matchers import MismatchError
 from testtools import TestCase
 from testscenarios.testcase import TestWithScenarios
-from tests.base import get_scenarios, JsonTestCase
+
+from jenkins_jobs import builder
+from tests.base import get_scenarios, JsonTestCase, YamlTestCase
+
+
+def _exclude_scenarios(input_filename):
+    return os.path.basename(input_filename).startswith("custom_")
 
 
 class TestCaseLocalYamlInclude(TestWithScenarios, TestCase, JsonTestCase):
@@ -26,4 +34,43 @@ class TestCaseLocalYamlInclude(TestWithScenarios, TestCase, JsonTestCase):
     modules XML parsing behaviour
     """
     fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
-    scenarios = get_scenarios(fixtures_path, 'yaml', 'json')
+    scenarios = get_scenarios(fixtures_path, 'yaml', 'json',
+                              filter_func=_exclude_scenarios)
+
+    def test_yaml_snippet(self):
+
+        if os.path.basename(self.in_filename).startswith("exception_"):
+            with ExpectedException(MismatchError):
+                super(TestCaseLocalYamlInclude, self).test_yaml_snippet()
+        else:
+            super(TestCaseLocalYamlInclude, self).test_yaml_snippet()
+
+
+class TestCaseLocalYamlAnchorAlias(TestWithScenarios, TestCase, YamlTestCase):
+    """
+    Verify yaml input is expanded to the expected yaml output when using yaml
+    anchors and aliases.
+    """
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+    scenarios = get_scenarios(fixtures_path, 'iyaml', 'oyaml')
+
+
+class TestCaseLocalYamlIncludeAnchors(TestCase):
+
+    fixtures_path = os.path.join(os.path.dirname(__file__), 'fixtures')
+
+    def test_multiple_same_anchor_in_multiple_toplevel_yaml(self):
+        """
+        Verify that anchors/aliases only span use of '!include' tag
+
+        To ensure that any yaml loaded by the include tag is in the same
+        space as the top level file, but individual top level yaml definitions
+        are treated by the yaml loader as independent.
+        """
+
+        files = ["custom_same_anchor-001-part1.yaml",
+                 "custom_same_anchor-001-part2.yaml"]
+
+        b = builder.Builder("http://example.com", "jenkins", None,
+                            plugins_list=[])
+        b.load_files([os.path.join(self.fixtures_path, f) for f in files])