Add load-branch to tenant configuration

Currently it is hard to test changes on trusted repositories. The
currently most common way is to duplicate jobs and roles and have base
jobs that use the test variants of the roles under test. This however
still imposes a risk of breaking things when trying to move the tested
changes to the productive roles and jobs. Further as a downstream
deployer it can be hard to follow e.g. zuul-jobs because those are not
tested in every downstream deployment. There is no easy way of
updating upstream repos like zuul-jobs. Thus most deployers either
just follow zuul-jobs and deal with eventual fall out from
incompatibilities or don't update it on a regularly basis.

This change aims to support an easier and risk free method of updating
repos like zuul-jobs. This can be done by defining branch filters for
a repo in the tenant config. This way one can have a test tenant using
the latest is greatest master branch while keeping a stable branch for
productuve tenants. This way updates can be tested in the test tenant
and if everything works one can merge this to the stable branch to
take the updates into production.

Change-Id: Id4b5e80c0b59e4075774e6ed0049b229173e8426

doc/source/reference/tenants.rst

@@ -130,6 +130,18 @@ configuration. Some examples of tenant definitions are:
       The items in the list follow the same format described in
       **untrusted-projects**.
 
+      .. attr:: <project>
+
+         The config-projects have an additional config option that
+         may be specified optionally.
+
+         .. attr:: load-branch
+            :default: master
+
+            Define which branch is loaded from a config project. By
+            default config projects load Zuul configuration only
+            from the master branch.
+
    .. attr:: untrusted-projects
 
       A list of projects to be treated as untrusted in this tenant.

releasenotes/notes/load-branch-03e4bb6577756f68.yaml

@@ -0,0 +1,6 @@
+---
+features:
+  - |
+    The tenant configuration now supports loading a different branch than
+    master from config projects.
+    See :attr:`tenant.config-projects.<project>.load-branch`.

tests/fixtures/config/tenant-parser/branch.yaml

@@ -0,0 +1,9 @@
+- tenant:
+    name: tenant-one
+    source:
+      gerrit:
+        config-projects:
+          - common-config:
+              load-branch: stable
+        untrusted-projects:
+          - org/project1

tests/unit/test_configloader.py

@@ -330,6 +330,41 @@ class TestTenantExcludeAll(TenantParserTestCase):
             "No error should have been accumulated")
 
 
+class TestTenantConfigBranches(ZuulTestCase):
+    tenant_config_file = 'config/tenant-parser/simple.yaml'
+
+    def _validate_job(self, job, branch):
+        tenant_one = self.sched.abide.tenants.get('tenant-one')
+        jobs = tenant_one.layout.getJobs(job)
+        self.assertEquals(len(jobs), 1)
+        self.assertIn(jobs[0].source_context.branch, branch)
+
+    def test_tenant_config_load_branch(self):
+        """
+        Tests that when specifying branches for a project only those branches
+        are parsed.
+        """
+        # Job must be defined in master
+        common_job = 'common-config-job'
+        self._validate_job(common_job, 'master')
+
+        self.log.debug('Creating branches')
+        self.create_branch('common-config', 'stable')
+        self.create_branch('common-config', 'feat_x')
+
+        self.sched.reconfigure(self.config)
+
+        # Job must be defined in master
+        self._validate_job(common_job, 'master')
+
+        # Reconfigure with load-branch stable for common-config
+        self.newTenantConfig('config/tenant-parser/branch.yaml')
+        self.sched.reconfigure(self.config)
+
+        # Now job must be defined on stable branch
+        self._validate_job(common_job, 'stable')
+
+
 class TestSplitConfig(ZuulTestCase):
     tenant_config_file = 'config/split-config/main.yaml'
 

zuul/configloader.py

@@ -1417,6 +1417,7 @@ class TenantParser(object):
         'shadow': to_list(str),
         'exclude-unprotected-branches': bool,
         'extra-config-paths': to_list(str),
+        'load-branch': str,
     }}
 
     project = vs.Any(str, project_dict)
@@ -1627,6 +1628,7 @@ class TenantParser(object):
             project_include = current_include
             shadow_projects = []
             project_exclude_unprotected_branches = None
+            project_load_branch = None
         else:
             project_name = list(conf.keys())[0]
             project = source.getProject(project_name)
@@ -1651,6 +1653,8 @@ class TenantParser(object):
                                             if not x.endswith('/')])
                 extra_config_dirs = tuple([x[:-1] for x in extra_config_paths
                                            if x.endswith('/')])
+            project_load_branch = conf[project_name].get(
+                'load-branch', None)
 
         tenant_project_config = model.TenantProjectConfig(project)
         tenant_project_config.load_classes = frozenset(project_include)
@@ -1659,6 +1663,7 @@ class TenantParser(object):
             project_exclude_unprotected_branches
         tenant_project_config.extra_config_files = extra_config_files
         tenant_project_config.extra_config_dirs = extra_config_dirs
+        tenant_project_config.load_branch = project_load_branch
 
         return tenant_project_config
 
@@ -1799,8 +1804,10 @@ class TenantParser(object):
         untrusted_projects_config = model.UnparsedConfig()
 
         for project in tenant.config_projects:
+            tpc = tenant.project_configs.get(project.canonical_name)
+            branch = tpc.load_branch if tpc.load_branch else 'master'
             branch_cache = abide.getUnparsedBranchCache(
-                project.canonical_name, 'master')
+                project.canonical_name, branch)
             tpc = tenant.project_configs[project.canonical_name]
             unparsed_branch_config = branch_cache.get(tpc)
 

zuul/model.py

@@ -3341,6 +3341,8 @@ class TenantProjectConfig(object):
         self.extra_config_files = ()
         # The list of paths to look for extra zuul config dirs
         self.extra_config_dirs = ()
+        # Load config from a different branch if this is a config project
+        self.load_branch = None
 
 
 class ProjectPipelineConfig(ConfigObject):