zuul
/
zuul
Code Issues Proposed changes

Add extra-config-paths tenant config option 

This is intended to address the case of the zuul-jobs repo, where
it would be convenient to include a project definition and job
definitions which test the roles in the zuul-jobs repo.  However,
we want the repo te be consumable by anyone, which means we can
not encode nodesets or required-projects in any jobs which may
be loaded by another Zuul.

To address this, this commit adds a feature which will allow us
to put those job and project definitions in a separate file or
directory which will not be loaded by default.  But in the Zuul
tenant of the OpenDev Zuul installation, we will configure the
system to load this secondary configuration location, so the
self-test jobs will be available.

Change-Id: Ic205d1f93f583514757a100471c47688d6641c53
This commit is contained in:
James E. Blair 2019-06-28 10:20:37 -07:00
parent 56eea2cb8c
commit eed9931d4c
9 changed files with 151 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
doc/source/admin/tenants.rst
View File

@ -183,6 +183,35 @@ configuration. Some examples of tenant definitions are:
processed. Defaults to the tenant wide setting of
exclude-unprotected-branches.
.. attr:: extra-config-paths
Normally Zuul loads in-repo configuration from the first
of these paths:
* zuul.yaml
* zuul.d/*
* .zuul.yaml
* .zuul.d/*
If this option is supplied then, after the normal process
completes, Zuul will also load any configuration found in
the files or paths supplied here. This can be a string or
a list. If a list of multiple items, Zuul will load
configuration from *all* of the items in the list (it will
not stop at the first extra configuration found).
Directories should be listed with a trailing ``/``. Example:
.. code-block:: yaml
extra-config-paths:
- zuul-extra.yaml
- zuul-extra.d/
This feature may be useful to allow a project that
primarily holds shared jobs or roles to include additional
in-repo configuration for its own testing (which may not
be relevant to other users of the project).
.. attr:: <project-group>
The items in the list are dictionaries with the following

12
releasenotes/notes/extra-config-paths-d8250b269a307873.yaml Normal file
View File

@ -0,0 +1,12 @@
---
features:
- |
A new option,
:attr:`tenant.untrusted-projects.<project>.extra-config-paths` has
been added to allow an admin to indicate that Zuul should load
extra configuration data from a specific project in a tenant.
This feature may be useful to allow a project that primarily holds
shared jobs or roles to include additional in-repo configuration
for its own testing (which may not be relevant to other users of
the project).

12
tests/fixtures/config/tenant-parser/extra.yaml vendored Normal file
View File

@ -0,0 +1,12 @@
- tenant:
name: tenant-one
source:
gerrit:
config-projects:
- common-config
untrusted-projects:
- org/project1
- org/project2:
extra-config-paths:
- extra.yaml
- extra.d/

2
tests/fixtures/config/tenant-parser/git/org_project2/extra.d/extra.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- job:
name: project2-extra-dir

2
tests/fixtures/config/tenant-parser/git/org_project2/extra.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- job:
name: project2-extra-file

31
tests/unit/test_configloader.py
View File

@ -416,3 +416,34 @@ class TestConfigConflict(ZuulTestCase):
['base', 'noop', 'trusted-zuul.yaml-job',
'untrusted-zuul.yaml-job'],
jobs)
class TestTenantExtra(TenantParserTestCase):
tenant_config_file = 'config/tenant-parser/extra.yaml'
def test_tenant_extra(self):
tenant = self.sched.abide.tenants.get('tenant-one')
self.assertTrue('project2-extra-file' in tenant.layout.jobs)
self.assertTrue('project2-extra-dir' in tenant.layout.jobs)
def test_dynamic_extra(self):
in_repo_conf = textwrap.dedent(
"""
- job:
name: project2-extra-file2
parent: common-config-job
- project:
name: org/project2
check:
jobs:
- project2-extra-file2
""")
file_dict = {'extra.yaml': in_repo_conf, '.zuul.yaml': ''}
A = self.fake_gerrit.addFakeChange('org/project2', 'master', 'A',
files=file_dict)
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([
dict(name='common-config-job', result='SUCCESS', changes='1,1'),
dict(name='project2-extra-file2', result='SUCCESS', changes='1,1'),
], ordered=False)

67
zuul/configloader.py
View File

@ -1365,6 +1365,7 @@ class TenantParser(object):
'exclude': to_list(classes),
'shadow': to_list(str),
'exclude-unprotected-branches': bool,
'extra-config-paths': to_list(str),
}}
project = vs.Any(str, project_dict)
@ -1556,6 +1557,9 @@ class TenantParser(object):
@staticmethod
def _getProject(source, conf, current_include):
extra_config_files = ()
extra_config_dirs = ()
if isinstance(conf, str):
# Return a project object whether conf is a dict or a str
project = source.getProject(conf)
@ -1579,12 +1583,21 @@ class TenantParser(object):
project_include = frozenset(project_include - project_exclude)
project_exclude_unprotected_branches = conf[project_name].get(
'exclude-unprotected-branches', None)
if conf[project_name].get('extra-config-paths') is not None:
extra_config_paths = as_list(
conf[project_name]['extra-config-paths'])
extra_config_files = tuple([x for x in extra_config_paths
if not x.endswith('/')])
extra_config_dirs = tuple([x[:-1] for x in extra_config_paths
if x.endswith('/')])
tenant_project_config = model.TenantProjectConfig(project)
tenant_project_config.load_classes = frozenset(project_include)
tenant_project_config.shadow_projects = shadow_projects
tenant_project_config.exclude_unprotected_branches = \
project_exclude_unprotected_branches
tenant_project_config.extra_config_files = extra_config_files
tenant_project_config.extra_config_dirs = extra_config_dirs
return tenant_project_config
@ -1670,8 +1683,9 @@ class TenantParser(object):
job = self.merger.getFiles(
project.source.connection.connection_name,
project.name, branch,
files=['zuul.yaml', '.zuul.yaml'],
dirs=['zuul.d', '.zuul.d'])
files=(['zuul.yaml', '.zuul.yaml'] +
list(tpc.extra_config_files)),
dirs=['zuul.d', '.zuul.d'] + list(tpc.extra_config_dirs))
self.log.debug("Submitting cat job %s for %s %s %s" % (
job, project.source.connection.connection_name,
project.name, branch))
@ -1689,19 +1703,25 @@ class TenantParser(object):
loaded = False
files = sorted(job.files.keys())
unparsed_config = model.UnparsedConfig()
for conf_root in ['zuul.yaml', 'zuul.d', '.zuul.yaml', '.zuul.d']:
tpc = tenant.project_configs[
job.source_context.project.canonical_name]
for conf_root in (
('zuul.yaml', 'zuul.d', '.zuul.yaml', '.zuul.d') +
tpc.extra_config_files + tpc.extra_config_dirs):
for fn in files:
fn_root = fn.split('/')[0]
if fn_root != conf_root or not job.files.get(fn):
continue
# Don't load from more than one configuration in a
# project-branch.
if loaded and loaded != conf_root:
self.log.warning(
"Multiple configuration files in %s" %
(job.source_context,))
continue
loaded = conf_root
# project-branch (unless an "extra" file/dir).
if (conf_root not in tpc.extra_config_files and
conf_root not in tpc.extra_config_dirs):
if (loaded and loaded != conf_root):
self.log.warning(
"Multiple configuration files in %s" %
(job.source_context,))
continue
loaded = conf_root
# Create a new source_context so we have unique filenames.
source_context = job.source_context.copy()
source_context.path = fn
@ -2117,6 +2137,8 @@ class ConfigLoader(object):
for branch in branches:
fns1 = []
fns2 = []
fns3 = []
fns4 = []
files_entry = files.connections.get(
project.source.connection.connection_name, {}).get(
project.name, {}).get(branch)
@ -2135,7 +2157,14 @@ class ConfigLoader(object):
fns1.append(fn)
if fn.startswith(".zuul.d/"):
fns2.append(fn)
fns = ["zuul.yaml"] + sorted(fns1) + [".zuul.yaml"] + sorted(fns2)
for ef in tpc.extra_config_files:
if fn.startswith(ef):
fns3.append(fn)
for ed in tpc.extra_config_dirs:
if fn.startswith(ed):
fns4.append(fn)
fns = (["zuul.yaml"] + sorted(fns1) + [".zuul.yaml"] +
sorted(fns2) + fns3 + sorted(fns4))
incdata = None
loaded = None
for fn in fns:
@ -2146,11 +2175,17 @@ class ConfigLoader(object):
fn, trusted)
# Prevent mixing configuration source
conf_root = fn.split('/')[0]
if loaded and loaded != conf_root:
self.log.warning(
"Multiple configuration in %s" % source_context)
continue
loaded = conf_root
# Don't load from more than one configuration in a
# project-branch (unless an "extra" file/dir).
if (conf_root not in tpc.extra_config_files and
conf_root not in tpc.extra_config_dirs):
if loaded and loaded != conf_root:
self.log.warning(
"Multiple configuration in %s" %
source_context)
continue
loaded = conf_root
incdata = self.tenant_parser.loadProjectYAML(
data, source_context, loading_errors)

11
zuul/manager/__init__.py
View File

@ -661,9 +661,14 @@ class PipelineManager(object):
if not build_set.ref:
build_set.setConfiguration()
if build_set.merge_state == build_set.NEW:
ready = self.scheduleMerge(item,
files=['zuul.yaml', '.zuul.yaml'],
dirs=['zuul.d', '.zuul.d'])
tenant = item.pipeline.tenant
tpc = tenant.project_configs[item.change.project.canonical_name]
ready = self.scheduleMerge(
item,
files=(['zuul.yaml', '.zuul.yaml'] +
list(tpc.extra_config_files)),
dirs=(['zuul.d', '.zuul.d'] +
list(tpc.extra_config_dirs)))
if build_set.files_state == build_set.NEW:
ready = self.scheduleFilesChanges(item)
if build_set.files_state == build_set.PENDING:

4
zuul/model.py
View File

@ -3167,6 +3167,10 @@ class TenantProjectConfig(object):
# be overridden by this one if not None.
self.exclude_unprotected_branches = None
self.parsed_branch_config = {} # branch -> ParsedConfig
# The list of paths to look for extra zuul config files
self.extra_config_files = ()
# The list of paths to look for extra zuul config dirs
self.extra_config_dirs = ()
class ProjectPipelineConfig(ConfigObject):