zuul
/
zuul
Code Issues Proposed changes

Merge "Permit config shadowing" into feature/zuulv3

This commit is contained in:
Jenkins 2017-07-11 03:11:10 +00:00 committed by Gerrit Code Review
parent 3be07391d8 6459db123a
commit 5434c40c41
15 changed files with 127 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
tests/base.py
View File

@ -2148,6 +2148,8 @@ class ZuulTestCase(BaseTestCase):
# Make sure we set up an RSA key for the project so that we
# don't spend time generating one:
if isinstance(project, dict):
project = list(project.keys())[0]
key_root = os.path.join(self.state_root, 'keys')
if not os.path.isdir(key_root):
os.mkdir(key_root, 0o700)

2
tests/fixtures/config/shadow/git/local-config/playbooks/base.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/shadow/git/local-config/playbooks/test2.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

25
tests/fixtures/config/shadow/git/local-config/zuul.yaml vendored Normal file
View File

@ -0,0 +1,25 @@
- pipeline:
name: check
manager: independent
trigger:
gerrit:
- event: patchset-created
success:
gerrit:
verified: 1
failure:
gerrit:
verified: -1
- job:
name: base
- job:
name: test2
- project:
name: org/project
check:
jobs:
- test1
- test2

1
tests/fixtures/config/shadow/git/org_project/README vendored Normal file
View File

@ -0,0 +1 @@
test

10
tests/fixtures/config/shadow/git/stdlib/.zuul.yaml vendored Normal file
View File

@ -0,0 +1,10 @@
- job:
name: base
- job:
name: test1
parent: base
- job:
name: test2
parent: base

1
tests/fixtures/config/shadow/git/stdlib/README vendored Normal file
View File

@ -0,0 +1 @@
test

2
tests/fixtures/config/shadow/git/stdlib/playbooks/base.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/shadow/git/stdlib/playbooks/test1.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

2
tests/fixtures/config/shadow/git/stdlib/playbooks/test2.yaml vendored Normal file
View File

@ -0,0 +1,2 @@
- hosts: all
tasks: []

10
tests/fixtures/config/shadow/main.yaml vendored Normal file
View File

@ -0,0 +1,10 @@
- tenant:
name: tenant-one
source:
gerrit:
config-projects:
- local-config
untrusted-projects:
- stdlib:
shadow: local-config
- org/project

24
tests/unit/test_model.py
View File

@ -40,7 +40,7 @@ class TestJob(BaseTestCase):
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
self.tenant = model.Tenant('tenant')
self.layout = model.Layout()
self.layout = model.Layout(self.tenant)
self.project = model.Project('project', self.source)
self.tpc = model.TenantProjectConfig(self.project)
self.tenant.addUntrustedProject(self.tpc)
@ -59,7 +59,7 @@ class TestJob(BaseTestCase):
@property
def job(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
job = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
@ -170,7 +170,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_configloader(self):
# TODO(jeblair): move this to a configloader test
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -333,8 +333,8 @@ class TestJob(BaseTestCase):
'playbooks/base'])
def test_job_auth_inheritance(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant = self.tenant
layout = self.layout
conf = yaml.safe_load('''
- secret:
@ -359,7 +359,7 @@ class TestJob(BaseTestCase):
secret = configloader.SecretParser.fromYaml(layout, conf)
layout.addSecret(secret)
base = configloader.JobParser.fromYaml(tenant, layout, {
base = configloader.JobParser.fromYaml(self.tenant, self.layout, {
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
@ -443,7 +443,7 @@ class TestJob(BaseTestCase):
def test_job_inheritance_job_tree(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
tpc = model.TenantProjectConfig(self.project)
tenant.addUntrustedProject(tpc)
@ -520,7 +520,7 @@ class TestJob(BaseTestCase):
def test_inheritance_keeps_matchers(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
layout = model.Layout(tenant)
pipeline = model.Pipeline('gate', layout)
layout.addPipeline(pipeline)
@ -571,11 +571,13 @@ class TestJob(BaseTestCase):
self.assertEqual([], item.getJobs())
def test_job_source_project(self):
tenant = model.Tenant('tenant')
layout = model.Layout()
tenant = self.tenant
layout = self.layout
base_project = model.Project('base_project', self.source)
base_context = model.SourceContext(base_project, 'master',
'test', True)
tpc = model.TenantProjectConfig(base_project)
tenant.addUntrustedProject(tpc)
base = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': base_context,
@ -587,6 +589,8 @@ class TestJob(BaseTestCase):
other_project = model.Project('other_project', self.source)
other_context = model.SourceContext(other_project, 'master',
'test', True)
tpc = model.TenantProjectConfig(other_project)
tenant.addUntrustedProject(tpc)
base2 = configloader.JobParser.fromYaml(tenant, layout, {
'_source_context': other_context,
'_start_mark': self.start_mark,

14
tests/unit/test_v3.py
View File

@ -630,3 +630,17 @@ class TestRoles(ZuulTestCase):
self.assertHistory([
dict(name='project-test', result='SUCCESS', changes='1,1 2,1'),
])
class TestShadow(ZuulTestCase):
tenant_config_file = 'config/shadow/main.yaml'
def test_shadow(self):
# Test that a repo is allowed to shadow another's job definitions.
A = self.fake_gerrit.addFakeChange('org/project', 'master', 'A')
self.fake_gerrit.addEvent(A.getPatchsetCreatedEvent(1))
self.waitUntilSettled()
self.assertHistory([
dict(name='test1', result='SUCCESS', changes='1,1'),
dict(name='test2', result='SUCCESS', changes='1,1'),
])

27
zuul/configloader.py
View File

@ -887,6 +887,7 @@ class TenantParser(object):
project_dict = {str: {
'include': to_list(classes),
'exclude': to_list(classes),
'shadow': to_list(str),
}}
project = vs.Any(str, project_dict)
@ -940,6 +941,10 @@ class TenantParser(object):
tenant.addConfigProject(tpc)
for tpc in untrusted_tpcs:
tenant.addUntrustedProject(tpc)
for tpc in config_tpcs + untrusted_tpcs:
TenantParser._resolveShadowProjects(tenant, tpc)
tenant.config_projects_config, tenant.untrusted_projects_config = \
TenantParser._loadTenantInRepoLayouts(merger, connections,
tenant.config_projects,
@ -953,6 +958,13 @@ class TenantParser(object):
connections)
return tenant
@staticmethod
def _resolveShadowProjects(tenant, tpc):
shadow_projects = []
for sp in tpc.shadow_projects:
shadow_projects.append(tenant.getProject(sp)[1])
tpc.shadow_projects = frozenset(shadow_projects)
@staticmethod
def _loadProjectKeys(project_key_dir, connection_name, project):
project.private_key_file = (
@ -1008,9 +1020,11 @@ class TenantParser(object):
# Return a project object whether conf is a dict or a str
project = source.getProject(conf)
project_include = current_include
shadow_projects = []
else:
project_name = list(conf.keys())[0]
project = source.getProject(project_name)
shadow_projects = as_list(conf[project_name].get('shadow', []))
project_include = frozenset(
as_list(conf[project_name].get('include', [])))
@ -1023,6 +1037,7 @@ class TenantParser(object):
tenant_project_config = model.TenantProjectConfig(project)
tenant_project_config.load_classes = frozenset(project_include)
tenant_project_config.shadow_projects = shadow_projects
return tenant_project_config
@ -1240,7 +1255,11 @@ class TenantParser(object):
continue
with configuration_exceptions('job', config_job):
job = JobParser.fromYaml(tenant, layout, config_job)
layout.addJob(job)
added = layout.addJob(job)
if not added:
TenantParser.log.debug(
"Skipped adding job %s which shadows an existing job" %
(job,))
if not skip_semaphores:
for config_semaphore in data.semaphores:
@ -1279,13 +1298,11 @@ class TenantParser(object):
@staticmethod
def _parseLayout(base, tenant, data, scheduler, connections):
layout = model.Layout()
layout = model.Layout(tenant)
TenantParser._parseLayoutItems(layout, tenant, data,
scheduler, connections)
layout.tenant = tenant
for pipeline in layout.pipelines.values():
pipeline.manager._postConfig(layout)
@ -1401,7 +1418,7 @@ class ConfigLoader(object):
for project in tenant.untrusted_projects:
self._loadDynamicProjectData(config, project, files, False)
layout = model.Layout()
layout = model.Layout(tenant)
# NOTE: the actual pipeline objects (complete with queues and
# enqueued items) are copied by reference here. This allows
# our shadow dynamic configuration to continue to interact

21
zuul/model.py
View File

@ -2009,6 +2009,7 @@ class TenantProjectConfig(object):
def __init__(self, project):
self.project = project
self.load_classes = set()
self.shadow_projects = set()
class ProjectConfig(object):
@ -2132,8 +2133,8 @@ class UnparsedTenantConfig(object):
class Layout(object):
"""Holds all of the Pipelines."""
def __init__(self):
self.tenant = None
def __init__(self, tenant):
self.tenant = tenant
self.project_configs = {}
self.project_templates = {}
self.pipelines = OrderedDict()
@ -2162,6 +2163,18 @@ class Layout(object):
prior_jobs = [j for j in self.getJobs(job.name) if
j.source_context.project !=
job.source_context.project]
# Unless the repo is permitted to shadow another. If so, and
# the job we are adding is from a repo that is permitted to
# shadow the one with the older jobs, skip adding this job.
job_project = job.source_context.project
job_tpc = self.tenant.project_configs[job_project.canonical_name]
skip_add = False
for prior_job in prior_jobs[:]:
prior_project = prior_job.source_context.project
if prior_project in job_tpc.shadow_projects:
prior_jobs.remove(prior_job)
skip_add = True
if prior_jobs:
raise Exception("Job %s in %s is not permitted to shadow "
"job %s in %s" % (
@ -2169,11 +2182,13 @@ class Layout(object):
job.source_context.project,
prior_jobs[0],
prior_jobs[0].source_context.project))
if skip_add:
return False
if job.name in self.jobs:
self.jobs[job.name].append(job)
else:
self.jobs[job.name] = [job]
return True
def addNodeSet(self, nodeset):
if nodeset.name in self.nodesets: