zuul
/
zuul
Code Issues Proposed changes

Add a zuul.buildset_refs variable 

This adds information about the changes associated with a
circular dependency queue item.  Currently the bundle_id can be
used to identify which of the items in zuul.items is related to
the current dependency cycle.  That variable is deprecated, so
zuul.buildset_refs can be used to replace that functionality.

Since it repeats some of the information at the top level (eg
zuul.change, zuul.project, etc), the code is refactored so they
can share the dictionary construction.  That is also used by
zuul.items.  This results in a few extra fields in zuul.items
now, such as the change message, but that is relatively
inconsequential, so is not called out in the release notes.

The src_dir is similarly included in all of these places.  In
writing this change it was discovered that
zuul.items.project.src_dir always used the golang scheme, but
zuul.project.src_dir used the correct per-job workspace scheme.
This has been corrected so that they both use the per-job scheme
now.

A significant reorganization of the job variable documentation is
included.  Previously we had a section with additional variables
for each item type, but since many of these are duplicated at the
top level, in the item list, and now in the refs list, that
structure became difficult to work with.  Instead, the
documentation for each of these sections now exhaustively lists
all of the possible variables.  This makes for some repitition,
but it also means that a user can see at a glance what variables
are available, and we can deep-link to each one.  To address the
variation between different item types, the variables that mutate
based on item type now contain a definition list indicating what
types they are valid for and their respective meanings.

Change-Id: Iab8f99d4c4f40c44d630120c458539060cc725b5
This commit is contained in:
James E. Blair 2024-03-13 16:07:25 -07:00
parent 6ccbdacdf2
commit 632839804c
7 changed files with 757 additions and 375 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

977
doc/source/job-content.rst
View File

File diff suppressed because it is too large Load Diff

12
releasenotes/notes/zuul-refs-variable-d569decc6e8ad4b8.yaml Normal file
View File

@ -0,0 +1,12 @@
---
features:
- |
Information about all of the changes (refs) associated with a
circular dependency queue item is now available in the
:var:`zuul.buildset_refs` variable.
fixes:
- |
The value of :var:`zuul.items.project.src_dir` did not take into
account the job's workspace scheme and was always constructed
using the golang scheme. That has been corrected so that it now
reflects the job's scheme.

13
tests/unit/test_merger_repo.py
View File

@ -18,6 +18,7 @@ import logging
import os
import shutil
from unittest import mock
from zuul.lib import yamlutil as yaml
import git
import testtools
@ -1339,6 +1340,18 @@ class TestOverlappingRepos(ZuulTestCase):
'subcomponent', '.git')
self.assertTrue(os.path.exists(jobdir_git_dir))
inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml')
with open(inv_path, 'r') as f:
inventory = yaml.safe_load(f)
zuul = inventory['all']['vars']['zuul']
self.assertEqual('src/component',
zuul['items'][0]['project']['src_dir'])
self.assertEqual('src/component',
zuul['projects']['review.example.com/component']
['src_dir'])
self.assertEqual('src/component',
zuul['buildset_refs'][0]['src_dir'])
class TestMergerUpgrade(ZuulTestCase):
tenant_config_file = 'config/single-tenant/main.yaml'

21
tests/unit/test_web.py
View File

@ -1237,6 +1237,16 @@ class TestWeb(BaseTestWeb):
'buildset': None,
'branch': 'master',
'ref': None,
'buildset_refs': [
{'branch': 'master',
'change_url': None,
'project': {
'canonical_hostname': 'review.example.com',
'canonical_name':
'review.example.com/org/project1',
'name': 'org/project1',
'short_name': 'project1'},
'src_dir': 'src/review.example.com/org/project1'}],
'pipeline': 'check',
'post_review': False,
'job': 'project-test1',
@ -1318,6 +1328,17 @@ class TestWeb(BaseTestWeb):
'src_dir': 'src/review.example.com/org/noop-project'},
'projects': {},
'ref': None,
'buildset_refs': [
{'branch': 'master',
'change_url': None,
'project': {
'canonical_hostname': 'review.example.com',
'canonical_name':
'review.example.com/org/noop-project',
'name': 'org/noop-project',
'short_name': 'noop-project'},
'src_dir':
'src/review.example.com/org/noop-project'}],
'tenant': 'tenant-one',
'timeout': None,
'voting': True}}

76
zuul/executor/common.py
View File

@ -12,11 +12,20 @@
# License for the specific language governing permissions and limitations
# under the License.
import copy
import os
from zuul.lib import strings
def make_src_dir(canonical_hostname, name, scheme):
return os.path.join('src',
strings.workspace_project_path(
canonical_hostname,
name,
scheme))
def construct_build_params(uuid, connections, job, item, pipeline,
dependent_changes=[], merger_items=[],
redact_secrets_and_keys=True):
@ -35,17 +44,19 @@ def construct_build_params(uuid, connections, job, item, pipeline,
short_name=change.project.name.split('/')[-1],
canonical_hostname=change.project.canonical_hostname,
canonical_name=change.project.canonical_name,
src_dir=os.path.join('src',
strings.workspace_project_path(
change.project.canonical_hostname,
change.project.name,
job.workspace_scheme)),
src_dir=make_src_dir(change.project.canonical_hostname,
change.project.name,
job.workspace_scheme),
)
zuul_params = dict(
# We override some values like project, so set the change fields
# first.
zuul_params = change.toDict()
zuul_params.update(dict(
build=uuid,
buildset=item.current_build_set.uuid,
ref=change.ref,
buildset_refs=[c.toDict() for c in item.changes],
pipeline=pipeline.name,
post_review=pipeline.post_review,
job=job.name,
@ -53,37 +64,25 @@ def construct_build_params(uuid, connections, job, item, pipeline,
tenant=tenant.name,
event_id=item.event.zuul_event_id if item.event else None,
jobtags=sorted(job.tags),
)
if hasattr(change, 'branch'):
zuul_params['branch'] = change.branch
if hasattr(change, 'tag'):
zuul_params['tag'] = change.tag
if hasattr(change, 'number'):
zuul_params['change'] = str(change.number)
if hasattr(change, 'url'):
zuul_params['change_url'] = change.url
if hasattr(change, 'patchset'):
zuul_params['patchset'] = str(change.patchset)
))
if hasattr(change, 'message'):
zuul_params['message'] = strings.b64encode(change.message)
zuul_params['change_message'] = change.message
if hasattr(change, 'topic'):
zuul_params['topic'] = change.topic
commit_id = None
if (hasattr(change, 'oldrev') and change.oldrev
and change.oldrev != '0' * 40):
zuul_params['oldrev'] = change.oldrev
commit_id = change.oldrev
if (hasattr(change, 'newrev') and change.newrev
and change.newrev != '0' * 40):
zuul_params['newrev'] = change.newrev
commit_id = change.newrev
if hasattr(change, 'commit_id'):
commit_id = change.commit_id
if commit_id:
zuul_params['commit_id'] = commit_id
zuul_params['projects'] = {} # Set below
# Fixup the src_dir for the items based on this job
dependent_changes = copy.deepcopy(dependent_changes)
for dep_change in dependent_changes:
dep_change['project']['src_dir'] = make_src_dir(
dep_change['project']['canonical_hostname'],
dep_change['project']['name'],
job.workspace_scheme)
# Fixup the src_dir for the refs based on this job
for r in zuul_params['buildset_refs']:
r['src_dir'] = make_src_dir(
r['project']['canonical_hostname'],
r['project']['name'],
job.workspace_scheme)
zuul_params['items'] = dependent_changes
zuul_params['child_jobs'] = [
x.name for x in item.current_build_set.job_graph.
@ -176,11 +175,10 @@ def construct_build_params(uuid, connections, job, item, pipeline,
# project.values() is easier for callers
canonical_name=p.canonical_name,
canonical_hostname=p.canonical_hostname,
src_dir=os.path.join('src',
strings.workspace_project_path(
p.canonical_hostname,
p.name,
job.workspace_scheme)),
src_dir=make_src_dir(
p.canonical_hostname,
p.name,
job.workspace_scheme),
required=(p in required_projects),
))

8
zuul/executor/server.py
View File

@ -2619,6 +2619,14 @@ class AnsibleJob(object):
if 'change_message' in zuul_vars:
zuul_vars['change_message'] = yaml.mark_strings_unsafe(
zuul_vars['change_message'])
for item in zuul_vars['items']:
if 'change_message' in item:
item['change_message'] = yaml.mark_strings_unsafe(
item['change_message'])
for ref in zuul_vars.get('buildset_refs', []):
if 'change_message' in ref:
ref['change_message'] = yaml.mark_strings_unsafe(
ref['change_message'])
with open(self.jobdir.zuul_vars, 'w') as zuul_vars_yaml:
zuul_vars_yaml.write(

25
zuul/model.py
View File

@ -20,7 +20,6 @@ import json
import hashlib
import logging
import math
import os
from functools import partial, total_ordering
import threading
@ -6177,8 +6176,18 @@ class Ref(object):
short_name=self.project.name.split('/')[-1],
canonical_hostname=self.project.canonical_hostname,
canonical_name=self.project.canonical_name,
src_dir=os.path.join('src', self.project.canonical_name),
)
d['change_url'] = self.url
commit_id = None
if self.oldrev and self.oldrev != '0' * 40:
d['oldrev'] = self.oldrev
commit_id = self.oldrev
if self.newrev and self.newrev != '0' * 40:
d['newrev'] = self.newrev
commit_id = self.newrev
if commit_id:
d['commit_id'] = commit_id
return d
@ -6211,6 +6220,12 @@ class Tag(Ref):
self.tag = None
self.containing_branches = []
def toDict(self):
# Render to a dict to use in passing json to the executor
d = super(Tag, self).toDict()
d["tag"] = self.tag
return d
def serialize(self):
d = super().serialize()
d["containing_branches"] = self.containing_branches
@ -6389,9 +6404,13 @@ class Change(Branch):
def toDict(self):
# Render to a dict to use in passing json to the executor
d = super(Change, self).toDict()
d.pop('oldrev', None)
d.pop('newrev', None)
d['change'] = str(self.number)
d['change_url'] = self.url
d['patchset'] = str(self.patchset)
d['commit_id'] = self.commit_id
d['change_message'] = self.message
d['topic'] = self.topic
return d