From 632839804ca46b6d7c971d97e0f2aba35a3bbf29 Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Wed, 13 Mar 2024 16:07:25 -0700 Subject: [PATCH] Add a zuul.buildset_refs variable This adds information about the changes associated with a circular dependency queue item. Currently the bundle_id can be used to identify which of the items in zuul.items is related to the current dependency cycle. That variable is deprecated, so zuul.buildset_refs can be used to replace that functionality. Since it repeats some of the information at the top level (eg zuul.change, zuul.project, etc), the code is refactored so they can share the dictionary construction. That is also used by zuul.items. This results in a few extra fields in zuul.items now, such as the change message, but that is relatively inconsequential, so is not called out in the release notes. The src_dir is similarly included in all of these places. In writing this change it was discovered that zuul.items.project.src_dir always used the golang scheme, but zuul.project.src_dir used the correct per-job workspace scheme. This has been corrected so that they both use the per-job scheme now. A significant reorganization of the job variable documentation is included. Previously we had a section with additional variables for each item type, but since many of these are duplicated at the top level, in the item list, and now in the refs list, that structure became difficult to work with. Instead, the documentation for each of these sections now exhaustively lists all of the possible variables. This makes for some repitition, but it also means that a user can see at a glance what variables are available, and we can deep-link to each one. To address the variation between different item types, the variables that mutate based on item type now contain a definition list indicating what types they are valid for and their respective meanings. Change-Id: Iab8f99d4c4f40c44d630120c458539060cc725b5 --- doc/source/job-content.rst | 977 ++++++++++++------ .../zuul-refs-variable-d569decc6e8ad4b8.yaml | 12 + tests/unit/test_merger_repo.py | 13 + tests/unit/test_web.py | 21 + zuul/executor/common.py | 76 +- zuul/executor/server.py | 8 + zuul/model.py | 25 +- 7 files changed, 757 insertions(+), 375 deletions(-) create mode 100644 releasenotes/notes/zuul-refs-variable-d569decc6e8ad4b8.yaml diff --git a/doc/source/job-content.rst b/doc/source/job-content.rst index cb5f6a8abf..97495b0eed 100644 --- a/doc/source/job-content.rst +++ b/doc/source/job-content.rst @@ -222,24 +222,594 @@ vary based on the pipeline's configuration. For example, when a new change is created, that change may be enqueued into the pipeline, while a tag may be enqueued into the pipeline when it is pushed. -Information about these items is available to jobs. All of the items -enqueued in a pipeline are git references, and therefore share some -attributes in common. But other attributes may vary based on the type -of item. +An item typically refers to a single git reference, but in the case of +a dependency cycle among changes, the item may be composed of multiple +changes. -In the case of circular dependencies, a queue item may have multiple -changes associated with it, and if jobs are deduplicated, then a -single build may be run for multiple changes. In these cases, Zuul -will select one of the queue item's changes to supply values for -variables related to the queue item's change. The selected change may -be the change that triggered the item to be enqueued, or it may not. -Beyond the fact that the change will be one of the item's changes, -this behavior should not be relied upon. +Information about these items is available to jobs. Since all of the +items enqueued in a pipeline represent one or more git references, the +different types of item share some attributes in common. But other +attributes may vary based on the type of ref. The different types of +ref are: + +Change + A change to the repository. Most often, this will be a git + reference which has not yet been merged into the repository (e.g., + a Gerrit change or a GitHub pull request). + +Branch + This represents a branch tip. This item may have been enqueued + because the branch was updated (via a change having merged, or a + direct push). Or it may have been enqueued by a timer for the + purpose of verifying the current condition of the branch. + +Tag + This represents a git tag. The item may have been enqueued because + a tag was created or deleted. + +Ref + This represents a git reference that is neither a change, branch, or + tag. + +If a build is running for a queue item with a single ref, the values +below are straightforward. Things are a little more complex if the +queue item represents multiple changes in a dependency cycle. In that +case, the same job may be run multiple times, each for a different +change in the cycle. If that happens, then many of the attributes +below (such as :var:`zuul.change` and :var:`zuul.project`, etc) will +refer to the particular change that is assigned to that build. +However, if a job is deduplicated, then one build is run for several +changes simultaneously. In that case, one of the changes which +triggered the job will arbitrarily be selected for those values. If +possible, Zuul will use the change that originally caused the item to +be enqueued, but that is not always possible, and that behavior should +not be relied upon. + +Job Ref +~~~~~~~ + +The following variables related to the job's selected ref (as +described above) are available: .. var:: zuul - All items provide the following information as Ansible variables - under the ``zuul`` key: + .. var:: project + + The job's project. If the job is running for a single change, + then this will be the project of that change. In the case of a + circular dependency queue item where this job is run more than + once for different changes in the item, this will be set to the + project of the particular change assigned to this build of the + job. If the job is deduplicated, then this is arbitrarily set + to one of the changes in the queue item that triggered the job. + + This is a data structure with the following fields: + + .. var:: name + + The name of the project, excluding hostname. E.g., `org/project`. + + .. var:: short_name + + The name of the project, excluding directories or + organizations. E.g., `project`. + + .. var:: canonical_hostname + + The canonical hostname where the project lives. E.g., + `git.example.com`. + + .. var:: canonical_name + + The full canonical name of the project including hostname. + E.g., `git.example.com/org/project`. + + .. var:: src_dir + + The path to the source code relative to the work dir. E.g., + `src/git.example.com/org/project`. + + .. var:: branch + + This field is present for the following item types: + + Branch + The item's branch (without the `refs/heads/` prefix). + + Change + The target branch of the change (without the `refs/heads/` + prefix). + + .. var:: change + + This field is present for the following item type: + + Change + The identifier for the change. + + .. var:: message + + The commit or pull request message of the change base64 encoded. Use the + `b64decode` filter in ansible when working with it. + + .. warning:: This variable is deprecated and will be removed in + a future version. Use :var:`zuul.change_message` + instead. + + .. var:: change_message + + This field is present for the following item type: + + Change + The commit or pull request message of the change. When + Zuul runs Ansible, this variable is tagged with the + ``!unsafe`` YAML tag so that Ansible will not interpolate + values into it. Note, however, that the `inventory.yaml` + file placed in the build's workspace for debugging and + inspection purposes does not inclued the ``!unsafe`` tag. + + .. var:: change_url + + This field is present for the following item type: + + Change + The URL to the source location of the given change. + E.g., `https://review.example.org/#/c/123456/` or + `https://github.com/example/example/pull/1234`. + + .. var:: patchset + + This field is present for the following item types: + + Change + The patchset identifier for the change. If a change is + revised, this will have a different value. + + .. var:: project + + The item's project. This is a data structure with the + following fields: + + .. var:: name + + The name of the project, excluding hostname. E.g., + `org/project`. + + .. var:: short_name + + The name of the project, excluding directories or + organizations. E.g., `project`. + + .. var:: canonical_hostname + + The canonical hostname where the project lives. E.g., + `git.example.com`. + + .. var:: canonical_name + + The full canonical name of the project including hostname. + E.g., `git.example.com/org/project`. + + .. var:: src_dir + + The path to the source code on the remote host, relative + to the home dir of the remote user. + E.g., `src/git.example.com/org/project`. + + .. var:: oldrev + + This field is present for the following item types: + + Branch + If the item was enqueued as the result of a change merging + or being pushed to the branch, the git sha of the old + revision will be included here. + + Tag + If the item was enqueued as the result of a tag being + deleted, the previous git sha of the tag will be included + here. If the tag was created, this variable will be + undefined. + + Ref + If the item was enqueued as the result of a ref being + deleted, the previous git sha of the ref will be included + here. If the ref was created, this variable will be + undefined. + + .. var:: newrev + + This field is present for the following item types: + + Branch + If the item was enqueued as the result of a change merging + or being pushed to the branch, the git sha of the new + revision will be included here. + + Tag + If the item was enqueued as the result of a tag being + created, the new git sha of the tag will be included here. + If the tag was deleted, this variable will be undefined. + + Ref + If the item was enqueued as the result of a ref being + created, the new git sha of the ref will be included here. + If the ref was deleted, this variable will be undefined. + + .. var:: commit_id + + This field is present for the following item types: + + Branch + The git sha of the branch. Identical to ``newrev`` or + ``oldrev`` if defined. + Tag + The git sha of the tag. Identical to ``newrev`` or + ``oldrev`` if defined. + Ref + The git sha of the ref. Identical to ``newrev`` or + ``oldrev`` if defined. + + .. var:: tag + + This field is present for the following item types: + + Tag + The name of the item's tag (without the `refs/tags/` prefix). + + .. var:: topic + + This field is present for the following item types: + + Change + The topic of the change (if any). + + .. var:: ref + + The git ref of the item. This will be the full path (e.g., + `refs/heads/master` or `refs/changes/...`). + + +Item +~~~~ + +The following variables related to the queue item are available: + +.. var:: zuul + + .. var:: items + :type: list + + .. note:: + + ``zuul.items`` conflicts with the ``items()`` builtin so the + variable can only be accessed with python dictionary like syntax, + e.g: ``zuul['items']`` + + A list of dictionaries, each representing a ref being tested + with this change. + + .. var:: branch + + This field is present for the following item types: + + Branch + The item's branch (without the `refs/heads/` prefix). + + Change + The target branch of the change (without the `refs/heads/` + prefix). + + .. var:: bundle_id + + This field is present for the following item type: + + Change + The id of the bundle if the change is in a circular + dependency cycle. + + Only available for items with more than one change. + + .. warning:: This variable is deprecated and will be removed in + a future version. Use :var:`zuul.buildset_refs` to + identify if the item is for a dependency cycle and + the associated changes instead. + + .. var:: change + + This field is present for the following item type: + + Change + The identifier for the change. + + .. var:: change_message + + This field is present for the following item type: + + Change + The commit or pull request message of the change. When + Zuul runs Ansible, this variable is tagged with the + ``!unsafe`` YAML tag so that Ansible will not interpolate + values into it. Note, however, that the `inventory.yaml` + file placed in the build's workspace for debugging and + inspection purposes does not inclued the ``!unsafe`` tag. + + .. var:: change_url + + This field is present for the following item type: + + Change + The URL to the source location of the given change. + E.g., `https://review.example.org/#/c/123456/` or + `https://github.com/example/example/pull/1234`. + + .. var:: patchset + + This field is present for the following item types: + + Change + The patchset identifier for the change. If a change is + revised, this will have a different value. + + .. var:: project + + The item's project. This is a data structure with the + following fields: + + .. var:: name + + The name of the project, excluding hostname. E.g., + `org/project`. + + .. var:: short_name + + The name of the project, excluding directories or + organizations. E.g., `project`. + + .. var:: canonical_hostname + + The canonical hostname where the project lives. E.g., + `git.example.com`. + + .. var:: canonical_name + + The full canonical name of the project including hostname. + E.g., `git.example.com/org/project`. + + .. var:: src_dir + + The path to the source code on the remote host, relative + to the home dir of the remote user. + E.g., `src/git.example.com/org/project`. + + .. var:: oldrev + + This field is present for the following item types: + + Branch + If the item was enqueued as the result of a change merging + or being pushed to the branch, the git sha of the old + revision will be included here. + + Tag + If the item was enqueued as the result of a tag being + deleted, the previous git sha of the tag will be included + here. If the tag was created, this variable will be + undefined. + + Ref + If the item was enqueued as the result of a ref being + deleted, the previous git sha of the ref will be included + here. If the ref was created, this variable will be + undefined. + + .. var:: newrev + + This field is present for the following item types: + + Branch + If the item was enqueued as the result of a change merging + or being pushed to the branch, the git sha of the new + revision will be included here. + + Tag + If the item was enqueued as the result of a tag being + created, the new git sha of the tag will be included here. + If the tag was deleted, this variable will be undefined. + + Ref + If the item was enqueued as the result of a ref being + created, the new git sha of the ref will be included here. + If the ref was deleted, this variable will be undefined. + + .. var:: commit_id + + This field is present for the following item types: + + Branch + The git sha of the branch. Identical to ``newrev`` or + ``oldrev`` if defined. + Tag + The git sha of the tag. Identical to ``newrev`` or + ``oldrev`` if defined. + Ref + The git sha of the ref. Identical to ``newrev`` or + ``oldrev`` if defined. + + .. var:: tag + + This field is present for the following item types: + + Tag + The name of the item's tag (without the `refs/tags/` prefix). + + .. var:: topic + + This field is present for the following item types: + + Change + The topic of the change (if any). + + .. var:: buildset_refs + :type: list + + A list of dictionaries, each representing a ref associated with + this queue item. Normally there is only one item in this list, + but if the queue item is a dependency cycle, each change in the + cycle will be present. + + .. var:: branch + + This field is present for the following item types: + + Branch + The item's branch (without the `refs/heads/` prefix). + + Change + The target branch of the change (without the `refs/heads/` + prefix). + + .. var:: change + + This field is present for the following item type: + + Change + The identifier for the change. + + .. var:: change_message + + This field is present for the following item type: + + Change + The commit or pull request message of the change. When + Zuul runs Ansible, this variable is tagged with the + ``!unsafe`` YAML tag so that Ansible will not interpolate + values into it. Note, however, that the `inventory.yaml` + file placed in the build's workspace for debugging and + inspection purposes does not inclued the ``!unsafe`` tag. + + .. var:: change_url + + This field is present for the following item type: + + Change + The URL to the source location of the given change. + E.g., `https://review.example.org/#/c/123456/` or + `https://github.com/example/example/pull/1234`. + + .. var:: patchset + + This field is present for the following item types: + + Change + The patchset identifier for the change. If a change is + revised, this will have a different value. + + .. var:: project + + The item's project. This is a data structure with the + following fields: + + .. var:: name + + The name of the project, excluding hostname. E.g., + `org/project`. + + .. var:: short_name + + The name of the project, excluding directories or + organizations. E.g., `project`. + + .. var:: canonical_hostname + + The canonical hostname where the project lives. E.g., + `git.example.com`. + + .. var:: canonical_name + + The full canonical name of the project including hostname. + E.g., `git.example.com/org/project`. + + .. var:: src_dir + + The path to the source code on the remote host, relative + to the home dir of the remote user. + E.g., `src/git.example.com/org/project`. + + .. var:: oldrev + + This field is present for the following item types: + + Branch + If the item was enqueued as the result of a change merging + or being pushed to the branch, the git sha of the old + revision will be included here. + + Tag + If the item was enqueued as the result of a tag being + deleted, the previous git sha of the tag will be included + here. If the tag was created, this variable will be + undefined. + + Ref + If the item was enqueued as the result of a ref being + deleted, the previous git sha of the ref will be included + here. If the ref was created, this variable will be + undefined. + + .. var:: newrev + + This field is present for the following item types: + + Branch + If the item was enqueued as the result of a change merging + or being pushed to the branch, the git sha of the new + revision will be included here. + + Tag + If the item was enqueued as the result of a tag being + created, the new git sha of the tag will be included here. + If the tag was deleted, this variable will be undefined. + + Ref + If the item was enqueued as the result of a ref being + created, the new git sha of the ref will be included here. + If the ref was deleted, this variable will be undefined. + + .. var:: commit_id + + This field is present for the following item types: + + Branch + The git sha of the branch. Identical to ``newrev`` or + ``oldrev`` if defined. + Tag + The git sha of the tag. Identical to ``newrev`` or + ``oldrev`` if defined. + Ref + The git sha of the ref. Identical to ``newrev`` or + ``oldrev`` if defined. + + .. var:: tag + + This field is present for the following item types: + + Tag + The name of the item's tag (without the `refs/tags/` prefix). + + .. var:: topic + + This field is present for the following item types: + + Change + The topic of the change (if any). + +Job +~~~ + +The following variables related to the job are available: + +.. var:: zuul .. var:: artifacts :type: list @@ -284,16 +854,16 @@ this behavior should not be relied upon. The UUID of the build. A build is a single execution of a job. When an item is enqueued into a pipeline, this usually results - in one build of each job configured for that item's project. - However, items may be re-enqueued in which case another build - may run. In dependent pipelines, the same job may run multiple - times for the same item as circumstances change ahead in the - queue. Each time a job is run, for whatever reason, it is - acompanied with a new unique id. + in one build of each job triggered by that item. However, items + may be re-enqueued in which case another build may run. In + dependent pipelines, the same job may run multiple times for the + same item as circumstances change ahead in the queue. Each time + a job is run, for whatever reason, it is acompanied with a new + unique id. .. var:: buildset - The build set UUID. When Zuul runs jobs for an item, the + The buildset UUID. When Zuul runs jobs for an item, the collection of those jobs is known as a buildset. If the configuration of items ahead in a dependent pipeline changes, Zuul creates a new buildset and restarts all of the jobs. @@ -303,11 +873,6 @@ this behavior should not be relied upon. A list of the first level dependent jobs to be run after this job has finished successfully. - .. var:: ref - - The git ref of the item. This will be the full path (e.g., - `refs/heads/master` or `refs/changes/...`). - .. var:: override_checkout If the job was configured to override the branch or tag checked @@ -354,40 +919,11 @@ this behavior should not be relied upon. The version of the Ansible community package release used for executing the job. - .. var:: project - - The item's project. This is a data structure with the following - fields: - - .. var:: name - - The name of the project, excluding hostname. E.g., `org/project`. - - .. var:: short_name - - The name of the project, excluding directories or - organizations. E.g., `project`. - - .. var:: canonical_hostname - - The canonical hostname where the project lives. E.g., - `git.example.com`. - - .. var:: canonical_name - - The full canonical name of the project including hostname. - E.g., `git.example.com/org/project`. - - .. var:: src_dir - - The path to the source code relative to the work dir. E.g., - `src/git.example.com/org/project`. - .. var:: projects :type: dict A dictionary of all projects prepared by Zuul for the item. It - includes, at least, the item's own project. It also includes + includes, at least, the item's own projects. It also includes the projects of any items this item depends on, as well as the projects that appear in :attr:`job.required-projects`. @@ -457,7 +993,6 @@ this behavior should not be relied upon. with_items: {{ zuul.projects.values() | list }} .. var:: playbook_context - :type: dict This dictionary contains information about the execution of each playbook in the job. This may be useful for understanding @@ -563,116 +1098,87 @@ this behavior should not be relied upon. git tags, these are simply free-form text fields that can be used by the job for reporting or classification purposes. - .. var:: items - :type: list + .. var:: resources - .. note:: + A job using a container build resources has access to a + resources variable that describes the resource. Resources is + a dictionary of group keys, each value consists of: - ``zuul.items`` conflicts with the ``items()`` builtin so the - variable can only be accessed with python dictionary like syntax, - e.g: ``zuul['items']`` + .. var:: namespace - A list of dictionaries, each representing an item being tested - with this change with the format: + The resource's namespace name. - .. var:: project + .. var:: context - The item's project. This is a data structure with the - following fields: + The kube config context name. - .. var:: name + .. var:: pod - The name of the project, excluding hostname. E.g., - `org/project`. + The name of the pod when the label defines a kubectl connection. - .. var:: short_name + Project or namespace resources might be used in a template as: - The name of the project, excluding directories or - organizations. E.g., `project`. + .. code-block:: yaml - .. var:: canonical_hostname + - hosts: localhost + tasks: + - name: Create a k8s resource + k8s_raw: + state: present + context: "{{ zuul.resources['node-name'].context }}" + namespace: "{{ zuul.resources['node-name'].namespace }}" - The canonical hostname where the project lives. E.g., - `git.example.com`. + Kubectl resources might be used in a template as: - .. var:: canonical_name + .. code-block:: yaml - The full canonical name of the project including hostname. - E.g., `git.example.com/org/project`. + - hosts: localhost + tasks: + - name: Copy src repos to the pod + command: > + oc rsync -q --progress=false + {{ zuul.executor.src_root }}/ + {{ zuul.resources['node-name'].pod }}:src/ + no_log: true - .. var:: src_dir +Working Directory +~~~~~~~~~~~~~~~~~ - The path to the source code on the remote host, relative - to the home dir of the remote user. - E.g., `src/git.example.com/org/project`. +Additionally, some information about the working directory and the +executor running the job is available: - .. var:: branch +.. var:: zuul - The target branch of the change (without the `refs/heads/` prefix). + .. var:: executor - .. var:: bundle_id + A number of values related to the executor running the job are + available: - The id of the bundle if the change is in a circular dependency cycle. + .. var:: hostname - .. var:: change + The hostname of the executor. - The identifier for the change. + .. var:: src_root - .. var:: change_url + The path to the source directory. - The URL to the source location of the given change. - E.g., `https://review.example.org/#/c/123456/` or - `https://github.com/example/example/pull/1234`. + .. var:: log_root - .. var:: patchset + The path to the logs directory. - The patchset identifier for the change. If a change is - revised, this will have a different value. + .. var:: work_root - .. var:: resources - :type: dict + The path to the working directory. - A job using a container build resources has access to a resources variable - that describes the resource. Resources is a dictionary of group keys, - each value consists of: + .. var:: inventory_file - .. var:: namespace - - The resource's namespace name. - - .. var:: context - - The kube config context name. - - .. var:: pod - - The name of the pod when the label defines a kubectl connection. - - Project or namespace resources might be used in a template as: - - .. code-block:: yaml - - - hosts: localhost - tasks: - - name: Create a k8s resource - k8s_raw: - state: present - context: "{{ zuul.resources['node-name'].context }}" - namespace: "{{ zuul.resources['node-name'].namespace }}" - - Kubectl resources might be used in a template as: - - .. code-block:: yaml - - - hosts: localhost - tasks: - - name: Copy src repos to the pod - command: > - oc rsync -q --progress=false - {{ zuul.executor.src_root }}/ - {{ zuul.resources['node-name'].pod }}:src/ - no_log: true + The path to the inventory. This variable is needed for jobs running + without a nodeset since Ansible doesn't set it for localhost; see + this `porting guide + `_. + The inventory file is only readable by jobs running in a + :term:`trusted execution context`. .. var:: zuul_success @@ -761,201 +1267,6 @@ this behavior should not be relied upon. A private IPv6 address of the node. -Change Items -~~~~~~~~~~~~ - -A change to the repository. Most often, this will be a git reference -which has not yet been merged into the repository (e.g., a gerrit -change or a GitHub pull request). The following additional variables -are available: - -.. var:: zuul - :hidden: - - .. var:: branch - - The target branch of the change (without the `refs/heads/` prefix). - - .. var:: change - - The identifier for the change. - - .. var:: patchset - - The patchset identifier for the change. If a change is revised, - this will have a different value. - - .. var:: change_url - - The URL to the source location of the given change. - E.g., `https://review.example.org/#/c/123456/` or - `https://github.com/example/example/pull/1234`. - - .. var:: message - - The commit or pull request message of the change base64 encoded. Use the - `b64decode` filter in ansible when working with it. - - .. warning:: This variable is deprecated and will be removed in - a future version. Use :var:`zuul.change_message` - instead. - - .. var:: change_message - - The commit or pull request message of the change. When Zuul - runs Ansible, this variable is tagged with the ``!unsafe`` YAML - tag so that Ansible will not interpolate values into it. Note, - however, that the `inventory.yaml` file placed in the build's - workspace for debugging and inspection purposes does not inclued - the ``!unsafe`` tag. - - .. var:: commit_id - - The git sha of the change. This may be the commit sha of the - current patchset revision or the tip of a pull request branch - depending on the source. Because of Zuul's speculative merge - process, this commit may not even appear in the prepared git - repos, so it should not be relied upon for git operations in - jobs. It is included here to support interfacing with systems - that identify a change by the commit. - - .. var:: topic - - The topic of the change (if any) - -Branch Items -~~~~~~~~~~~~ - -This represents a branch tip. This item may have been enqueued -because the branch was updated (via a change having merged, or a -direct push). Or it may have been enqueued by a timer for the purpose -of verifying the current condition of the branch. The following -additional variables are available: - -.. var:: zuul - :hidden: - - .. var:: branch - - The name of the item's branch (without the `refs/heads/` - prefix). - - .. var:: oldrev - - If the item was enqueued as the result of a change merging or - being pushed to the branch, the git sha of the old revision will - be included here. Otherwise, this variable will be undefined. - - .. var:: newrev - - If the item was enqueued as the result of a change merging or - being pushed to the branch, the git sha of the new revision will - be included here. Otherwise, this variable will be undefined. - - .. var:: commit_id - - The git sha of the branch. Identical to ``newrev`` or - ``oldrev`` if defined. - -Tag Items -~~~~~~~~~ - -This represents a git tag. The item may have been enqueued because a -tag was created or deleted. The following additional variables are -available: - -.. var:: zuul - :hidden: - - .. var:: tag - - The name of the item's tag (without the `refs/tags/` prefix). - - .. var:: oldrev - - If the item was enqueued as the result of a tag being deleted, - the previous git sha of the tag will be included here. If the - tag was created, this variable will be undefined. - - .. var:: newrev - - If the item was enqueued as the result of a tag being created, - the new git sha of the tag will be included here. If the tag - was deleted, this variable will be undefined. - - .. var:: commit_id - - The git sha of the branch. Identical to ``newrev`` or - ``oldrev`` if defined. - -Ref Items -~~~~~~~~~ - -This represents a git reference that is neither a change, branch, or -tag. Note that all items include a `ref` attribute which may be used -to identify the ref. The following additional variables are -available: - -.. var:: zuul - :hidden: - - .. var:: oldrev - - If the item was enqueued as the result of a ref being deleted, - the previous git sha of the ref will be included here. If the - ref was created, this variable will be undefined. - - .. var:: newrev - - If the item was enqueued as the result of a ref being created, - the new git sha of the ref will be included here. If the ref - was deleted, this variable will be undefined. - - .. var:: commit_id - - The git sha of the branch. Identical to ``newrev`` or - ``oldrev`` if defined. - -Working Directory -~~~~~~~~~~~~~~~~~ - -Additionally, some information about the working directory and the -executor running the job is available: - -.. var:: zuul - :hidden: - - .. var:: executor - - A number of values related to the executor running the job are - available: - - .. var:: hostname - - The hostname of the executor. - - .. var:: src_root - - The path to the source directory. - - .. var:: log_root - - The path to the logs directory. - - .. var:: work_root - - The path to the working directory. - - .. var:: inventory_file - - The path to the inventory. This variable is needed for jobs running - without a nodeset since Ansible doesn't set it for localhost; see - this `porting guide - `_. - - The inventory file is only readable by jobs running in a - :term:`trusted execution context`. - SSH Keys -------- diff --git a/releasenotes/notes/zuul-refs-variable-d569decc6e8ad4b8.yaml b/releasenotes/notes/zuul-refs-variable-d569decc6e8ad4b8.yaml new file mode 100644 index 0000000000..b6c332f90a --- /dev/null +++ b/releasenotes/notes/zuul-refs-variable-d569decc6e8ad4b8.yaml @@ -0,0 +1,12 @@ +--- +features: + - | + Information about all of the changes (refs) associated with a + circular dependency queue item is now available in the + :var:`zuul.buildset_refs` variable. +fixes: + - | + The value of :var:`zuul.items.project.src_dir` did not take into + account the job's workspace scheme and was always constructed + using the golang scheme. That has been corrected so that it now + reflects the job's scheme. diff --git a/tests/unit/test_merger_repo.py b/tests/unit/test_merger_repo.py index 82ca5fb712..ff93ae6989 100644 --- a/tests/unit/test_merger_repo.py +++ b/tests/unit/test_merger_repo.py @@ -18,6 +18,7 @@ import logging import os import shutil from unittest import mock +from zuul.lib import yamlutil as yaml import git import testtools @@ -1339,6 +1340,18 @@ class TestOverlappingRepos(ZuulTestCase): 'subcomponent', '.git') self.assertTrue(os.path.exists(jobdir_git_dir)) + inv_path = os.path.join(build.jobdir.root, 'ansible', 'inventory.yaml') + with open(inv_path, 'r') as f: + inventory = yaml.safe_load(f) + zuul = inventory['all']['vars']['zuul'] + self.assertEqual('src/component', + zuul['items'][0]['project']['src_dir']) + self.assertEqual('src/component', + zuul['projects']['review.example.com/component'] + ['src_dir']) + self.assertEqual('src/component', + zuul['buildset_refs'][0]['src_dir']) + class TestMergerUpgrade(ZuulTestCase): tenant_config_file = 'config/single-tenant/main.yaml' diff --git a/tests/unit/test_web.py b/tests/unit/test_web.py index d610302edd..c801531dbf 100644 --- a/tests/unit/test_web.py +++ b/tests/unit/test_web.py @@ -1237,6 +1237,16 @@ class TestWeb(BaseTestWeb): 'buildset': None, 'branch': 'master', 'ref': None, + 'buildset_refs': [ + {'branch': 'master', + 'change_url': None, + 'project': { + 'canonical_hostname': 'review.example.com', + 'canonical_name': + 'review.example.com/org/project1', + 'name': 'org/project1', + 'short_name': 'project1'}, + 'src_dir': 'src/review.example.com/org/project1'}], 'pipeline': 'check', 'post_review': False, 'job': 'project-test1', @@ -1318,6 +1328,17 @@ class TestWeb(BaseTestWeb): 'src_dir': 'src/review.example.com/org/noop-project'}, 'projects': {}, 'ref': None, + 'buildset_refs': [ + {'branch': 'master', + 'change_url': None, + 'project': { + 'canonical_hostname': 'review.example.com', + 'canonical_name': + 'review.example.com/org/noop-project', + 'name': 'org/noop-project', + 'short_name': 'noop-project'}, + 'src_dir': + 'src/review.example.com/org/noop-project'}], 'tenant': 'tenant-one', 'timeout': None, 'voting': True}} diff --git a/zuul/executor/common.py b/zuul/executor/common.py index e79fedc8e7..decbcf9dbb 100644 --- a/zuul/executor/common.py +++ b/zuul/executor/common.py @@ -12,11 +12,20 @@ # License for the specific language governing permissions and limitations # under the License. +import copy import os from zuul.lib import strings +def make_src_dir(canonical_hostname, name, scheme): + return os.path.join('src', + strings.workspace_project_path( + canonical_hostname, + name, + scheme)) + + def construct_build_params(uuid, connections, job, item, pipeline, dependent_changes=[], merger_items=[], redact_secrets_and_keys=True): @@ -35,17 +44,19 @@ def construct_build_params(uuid, connections, job, item, pipeline, short_name=change.project.name.split('/')[-1], canonical_hostname=change.project.canonical_hostname, canonical_name=change.project.canonical_name, - src_dir=os.path.join('src', - strings.workspace_project_path( - change.project.canonical_hostname, - change.project.name, - job.workspace_scheme)), + src_dir=make_src_dir(change.project.canonical_hostname, + change.project.name, + job.workspace_scheme), ) - zuul_params = dict( + # We override some values like project, so set the change fields + # first. + zuul_params = change.toDict() + zuul_params.update(dict( build=uuid, buildset=item.current_build_set.uuid, ref=change.ref, + buildset_refs=[c.toDict() for c in item.changes], pipeline=pipeline.name, post_review=pipeline.post_review, job=job.name, @@ -53,37 +64,25 @@ def construct_build_params(uuid, connections, job, item, pipeline, tenant=tenant.name, event_id=item.event.zuul_event_id if item.event else None, jobtags=sorted(job.tags), - ) - if hasattr(change, 'branch'): - zuul_params['branch'] = change.branch - if hasattr(change, 'tag'): - zuul_params['tag'] = change.tag - if hasattr(change, 'number'): - zuul_params['change'] = str(change.number) - if hasattr(change, 'url'): - zuul_params['change_url'] = change.url - if hasattr(change, 'patchset'): - zuul_params['patchset'] = str(change.patchset) + )) if hasattr(change, 'message'): zuul_params['message'] = strings.b64encode(change.message) - zuul_params['change_message'] = change.message - if hasattr(change, 'topic'): - zuul_params['topic'] = change.topic - commit_id = None - if (hasattr(change, 'oldrev') and change.oldrev - and change.oldrev != '0' * 40): - zuul_params['oldrev'] = change.oldrev - commit_id = change.oldrev - if (hasattr(change, 'newrev') and change.newrev - and change.newrev != '0' * 40): - zuul_params['newrev'] = change.newrev - commit_id = change.newrev - if hasattr(change, 'commit_id'): - commit_id = change.commit_id - if commit_id: - zuul_params['commit_id'] = commit_id - zuul_params['projects'] = {} # Set below + + # Fixup the src_dir for the items based on this job + dependent_changes = copy.deepcopy(dependent_changes) + for dep_change in dependent_changes: + dep_change['project']['src_dir'] = make_src_dir( + dep_change['project']['canonical_hostname'], + dep_change['project']['name'], + job.workspace_scheme) + # Fixup the src_dir for the refs based on this job + for r in zuul_params['buildset_refs']: + r['src_dir'] = make_src_dir( + r['project']['canonical_hostname'], + r['project']['name'], + job.workspace_scheme) + zuul_params['items'] = dependent_changes zuul_params['child_jobs'] = [ x.name for x in item.current_build_set.job_graph. @@ -176,11 +175,10 @@ def construct_build_params(uuid, connections, job, item, pipeline, # project.values() is easier for callers canonical_name=p.canonical_name, canonical_hostname=p.canonical_hostname, - src_dir=os.path.join('src', - strings.workspace_project_path( - p.canonical_hostname, - p.name, - job.workspace_scheme)), + src_dir=make_src_dir( + p.canonical_hostname, + p.name, + job.workspace_scheme), required=(p in required_projects), )) diff --git a/zuul/executor/server.py b/zuul/executor/server.py index b5ac7214cd..3eb70e6469 100644 --- a/zuul/executor/server.py +++ b/zuul/executor/server.py @@ -2619,6 +2619,14 @@ class AnsibleJob(object): if 'change_message' in zuul_vars: zuul_vars['change_message'] = yaml.mark_strings_unsafe( zuul_vars['change_message']) + for item in zuul_vars['items']: + if 'change_message' in item: + item['change_message'] = yaml.mark_strings_unsafe( + item['change_message']) + for ref in zuul_vars.get('buildset_refs', []): + if 'change_message' in ref: + ref['change_message'] = yaml.mark_strings_unsafe( + ref['change_message']) with open(self.jobdir.zuul_vars, 'w') as zuul_vars_yaml: zuul_vars_yaml.write( diff --git a/zuul/model.py b/zuul/model.py index 063e7ccf25..b6ab9f913b 100644 --- a/zuul/model.py +++ b/zuul/model.py @@ -20,7 +20,6 @@ import json import hashlib import logging import math -import os from functools import partial, total_ordering import threading @@ -6177,8 +6176,18 @@ class Ref(object): short_name=self.project.name.split('/')[-1], canonical_hostname=self.project.canonical_hostname, canonical_name=self.project.canonical_name, - src_dir=os.path.join('src', self.project.canonical_name), ) + d['change_url'] = self.url + + commit_id = None + if self.oldrev and self.oldrev != '0' * 40: + d['oldrev'] = self.oldrev + commit_id = self.oldrev + if self.newrev and self.newrev != '0' * 40: + d['newrev'] = self.newrev + commit_id = self.newrev + if commit_id: + d['commit_id'] = commit_id return d @@ -6211,6 +6220,12 @@ class Tag(Ref): self.tag = None self.containing_branches = [] + def toDict(self): + # Render to a dict to use in passing json to the executor + d = super(Tag, self).toDict() + d["tag"] = self.tag + return d + def serialize(self): d = super().serialize() d["containing_branches"] = self.containing_branches @@ -6389,9 +6404,13 @@ class Change(Branch): def toDict(self): # Render to a dict to use in passing json to the executor d = super(Change, self).toDict() + d.pop('oldrev', None) + d.pop('newrev', None) d['change'] = str(self.number) - d['change_url'] = self.url d['patchset'] = str(self.patchset) + d['commit_id'] = self.commit_id + d['change_message'] = self.message + d['topic'] = self.topic return d