prepare-workspace-git: Add ability to define synced pojects

The prepare_workspace_sync_required_projects_only variable allows
users to define which projects to sync to the node. This can prevent
syncing of unnecessary repositories. For some builds e.g. the
depends-on repositories dont need to be synced. The projects are
filtered based on the 'required' flag present in each zuul.project
entry and the required projects list also does not contain projects
which are present due to Depends-On or gate queue sequencing.
Having unnecessary repos in the workspace can for example also break
the analysis phase of bazel.

Change-Id: I3cc36cbfc60c81956caf5137da63973aeade4e21
Co-Authored-By: James E. Blair <jim@acmegating.com>
Co-Authored-By: Bernhard Berg <bernhard.berg@bearingpoint.com>
This commit is contained in:
Lukas Kranz 2023-07-07 10:00:12 +02:00 committed by James E. Blair
parent 7761396303
commit 5eca8feda9
11 changed files with 153 additions and 10 deletions

View File

@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the
The root of the cached repos. The root of the cached repos.
.. zuul:rolevar:: prepare_workspace_sync_required_projects_only
:type: bool
:default: False
A flag which if set to true, filters the to be synchronized project
list to only use projects which are required by the job.
.. zuul:rolevar:: mirror_workspace_quiet .. zuul:rolevar:: mirror_workspace_quiet
:default: false :default: false

View File

@ -1,3 +1,4 @@
cached_repos_root: /opt/git cached_repos_root: /opt/git
mirror_workspace_quiet: false mirror_workspace_quiet: false
zuul_workspace_root: "{{ ansible_user_dir }}" zuul_workspace_root: "{{ ansible_user_dir }}"
prepare_workspace_sync_required_projects_only: false

View File

@ -1,3 +1,20 @@
- name: Filter zuul projects if sync-only-required-projects flag is set
set_fact:
_zuul_projects: >
{{ _zuul_projects | default({}) |
combine({ zj_project.key : zj_project.value }) }}
with_dict: "{{ zuul.projects }}"
loop_control:
loop_var: zj_project
when:
- prepare_workspace_sync_required_projects_only
- zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required
- name: Don't filter zuul projects if flag is false
set_fact:
_zuul_projects: "{{ zuul.projects }}"
when: not prepare_workspace_sync_required_projects_only
# Do all the steps in a single shell script. This reduces the number of times # Do all the steps in a single shell script. This reduces the number of times
# ansible must loop over the list of projects which reduces the amount of # ansible must loop over the list of projects which reduces the amount of
# task startup time we incur. # task startup time we incur.
@ -17,7 +34,7 @@
git remote add origin file:///dev/null git remote add origin file:///dev/null
args: args:
creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}" creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}"
with_items: "{{ zuul.projects.values() | list }}" with_items: "{{ _zuul_projects.values() }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
# We're using git in a shell script because it is faster and the module # We're using git in a shell script because it is faster and the module
@ -31,7 +48,7 @@
value: ignore value: ignore
scope: local scope: local
repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
with_dict: "{{ zuul.projects }}" with_dict: "{{ _zuul_projects }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
@ -46,7 +63,7 @@
chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
environment: environment:
GIT_ALLOW_PROTOCOL: ext:ssh GIT_ALLOW_PROTOCOL: ext:ssh
with_dict: "{{ zuul.projects }}" with_dict: "{{ _zuul_projects }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
delegate_to: localhost delegate_to: localhost
@ -80,7 +97,7 @@
git log --pretty=oneline -1 git log --pretty=oneline -1
args: args:
chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
with_dict: "{{ zuul.projects }}" with_dict: "{{ _zuul_projects }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
# ANSIBLE0006: Skip linting since it triggers on the "git" command, # ANSIBLE0006: Skip linting since it triggers on the "git" command,

View File

@ -16,6 +16,13 @@ The cached repos need to be placed using the canonical name under the
The root of the cached repos. The root of the cached repos.
.. zuul:rolevar:: prepare_workspace_sync_required_projects_only
:type: bool
:default: False
A flag which if set to true, filters the to be synchronized project
list to only use projects which are required by the job.
.. zuul:rolevar:: mirror_workspace_quiet .. zuul:rolevar:: mirror_workspace_quiet
:default: false :default: false

View File

@ -1,3 +1,4 @@
cached_repos_root: /opt/git cached_repos_root: /opt/git
mirror_workspace_quiet: false mirror_workspace_quiet: false
zuul_workspace_root: "{{ ansible_user_dir }}" zuul_workspace_root: "{{ ansible_user_dir }}"
prepare_workspace_sync_required_projects_only: false

View File

@ -1,3 +1,20 @@
- name: Filter zuul projects if sync-only-required-projects flag is set
set_fact:
_zuul_projects: >
{{ _zuul_projects | default({}) |
combine({ zj_project.key : zj_project.value }) }}
with_dict: "{{ zuul.projects }}"
loop_control:
loop_var: zj_project
when:
- prepare_workspace_sync_required_projects_only
- zj_project.value.canonical_name == zuul.project.canonical_name or zj_project.value.required
- name: Don't filter zuul projects if flag is false
set_fact:
_zuul_projects: "{{ zuul.projects }}"
when: not prepare_workspace_sync_required_projects_only
# Do all the steps in a single shell script. This reduces the number of times # Do all the steps in a single shell script. This reduces the number of times
# ansible must loop over the list of projects which reduces the amount of # ansible must loop over the list of projects which reduces the amount of
# task startup time we incur. # task startup time we incur.
@ -17,7 +34,7 @@
git remote add origin file:///dev/null git remote add origin file:///dev/null
args: args:
creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}" creates: "{{ zuul_workspace_root }}/{{ zj_project.src_dir }}"
with_items: "{{ zuul.projects.values() | list }}" with_items: "{{ _zuul_projects.values() }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
# We're using git in a shell script because it is faster and the module # We're using git in a shell script because it is faster and the module
@ -31,7 +48,7 @@
value: ignore value: ignore
scope: local scope: local
repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
with_dict: "{{ zuul.projects }}" with_dict: "{{ _zuul_projects }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
@ -46,7 +63,7 @@
chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}"
environment: environment:
GIT_ALLOW_PROTOCOL: ext:ssh GIT_ALLOW_PROTOCOL: ext:ssh
with_dict: "{{ zuul.projects }}" with_dict: "{{ _zuul_projects }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
delegate_to: localhost delegate_to: localhost
@ -80,7 +97,7 @@
git log --pretty=oneline -1 git log --pretty=oneline -1
args: args:
chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}"
with_dict: "{{ zuul.projects }}" with_dict: "{{ _zuul_projects }}"
loop_control: loop_control:
loop_var: zj_project loop_var: zj_project
# ANSIBLE0006: Skip linting since it triggers on the "git" command, # ANSIBLE0006: Skip linting since it triggers on the "git" command,

View File

@ -6,6 +6,7 @@
# Note: set-zuul-log-path-fact is tested by emit-job-header.yaml # Note: set-zuul-log-path-fact is tested by emit-job-header.yaml
- import_playbook: emit-job-header.yaml - import_playbook: emit-job-header.yaml
- import_playbook: ensure-output-dirs.yaml - import_playbook: ensure-output-dirs.yaml
- import_playbook: prepare-workspace-git-required-projects-only.yaml
- import_playbook: prepare-workspace-git.yaml - import_playbook: prepare-workspace-git.yaml
- import_playbook: configure-mirrors.yaml - import_playbook: configure-mirrors.yaml
- import_playbook: fetch-zuul-cloner.yaml - import_playbook: fetch-zuul-cloner.yaml

View File

@ -0,0 +1,6 @@
- name: Test the prepare-workspace-git role
hosts: all
roles:
- role: prepare-workspace-git
vars:
prepare_workspace_sync_required_projects_only: true

View File

@ -0,0 +1,72 @@
- name: Prepare to test the prepare-workspace-git role with sync required only
hosts: all
tasks:
- name: Delete remote source directory to start with clean state
file:
state: absent
path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
with_dict: "{{ zuul.projects }}"
# We need to override the zuul.projects variable, and that is not
# possible in a Zuul job. So we use a nested Ansible to perform this
# test.
- name: Test the prepare-workspace-git role with sync required only
hosts: localhost
vars:
# Mutate the zuul vars supplied to this test job to simulate a
# repo being included as non-required (i.e., a depends-on).
zuul_mod:
projects:
opendev.org/zuul/project-config:
required: false
tasks:
- name: Create nested zuul vars
set_fact:
nested_zuul:
zuul: "{{ zuul | combine(zuul_mod, recursive=true) }}"
- name: Write nested zuul vars
copy:
content: '{{ nested_zuul | to_nice_yaml(indent=2) }}'
dest: "{{ zuul.executor.work_root }}/nested-zuul-vars.yaml"
- name: Run nested Ansible
command: >-
{{ ansible_playbook_python | dirname}}/ansible-playbook
-vvv
-e @{{ zuul.executor.work_root }}/nested-zuul-vars.yaml
-e zuul_execution_phase=nested
-e zuul_execution_phase_index=0
-e zuul_execution_canonical_name_and_path=opendev.org/zuul/zuul-jobs/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml
-e zuul_execution_trusted=False
-e zuul_execution_branch={{zuul_execution_branch}}
{{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/test-playbooks/base-roles/prepare-workspace-git-required-projects-only-inner.yaml
environment:
ANSIBLE_ROLES_PATH: "{{ zuul.executor.work_root }}/{{ zuul.projects['opendev.org/zuul/zuul-jobs'].src_dir }}/roles"
- name: Verify the prepare-workspace-git role with sync required only
hosts: all
tasks:
# opendev/base-jobs is in 'required-projects'.
# Also check that the project being tested is being prepared.
# We're checking them explicitly rather than with_items on zuul.projects
# in case there is a regression which would take an item out.
- name: Check that opendev/base-jobs was prepared
stat:
path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
register: base_jobs
- name: Check that zuul/project-config was not prepared
stat:
path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
register: project_config
- name: Check this project was prepared
stat:
path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
register: self_config
- name: Validate that required projects have been prepared
assert:
that:
- base_jobs.stat.exists
- not project_config.stat.exists
- self_config.stat.exists

View File

@ -1,5 +1,12 @@
- name: Test the prepare-workspace-git role - name: Test the prepare-workspace-git role
hosts: all hosts: all
pre_tasks:
- name: Delete remote source directory to start with clean state
file:
state: absent
path: "{{ ansible_user_dir }}/{{ item.value.src_dir }}"
with_dict: "{{ zuul.projects }}"
roles: roles:
- role: prepare-workspace-git - role: prepare-workspace-git
post_tasks: post_tasks:
@ -12,6 +19,11 @@
path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs" path: "{{ ansible_user_dir }}/src/opendev.org/opendev/base-jobs"
register: base_jobs register: base_jobs
- name: Check that zuul/project-config was prepared
stat:
path: "{{ ansible_user_dir }}/src/opendev.org/zuul/project-config"
register: project_config
- name: Check this project was prepared - name: Check this project was prepared
stat: stat:
path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}"
@ -21,4 +33,5 @@
assert: assert:
that: that:
- base_jobs.stat.exists - base_jobs.stat.exists
- project_config.stat.exists
- self_config.stat.exists - self_config.stat.exists

View File

@ -116,10 +116,11 @@
tags: all-platforms tags: all-platforms
abstract: true abstract: true
run: test-playbooks/base-roles/base.yaml run: test-playbooks/base-roles/base.yaml
# Testing of fetch-zuul-cloner and use-cached-repos need this repo # Testing of fetch-zuul-cloner and prepare-workspace-git need
# in required-projects # these repos in required-projects
required-projects: required-projects:
- opendev/base-jobs - opendev/base-jobs
- zuul/project-config
files: files:
- ^roles/configure-mirrors/.* - ^roles/configure-mirrors/.*
- ^roles/emit-job-header/.* - ^roles/emit-job-header/.*