From ce2bea51d46a653e739291d7ee674ed688ed33eb Mon Sep 17 00:00:00 2001 From: Lukas Kranz Date: Thu, 13 Jul 2023 06:49:06 +0200 Subject: [PATCH] Deprecate mirror-workspace-git-repos This change is preparation for https://review.opendev.org/c/zuul/zuul-jobs/+/887917 In the beginning, there was only prepare-workspace[0] which rsynced repos. Then we added mirror-workspace-git[1] to make it more efficient by using git operation, but it required some openstack-specific code in project-config to work. Then we added prepare-workspace-git[2] which completed the git-based sync solution by locating everything requried in zuul-jobs. It used mirror-workspace-git by reference and added this TODO: # TODO(tobiash): we might want to deprecate the role mirror-workspace-git-repos # and move it here. This change completes that TODO by moving the mirror-workspace-git-repos code into prepare-workspace-git and places the repo in a sensible and maintainable state with two simple and good options: * prepare-workspace (rsync) * prepare-workspace-git (git) In the unlikely event anyone is still using mirror-workspace-git-repos standalone (OpenStack/OpenDev is not, and that solution was haphazard as described above) they would be well served by a notification that there is a better alternative which is what most of the community actually uses now. [0] cfffd4431b8efc2f4df1999ecb89384a29c59238 [1] 348598e96aac742954fa326a15a4ef8fd7f71b8b [2] 7cee7156bcde8bc396ac4b6581bf2cae02eea0e9 Change-Id: Ib80e0447d49363182fd0d4c4d4e269841bc3aa95 --- doc/source/deprecated-roles.rst | 1 + doc/source/general-roles.rst | 1 - roles/mirror-workspace-git-repos/README.rst | 3 + roles/prepare-workspace-git/README.rst | 6 ++ .../prepare-workspace-git/defaults/main.yaml | 1 + roles/prepare-workspace-git/tasks/main.yaml | 67 +++++++++++++++++-- roles/test-prepare-workspace-git/README.rst | 6 ++ .../defaults/main.yaml | 1 + .../tasks/main.yaml | 67 +++++++++++++++++-- 9 files changed, 142 insertions(+), 11 deletions(-) diff --git a/doc/source/deprecated-roles.rst b/doc/source/deprecated-roles.rst index 593ff252e..fa629ce69 100644 --- a/doc/source/deprecated-roles.rst +++ b/doc/source/deprecated-roles.rst @@ -2,6 +2,7 @@ Deprecated and Test Roles ========================= .. zuul:autorole:: fetch-zuul-cloner +.. zuul:autorole:: mirror-workspace-git-repos .. zuul:autorole:: test-mirror-workspace-git-repos .. zuul:autorole:: test-upload-logs-swift .. zuul:autorole:: test-prepare-workspace-git diff --git a/doc/source/general-roles.rst b/doc/source/general-roles.rst index cbc9d9255..cb278a343 100644 --- a/doc/source/general-roles.rst +++ b/doc/source/general-roles.rst @@ -28,7 +28,6 @@ General Purpose Roles .. zuul:autorole:: intercept-job .. zuul:autorole:: log-inventory .. zuul:autorole:: markdownlint -.. zuul:autorole:: mirror-workspace-git-repos .. zuul:autorole:: multi-node-bridge .. zuul:autorole:: multi-node-firewall .. zuul:autorole:: multi-node-hosts-file diff --git a/roles/mirror-workspace-git-repos/README.rst b/roles/mirror-workspace-git-repos/README.rst index 118856bc0..452fdb6f8 100644 --- a/roles/mirror-workspace-git-repos/README.rst +++ b/roles/mirror-workspace-git-repos/README.rst @@ -1,5 +1,8 @@ Mirror the local git repos to remote nodes +.. warning:: This role is deprecated. Use + :zuul:role:`prepare-workspace-git` instead. + This role uses git operations (unlike :zuul:role:`prepare-workspace` which uses rsync) to mirror the local prepared git repos to the remote nodes. This may be useful if the remote node already has a copy of diff --git a/roles/prepare-workspace-git/README.rst b/roles/prepare-workspace-git/README.rst index 4e23c5761..1cc5ce270 100644 --- a/roles/prepare-workspace-git/README.rst +++ b/roles/prepare-workspace-git/README.rst @@ -16,6 +16,12 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: mirror_workspace_quiet + :default: false + + If `true` git operations will be silenced and won't print every + changed reference. + .. zuul:rolevar:: zuul_workspace_root :default: "{{ ansible_user_dir }}" diff --git a/roles/prepare-workspace-git/defaults/main.yaml b/roles/prepare-workspace-git/defaults/main.yaml index 032db3c2a..f6e5da9bb 100644 --- a/roles/prepare-workspace-git/defaults/main.yaml +++ b/roles/prepare-workspace-git/defaults/main.yaml @@ -1,2 +1,3 @@ cached_repos_root: /opt/git +mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" diff --git a/roles/prepare-workspace-git/tasks/main.yaml b/roles/prepare-workspace-git/tasks/main.yaml index d1ff3717d..d57ef11f1 100644 --- a/roles/prepare-workspace-git/tasks/main.yaml +++ b/roles/prepare-workspace-git/tasks/main.yaml @@ -25,8 +25,65 @@ tags: - skip_ansible_lint -# TODO(tobiash): we might want to deprecate the role mirror-workspace-git-repos -# and move it here. -- name: Synchronize repos - import_role: - name: mirror-workspace-git-repos +- name: Allow pushing to non-bare repo + git_config: + name: receive.denyCurrentBranch + value: ignore + scope: local + repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + +- name: Synchronize src repos to workspace directory + command: |- + {% if ansible_connection == "kubectl" %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl --context {{ zuul.resources[inventory_hostname].context }} -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + {% else %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }} + {% endif %} + args: + chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" + environment: + GIT_ALLOW_PROTOCOL: ext:ssh + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + delegate_to: localhost + # We occasionally see git pushes in the middle of this loop fail then + # subsequent pushes for other repos succeed. The entire loop ends up + # failing because one of the pushes failed. Mitigate this by retrying + # on failure. + register: git_push + until: git_push is success + retries: 3 + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but push is not supported by ansible git module. + tags: + - skip_ansible_lint + +# Do this as a multi-line shell so that we can do the loop once +- name: Update remote repository state correctly + shell: | + set -eu + + # Reset is needed because we pushed to a non-bare repo + git reset --hard + # Clean is needed because we pushed to a non-bare repo + git clean -xdf + # Undo the config setting we did above + git config --local --unset receive.denyCurrentBranch + # checkout the branch matching the branch set up by the executor + git checkout {% if mirror_workspace_quiet %}--quiet{% endif %} {{ zj_project.value.checkout }} + # put out a status line with the current HEAD + echo "{{ zj_project.value.canonical_name }} checked out to:" + git log --pretty=oneline -1 + args: + chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but we prefer the shell above + tags: + - skip_ansible_lint diff --git a/roles/test-prepare-workspace-git/README.rst b/roles/test-prepare-workspace-git/README.rst index 4e23c5761..1cc5ce270 100644 --- a/roles/test-prepare-workspace-git/README.rst +++ b/roles/test-prepare-workspace-git/README.rst @@ -16,6 +16,12 @@ The cached repos need to be placed using the canonical name under the The root of the cached repos. +.. zuul:rolevar:: mirror_workspace_quiet + :default: false + + If `true` git operations will be silenced and won't print every + changed reference. + .. zuul:rolevar:: zuul_workspace_root :default: "{{ ansible_user_dir }}" diff --git a/roles/test-prepare-workspace-git/defaults/main.yaml b/roles/test-prepare-workspace-git/defaults/main.yaml index 032db3c2a..f6e5da9bb 100644 --- a/roles/test-prepare-workspace-git/defaults/main.yaml +++ b/roles/test-prepare-workspace-git/defaults/main.yaml @@ -1,2 +1,3 @@ cached_repos_root: /opt/git +mirror_workspace_quiet: false zuul_workspace_root: "{{ ansible_user_dir }}" diff --git a/roles/test-prepare-workspace-git/tasks/main.yaml b/roles/test-prepare-workspace-git/tasks/main.yaml index 839dbdb04..d57ef11f1 100644 --- a/roles/test-prepare-workspace-git/tasks/main.yaml +++ b/roles/test-prepare-workspace-git/tasks/main.yaml @@ -25,8 +25,65 @@ tags: - skip_ansible_lint -# TODO(tobiash): we might want to deprecate the role mirror-workspace-git-repos -# and move it here. -- name: Synchronize repos - import_role: - name: test-mirror-workspace-git-repos +- name: Allow pushing to non-bare repo + git_config: + name: receive.denyCurrentBranch + value: ignore + scope: local + repo: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + +- name: Synchronize src repos to workspace directory + command: |- + {% if ansible_connection == "kubectl" %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror "ext::kubectl --context {{ zuul.resources[inventory_hostname].context }} -n {{ zuul.resources[inventory_hostname].namespace }} exec -i {{ zuul.resources[inventory_hostname].pod }} -- %S {{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + {% else %} + git push {% if mirror_workspace_quiet %}--quiet{% endif %} --mirror git+ssh://{{ ansible_user }}@{{ ansible_host | ipwrap }}:{{ ansible_port }}/{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }} + {% endif %} + args: + chdir: "{{ zuul.executor.work_root }}/{{ zj_project.value.src_dir }}" + environment: + GIT_ALLOW_PROTOCOL: ext:ssh + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + delegate_to: localhost + # We occasionally see git pushes in the middle of this loop fail then + # subsequent pushes for other repos succeed. The entire loop ends up + # failing because one of the pushes failed. Mitigate this by retrying + # on failure. + register: git_push + until: git_push is success + retries: 3 + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but push is not supported by ansible git module. + tags: + - skip_ansible_lint + +# Do this as a multi-line shell so that we can do the loop once +- name: Update remote repository state correctly + shell: | + set -eu + + # Reset is needed because we pushed to a non-bare repo + git reset --hard + # Clean is needed because we pushed to a non-bare repo + git clean -xdf + # Undo the config setting we did above + git config --local --unset receive.denyCurrentBranch + # checkout the branch matching the branch set up by the executor + git checkout {% if mirror_workspace_quiet %}--quiet{% endif %} {{ zj_project.value.checkout }} + # put out a status line with the current HEAD + echo "{{ zj_project.value.canonical_name }} checked out to:" + git log --pretty=oneline -1 + args: + chdir: "{{ zuul_workspace_root }}/{{ zj_project.value.src_dir }}" + with_dict: "{{ zuul.projects }}" + loop_control: + loop_var: zj_project + # ANSIBLE0006: Skip linting since it triggers on the "git" command, + # but we prefer the shell above + tags: + - skip_ansible_lint