Make git clone process idempotent

This patch makes the git clone process idempotent.

Using asynchronous Ansible tasks to achieve the goal
also resolves a race condition seen when using the
selective git clone process with a small number of
git sources.

Ansible handles the asynchronous tasks better than
the bash job control 'wait' command.

The test to validate that a missing 'origin' remote
does not break the git clone process was removed due
to https://github.com/ansible/ansible/issues/19290

Closes-Bug: #1649329
Change-Id: I3e68f155190461470adb7e46d904705d991973af

tasks/repo_clone_git.yml

@@ -13,10 +13,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-- name: Create clone process script
-  template:
-    src: "op-clone-script.sh.j2"
-    dest: "/opt/op-clone-script.sh"
+# TODO (odyssey4me): Remove in Pike
+- name: Remove clone process script (from Newton and before)
+  file:
+    path: "/opt/op-clone-script.sh"
+    state: absent
   tags:
     - repo-clone-repos
 
@@ -37,10 +38,33 @@
   tags:
     - repo-clone-repos
 
-- name: Run clone process script
-  command: "bash /opt/op-clone-script.sh"
-  changed_when: false
+- name: Clone git repositories asynchronously
   become: yes
   become_user: "{{ repo_build_service_user_name }}"
+  git:
+    repo: "{{ item['url'] }}"
+    dest: "{{ repo_build_git_dir }}/{{ item['name'] }}"
+    version: "{{ item['version'] }}"
+    force: yes
+  with_items: "{{ local_packages.results.0.item.remote_package_parts }}"
+  when: >
+    (not repo_build_git_selective | bool) or
+    ((groups[item['project_group']] is defined) and (groups[item['project_group']] | length > 0))
+  register: _git_clone
+  async: 1800
+  poll: 0
   tags:
     - repo-clone-repos
+
+- name: Wait for git clones to complete
+  become: yes
+  become_user: "{{ repo_build_service_user_name }}"
+  async_status:
+    jid: "{{ item['ansible_job_id'] }}"
+  register: _git_jobs
+  until: "{{ _git_jobs['finished'] | bool }}"
+  delay: 5
+  retries: 360
+  with_items: "{{ _git_clone['results'] }}"
+  when:
+    - item['ansible_job_id'] is defined

templates/op-clone-script.sh.j2

@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-set -vea
-
-# Clone a git repository / Update git repository.
-function git_clone {
-    repo="$1"
-    dest="$2"
-    version="$3"
-
-    # If the git repository directory is locally found the function will ensure
-    # the origin remote matches the expected URL. If the remote URL does not
-    # match then it will be set to the expected value.
-    # Once the remotes are satisfied all refs will be fetched.
-    if [ -d "$dest/.git" ];then
-        pushd $dest
-            if ! git remote -v | grep -q "^origin"; then
-                git remote add origin "${repo}"
-            elif ! git remote -v | grep -q "^origin.* ${repo}"; then
-                git remote set-url origin "${repo}"
-            fi
-            if [ "${version}" == "master" ]; then
-              # For the version 'master', always do a fetch
-              git fetch --all
-            elif ! git checkout ${version}; then
-              # Only do a fetch if the repo doesn't have the version we need
-              git fetch --all
-            fi
-            git clean -f -d
-            git gc --auto
-        popd
-
-    # If the local target directory is not a valid git repository it will be removed and cloned.
-    elif [ -d "$dest" ];then
-        rm -rf "$dest"
-        git clone "$repo" "$dest"
-    else  # If the local target directory does not exist it will be cloned.
-        git clone "$repo" "$dest"
-    fi
-
-    # The version passed to the function will be checked out.
-    pushd "$dest"
-      git checkout $version
-    popd
-}
-
-if [ -f /etc/environment ]; then
-    source /etc/environment
-fi
-
-PID=()
-# Run the git clone. This will loop over all of the package parts and clone all known repositories.
-{% for item in local_packages.results.0.item.remote_package_parts %}
-{%   set _host_group = item['project_group'] | default('all') %}
-{%   if ((groups[_host_group] is defined) and (groups[_host_group] | length > 0)) %}
-{%     set _clone_repo = True %}
-{%   else %}
-{%     set _clone_repo = False %}
-{%   endif %}
-{%   if (not repo_build_git_selective | bool) or (_clone_repo | bool) %}
-#  Clone ops are done in parallel at a count of the known "ansible_processor_count" or using a default of 5.
-git_clone "{{ item['url'] }}" "{{ repo_build_git_dir }}/{{ item['name'] }}" "{{ item['version'] }}" &
-pid[{{ loop.index }}]=$!
-{%     if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %}
-for job_pid in ${!pid[@]}; do
-    wait ${pid[$job_pid]} || exit 99
-done
-{%     endif %}
-{%   endif %}
-{% endfor %}

tests/test-install-server.yml

@@ -60,10 +60,5 @@
       args:
         chdir: "/var/www/repo/openstackgit/keystone"
 
-    - name: Intentionally remote the tempest repo remote origin
-      shell: "git remote remove origin"
-      args:
-        chdir: "/var/www/repo/openstackgit/tempest"
-
   vars_files:
     - test-vars.yml