openstack
/
openstack-ansible-tests
Code Issues Proposed changes

Implement zuul v2/v3 role fetch shim 

In order to be able to simultaneously work with the
differing implementations of zuul-cloner in zuul v2
and zuul v3 we implement a detection routine to
behave appropriately for each version.

This is very much a band-aid just to help us transition
because once the transition to zuul v3 is complete we
should be able to ditch this whole playbook for
something built into zuul v3, and use something very
simple for executing tests when not using zuul.

Change-Id: Ic824e7ca7c564d5de7a27c77a79014ad274afade
This commit is contained in:
Jesse Pretorius 2017-10-13 16:40:59 +01:00
parent f4345164aa
commit b58a2c0287
4 changed files with 215 additions and 85 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

249
get-ansible-role-requirements.yml
View File

@ -24,6 +24,31 @@
gather_facts: false
tasks:
- name: Remove target role directories if they are not git repositories
shell: |
EXIT_CODE=0
{% for role in osa_roles %}
{% if role['scm'] == "git" or role['scm'] is undefined %}
ROLE_REPO_PATH="{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ role['name'] | default(role['src'] | basename) }}"
if [[ -e ${ROLE_REPO_PATH} ]] && [[ ! -d "${ROLE_REPO_PATH}/.git" ]]; then
echo "${ROLE_REPO_PATH} is not a git repo, deleting..."
rm -rf "${ROLE_REPO_PATH}"
EXIT_CODE=2
fi
{% endif %}
{% endfor %}
exit ${EXIT_CODE}
args:
executable: /bin/bash
register: existing_dir_cleanup
changed_when: existing_dir_cleanup.rc == 2
failed_when: existing_dir_cleanup.rc not in [0,2]
- name: Create the ansible role directory
file:
dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}"
state: directory
- name: Check whether zuul-cloner is installed and provide the path to it
shell: |
which zuul-cloner || { [[ -x /usr/zuul-env/bin/zuul-cloner ]] && echo '/usr/zuul-env/bin/zuul-cloner'; }
@ -33,88 +58,162 @@
failed_when: false
register: _zuul_cloner_check
- name: Remove target directory if required
shell: |
if [[ ! -d "{{ item.path | default(homedir + '/.ansible/roles') }}/{{ item.name | default(item.src | basename) }}/.git" ]]; then
rm -rf "{{ item.path | default(homedir + '/.ansible/roles') }}/{{ item.name | default(item.src | basename) }}"
fi
args:
executable: /bin/bash
when:
- item.scm == "git" or item.scm is undefined
with_items: "{{ roles }}"
- name: Prepare git clone list
- name: Set fact to decide which version of zuul to cater to, if any
set_fact:
git_roles: >
{%- set filtered_role_list = [] %}
{%- for role in roles %}
{%- if (not role.src | match(".*git.openstack.org.*")) or
(role.name | match("os_previous_.*")) or
(role.name | match("previous_.*")) %}
{%- set _ = filtered_role_list.append(role) %}
zuul_version: >-
{#- This tells us that we are running under zuul v2 or v3 #}
{%- if _zuul_cloner_check.rc == 0 %}
{#- So now we check for an environment variable that's unique to zuul v2 #}
{#- ref: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/ansible/filter/zuul_filters.py?h=feature/zuulv3#n17 #}
{%- if lookup('env', 'ZUUL_REF') != "" %}
{%- set version_output="2" %}
{%- else %}
{%- set version_output="3" %}
{%- endif %}
{%- endfor %}
{{- filtered_role_list -}}
zuul_roles: >
{%- set filtered_role_list = [] %}
{%- for role in roles %}
{%- if (role.src | match(".*git.openstack.org.*")) and (not role.name | match("os_previous_.*")) %}
{%- set role_src_cleaned = role.src | regex_replace('https://git.openstack.org/', '') %}
{%- set _ = filtered_role_list.append(role_src_cleaned) %}
{%- endif %}
{%- endfor %}
{{- filtered_role_list -}}
openstack_repo_list: >
{%- set filtered_repo_list = [] %}
{%- set repo_list = lookup('env', 'ZUUL_CHANGES').split('^') %}
{%- for repo in repo_list %}
{%- set repo_cleaned = repo | regex_replace(':.*$', '') %}
{%- if not repo_cleaned | match("^openstack/openstack-ansible.*") %}
{%- set _ = filtered_repo_list.append(repo_cleaned) %}
{%- endif %}
{%- endfor %}
{{- filtered_repo_list -}}
when:
- _zuul_cloner_check.rc == 0
{%- else %}
{%- set version_output="none" %}
{%- endif %}
{{- version_output }}
- name: Clone git repos (with git)
git:
repo: "{{ item.src }}"
dest: "{{ homedir }}/.ansible/roles/{{ item.name | default(item.src | basename) }}"
version: "{{ item.version | default('master') }}"
update: true
force: true
when:
- item.scm == "git" or item.scm is undefined
with_items: "{{ git_roles | default(roles) }}"
- name: Clone git repos (no zuul)
block:
- name: Prepare git clone list
set_fact:
git_roles: "{{ osa_roles }}"
- name: Create clone map
copy:
content: |
clonemap:
- name: 'openstack/openstack-ansible-tests'
dest: '{{ lookup("env", "WORKING_DIR") }}/tests/common'
- name: 'openstack/openstack-ansible-(?!tests)(.*)'
dest: '{{ homedir }}/.ansible/roles/\1'
- name: 'openstack/(?!openstack-ansible)(.*)'
dest: '{{ homedir }}/git/openstack/\1'
dest: "{{ homedir }}/.ansible/clonemap.yml"
- name: Clone git repos
git:
repo: "{{ item['src'] }}"
dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item['name'] | default(item['src'] | basename) }}"
version: "{{ item['version'] | default('master') }}"
update: true
force: true
with_items: "{{ git_roles }}"
when:
- item['scm'] == "git" or item['scm'] is undefined
when:
- _zuul_cloner_check.rc == 0
- zuul_version == "none"
- name: Clone git repos (with zuul-cloner)
shell: |
{{ _zuul_cloner_check.stdout }} \
-m {{ homedir }}/.ansible/clonemap.yml \
--cache-dir /opt/git \
git://git.openstack.org \
{% for repo in zuul_roles + openstack_repo_list %}
{{ repo }} \
{% endfor %}
- name: Clone git repos (zuul v2)
block:
- name: Prepare git clone list
set_fact:
git_roles: >
{%- set filtered_role_list = [] %}
{%- for role in osa_roles %}
{%- if (not role.src | match(".*git.openstack.org.*")) or
(role.name | match("os_previous_.*")) or
(role.name | match("previous_.*")) %}
{%- set _ = filtered_role_list.append(role) %}
{%- endif %}
{%- endfor %}
{{- filtered_role_list -}}
zuul_roles: >
{%- set filtered_role_list = [] %}
{%- for role in osa_roles %}
{%- if (role.src | match(".*git.openstack.org.*")) and (not role.name | match("os_previous_.*")) %}
{%- set role_src_cleaned = role.src | regex_replace('https://git.openstack.org/', '') %}
{%- set _ = filtered_role_list.append(role_src_cleaned) %}
{%- endif %}
{%- endfor %}
{{- filtered_role_list -}}
openstack_repo_list: >
{%- set filtered_repo_list = [] %}
{%- set repo_list = lookup('env', 'ZUUL_CHANGES').split('^') %}
{%- for repo in repo_list %}
{%- set repo_cleaned = repo | regex_replace(':.*$', '') %}
{%- if not repo_cleaned | match("^openstack/openstack-ansible.*") %}
{%- set _ = filtered_repo_list.append(repo_cleaned) %}
{%- endif %}
{%- endfor %}
{{- filtered_repo_list -}}
- name: Clone git repos (non openstack)
git:
repo: "{{ item['src'] }}"
dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item['name'] | default(item['src'] | basename) }}"
version: "{{ item['version'] | default('master') }}"
update: true
force: true
with_items: "{{ git_roles }}"
when:
- item['scm'] == "git" or item['scm'] is undefined
- name: Create clone map
copy:
content: |
clonemap:
- name: 'openstack/openstack-ansible-tests'
dest: '{{ lookup("env", "WORKING_DIR") }}/tests/common'
- name: 'openstack/openstack-ansible-(?!tests)(.*)'
dest: '{{ lookup("env", "ANSIBLE_ROLE_DIR") }}/\1'
- name: 'openstack/(?!openstack-ansible)(.*)'
dest: '{{ homedir }}/git/openstack/\1'
dest: "{{ homedir }}/.ansible/clonemap.yml"
- name: Clone git repos (with zuul-cloner)
shell: |
{{ _zuul_cloner_check.stdout }} \
-m {{ homedir }}/.ansible/clonemap.yml \
--cache-dir /opt/git \
git://git.openstack.org \
{% for repo in zuul_roles + openstack_repo_list %}
{{ repo }} \
{% endfor %}
when:
- _zuul_cloner_check.rc == 0
- zuul_version == "2"
- name: Clone git repos (zuul v3)
block:
- name: Find repositories already cloned by zuul
command: "find {{ zuul_git_src_dir }} -type d -maxdepth 2"
register: zuul_src_folder
- name: Simplify the given src repository list
set_fact:
zuul_src_repo_list: >
{%- set filtered_repo_list = [] %}
{%- for folder_path in zuul_src_folder['stdout_lines'] %}
{%- if folder_path | match("^" ~ zuul_git_src_dir ~ "/openstack/openstack-ansible-(?!tests).*") %}
{%- set repo_cleaned = folder_path | regex_replace('^' ~ zuul_git_src_dir ~ '/', '') %}
{%- set _ = filtered_repo_list.append(repo_cleaned) %}
{%- endif %}
{%- endfor %}
{{- filtered_repo_list -}}
- name: Prepare git clone list
set_fact:
git_roles: >
{%- set filtered_role_list = [] %}
{%- for role in osa_roles %}
{%- if role['src'] | regex_replace('https://git.openstack.org/', '') not in zuul_src_repo_list %}
{%- set _ = filtered_role_list.append(role) %}
{%- endif %}
{%- endfor %}
{{- filtered_role_list -}}
- name: Link the zuul provided roles
file:
src: "{{ zuul_git_src_dir }}/{{ item }}"
dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item | regex_replace('openstack/openstack-ansible-', '') }}"
state: link
force: yes
with_items: "{{ zuul_src_repo_list }}"
- name: Clone the remaining git repos
git:
repo: "{{ item['src'] }}"
dest: "{{ lookup('env', 'ANSIBLE_ROLE_DIR') }}/{{ item['name'] | default(item['src'] | basename) }}"
version: "{{ item['version'] | default('master') }}"
update: true
force: true
with_items: "{{ git_roles }}"
when:
- item['scm'] == "git" or item['scm'] is undefined
when:
- zuul_version == "3"
vars:
role_file: "{{ toxinidir }}/tests/ansible-role-requirements.yml"
roles: "{{ lookup('file', role_file) | from_yaml }}"
osa_roles: "{{ lookup('file', role_file) | from_yaml }}"
zuul_git_src_dir: "/home/zuul/src/git.openstack.org"

22
test-ansible-env-prep.sh
View File

@ -94,7 +94,11 @@ mkdir -p "${ANSIBLE_LOG_DIR}"
# If zuul-cloner is present, use it so that we
# also include any dependent patches from the
# plugins repo noted in the commit message.
if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
# We only want to use zuul-cloner if we detect
# zuul v2 running, so we check for the presence
# of the ZUUL_REF environment variable.
# ref: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/ansible/filter/zuul_filters.py?h=feature/zuulv3#n17
if [[ -x /usr/zuul-env/bin/zuul-cloner ]] && [[ "${ZUUL_REF:-none}" != "none" ]]; then
# Prepare the clonemap for zuul-cloner to use
create_plugins_clonemap
@ -117,8 +121,11 @@ if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
else
if [[ ! -d "${ANSIBLE_PLUGIN_DIR}" ]]; then
# The plugins repo doesn't need a clone, we can just
# symlink it.
if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-plugins" ]]; then
# symlink it. As zuul v3 clones into a folder called
# 'workspace' we have to use one of its environment
# variables to determine the project name.
if [[ "${ZUUL_SHORT_PROJECT_NAME:-none}" == "openstack-ansible-plugins" ]] ||\
[[ "$(basename ${WORKING_DIR})" == "openstack-ansible-plugins" ]]; then
ln -s ${WORKING_DIR} "${ANSIBLE_PLUGIN_DIR}"
else
git clone \
@ -129,8 +136,11 @@ else
if [[ ! -d "${OSA_OPS_DIR}" ]]; then
# The ops repo doesn't need a clone, we can just
# symlink it.
if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-ops" ]]; then
# symlink it. As zuul v3 clones into a folder called
# 'workspace' we have to use one of its environment
# variables to determine the project name.
if [[ "${ZUUL_SHORT_PROJECT_NAME:-none}" == "openstack-ansible-ops" ]] ||\
[[ "$(basename ${WORKING_DIR})" == "openstack-ansible-ops" ]]; then
ln -s ${WORKING_DIR} "${OSA_OPS_DIR}"
else
git clone \
@ -146,7 +156,7 @@ if [ ! -d "${ANSIBLE_ROLE_DIR}" ] && [ -f "${ANSIBLE_ROLE_REQUIREMENTS_PATH}" ];
ansible-playbook -i ${ANSIBLE_INVENTORY} \
${COMMON_TESTS_PATH}/get-ansible-role-requirements.yml \
-e "toxinidir=${WORKING_DIR} homedir=${TESTING_HOME}" \
-v
-vvv
fi

13
tests/tests-repo-clone.sh
View File

@ -52,7 +52,11 @@ EOF
# If zuul-cloner is present, use it so that we
# also include any dependent patches from the
# tests repo noted in the commit message.
if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
# We only want to use zuul-cloner if we detect
# zuul v2 running, so we check for the presence
# of the ZUUL_REF environment variable.
# ref: http://git.openstack.org/cgit/openstack-infra/zuul/tree/zuul/ansible/filter/zuul_filters.py?h=feature/zuulv3#n17
if [[ -x /usr/zuul-env/bin/zuul-cloner ]] && [[ "${ZUUL_REF:-none}" != "none" ]]; then
# Prepare the clonemap for zuul-cloner to use
create_tests_clonemap
@ -74,8 +78,11 @@ if [[ -x /usr/zuul-env/bin/zuul-cloner ]]; then
elif [[ ! -d tests/common ]]; then
# The tests repo doesn't need a clone, we can just
# symlink it.
if [[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
# symlink it. As zuul v3 clones into a folder called
# 'workspace' we have to use one of its environment
# variables to determine the project name.
if [[ "${ZUUL_SHORT_PROJECT_NAME:-none}" == "openstack-ansible-tests" ]] ||\
[[ "$(basename ${WORKING_DIR})" == "openstack-ansible-tests" ]]; then
ln -s ${WORKING_DIR} ${WORKING_DIR}/tests/common
else
git clone \

16
tox.ini
View File

@ -1,7 +1,7 @@
[tox]
minversion = 2.0
skipsdist = True
envlist = docs,linters,functional
envlist = docs,linters,func
[testenv]
@ -102,6 +102,20 @@ commands =
bash -c "{toxinidir}/tests/common/test-ansible-lint.sh"
# This is here due to a bug in the ubuntu-xenial
# job which does not do the translation from
# 'func' to 'functional' for us.
# TODO(odyssey4me):
# Remove this once https://review.openstack.org/512351
# has been merged.
[testenv:func]
deps =
{[testenv:ansible]deps}
commands =
bash -c "{toxinidir}/tests/tests-repo-clone.sh"
bash -c "{toxinidir}/tests/common/test-ansible-functional.sh"
[testenv:functional]
deps =
{[testenv:ansible]deps}