Ensure build/install works in serial/parallel execution

When executing the role in parallel, there is a race condition
due to all targets trying to build simultaneously and then all
trying to copy their packages back to the same folder on the
deployment host.

In this patch we implement the use of a dynamic group based on
the venv_reuse_download_subfolder which, by default, combines
the architecture and distribution data. We then use this group
to ensure that only the first of each group does the build and
the others consume the first host's package.

The conditional in tasks/python_venv_install.yml checking
whether the src venv is present was unnecessary given the same
conditional is present in tasks/main.yml. It interferes with
the new parallel execution capability, so it's been removed.

To validate whether builds happen properly, whether in serial
or parallel, and to also implement a way to implement tests for
future validations, we implement a test system using multiple
containers while allowing localhost to be the deployment host
only.

Change-Id: I29d230c16bee3e2d7e434f98c05af4a6a53ff10a
This commit is contained in:
Jesse Pretorius 2018-03-28 13:09:19 +01:00
parent 5824d2f849
commit 4417b2ceef
12 changed files with 311 additions and 20 deletions

View File

@ -17,16 +17,28 @@
tags:
- always
# To ensure that only the first host in every distro/architecture
# group does the build, and the others make use of that build,
# without forcing the use of serialisation, we use a dynamic
# group based on the venv_reuse_download_subfolder which combines
# the distribution/architecture and is therefore unique for mixed
# environments.
- name: Add hosts to dynamic inventory group to ensure build/install serialization
group_by:
key: "{{ venv_reuse_download_subfolder }}"
- include_tasks: "python_venv_build.yml"
when:
- (not _src_venv_present.stat.exists | bool) or
(not venv_reuse_enable | bool)
- inventory_hostname == groups[venv_reuse_download_subfolder][0]
- include_tasks: "python_venv_install.yml"
when:
- venv_reuse_enable | bool
- not venv_reuse_build_only | bool
- _src_venv_present.stat.exists | bool
- (_src_venv_present.stat.exists | bool) or
inventory_hostname != groups[venv_reuse_download_subfolder][0]
- include_tasks: "python_venv_set_facts.yml"
when:

View File

@ -18,8 +18,6 @@
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.checksum"
dest: "{{ venv_destination_path | dirname }}/"
register: _venv_checksum_copy
when:
- _src_venv_present.stat.exists | bool
- name: Remove existing venv on target host if it is changing
file:

View File

@ -0,0 +1,20 @@
- name: apt_package_pinning
src: https://git.openstack.org/openstack/openstack-ansible-apt_package_pinning
scm: git
version: master
- name: pip_install
src: https://git.openstack.org/openstack/openstack-ansible-pip_install
scm: git
version: master
- name: openstack_hosts
src: https://git.openstack.org/openstack/openstack-ansible-openstack_hosts
scm: git
version: master
- name: lxc_hosts
src: https://git.openstack.org/openstack/openstack-ansible-lxc_hosts
scm: git
version: master
- name: lxc_container_create
src: https://git.openstack.org/openstack/openstack-ansible-lxc_container_create
scm: git
version: master

View File

@ -0,0 +1,31 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
container_name: "{{ inventory_hostname }}"
container_networks:
management_address:
address: "{{ ansible_host }}"
bridge: "br-mgmt"
interface: "eth1"
netmask: "255.255.252.0"
type: "veth"
physical_host: localhost
properties:
service_name: "{{ inventory_hostname }}"
# NOTE(cloudnull): The lxc-openstack AA profile for is used to ensure general
# container functionality typical to the integrated build.
lxc_container_config_list:
- "lxc.aa_profile=lxc-openstack"

View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_host: 10.1.0.2
ansible_become: True
ansible_user: root

View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_host: 10.1.0.3
ansible_become: True
ansible_user: root

View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_host: 10.1.0.4
ansible_become: True
ansible_user: root

View File

@ -0,0 +1,18 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
ansible_host: 10.1.0.5
ansible_become: True
ansible_user: root

View File

@ -1,5 +1,5 @@
---
# Copyright 2016, Rackspace US, Inc.
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -13,4 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
bridges:
- name: "br-mgmt"
ip_addr: "10.1.0.1"
ansible_python_interpreter: "/usr/bin/python2"

View File

@ -1,2 +1,20 @@
[all]
localhost
container1
container2
container3
container4
[all_containers]
container1
container2
container3
container4
[test1]
container1
container2
[test2]
container3
container4

138
tests/test-functional.yml Normal file
View File

@ -0,0 +1,138 @@
---
# Copyright 2018, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Clean up from previous tests
hosts: "{{ target_host_group_name }}:localhost"
become: yes
any_errors_fatal: yes
tasks:
- name: Clean up files/folders from previous tests
file:
path: "{{ item }}"
state: absent
with_items:
- "{{ lookup('env', 'HOME') | default('/opt', true) }}/venvs"
- "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache"
- name: Clean up facts from previous tests
ini_file:
path: "/etc/ansible/facts.d/openstack_ansible.fact"
section: "{{ target_host_group_name }}"
state: absent
- name: Refresh the inventory to clear the added groups
meta: refresh_inventory
- name: Execute role test and verify target host content
hosts: "{{ target_host_group_name }}"
become: yes
any_errors_fatal: yes
serial: "{{ target_serial }}"
vars:
base_directory: "{{ lookup('env', 'HOME') | default('/opt', true) }}"
tasks:
- name: Execute role
include_role:
name: "python_venv_build"
private: yes
vars:
venv_pip_packages:
- "Jinja2==2.10"
venv_destination_path: "{{ base_directory }}/venvs/{{ target_host_group_name }}"
venv_facts_when_changed:
- section: "{{ target_host_group_name }}"
option: "test"
value: True
- name: refresh local facts
setup:
filter: ansible_local
gather_subset: "!all"
- name: Show the ansible_local facts
debug:
var: ansible_local
- name: Verify that the facts were set
assert:
that:
- ansible_local['openstack_ansible'][target_host_group_name]['test'] | bool
- name: Find files/folders on targets
find:
file_type: directory
get_checksum: no
recurse: no
paths:
- "{{ base_directory }}/venvs"
register: _target_folders
- name: Compile the folder list from the targets
set_fact:
_target_folder_list: >-
{%- set folder_list = [] %}
{%- for item in _target_folders['files'] %}
{%- set _ = folder_list.append(item['path']) %}
{%- endfor %}
{{ folder_list }}
- name: Show the files/folder from the targets
debug:
var: _target_folder_list
- name: Verify the folder list from the targets
assert:
that:
- "{{ base_directory ~ '/venvs/' ~ target_host_group_name in _target_folder_list }}"
- name: Verify deploy host content
hosts: localhost
connection: local
become: yes
any_errors_fatal: yes
vars:
base_directory: "{{ lookup('env', 'HOME') | default('/opt', true) }}"
sub_directory: "{{ (ansible_distribution | lower) | replace(' ', '_') }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"
package_file_path: "{{ base_directory }}/cache/venvs/{{ sub_directory }}/{{ target_host_group_name }}"
tasks:
- name: Find files/folders on deploy host
find:
file_type: any
get_checksum: no
recurse: yes
paths:
- "{{ base_directory }}/cache"
register: _localhost_folders
- name: Compile the folder list from the deploy host
set_fact:
_localhost_folder_list: >-
{%- set folder_list = [] %}
{%- for item in _localhost_folders['files'] %}
{%- set _ = folder_list.append(item['path']) %}
{%- endfor %}
{{ folder_list }}
- name: Show the files/folders from the deploy host
debug:
var: _localhost_folder_list
- name: Verify the folder list from the deploy host
assert:
that:
- "{{ package_file_path ~ '.tgz' in _localhost_folder_list }}"
- "{{ package_file_path ~ '.checksum' in _localhost_folder_list }}"

View File

@ -13,19 +13,17 @@
# See the License for the specific language governing permissions and
# limitations under the License.
- hosts: localhost
connection: local
user: root
become: true
tasks:
- name: Execute role
include_role:
name: "python_venv_build"
private: yes
vars:
venv_pip_packages: "jinja==1.2"
venv_destination_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/test_venv"
venv_facts_when_changed:
- section: "test"
option: "test_fact"
value: True
- name: Prepare the host/containers
include: common/test-setup-host.yml
- name: Verify that the role works with a serial playbook
include: test-functional.yml
vars:
target_host_group_name: test1
target_serial: 1
- name: Verify that the role works with a parallel playbook
include: test-functional.yml
vars:
target_host_group_name: test2
target_serial: "100%"