Use wheels when building venvs

In order to speed up the venv build process, especially
when building multiple venvs, we implement the ability
to build the wheels prior to building the venvs. Once
the wheels are built, they are pulled back to the deploy
host so that they can be re-used for the next venv build
on another host.

By default we ensure that the location separates wheels
per distribution and architecture to prevent re-use of
wheels between them which can sometimes cause problems.

Change-Id: I0c99ea8e0b57130511704659a12c904b98ba5bcd
This commit is contained in:
Jesse Pretorius 2018-03-28 17:48:25 +01:00 committed by Jesse Pretorius (odyssey4me)
parent 22e1811796
commit 24dda60d55
5 changed files with 71 additions and 15 deletions

View File

@ -65,20 +65,35 @@ venv_reuse_enable: yes
# the deploy host can be pre-populated with the venvs.
venv_reuse_build_only: no
# Enable the use of wheels to speed up the venv
# build process, especially when building multiple
# venvs.
venv_reuse_build_wheels: yes
# The path where a built venv should be stored on the
# deployment host. By default, ensure that the location
# separates venvs per distribution and architecture to
# prevent re-use of venvs between them.
venv_reuse_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/venvs/{{ venv_reuse_download_subfolder }}"
venv_reuse_download_venv_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/venvs/{{ venv_reuse_download_subfolder }}"
# The path where built wheels should be stored on the
# deployment host. By default, ensure that the location
# separates wheels per distribution and architecture to
# prevent reuse of wheels between them.
venv_reuse_download_wheel_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/wheels/{{ venv_reuse_download_subfolder }}"
# NOTE(hwoarang): ansible_distribution may return a string with spaces
# such as "openSUSE Leap" so we need to replace the space with underscore
# in order to create a more sensible repo name for the distro.
venv_reuse_download_subfolder: "{{ (ansible_distribution | lower) | replace(' ', '_') }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"
# The owner of the venv_reuse_download_path
# The owner of the venv_reuse_download_venv_path
venv_reuse_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
# The path where the wheels are cached on the target host
# for speeding up the build process.
venv_wheel_destination_path: "/var/cache/pip/wheels"
# The facts to set when the venv changes during a
# build, or the installation of a venv.
# Eg:

View File

@ -36,10 +36,41 @@
retries: 5
delay: 2
- name: Create venv directory on the target host
- name: Create wheel/venv directories on the target host
file:
path: "{{ venv_destination_path }}"
path: "{{ item }}"
state: directory
with_items:
- "{{ venv_destination_path }}"
- "{{ venv_wheel_destination_path }}"
- name: Build and synchronize wheels when venv_reuse_build_wheels is enabled
when: venv_reuse_build_wheels | bool
block:
- name: Copy any existing wheels to the target host for re-use
synchronize:
src: "{{ venv_reuse_download_wheel_path }}/"
dest: "{{ venv_wheel_destination_path }}"
owner: no
group: no
- name: Build wheels for the packages to be installed into the venv
command: >-
pip wheel
--wheel-dir {{ venv_wheel_destination_path }}/
--find-links {{ venv_wheel_destination_path }}/
--log /var/log/python_wheel_build.log
{{ venv_pip_install_args }}
{{ venv_pip_packages | join(' ') }}
- name: Copy any new wheels from the target host to the deploy host for re-use
synchronize:
mode: pull
src: "{{ venv_wheel_destination_path }}/"
dest: "{{ venv_reuse_download_wheel_path }}"
owner: no
group: no
#TODO(odyssey4me):
# Split the venv build into multiple parts:
@ -54,7 +85,10 @@
state: "{{ pip_package_state }}"
virtualenv: "{{ venv_destination_path }}"
virtualenv_site_packages: "no"
extra_args: "{{ venv_pip_install_args }}"
extra_args: >-
--find-links {{ venv_wheel_destination_path }}/
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
register: _install_venv_pip_packages
until: _install_venv_pip_packages | success
retries: 5
@ -104,9 +138,9 @@
flat: yes
with_items:
- src: "{{ venv_destination_path }}.tgz"
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.tgz"
dest: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.tgz"
- src: "{{ venv_destination_path }}.checksum"
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.checksum"
dest: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.checksum"
when:
- _venv_package_build is mapping
- _venv_package_build | changed

View File

@ -15,7 +15,7 @@
- name: Copy the venv checksum file to the target host
copy:
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.checksum"
src: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.checksum"
dest: "{{ venv_destination_path | dirname }}/"
register: _venv_checksum_copy
@ -35,7 +35,7 @@
- name: Unarchive pre-built venv
unarchive:
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.tgz"
src: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.tgz"
dest: "{{ venv_destination_path }}"
remote_src: no
when:

View File

@ -32,19 +32,22 @@
- venv_reuse_build_only | bool
- not venv_reuse_enable | bool
- name: Ensure that venv_reuse_download_path exists on the deployment host
- name: Ensure that venv_reuse paths exists on the deployment host
file:
path: "{{ venv_reuse_download_path }}"
path: "{{ item }}"
state: directory
owner: "{{ venv_reuse_download_path_owner }}"
delegate_to: localhost
run_once: yes
with_items:
- "{{ venv_reuse_download_venv_path }}"
- "{{ venv_reuse_download_wheel_path }}"
when:
- venv_reuse_enable | bool
- name: Check if venv tgz is present on the deployment host
stat:
path: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.tgz"
path: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.tgz"
get_attributes: no
get_checksum: no
get_md5: no

View File

@ -20,11 +20,15 @@
tasks:
- name: Clean up files/folders from previous tests
file:
path: "{{ item }}"
path: "{{ item.path }}"
state: absent
when: item.condition | default(omit)
with_items:
- "{{ lookup('env', 'HOME') | default('/opt', true) }}/venvs"
- "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache"
- path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/venvs"
- path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/venvs"
- path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/wheels"
condition: "{{ target_host_group_name == 'test1' }}"
- path: "/var/cache/pip"
- name: Clean up facts from previous tests
ini_file: