Use wheels when building venvs
In order to speed up the venv build process, especially when building multiple venvs, we implement the ability to build the wheels prior to building the venvs. Once the wheels are built, they are pulled back to the deploy host so that they can be re-used for the next venv build on another host. By default we ensure that the location separates wheels per distribution and architecture to prevent re-use of wheels between them which can sometimes cause problems. Change-Id: I0c99ea8e0b57130511704659a12c904b98ba5bcd
This commit is contained in:
parent
22e1811796
commit
24dda60d55
|
@ -65,20 +65,35 @@ venv_reuse_enable: yes
|
|||
# the deploy host can be pre-populated with the venvs.
|
||||
venv_reuse_build_only: no
|
||||
|
||||
# Enable the use of wheels to speed up the venv
|
||||
# build process, especially when building multiple
|
||||
# venvs.
|
||||
venv_reuse_build_wheels: yes
|
||||
|
||||
# The path where a built venv should be stored on the
|
||||
# deployment host. By default, ensure that the location
|
||||
# separates venvs per distribution and architecture to
|
||||
# prevent re-use of venvs between them.
|
||||
venv_reuse_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/venvs/{{ venv_reuse_download_subfolder }}"
|
||||
venv_reuse_download_venv_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/venvs/{{ venv_reuse_download_subfolder }}"
|
||||
|
||||
# The path where built wheels should be stored on the
|
||||
# deployment host. By default, ensure that the location
|
||||
# separates wheels per distribution and architecture to
|
||||
# prevent reuse of wheels between them.
|
||||
venv_reuse_download_wheel_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/wheels/{{ venv_reuse_download_subfolder }}"
|
||||
|
||||
# NOTE(hwoarang): ansible_distribution may return a string with spaces
|
||||
# such as "openSUSE Leap" so we need to replace the space with underscore
|
||||
# in order to create a more sensible repo name for the distro.
|
||||
venv_reuse_download_subfolder: "{{ (ansible_distribution | lower) | replace(' ', '_') }}-{{ ansible_distribution_version.split('.')[:2] | join('.') }}-{{ ansible_architecture | lower }}"
|
||||
|
||||
# The owner of the venv_reuse_download_path
|
||||
# The owner of the venv_reuse_download_venv_path
|
||||
venv_reuse_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
|
||||
|
||||
# The path where the wheels are cached on the target host
|
||||
# for speeding up the build process.
|
||||
venv_wheel_destination_path: "/var/cache/pip/wheels"
|
||||
|
||||
# The facts to set when the venv changes during a
|
||||
# build, or the installation of a venv.
|
||||
# Eg:
|
||||
|
|
|
@ -36,10 +36,41 @@
|
|||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Create venv directory on the target host
|
||||
- name: Create wheel/venv directories on the target host
|
||||
file:
|
||||
path: "{{ venv_destination_path }}"
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
with_items:
|
||||
- "{{ venv_destination_path }}"
|
||||
- "{{ venv_wheel_destination_path }}"
|
||||
|
||||
- name: Build and synchronize wheels when venv_reuse_build_wheels is enabled
|
||||
when: venv_reuse_build_wheels | bool
|
||||
block:
|
||||
|
||||
- name: Copy any existing wheels to the target host for re-use
|
||||
synchronize:
|
||||
src: "{{ venv_reuse_download_wheel_path }}/"
|
||||
dest: "{{ venv_wheel_destination_path }}"
|
||||
owner: no
|
||||
group: no
|
||||
|
||||
- name: Build wheels for the packages to be installed into the venv
|
||||
command: >-
|
||||
pip wheel
|
||||
--wheel-dir {{ venv_wheel_destination_path }}/
|
||||
--find-links {{ venv_wheel_destination_path }}/
|
||||
--log /var/log/python_wheel_build.log
|
||||
{{ venv_pip_install_args }}
|
||||
{{ venv_pip_packages | join(' ') }}
|
||||
|
||||
- name: Copy any new wheels from the target host to the deploy host for re-use
|
||||
synchronize:
|
||||
mode: pull
|
||||
src: "{{ venv_wheel_destination_path }}/"
|
||||
dest: "{{ venv_reuse_download_wheel_path }}"
|
||||
owner: no
|
||||
group: no
|
||||
|
||||
#TODO(odyssey4me):
|
||||
# Split the venv build into multiple parts:
|
||||
|
@ -54,7 +85,10 @@
|
|||
state: "{{ pip_package_state }}"
|
||||
virtualenv: "{{ venv_destination_path }}"
|
||||
virtualenv_site_packages: "no"
|
||||
extra_args: "{{ venv_pip_install_args }}"
|
||||
extra_args: >-
|
||||
--find-links {{ venv_wheel_destination_path }}/
|
||||
--log /var/log/python_venv_build.log
|
||||
{{ venv_pip_install_args }}
|
||||
register: _install_venv_pip_packages
|
||||
until: _install_venv_pip_packages | success
|
||||
retries: 5
|
||||
|
@ -104,9 +138,9 @@
|
|||
flat: yes
|
||||
with_items:
|
||||
- src: "{{ venv_destination_path }}.tgz"
|
||||
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.tgz"
|
||||
dest: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.tgz"
|
||||
- src: "{{ venv_destination_path }}.checksum"
|
||||
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.checksum"
|
||||
dest: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.checksum"
|
||||
when:
|
||||
- _venv_package_build is mapping
|
||||
- _venv_package_build | changed
|
||||
|
|
|
@ -15,7 +15,7 @@
|
|||
|
||||
- name: Copy the venv checksum file to the target host
|
||||
copy:
|
||||
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.checksum"
|
||||
src: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.checksum"
|
||||
dest: "{{ venv_destination_path | dirname }}/"
|
||||
register: _venv_checksum_copy
|
||||
|
||||
|
@ -35,7 +35,7 @@
|
|||
|
||||
- name: Unarchive pre-built venv
|
||||
unarchive:
|
||||
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.tgz"
|
||||
src: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.tgz"
|
||||
dest: "{{ venv_destination_path }}"
|
||||
remote_src: no
|
||||
when:
|
||||
|
|
|
@ -32,19 +32,22 @@
|
|||
- venv_reuse_build_only | bool
|
||||
- not venv_reuse_enable | bool
|
||||
|
||||
- name: Ensure that venv_reuse_download_path exists on the deployment host
|
||||
- name: Ensure that venv_reuse paths exists on the deployment host
|
||||
file:
|
||||
path: "{{ venv_reuse_download_path }}"
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: "{{ venv_reuse_download_path_owner }}"
|
||||
delegate_to: localhost
|
||||
run_once: yes
|
||||
with_items:
|
||||
- "{{ venv_reuse_download_venv_path }}"
|
||||
- "{{ venv_reuse_download_wheel_path }}"
|
||||
when:
|
||||
- venv_reuse_enable | bool
|
||||
|
||||
- name: Check if venv tgz is present on the deployment host
|
||||
stat:
|
||||
path: "{{ venv_reuse_download_path }}/{{ venv_destination_path | basename }}.tgz"
|
||||
path: "{{ venv_reuse_download_venv_path }}/{{ venv_destination_path | basename }}.tgz"
|
||||
get_attributes: no
|
||||
get_checksum: no
|
||||
get_md5: no
|
||||
|
|
|
@ -20,11 +20,15 @@
|
|||
tasks:
|
||||
- name: Clean up files/folders from previous tests
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
path: "{{ item.path }}"
|
||||
state: absent
|
||||
when: item.condition | default(omit)
|
||||
with_items:
|
||||
- "{{ lookup('env', 'HOME') | default('/opt', true) }}/venvs"
|
||||
- "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache"
|
||||
- path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/venvs"
|
||||
- path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/venvs"
|
||||
- path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/wheels"
|
||||
condition: "{{ target_host_group_name == 'test1' }}"
|
||||
- path: "/var/cache/pip"
|
||||
|
||||
- name: Clean up facts from previous tests
|
||||
ini_file:
|
||||
|
|
Loading…
Reference in New Issue