Apply constraints when building the venv
Currently the role expects all constraints to be applied in pip arguments provided to the role, which means that there is some pre-processing required outside the role for this to happen. In order to pave the way to replace repo_build with this role, we need to be able to apply constraints and maintain idempotency even when building from a git source. To achieve this we ensure that we build the wheels in a temporary location, then use the resulting wheels for the specific service to build a service-specific set of requirements and constraints. To enable idempotency, we only rebuild the wheels if the requirements/constraints change. We use slurp to collect the constraints and re-implement them when installing the venv on the target hosts. This prevents us having to inform the venv build role about the repo server URL. We may change this at a later date in order to facilitate a centralised repo server for multiple regions. Change-Id: I7c467e3a9e6627b75664b94f6b8e3232975171a7
This commit is contained in:
parent
0e44d4230a
commit
8f5ea40a5b
|
@ -56,6 +56,13 @@ venv_default_pip_packages: []
|
|||
# into the venv.
|
||||
venv_pip_packages: []
|
||||
|
||||
# A list of constraints to be applied when building
|
||||
# or installing python packages.
|
||||
venv_build_constraints: []
|
||||
|
||||
# Arguments to pass to pip when building the wheels
|
||||
venv_pip_build_args: ""
|
||||
|
||||
# Arguments to pass to pip when installing into the venv
|
||||
venv_pip_install_args: ""
|
||||
|
||||
|
@ -93,6 +100,12 @@ venv_build_host: "{{ venv_build_targets[ansible_distribution_version][ansible_ar
|
|||
# build host for the purpose of building the wheels.
|
||||
venv_build_host_venv_path: "/openstack/venvs/wheel-builder"
|
||||
|
||||
# The path where the requirements/constraints are stored
|
||||
# on the build host in order to ensure the build process
|
||||
# is idempotent.
|
||||
venv_build_host_requirements_path: >-
|
||||
/var/www/repo/os-releases/{{ openstack_release | default('master') }}
|
||||
|
||||
# The path where the wheels are cached on the build host
|
||||
# for speeding up the build process.
|
||||
# TODO(odyssey4me):
|
||||
|
|
|
@ -57,6 +57,37 @@
|
|||
args:
|
||||
creates: "{{ venv_install_destination_path }}/bin/activate"
|
||||
|
||||
# Note(odyssey4me):
|
||||
# This requirements file is not used for anything except to determine
|
||||
# if requirements have changed. This helps reduce the execution time
|
||||
# of the role and to make the role execution idempotent. If not for
|
||||
# the conditional when installing the packages, any git constraints
|
||||
# would result in the package for that constraint always being
|
||||
# reinstalled.
|
||||
- name: Build requirements file for the venv
|
||||
copy:
|
||||
dest: "{{ venv_install_destination_path }}/requirements.txt"
|
||||
content: |
|
||||
{% for item in (venv_default_pip_packages | union(venv_pip_packages)) | sort %}
|
||||
{{ item }}
|
||||
{% endfor %}
|
||||
register: _requirement_file
|
||||
|
||||
- name: Build constraints file for the venv
|
||||
copy:
|
||||
dest: "{{ venv_install_destination_path }}/constraints.txt"
|
||||
content: |
|
||||
{% if (venv_build_host != inventory_hostname) and
|
||||
(_constraints_file_slurp is defined) and
|
||||
(_constraints_file_slurp.content is defined) %}
|
||||
{{ _constraints_file_slurp.content | b64decode }}
|
||||
{% else %}
|
||||
{% for item in venv_build_constraints %}
|
||||
{{ item }}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
register: _constraint_file
|
||||
|
||||
- name: Upgrade pip/setuptools/wheel to the versions we want
|
||||
pip:
|
||||
name:
|
||||
|
@ -75,13 +106,15 @@
|
|||
|
||||
- name: Install python packages into the venv
|
||||
pip:
|
||||
name: "{{ venv_default_pip_packages + venv_pip_packages }}"
|
||||
name: "{{ (venv_default_pip_packages + venv_pip_packages) | unique | sort }}"
|
||||
state: "{{ venv_pip_package_state }}"
|
||||
virtualenv: "{{ venv_install_destination_path }}"
|
||||
extra_args: >-
|
||||
--constraint {{ venv_install_destination_path }}/constraints.txt
|
||||
--pre
|
||||
--log /var/log/python_venv_build.log
|
||||
{{ venv_pip_install_args }}
|
||||
when: (_requirement_file is changed) or (_constraint_file is changed)
|
||||
register: _install_venv_pip_packages
|
||||
until: _install_venv_pip_packages is success
|
||||
retries: 5
|
||||
|
|
|
@ -33,10 +33,15 @@
|
|||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Ensure a fresh venv_build_host_venv_path if venv_rebuild is enabled
|
||||
- name: Clean up paths and files if venv_rebuild is enabled
|
||||
file:
|
||||
path: "{{ venv_build_host_venv_path }}"
|
||||
path: "{{ item }}"
|
||||
state: absent
|
||||
with_items:
|
||||
- "{{ venv_build_host_wheel_path }}"
|
||||
- "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt"
|
||||
- "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt"
|
||||
- "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt"
|
||||
when:
|
||||
- venv_rebuild | bool
|
||||
|
||||
|
@ -45,6 +50,11 @@
|
|||
path: "{{ venv_build_host_wheel_path }}"
|
||||
state: directory
|
||||
|
||||
- name: Create requirements/constraints file directory on the build host
|
||||
file:
|
||||
path: "{{ venv_build_host_requirements_path }}"
|
||||
state: directory
|
||||
|
||||
# NOTE(odyssey4me):
|
||||
# Not using --always-copy for CentOS/SuSE due to
|
||||
# https://github.com/pypa/virtualenv/issues/565
|
||||
|
@ -58,6 +68,24 @@
|
|||
args:
|
||||
creates: "{{ venv_build_host_venv_path }}/bin/activate"
|
||||
|
||||
- name: Build requirements file for the venv
|
||||
copy:
|
||||
dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt"
|
||||
content: |
|
||||
{% for item in (venv_default_pip_packages | union(venv_pip_packages)) | sort %}
|
||||
{{ item }}
|
||||
{% endfor %}
|
||||
register: _requirement_file
|
||||
|
||||
- name: Build constraints file for the venv
|
||||
copy:
|
||||
dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt"
|
||||
content: |
|
||||
{% for item in venv_build_constraints %}
|
||||
{{ item }}
|
||||
{% endfor %}
|
||||
register: _constraint_file
|
||||
|
||||
- name: Upgrade the wheel build virtualenv pip/setuptools/wheel to the versions we want
|
||||
pip:
|
||||
name:
|
||||
|
@ -67,25 +95,61 @@
|
|||
state: "{{ venv_pip_package_state }}"
|
||||
virtualenv: "{{ venv_build_host_venv_path }}"
|
||||
extra_args: >-
|
||||
--constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt
|
||||
--find-links {{ venv_build_host_wheel_path }}/
|
||||
--log /var/log/python_venv_build.log
|
||||
{{ venv_pip_install_args }}
|
||||
{{ venv_pip_build_args }}
|
||||
register: _update_virtualenv_packages
|
||||
until: _update_virtualenv_packages is success
|
||||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Build wheels for the packages to be installed into the venv
|
||||
command: >-
|
||||
{{ venv_build_host_venv_path }}/bin/pip wheel
|
||||
--wheel-dir {{ venv_build_host_wheel_path }}/
|
||||
--find-links {{ venv_build_host_wheel_path }}/
|
||||
--log /var/log/python_wheel_build.log
|
||||
{{ venv_pip_install_args }}
|
||||
{{ (venv_default_pip_packages + venv_pip_packages) | join(' ') }}
|
||||
register: _build_python_wheels
|
||||
until: _build_python_wheels is success
|
||||
changed_when: (_build_python_wheels.stdout.find('Successfully built') != -1) or
|
||||
(_build_python_wheels.stdout | regex_search('Saved \S*\.whl'))
|
||||
retries: 5
|
||||
delay: 2
|
||||
- name: Build wheels and constraints file
|
||||
when: (_requirement_file is changed) or (_constraint_file is changed)
|
||||
block:
|
||||
- name: Clean up temporary wheel build path
|
||||
file:
|
||||
path: "/tmp/{{ venv_install_destination_path | basename }}"
|
||||
state: absent
|
||||
|
||||
- name: Build wheels for the packages to be installed into the venv
|
||||
command: >-
|
||||
{{ venv_build_host_venv_path }}/bin/pip wheel
|
||||
--requirement {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt
|
||||
--constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt
|
||||
--wheel-dir /tmp/{{ venv_install_destination_path | basename }}/
|
||||
--find-links {{ venv_build_host_wheel_path }}/
|
||||
--log /var/log/python_wheel_build.log
|
||||
{{ venv_pip_build_args }}
|
||||
register: _build_python_wheels
|
||||
until: _build_python_wheels is success
|
||||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Index built wheels
|
||||
find:
|
||||
paths: "/tmp/{{ venv_install_destination_path | basename }}"
|
||||
file_type: file
|
||||
register: _built_wheels
|
||||
|
||||
- name: Move built wheels to common wheel path
|
||||
shell: >-
|
||||
mv /tmp/{{ venv_install_destination_path | basename }}/* {{ venv_build_host_wheel_path }}/
|
||||
args:
|
||||
executable: /bin/bash
|
||||
|
||||
- name: Build constraints file for installation purposes
|
||||
copy:
|
||||
content: |
|
||||
{% for file_data in _built_wheels['files'] %}
|
||||
{% set file_name = file_data['path'] | basename %}
|
||||
{{ file_name.split('-')[0] | lower }}=={{ (file_name.split('-')[1].split('_')) | join('.post') | lower }}
|
||||
{% endfor %}
|
||||
dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt"
|
||||
|
||||
- name: Slurp up the constraints file for later re-deployment
|
||||
slurp:
|
||||
src: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt"
|
||||
register: _constraints_file_slurp
|
||||
tags:
|
||||
- install
|
||||
|
|
Loading…
Reference in New Issue