openstack
/
ansible-role-python_venv_build
Code Issues Proposed changes

Apply constraints when building the venv 

Currently the role expects all constraints to be applied
in pip arguments provided to the role, which means that
there is some pre-processing required outside the role
for this to happen.

In order to pave the way to replace repo_build with this
role, we need to be able to apply constraints and maintain
idempotency even when building from a git source.

To achieve this we ensure that we build the wheels in a
temporary location, then use the resulting wheels for the
specific service to build a service-specific set of
requirements and constraints. To enable idempotency, we
only rebuild the wheels if the requirements/constraints
change.

We use slurp to collect the constraints and re-implement
them when installing the venv on the target hosts. This
prevents us having to inform the venv build role about
the repo server URL. We may change this at a later date
in order to facilitate a centralised repo server for
multiple regions.

Change-Id: I7c467e3a9e6627b75664b94f6b8e3232975171a7
This commit is contained in:
Jesse Pretorius 2018-11-27 14:28:15 +00:00
parent 0e44d4230a
commit 8f5ea40a5b
3 changed files with 128 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
defaults/main.yml
View File

@ -56,6 +56,13 @@ venv_default_pip_packages: []
# into the venv.
venv_pip_packages: []
# A list of constraints to be applied when building
# or installing python packages.
venv_build_constraints: []
# Arguments to pass to pip when building the wheels
venv_pip_build_args: ""
# Arguments to pass to pip when installing into the venv
venv_pip_install_args: ""
@ -93,6 +100,12 @@ venv_build_host: "{{ venv_build_targets[ansible_distribution_version][ansible_ar
# build host for the purpose of building the wheels.
venv_build_host_venv_path: "/openstack/venvs/wheel-builder"
# The path where the requirements/constraints are stored
# on the build host in order to ensure the build process
# is idempotent.
venv_build_host_requirements_path: >-
/var/www/repo/os-releases/{{ openstack_release | default('master') }}
# The path where the wheels are cached on the build host
# for speeding up the build process.
# TODO(odyssey4me):

35
tasks/python_venv_install.yml
View File

@ -57,6 +57,37 @@
args:
creates: "{{ venv_install_destination_path }}/bin/activate"
# Note(odyssey4me):
# This requirements file is not used for anything except to determine
# if requirements have changed. This helps reduce the execution time
# of the role and to make the role execution idempotent. If not for
# the conditional when installing the packages, any git constraints
# would result in the package for that constraint always being
# reinstalled.
- name: Build requirements file for the venv
copy:
dest: "{{ venv_install_destination_path }}/requirements.txt"
content: |
{% for item in (venv_default_pip_packages | union(venv_pip_packages)) | sort %}
{{ item }}
{% endfor %}
register: _requirement_file
- name: Build constraints file for the venv
copy:
dest: "{{ venv_install_destination_path }}/constraints.txt"
content: |
{% if (venv_build_host != inventory_hostname) and
(_constraints_file_slurp is defined) and
(_constraints_file_slurp.content is defined) %}
{{ _constraints_file_slurp.content | b64decode }}
{% else %}
{% for item in venv_build_constraints %}
{{ item }}
{% endfor %}
{% endif %}
register: _constraint_file
- name: Upgrade pip/setuptools/wheel to the versions we want
pip:
name:
@ -75,13 +106,15 @@
- name: Install python packages into the venv
pip:
name: "{{ venv_default_pip_packages + venv_pip_packages }}"
name: "{{ (venv_default_pip_packages + venv_pip_packages) | unique | sort }}"
state: "{{ venv_pip_package_state }}"
virtualenv: "{{ venv_install_destination_path }}"
extra_args: >-
--constraint {{ venv_install_destination_path }}/constraints.txt
--pre
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
when: (_requirement_file is changed) or (_constraint_file is changed)
register: _install_venv_pip_packages
until: _install_venv_pip_packages is success
retries: 5

98
tasks/python_venv_wheel_build.yml
View File

@ -33,10 +33,15 @@
retries: 5
delay: 2
- name: Ensure a fresh venv_build_host_venv_path if venv_rebuild is enabled
- name: Clean up paths and files if venv_rebuild is enabled
file:
path: "{{ venv_build_host_venv_path }}"
path: "{{ item }}"
state: absent
with_items:
- "{{ venv_build_host_wheel_path }}"
- "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt"
- "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt"
- "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt"
when:
- venv_rebuild | bool
@ -45,6 +50,11 @@
path: "{{ venv_build_host_wheel_path }}"
state: directory
- name: Create requirements/constraints file directory on the build host
file:
path: "{{ venv_build_host_requirements_path }}"
state: directory
# NOTE(odyssey4me):
# Not using --always-copy for CentOS/SuSE due to
# https://github.com/pypa/virtualenv/issues/565
@ -58,6 +68,24 @@
args:
creates: "{{ venv_build_host_venv_path }}/bin/activate"
- name: Build requirements file for the venv
copy:
dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt"
content: |
{% for item in (venv_default_pip_packages | union(venv_pip_packages)) | sort %}
{{ item }}
{% endfor %}
register: _requirement_file
- name: Build constraints file for the venv
copy:
dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt"
content: |
{% for item in venv_build_constraints %}
{{ item }}
{% endfor %}
register: _constraint_file
- name: Upgrade the wheel build virtualenv pip/setuptools/wheel to the versions we want
pip:
name:
@ -67,25 +95,61 @@
state: "{{ venv_pip_package_state }}"
virtualenv: "{{ venv_build_host_venv_path }}"
extra_args: >-
--constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt
--find-links {{ venv_build_host_wheel_path }}/
--log /var/log/python_venv_build.log
{{ venv_pip_install_args }}
{{ venv_pip_build_args }}
register: _update_virtualenv_packages
until: _update_virtualenv_packages is success
retries: 5
delay: 2
- name: Build wheels for the packages to be installed into the venv
command: >-
{{ venv_build_host_venv_path }}/bin/pip wheel
--wheel-dir {{ venv_build_host_wheel_path }}/
--find-links {{ venv_build_host_wheel_path }}/
--log /var/log/python_wheel_build.log
{{ venv_pip_install_args }}
{{ (venv_default_pip_packages + venv_pip_packages) | join(' ') }}
register: _build_python_wheels
until: _build_python_wheels is success
changed_when: (_build_python_wheels.stdout.find('Successfully built') != -1) or
(_build_python_wheels.stdout | regex_search('Saved \S*\.whl'))
retries: 5
delay: 2
- name: Build wheels and constraints file
when: (_requirement_file is changed) or (_constraint_file is changed)
block:
- name: Clean up temporary wheel build path
file:
path: "/tmp/{{ venv_install_destination_path | basename }}"
state: absent
- name: Build wheels for the packages to be installed into the venv
command: >-
{{ venv_build_host_venv_path }}/bin/pip wheel
--requirement {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt
--constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt
--wheel-dir /tmp/{{ venv_install_destination_path | basename }}/
--find-links {{ venv_build_host_wheel_path }}/
--log /var/log/python_wheel_build.log
{{ venv_pip_build_args }}
register: _build_python_wheels
until: _build_python_wheels is success
retries: 5
delay: 2
- name: Index built wheels
find:
paths: "/tmp/{{ venv_install_destination_path | basename }}"
file_type: file
register: _built_wheels
- name: Move built wheels to common wheel path
shell: >-
mv /tmp/{{ venv_install_destination_path | basename }}/* {{ venv_build_host_wheel_path }}/
args:
executable: /bin/bash
- name: Build constraints file for installation purposes
copy:
content: |
{% for file_data in _built_wheels['files'] %}
{% set file_name = file_data['path'] | basename %}
{{ file_name.split('-')[0] | lower }}=={{ (file_name.split('-')[1].split('_')) | join('.post') | lower }}
{% endfor %}
dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt"
- name: Slurp up the constraints file for later re-deployment
slurp:
src: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt"
register: _constraints_file_slurp
tags:
- install