From cce10ac38ebb8c7c0dbfcfcd4d3b921658c50eae Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Wed, 17 Apr 2019 12:14:12 +0100 Subject: [PATCH] Implement global constraints In the previous repo build process, we had global constraints which override upper constraints and anything set in the roles. This was essential for two purposes: 1. To enable us to pin things that were not in upper constraints. eg: pip, setuptools, wheel 2. To enable us to pin things which were in upper constraints, but broken. This would usually be a temporary measure until upper constraints was fixed. This patch implements a new variable 'venv_build_global_constraints' which is a list of constraints to be applied globally for all venvs. This list will be used to produce a file in the venv suffixed with '-global-constraints.txt' and will be used on the pip command line when building the wheels and when installing packages. We also ensure that all constraints are used when both building and installing pip, setuptools and wheel into the venv. Change-Id: I9ae3ef19c863b9237a51d2fcd6f4ebce1a9ebad7 --- defaults/main.yml | 5 +++++ tasks/python_venv_install.yml | 15 ++++++++++++++- tasks/python_venv_wheel_build.yml | 14 +++++++++++++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 5502f86..f7b3bf6 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -66,6 +66,11 @@ venv_pip_upgrade_noconf: false # or installing python packages. venv_build_constraints: [] +# A list of pip constraints to be applied as global +# constraints ahead of the list in venv_build_constraints. +# This is useful for global pins across all venvs. +venv_build_global_constraints: [] + # Arguments to pass to pip when building the wheels venv_pip_build_args: "" diff --git a/tasks/python_venv_install.yml b/tasks/python_venv_install.yml index fbb9061..d526ed8 100644 --- a/tasks/python_venv_install.yml +++ b/tasks/python_venv_install.yml @@ -73,6 +73,15 @@ {% endfor %} register: _requirement_file +- name: Build global constraints file for the venv + copy: + dest: "{{ venv_install_destination_path }}/global-constraints.txt" + content: | + {% for item in venv_build_global_constraints %} + {{ item }} + {% endfor %} + register: _global_constraint_file + - name: Build constraints file for the venv copy: dest: "{{ venv_install_destination_path }}/constraints.txt" @@ -97,6 +106,8 @@ state: "{{ venv_pip_package_state }}" virtualenv: "{{ venv_install_destination_path }}" extra_args: >- + --constraint {{ venv_install_destination_path }}/global-constraints.txt + --constraint {{ venv_install_destination_path }}/constraints.txt --log /var/log/python_venv_build.log {{ venv_default_pip_install_args }} {{ venv_pip_install_args }} @@ -116,12 +127,13 @@ state: "{{ venv_pip_package_state }}" virtualenv: "{{ venv_install_destination_path }}" extra_args: >- + --constraint {{ venv_install_destination_path }}/global-constraints.txt --constraint {{ venv_install_destination_path }}/constraints.txt --pre --log /var/log/python_venv_build.log {{ venv_default_pip_install_args }} {{ venv_pip_install_args }} - when: (_requirement_file is changed) or (_constraint_file is changed) + when: (_requirement_file is changed) or (_global_constraint_file is changed) or (_constraint_file is changed) register: _install_venv_pip_packages until: _install_venv_pip_packages is success retries: 5 @@ -134,6 +146,7 @@ state: absent with_items: - "{{ venv_install_destination_path }}/constraints.txt" + - "{{ venv_install_destination_path }}/global-constraints.txt" - "{{ venv_install_destination_path }}/requirements.txt" - fail: msg: > diff --git a/tasks/python_venv_wheel_build.yml b/tasks/python_venv_wheel_build.yml index 02bfef6..760d384 100644 --- a/tasks/python_venv_wheel_build.yml +++ b/tasks/python_venv_wheel_build.yml @@ -47,6 +47,7 @@ with_items: - "{{ venv_build_host_wheel_path }}" - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt" + - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt" - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt" - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt" when: @@ -84,6 +85,15 @@ {% endfor %} register: _requirement_file + - name: Build global constraints file for the venv + copy: + dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt" + content: | + {% for item in venv_build_global_constraints %} + {{ item }} + {% endfor %} + register: _global_constraint_file + - name: Build constraints file for the venv copy: dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt" @@ -102,6 +112,7 @@ state: "{{ venv_pip_package_state }}" virtualenv: "{{ venv_build_host_venv_path }}" extra_args: >- + --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt --find-links {{ venv_build_host_wheel_path }}/ --log /var/log/python_venv_build.log @@ -114,7 +125,7 @@ delay: 2 - name: Build wheels and constraints file - when: (_requirement_file is changed) or (_constraint_file is changed) + when: (_requirement_file is changed) or (_global_constraint_file is changed) or (_constraint_file is changed) block: - name: Clean up temporary wheel build path file: @@ -125,6 +136,7 @@ command: >- {{ venv_build_host_venv_path }}/bin/pip wheel --requirement {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt + --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-global-constraints.txt --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt --wheel-dir /tmp/{{ venv_install_destination_path | basename }}/ --find-links {{ venv_build_host_wheel_path }}/