From 8f5ea40a5b5ac20f706c08f958a7946d8b1581ba Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Tue, 27 Nov 2018 14:28:15 +0000 Subject: [PATCH] Apply constraints when building the venv Currently the role expects all constraints to be applied in pip arguments provided to the role, which means that there is some pre-processing required outside the role for this to happen. In order to pave the way to replace repo_build with this role, we need to be able to apply constraints and maintain idempotency even when building from a git source. To achieve this we ensure that we build the wheels in a temporary location, then use the resulting wheels for the specific service to build a service-specific set of requirements and constraints. To enable idempotency, we only rebuild the wheels if the requirements/constraints change. We use slurp to collect the constraints and re-implement them when installing the venv on the target hosts. This prevents us having to inform the venv build role about the repo server URL. We may change this at a later date in order to facilitate a centralised repo server for multiple regions. Change-Id: I7c467e3a9e6627b75664b94f6b8e3232975171a7 --- defaults/main.yml | 13 ++++ tasks/python_venv_install.yml | 35 ++++++++++- tasks/python_venv_wheel_build.yml | 98 +++++++++++++++++++++++++------ 3 files changed, 128 insertions(+), 18 deletions(-) diff --git a/defaults/main.yml b/defaults/main.yml index 4dc6759..e625e80 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -56,6 +56,13 @@ venv_default_pip_packages: [] # into the venv. venv_pip_packages: [] +# A list of constraints to be applied when building +# or installing python packages. +venv_build_constraints: [] + +# Arguments to pass to pip when building the wheels +venv_pip_build_args: "" + # Arguments to pass to pip when installing into the venv venv_pip_install_args: "" @@ -93,6 +100,12 @@ venv_build_host: "{{ venv_build_targets[ansible_distribution_version][ansible_ar # build host for the purpose of building the wheels. venv_build_host_venv_path: "/openstack/venvs/wheel-builder" +# The path where the requirements/constraints are stored +# on the build host in order to ensure the build process +# is idempotent. +venv_build_host_requirements_path: >- + /var/www/repo/os-releases/{{ openstack_release | default('master') }} + # The path where the wheels are cached on the build host # for speeding up the build process. # TODO(odyssey4me): diff --git a/tasks/python_venv_install.yml b/tasks/python_venv_install.yml index a11fd93..96bb734 100644 --- a/tasks/python_venv_install.yml +++ b/tasks/python_venv_install.yml @@ -57,6 +57,37 @@ args: creates: "{{ venv_install_destination_path }}/bin/activate" +# Note(odyssey4me): +# This requirements file is not used for anything except to determine +# if requirements have changed. This helps reduce the execution time +# of the role and to make the role execution idempotent. If not for +# the conditional when installing the packages, any git constraints +# would result in the package for that constraint always being +# reinstalled. +- name: Build requirements file for the venv + copy: + dest: "{{ venv_install_destination_path }}/requirements.txt" + content: | + {% for item in (venv_default_pip_packages | union(venv_pip_packages)) | sort %} + {{ item }} + {% endfor %} + register: _requirement_file + +- name: Build constraints file for the venv + copy: + dest: "{{ venv_install_destination_path }}/constraints.txt" + content: | + {% if (venv_build_host != inventory_hostname) and + (_constraints_file_slurp is defined) and + (_constraints_file_slurp.content is defined) %} + {{ _constraints_file_slurp.content | b64decode }} + {% else %} + {% for item in venv_build_constraints %} + {{ item }} + {% endfor %} + {% endif %} + register: _constraint_file + - name: Upgrade pip/setuptools/wheel to the versions we want pip: name: @@ -75,13 +106,15 @@ - name: Install python packages into the venv pip: - name: "{{ venv_default_pip_packages + venv_pip_packages }}" + name: "{{ (venv_default_pip_packages + venv_pip_packages) | unique | sort }}" state: "{{ venv_pip_package_state }}" virtualenv: "{{ venv_install_destination_path }}" extra_args: >- + --constraint {{ venv_install_destination_path }}/constraints.txt --pre --log /var/log/python_venv_build.log {{ venv_pip_install_args }} + when: (_requirement_file is changed) or (_constraint_file is changed) register: _install_venv_pip_packages until: _install_venv_pip_packages is success retries: 5 diff --git a/tasks/python_venv_wheel_build.yml b/tasks/python_venv_wheel_build.yml index b5c90f0..e4d5a6b 100644 --- a/tasks/python_venv_wheel_build.yml +++ b/tasks/python_venv_wheel_build.yml @@ -33,10 +33,15 @@ retries: 5 delay: 2 - - name: Ensure a fresh venv_build_host_venv_path if venv_rebuild is enabled + - name: Clean up paths and files if venv_rebuild is enabled file: - path: "{{ venv_build_host_venv_path }}" + path: "{{ item }}" state: absent + with_items: + - "{{ venv_build_host_wheel_path }}" + - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt" + - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt" + - "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt" when: - venv_rebuild | bool @@ -45,6 +50,11 @@ path: "{{ venv_build_host_wheel_path }}" state: directory + - name: Create requirements/constraints file directory on the build host + file: + path: "{{ venv_build_host_requirements_path }}" + state: directory + # NOTE(odyssey4me): # Not using --always-copy for CentOS/SuSE due to # https://github.com/pypa/virtualenv/issues/565 @@ -58,6 +68,24 @@ args: creates: "{{ venv_build_host_venv_path }}/bin/activate" + - name: Build requirements file for the venv + copy: + dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt" + content: | + {% for item in (venv_default_pip_packages | union(venv_pip_packages)) | sort %} + {{ item }} + {% endfor %} + register: _requirement_file + + - name: Build constraints file for the venv + copy: + dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt" + content: | + {% for item in venv_build_constraints %} + {{ item }} + {% endfor %} + register: _constraint_file + - name: Upgrade the wheel build virtualenv pip/setuptools/wheel to the versions we want pip: name: @@ -67,25 +95,61 @@ state: "{{ venv_pip_package_state }}" virtualenv: "{{ venv_build_host_venv_path }}" extra_args: >- + --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt --find-links {{ venv_build_host_wheel_path }}/ --log /var/log/python_venv_build.log - {{ venv_pip_install_args }} + {{ venv_pip_build_args }} register: _update_virtualenv_packages until: _update_virtualenv_packages is success retries: 5 delay: 2 - - name: Build wheels for the packages to be installed into the venv - command: >- - {{ venv_build_host_venv_path }}/bin/pip wheel - --wheel-dir {{ venv_build_host_wheel_path }}/ - --find-links {{ venv_build_host_wheel_path }}/ - --log /var/log/python_wheel_build.log - {{ venv_pip_install_args }} - {{ (venv_default_pip_packages + venv_pip_packages) | join(' ') }} - register: _build_python_wheels - until: _build_python_wheels is success - changed_when: (_build_python_wheels.stdout.find('Successfully built') != -1) or - (_build_python_wheels.stdout | regex_search('Saved \S*\.whl')) - retries: 5 - delay: 2 + - name: Build wheels and constraints file + when: (_requirement_file is changed) or (_constraint_file is changed) + block: + - name: Clean up temporary wheel build path + file: + path: "/tmp/{{ venv_install_destination_path | basename }}" + state: absent + + - name: Build wheels for the packages to be installed into the venv + command: >- + {{ venv_build_host_venv_path }}/bin/pip wheel + --requirement {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-requirements.txt + --constraint {{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-source-constraints.txt + --wheel-dir /tmp/{{ venv_install_destination_path | basename }}/ + --find-links {{ venv_build_host_wheel_path }}/ + --log /var/log/python_wheel_build.log + {{ venv_pip_build_args }} + register: _build_python_wheels + until: _build_python_wheels is success + retries: 5 + delay: 2 + + - name: Index built wheels + find: + paths: "/tmp/{{ venv_install_destination_path | basename }}" + file_type: file + register: _built_wheels + + - name: Move built wheels to common wheel path + shell: >- + mv /tmp/{{ venv_install_destination_path | basename }}/* {{ venv_build_host_wheel_path }}/ + args: + executable: /bin/bash + + - name: Build constraints file for installation purposes + copy: + content: | + {% for file_data in _built_wheels['files'] %} + {% set file_name = file_data['path'] | basename %} + {{ file_name.split('-')[0] | lower }}=={{ (file_name.split('-')[1].split('_')) | join('.post') | lower }} + {% endfor %} + dest: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt" + + - name: Slurp up the constraints file for later re-deployment + slurp: + src: "{{ venv_build_host_requirements_path }}/{{ venv_install_destination_path | basename }}-constraints.txt" + register: _constraints_file_slurp + tags: + - install