Separate build and install stage
In order to ensure that the build tasks are entirely skipped when a package venv is re-used, the build and install stages are split. The ability to re-use venvs is also now able to be toggled. Disabling this feature would set the build to always happen, catering to environments where a service venv is always deployed to the same folder (eg: stateless hypervisors with squashfs partitions). The ability to set constraints, etc is changed to a generalised set of arguments that can be passed to the pip install task.
This commit is contained in:
Jesse Pretorius
parent
afafacfba5
commit
d5a9f025b8
|
|
@ -13,7 +13,20 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# The list of distribution packages to install
|
||||
#
|
||||
# Required variables
|
||||
#
|
||||
|
||||
# The path where venvs are extracted to
|
||||
# on the target host during an install, for example:
|
||||
# venv_destination_path: "/openstack/venvs/myvenv"
|
||||
|
||||
#
|
||||
# Optional variables
|
||||
#
|
||||
|
||||
# Distribution packages which must be installed
|
||||
# on the host for the purpose of building the venv.
|
||||
distro_package_list: []
|
||||
|
||||
# Set the package install state for packages
|
||||
|
|
@ -22,34 +35,46 @@ distro_package_state: "latest"
|
|||
|
||||
# The time in seconds that the distribution package
|
||||
# cache is valid for. This is only used by the apt
|
||||
# package manager
|
||||
# and zypper package managers.
|
||||
distro_cache_valid_time: 600
|
||||
|
||||
# Python packages which must be installed
|
||||
# on to the host
|
||||
# on to the host for the purpose of building
|
||||
# the venv.
|
||||
host_pip_packages: []
|
||||
|
||||
# Arguments to pass to the installation
|
||||
# of pip packages on the host.
|
||||
host_pip_install_args: ""
|
||||
|
||||
# Python packages which must be installed
|
||||
# into the venv
|
||||
# into the venv.
|
||||
venv_pip_packages: []
|
||||
|
||||
# General pip install constraints
|
||||
pip_install_constraints: ""
|
||||
# Arguments to pass to the venv build
|
||||
venv_pip_install_args: ""
|
||||
|
||||
# Specific constraints for the venv
|
||||
pip_install_venv_constraints: ""
|
||||
# Enable the reuse of venvs across multiple hosts.
|
||||
# This sets the build process to copy the venv to
|
||||
# the deployment host once it's built, then to
|
||||
# re-use the venv in subsequent deployments.
|
||||
venv_reuse_enable: yes
|
||||
|
||||
# General pip install extra options
|
||||
# This is especially useful for proxy options
|
||||
pip_install_options: ""
|
||||
# The path where a built venv should be stored on the
|
||||
# deployment host.
|
||||
venv_reuse_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache"
|
||||
|
||||
# The path where venvs are stored on the
|
||||
# deployment host
|
||||
venv_download_path: "{{ lookup('env', 'HOME') | default('/opt', true) }}/cache/files"
|
||||
# The owner of the venv_reuse_download_path
|
||||
venv_reuse_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
|
||||
|
||||
# The owner of the venv_download_path
|
||||
venv_download_path_owner: "{{ lookup('env', 'USER') | default('root', true) }}"
|
||||
# The facts to set when the venv changes during a
|
||||
# build, or the installation of a venv.
|
||||
# Eg:
|
||||
# set_facts_when_changed:
|
||||
# - section: glance
|
||||
# option: venv_tag
|
||||
# value: "{{ glance_venv_tag }}"
|
||||
venv_facts_when_changed: []
|
||||
|
||||
# The path where venvs are extracted to
|
||||
# on the target host, for example:
|
||||
# venv_destination_path: "/openstack/venvs/myvenv"
|
||||
# The INI file name to use for the fact setting.
|
||||
venv_facts_dest: "openstack_ansible"
|
||||
|
|
|
|||
|
|
@ -14,9 +14,5 @@
|
|||
# limitations under the License.
|
||||
|
||||
- meta: noop
|
||||
listen: Manage LB
|
||||
when: false
|
||||
|
||||
- meta: noop
|
||||
listen: Restart services
|
||||
listen: venv changed
|
||||
when: false
|
||||
|
|
|
|||
199
tasks/main.yml
199
tasks/main.yml
|
|
@ -13,199 +13,20 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# TODO(odyssey4me):
|
||||
# 1. Cater for Logan's model where we must just extract
|
||||
# over the existing folder without wiping it out first.
|
||||
# 2. Also, wiping the directory out from under it probably
|
||||
# wrecks the service while it's running. We should figure
|
||||
# out a better way of atomically replacing the venv without
|
||||
# ripping the folder out from under it.
|
||||
|
||||
- name: Install distro packages
|
||||
package:
|
||||
name: "{{ distro_package_list }}"
|
||||
state: "{{ distro_package_state }}"
|
||||
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
|
||||
cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}"
|
||||
register: _install_distro_packages
|
||||
until: _install_distro_packages | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Install required pip packages on the host
|
||||
pip:
|
||||
name: "{{ host_pip_packages }}"
|
||||
state: latest
|
||||
extra_args: >-
|
||||
{{ pip_install_constraints }}
|
||||
{{ pip_install_options }}
|
||||
register: _install_host_pip_packages
|
||||
until: _install_host_pip_packages | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Ensure that venv_download_path exists on the deployment host
|
||||
file:
|
||||
path: "{{ venv_download_path }}/{{ venv_destination_path | dirname }}"
|
||||
state: directory
|
||||
owner: "{{ venv_download_path_owner }}"
|
||||
delegate_to: localhost
|
||||
run_once: yes
|
||||
|
||||
- name: Check if venv is present on the deployment host
|
||||
stat:
|
||||
path: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
|
||||
get_attributes: no
|
||||
get_checksum: no
|
||||
get_md5: no
|
||||
get_mime: no
|
||||
register: _venv_tgz
|
||||
delegate_to: localhost
|
||||
run_once: yes
|
||||
|
||||
- name: Copy the venv checksum file to the target host
|
||||
copy:
|
||||
src: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum"
|
||||
dest: "{{ venv_destination_path | dirname }}"
|
||||
register: _venv_checksum_copy
|
||||
when:
|
||||
- _venv_tgz.stat.exists | bool
|
||||
|
||||
# TODO(odyssey4me):
|
||||
# 1. Cater for Logan's model where we must just extract
|
||||
# over the existing folder without wiping it out first.
|
||||
# 2. Also, removing it like this probably wrecks the service
|
||||
# while it's running. We should figure out a better way
|
||||
# of atomically replacing the venv without ripping the
|
||||
# folder out from under it.
|
||||
|
||||
# Due to our Ansible strategy, a skipped task does not have
|
||||
# a dictionary result. As such we validate that the register
|
||||
# is a mapping (dict).
|
||||
- name: Remove existing venv on target host if it is changing
|
||||
file:
|
||||
path: "{{ venv_destination_path }}"
|
||||
state: absent
|
||||
when:
|
||||
- _venv_checksum_copy is mapping
|
||||
- _venv_checksum_copy | changed
|
||||
|
||||
- name: Create venv directory on the target host
|
||||
file:
|
||||
path: "{{ venv_destination_path }}"
|
||||
state: directory
|
||||
register: _create_venv_dir
|
||||
|
||||
# Due to our Ansible strategy, a skipped task does not have
|
||||
# a dictionary result. As such we validate that the register
|
||||
# is a mapping (dict).
|
||||
- name: Unarchive pre-built venv
|
||||
unarchive:
|
||||
src: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
|
||||
dest: "{{ venv_destination_path }}"
|
||||
remote_src: no
|
||||
when:
|
||||
- _venv_checksum_copy is mapping
|
||||
- _venv_checksum_copy | changed
|
||||
notify:
|
||||
- Manage LB
|
||||
- Restart services
|
||||
|
||||
#TODO(odyssey4me):
|
||||
# Split the venv build into multiple parts:
|
||||
# 1. Create the venv without pip, setuptools, wheel
|
||||
# 2. Use get-pip.py to install the right versions
|
||||
# of pip, setuptools, wheel into the venv
|
||||
# 3. Install the packages into the venv
|
||||
|
||||
- name: Build venv
|
||||
pip:
|
||||
name: "{{ venv_pip_packages }}"
|
||||
state: latest
|
||||
virtualenv: "{{ venv_destination_path }}"
|
||||
virtualenv_site_packages: "no"
|
||||
extra_args: >-
|
||||
{{ pip_install_venv_constraints }}
|
||||
{{ pip_install_constraints }}
|
||||
{{ pip_install_options }}
|
||||
register: _install_venv_pip_packages
|
||||
until: _install_venv_pip_packages | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
when:
|
||||
- not _venv_tgz.stat.exists | bool
|
||||
notify:
|
||||
- Manage LB
|
||||
- Restart services
|
||||
|
||||
# Due to our Ansible strategy, a skipped task does not have
|
||||
# a dictionary result. As such we validate that the register
|
||||
# is a mapping (dict).
|
||||
- name: Update virtualenv python and paths
|
||||
shell: |
|
||||
sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/*
|
||||
virtualenv {{ venv_destination_path }}
|
||||
when:
|
||||
- _venv_checksum_copy is mapping
|
||||
- _venv_checksum_copy | changed
|
||||
- include_tasks: "python_venv_preflight.yml"
|
||||
tags:
|
||||
- skip_ansible_lint
|
||||
- always
|
||||
|
||||
# Due to our Ansible strategy, a skipped task does not have
|
||||
# a dictionary result. As such we validate that the register
|
||||
# is a mapping (dict).
|
||||
- name: Clean up the virtualenv before packaging
|
||||
shell: |
|
||||
find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete
|
||||
- include_tasks: "python_venv_build.yml"
|
||||
when:
|
||||
- _install_venv_pip_packages is mapping
|
||||
- _install_venv_pip_packages | changed
|
||||
- (not _src_venv_present.stat.exists | bool) or
|
||||
(not venv_reuse_enable | bool)
|
||||
|
||||
# Note(odyssey4me):
|
||||
# We purposefully use shel instead of the archive module
|
||||
# here. The archive module's output is far too verbose to
|
||||
# be practical when debugging.
|
||||
#
|
||||
# Due to our Ansible strategy, a skipped task does not have
|
||||
# a dictionary result. As such we validate that the register
|
||||
# is a mapping (dict).
|
||||
- name: Package venv
|
||||
shell: |
|
||||
tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' .
|
||||
args:
|
||||
chdir: "{{ venv_destination_path }}"
|
||||
executable: /bin/bash
|
||||
warn: no
|
||||
register: _venv_package_build
|
||||
- include_tasks: "python_venv_install.yml"
|
||||
when:
|
||||
- _install_venv_pip_packages is mapping
|
||||
- _install_venv_pip_packages | changed
|
||||
- venv_reuse_enable | bool
|
||||
- _src_venv_present.stat.exists | bool
|
||||
|
||||
# Due to our Ansible strategy, a skipped task does not have
|
||||
# a dictionary result. As such we validate that the register
|
||||
# is a mapping (dict).
|
||||
- name: Prepare checksum for packaged venv
|
||||
shell: |
|
||||
sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
- include_tasks: "python_venv_set_facts.yml"
|
||||
when:
|
||||
- _venv_package_build is mapping
|
||||
- _venv_package_build | changed
|
||||
|
||||
# Due to our Ansible strategy, a skipped task does not have
|
||||
# a dictionary result. As such we validate that the register
|
||||
# is a mapping (dict).
|
||||
- name: Copy the packaged venv and checksum file to the deployment host
|
||||
fetch:
|
||||
src: "{{ item.src }}"
|
||||
dest: "{{ item.dest }}"
|
||||
flat: yes
|
||||
with_items:
|
||||
- src: "{{ venv_destination_path }}.tgz"
|
||||
dest: "{{ venv_download_path }}/{{ venv_destination_path }}.tgz"
|
||||
- src: "{{ venv_destination_path }}.checksum"
|
||||
dest: "{{ venv_download_path }}/{{ venv_destination_path }}.checksum"
|
||||
when:
|
||||
- _venv_package_build is mapping
|
||||
- _venv_package_build | changed
|
||||
- venv_facts_when_changed != []
|
||||
|
|
|
|||
|
|
@ -0,0 +1,111 @@
|
|||
---
|
||||
# Copyright 2018, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Install distro packages for venv build
|
||||
package:
|
||||
name: "{{ distro_package_list }}"
|
||||
state: "{{ distro_package_state }}"
|
||||
update_cache: "{{ (ansible_pkg_mgr in ['apt', 'zypper']) | ternary('yes', omit) }}"
|
||||
cache_valid_time: "{{ (ansible_pkg_mgr == 'apt') | ternary(distro_cache_valid_time, omit) }}"
|
||||
register: _install_distro_packages
|
||||
until: _install_distro_packages | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Install pip packages on the host for venv build
|
||||
pip:
|
||||
name: "{{ host_pip_packages }}"
|
||||
state: latest
|
||||
extra_args: "{{ host_pip_install_args }}"
|
||||
register: _install_host_pip_packages
|
||||
until: _install_host_pip_packages | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
|
||||
- name: Create venv directory on the target host
|
||||
file:
|
||||
path: "{{ venv_destination_path }}"
|
||||
state: directory
|
||||
|
||||
#TODO(odyssey4me):
|
||||
# Split the venv build into multiple parts:
|
||||
# 1. Create the venv without pip, setuptools, wheel
|
||||
# 2. Use get-pip.py to install the right versions
|
||||
# of pip, setuptools, wheel into the venv
|
||||
# 3. Install the packages into the venv
|
||||
|
||||
- name: Build venv
|
||||
pip:
|
||||
name: "{{ venv_pip_packages }}"
|
||||
state: latest
|
||||
virtualenv: "{{ venv_destination_path }}"
|
||||
virtualenv_site_packages: "no"
|
||||
extra_args: "{{ venv_pip_install_args }}"
|
||||
register: _install_venv_pip_packages
|
||||
until: _install_venv_pip_packages | success
|
||||
retries: 5
|
||||
delay: 2
|
||||
notify:
|
||||
- venv changed
|
||||
|
||||
- name: Package the venv when venv_reuse_enable is enabled
|
||||
when: venv_reuse_enable | bool
|
||||
block:
|
||||
|
||||
- name: Clean up the virtualenv before packaging
|
||||
shell: |
|
||||
find {{ venv_destination_path }}/bin -type f -name '*.pyc' -delete
|
||||
when:
|
||||
- _install_venv_pip_packages is mapping
|
||||
- _install_venv_pip_packages | changed
|
||||
|
||||
# Note(odyssey4me):
|
||||
# We purposefully use shell instead of the archive module
|
||||
# here. The archive module's output is far too verbose to
|
||||
# be practical when debugging.
|
||||
- name: Package venv
|
||||
shell: |
|
||||
tar czf '{{ venv_destination_path }}.tgz' -C '{{ venv_destination_path }}' .
|
||||
args:
|
||||
chdir: "{{ venv_destination_path }}"
|
||||
executable: /bin/bash
|
||||
warn: no
|
||||
register: _venv_package_build
|
||||
when:
|
||||
- _install_venv_pip_packages is mapping
|
||||
- _install_venv_pip_packages | changed
|
||||
|
||||
- name: Prepare checksum for packaged venv
|
||||
shell: |
|
||||
sha1sum '{{ venv_destination_path }}.tgz' | awk '{print $1}' > '{{ venv_destination_path }}.checksum'
|
||||
args:
|
||||
executable: /bin/bash
|
||||
when:
|
||||
- _venv_package_build is mapping
|
||||
- _venv_package_build | changed
|
||||
|
||||
- name: Copy the packaged venv and checksum file to the deployment host
|
||||
fetch:
|
||||
src: "{{ item.src }}"
|
||||
dest: "{{ item.dest }}"
|
||||
flat: yes
|
||||
with_items:
|
||||
- src: "{{ venv_destination_path }}.tgz"
|
||||
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
|
||||
- src: "{{ venv_destination_path }}.checksum"
|
||||
dest: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum"
|
||||
when:
|
||||
- _venv_package_build is mapping
|
||||
- _venv_package_build | changed
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
---
|
||||
# Copyright 2018, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Copy the venv checksum file to the target host
|
||||
copy:
|
||||
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.checksum"
|
||||
dest: "{{ venv_destination_path | dirname }}"
|
||||
register: _venv_checksum_copy
|
||||
- _src_venv_present.stat.exists | bool
|
||||
|
||||
- name: Remove existing venv on target host if it is changing
|
||||
file:
|
||||
path: "{{ venv_destination_path }}"
|
||||
state: absent
|
||||
- _venv_checksum_copy is mapping
|
||||
- _venv_checksum_copy | changed
|
||||
|
||||
- name: Create venv directory on the target host
|
||||
file:
|
||||
path: "{{ venv_destination_path }}"
|
||||
state: directory
|
||||
|
||||
- name: Unarchive pre-built venv
|
||||
unarchive:
|
||||
src: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
|
||||
dest: "{{ venv_destination_path }}"
|
||||
remote_src: no
|
||||
- _venv_checksum_copy is mapping
|
||||
- _venv_checksum_copy | changed
|
||||
notify:
|
||||
- venv changed
|
||||
|
||||
- name: Update virtualenv python and paths
|
||||
shell: |
|
||||
sed -si '1s/^.*python.*$/#!{{ (venv_destination_path ~ '/bin') | replace ('/','\/') }}\/python/' {{ venv_destination_path }}/bin/*
|
||||
virtualenv {{ venv_destination_path }}
|
||||
when:
|
||||
- _venv_checksum_copy is mapping
|
||||
- _venv_checksum_copy | changed
|
||||
tags:
|
||||
- skip_ansible_lint
|
||||
|
|
@ -0,0 +1,43 @@
|
|||
---
|
||||
# Copyright 2018, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Verify that venv_destination_path has been provided
|
||||
fail:
|
||||
msg: |
|
||||
The variable venv_destination_path is required and
|
||||
has not been set
|
||||
when:
|
||||
- venv_destination_path is not defined
|
||||
|
||||
- name: Check if venv tgz is present on the deployment host
|
||||
stat:
|
||||
path: "{{ venv_reuse_download_path }}/{{ venv_destination_path }}.tgz"
|
||||
get_attributes: no
|
||||
get_checksum: no
|
||||
get_md5: no
|
||||
get_mime: no
|
||||
register: _src_venv_present
|
||||
delegate_to: localhost
|
||||
run_once: yes
|
||||
|
||||
- name: Ensure that venv_reuse_download_path exists on the deployment host
|
||||
file:
|
||||
path: "{{ venv_reuse_download_path }}/{{ venv_destination_path | dirname }}"
|
||||
state: directory
|
||||
owner: "{{ venv_reuse_download_path_owner }}"
|
||||
delegate_to: localhost
|
||||
run_once: yes
|
||||
when:
|
||||
- venv_reuse_enable | bool
|
||||
|
|
@ -0,0 +1,34 @@
|
|||
---
|
||||
# Copyright 2018, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Ensure local facts folder exists
|
||||
file:
|
||||
path: /etc/ansible/facts.d
|
||||
state: directory
|
||||
|
||||
- name: Record the necessary facts
|
||||
ini_file:
|
||||
dest: "/etc/ansible/facts.d/{{ venv_facts_dest }}.fact"
|
||||
section: "{{ item.section }}"
|
||||
option: "{{ item.option }}"
|
||||
value: "{{ item.value }}"
|
||||
with_items: "{{ venv_facts_when_changed }}"
|
||||
when:
|
||||
- (_venv_checksum_copy is defined and
|
||||
_venv_checksum_copy is mapping and
|
||||
_venv_checksum_copy | changed) or
|
||||
(_install_venv_pip_packages is defined and
|
||||
_install_venv_pip_packages is mapping and
|
||||
_install_venv_pip_packages | changed)
|
||||
Loading…
Reference in New Issue