From 27ec4a229d4d5cab2e0d16b3fde3ebc8f463fd67 Mon Sep 17 00:00:00 2001 From: Jesse Pretorius Date: Fri, 5 May 2017 18:12:14 +0100 Subject: [PATCH] Make venv build process idempotent The venv build process currently executes on every build, destroying all existing venvs for the distribution/architecture of the repo server it executes on, then rebuilds them. It is also terribly difficult to troubleshoot due to the fact that it is executed in parallel through a single bash script using backgrounded processes. This patch breaks the build process up into two parts - the per-venv options files, and the script that uses the options to build the venvs. With this breakdown we're able to do the following: - Only execute a venv rebuild if the venv options (indexes, requirements) have changed. - Use the Ansible asynchonous execution to execute parallel venv builds. As a very welcome side-effect, this also means that the venv build execution provides individual output for success or failures, making it much easier to see what went wrong when failing. As part of the patch, the removal of the *.in, *.txt and *.html files on each wheel build is taken out. This is to protect the venv options files. The removal of those files was unnecessary anyway as they're templated and therefore replaced by Ansible if they need to be changed. Change-Id: I063c3addb6fbabb01d620be33aac2cab29a02750 --- defaults/main.yml | 10 +- tasks/main.yml | 2 +- tasks/repo_pre_build.yml | 3 - tasks/repo_venv.yml | 18 ---- tasks/repo_venv_build.yml | 80 +++++++++------- tasks/repo_venv_post_build.yml | 23 ----- tasks/repo_venv_pre_build.yml | 59 ------------ templates/op-venv-script.sh.j2 | 128 ------------------------- templates/venv-build-options.txt.j2 | 141 ++++++++++++++++++++++++++++ templates/venv-build-script.sh.j2 | 93 ++++++++++++++++++ vars/main.yml | 37 ++++++++ 11 files changed, 325 insertions(+), 269 deletions(-) delete mode 100644 tasks/repo_venv.yml delete mode 100644 tasks/repo_venv_post_build.yml delete mode 100644 tasks/repo_venv_pre_build.yml delete mode 100644 templates/op-venv-script.sh.j2 create mode 100644 templates/venv-build-options.txt.j2 create mode 100644 templates/venv-build-script.sh.j2 create mode 100644 vars/main.yml diff --git a/defaults/main.yml b/defaults/main.yml index a6276b7..c0f799e 100644 --- a/defaults/main.yml +++ b/defaults/main.yml @@ -55,6 +55,12 @@ repo_build_wheel_selective: "{{ true if repo_build_venv_selective | bool else fa # Toggle whether venvs should be built selectively or not repo_build_venv_selective: yes +# Toggle whether a venv rebuild should be forced +repo_build_venv_rebuild: no + +# Timeout (in minutes) for a venv build +repo_build_venv_timeout: 30 + # Optionally set this to change the default index from pypi to an alternative #repo_build_pip_default_index: "https://pypi.python.org/simple" @@ -76,11 +82,11 @@ repo_build_timeout: 120 repo_build_concurrency: "{{ (((ansible_processor_vcpus|default(1)) | int) > 4) | ternary(8, 4) }}" repo_build_venv_build_dir: "/tmp/openstack-venv-builder" repo_build_venv_dir: "{{ repo_build_base_path }}/venvs/{{ repo_build_release_tag }}/{{ repo_build_os_distro_version }}" -repo_build_venv_pip_install_options: > +repo_build_venv_pip_install_options: >- --timeout 120 --find-links {{ repo_build_release_path }} --log /var/log/repo/repo_venv_builder.log -repo_build_venv_command_options: > +repo_build_venv_command_options: >- {{ virtualenv_bin }} --always-copy --extra-search-dir {{ repo_build_release_path }} diff --git a/tasks/main.yml b/tasks/main.yml index b1a2de8..4c3c835 100644 --- a/tasks/main.yml +++ b/tasks/main.yml @@ -31,7 +31,7 @@ - include: repo_post_build.yml # Venv building -- include: repo_venv.yml +- include: repo_venv_build.yml tags: - repo-build-venvs diff --git a/tasks/repo_pre_build.yml b/tasks/repo_pre_build.yml index 16136e5..bdc2a61 100644 --- a/tasks/repo_pre_build.yml +++ b/tasks/repo_pre_build.yml @@ -30,9 +30,6 @@ patterns: - "*{{ ansible_architecture | lower }}.whl" - "*none-any.whl" - - "*.in" - - "*.html" - - "*.txt" register: os_release_files tags: - repo-clean-workspace diff --git a/tasks/repo_venv.yml b/tasks/repo_venv.yml deleted file mode 100644 index 12cd64b..0000000 --- a/tasks/repo_venv.yml +++ /dev/null @@ -1,18 +0,0 @@ ---- -# Copyright 2015, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- include: repo_venv_pre_build.yml -- include: repo_venv_build.yml -- include: repo_venv_post_build.yml diff --git a/tasks/repo_venv_build.yml b/tasks/repo_venv_build.yml index ef61700..2ff8f92 100644 --- a/tasks/repo_venv_build.yml +++ b/tasks/repo_venv_build.yml @@ -13,6 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +- name: Create venv directories + file: + path: "{{ item }}" + state: "directory" + owner: "{{ repo_build_service_user_name }}" + mode: 02755 + with_items: + - "{{ repo_build_venv_build_dir }}/venvs" + - "{{ repo_build_venv_dir }}" + - name: Get venv command path command: which virtualenv changed_when: false @@ -26,21 +36,6 @@ tags: - always -- name: Check for created venvs - command: > - ls -1 "{{ repo_build_venv_dir }}/" - changed_when: false - register: created_venvs - tags: - - repo-create-venv - -- name: Set existing venv fact - set_fact: - existing_venvs: "{{ created_venvs.stdout_lines }}" - tags: - - repo-command-bin - - repo-create-venv - # This is removed so that virtual env is not installing unknown # packages as assumed wheels, IE pip8 - name: Ensure virtualenv_support is absent @@ -50,28 +45,43 @@ with_items: - "/usr/local/lib/python2.7/dist-packages/virtualenv_support" - "/usr/local/lib/python2.7/site-packages/virtualenv_support" - tags: - - repo-create-venv -- name: Create venv process script +- name: Create venv build script template: - src: "op-venv-script.sh.j2" - dest: "/opt/op-venv-script.sh" - tags: - - repo-create-venv + src: "venv-build-script.sh.j2" + dest: "/opt/venv-build-script.sh" + mode: "0755" -- name: Run venv process script - command: "bash /opt/op-venv-script.sh" - changed_when: false - tags: - - repo-create-venv - - repo-venv-compress-archive - - repo-create-venv-archive - - repo-create-venv-checksum +- name: Create venv build options files + template: + src: "venv-build-options.txt.j2" + dest: "{{ repo_build_release_path }}/venv-build-options-{{ item['role_name'] }}.txt" + with_items: "{{ filtered_venv_role_python_requirements }}" + register: _create_venv_options_files -- name: Remove venv process script - file: - path: "/opt/op-venv-script.sh" - state: absent +- name: Execute the venv build scripts asynchonously + shell: "/opt/venv-build-script.sh {{ repo_build_release_path }}/venv-build-options-{{ item['item']['role_name'] }}.txt" + args: + executable: "/bin/bash" + when: (item | changed) or (repo_build_venv_rebuild | bool) + with_items: + - "{{ _create_venv_options_files.results }}" + register: _build_venv + async: "{{ repo_build_venv_timeout * 60 }}" + poll: 0 + # This task requires the use of the shell module, so we skip lint + # to avoid: + # ANSIBLE0013 Use shell only when shell functionality is required tags: - - repo-create-venv + - skip_ansible_lint + +- name: Wait for the venvs builds to complete + async_status: + jid: "{{ item['ansible_job_id'] }}" + register: _venv_jobs + until: _venv_jobs['finished'] | bool + delay: 10 + retries: "{{ repo_build_venv_timeout * 6 }}" + with_items: "{{ _build_venv['results'] }}" + when: + - item['ansible_job_id'] is defined diff --git a/tasks/repo_venv_post_build.yml b/tasks/repo_venv_post_build.yml deleted file mode 100644 index a28ca5b..0000000 --- a/tasks/repo_venv_post_build.yml +++ /dev/null @@ -1,23 +0,0 @@ ---- -# Copyright 2015, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Cleanup venv directory - file: - path: "{{ item }}" - state: absent - with_items: - - "{{ repo_build_venv_build_dir }}" - tags: - - repo-cleanup-venv-location diff --git a/tasks/repo_venv_pre_build.yml b/tasks/repo_venv_pre_build.yml deleted file mode 100644 index a8b0763..0000000 --- a/tasks/repo_venv_pre_build.yml +++ /dev/null @@ -1,59 +0,0 @@ ---- -# Copyright 2015, Rackspace US, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -- name: Make sure old venv build directories are clean - file: - path: "{{ item }}" - state: "absent" - with_items: - - "{{ repo_build_venv_build_dir }}" - tags: - - repo-create-venv-location - - repo-venv-compress-archive - - repo-create-venv-archive - -- name: Register arch base venvs - command: "find {{ repo_build_venv_dir }}/ -name '*{{ ansible_architecture | lower }}*'" - args: - removes: "{{ repo_build_venv_dir }}" - register: base_venvs - tags: - - repo-create-venv-location - - repo-venv-compress-archive - - repo-create-venv-archive - -- name: Destroy base venvs to rebuild them - file: - path: "{{ item }}" - state: "absent" - with_items: "{{ base_venvs.stdout_lines|default([]) }}" - tags: - - repo-create-venv-location - - repo-venv-compress-archive - - repo-create-venv-archive - -- name: Create venv directory - file: - path: "{{ item }}" - state: "directory" - owner: "{{ repo_build_service_user_name }}" - mode: 02755 - with_items: - - "{{ repo_build_venv_build_dir }}/venvs" - - "{{ repo_build_venv_dir }}" - tags: - - repo-create-venv-location - - repo-venv-compress-archive - - repo-create-venv-archive diff --git a/templates/op-venv-script.sh.j2 b/templates/op-venv-script.sh.j2 deleted file mode 100644 index 4795123..0000000 --- a/templates/op-venv-script.sh.j2 +++ /dev/null @@ -1,128 +0,0 @@ -#!/usr/local/env bash - -set -ev - -function venv_create { - VENV_PATH="$1" - VENV_FILE="$2" - ROLE_VENV_WITH_INDEX="$3" - VENV_VALUES="$4" - - # If the venv working directory already exists remove it - [[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}" - - # If the pip build directory already exists remove it - [[ -d "/tmp/${VENV_FILE}" ]] && rm -rf "/tmp/${VENV_FILE}" - - # Create the virtualenv shell - {{ repo_build_venv_command_options.split() | join(' ') }} "${VENV_PATH}" - - # Create the pip build directory - mkdir -p "/tmp/${VENV_FILE}" - - # Activate the python virtual environment for good measure - source "${VENV_PATH}/bin/activate" - - # Run the pip install within the venv and specify a specific build directory which - # resolves pip locking issues when run in parallel. - {% set pip_command = [] %} - {% set _ = pip_command.append('${VENV_PATH}/bin/pip install') %} - {% set _ = pip_command.append('--build "/tmp/${VENV_FILE}"') %} - {% set _ = pip_command.append(repo_build_venv_pip_install_options.strip()) %} - - if [ "${ROLE_VENV_WITH_INDEX}" = false ]; then - {{ pip_command | join(' ') }} --no-index ${VENV_VALUES} - - {% if repo_build_pip_default_index is defined %} - {% set _ = pip_command.append('--index-url ' + repo_build_pip_default_index) %} - {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_default_index | netloc_no_port) %} - {% endif %} - {% if repo_build_pip_extra_index is defined %} - {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_index) %} - {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_index | netloc_no_port) %} - {% endif %} - {% if repo_build_pip_extra_indexes is defined %} - {% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_indexes | join(' --extra-index-url ')) %} - {% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ')) %} - {% endif %} - - else - {{ pip_command | join(' ') }} ${VENV_VALUES} - - fi - - # Deactivate the venv for good measure - deactivate - - # Find and remove all of the python pyc files - find "${VENV_PATH}" -type f -name '*.pyc' -delete - - # Create the archive - tar czf "${VENV_FILE}.tgz" -C "${VENV_PATH}" . - - # Create a checksum file for the archive - sha1sum "${VENV_FILE}.tgz" | awk '{print $1}' > "${VENV_FILE}.checksum" - - # Remove the working directories - rm -rf "${VENV_PATH}" - rm -rf "/tmp/${VENV_FILE}" -} - -# First operation is to sort and set the known os_* roles and create a unique dict. -# NOTE: this is a Jinja loop and will not be rendered within the script. For debugging -# purposes the group data will be rendered as a comment. -{% set os_group = {} %} -{% for role_name, role_data in local_packages.results.0.item.role_requirements.items() %} -{% set _host_group = role_data['project_group'] %} -{% if ("os_" in role_name) and ((not repo_build_venv_selective | bool) or ((groups[_host_group] is defined) and (groups[_host_group] | length > 0))) %} -{% set requirement_list = [] %} -{% for requirement_key, requirement_data in role_data.items() %} -{% if '_pip_packages' in requirement_key and 'requires' not in requirement_key and 'optional' not in requirement_key and 'proprietary' not in requirement_key %} -{% for requirement in requirement_data %} -{% set _ = requirement_list.append(requirement) %} -{% endfor %} -{% endif %} -{% endfor %} -# venv to build {{ role_name }} -# * packages within the {{ role_name }} venv: {{ requirement_list }} -{% set _ = os_group.update({role_name: requirement_list}) %} -{% endif %} -{% endfor %} - -PID=() -# Run the venv create. This will loop over all of the os_group role packages and create a python virtual env. -# Venv creation is done parallel at a count of the known "ansible_processor_count" or using a default of 5. -# This loop will enter the venv build directory and create tagged venvs in a distribution directory -# If the venv archive already exists the creation process will be skipped -pushd "{{ repo_build_venv_dir }}" -{% for key, value in os_group.items() %} - -{% set venvwithindex = [] %} -{% for clone_item in local_packages.results.0.item.remote_package_parts %} -{% if clone_item['name'] == (key | replace('os_', '')) and 'venvwithindex=True' in clone_item['original'] %} -{% set _ = venvwithindex.append(true) %} -{% endif %} -{% endfor %} -{% if venvwithindex %} -ROLE_VENV_WITH_INDEX=true -{% else %} -ROLE_VENV_WITH_INDEX=false -{% endif %} -ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}" -ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}" -if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then - venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "${ROLE_VENV_WITH_INDEX}" "{{ value | join(' ') }}" & - pid[{{ loop.index }}]=$! -fi -unset ROLE_VENV_PATH -unset ROLE_VENV_FILE -unset ROLE_VENV_WITH_INDEX -{% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %} -for job_pid in ${!pid[@]}; do - wait ${pid[$job_pid]} || exit 99 -done - -{% endif %} -{% endfor %} -popd - diff --git a/templates/venv-build-options.txt.j2 b/templates/venv-build-options.txt.j2 new file mode 100644 index 0000000..61ef7ab --- /dev/null +++ b/templates/venv-build-options.txt.j2 @@ -0,0 +1,141 @@ +# The purpose of this file is to provide the configuration options to the venv +# build script for each venv. These options are all set out in a file in order +# to enable the idempotence of the venv build process. + +# The working directory for the venv build +ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ item['role_name'] | replace('os_', '') }}" + +# The name used for the venv working directory, and resulting compressed venv +ROLE_VENV_FILE="{{ item['role_name'] | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}" + +# The index options used by pip when building the venvs +PIP_INDEX_OPTIONS="" +{# #} +{# In order to build venvs with packages not generally available in the #} +{# repo through the wheel build, the boolean 'venvwithindex' flag provided #} +{# in the remote_package_parts data enables the use of additional indexes. #} +{# This is useful when building venvs which require different constraints. #} +{# #} +{% for clone_item in local_packages.results.0.item.remote_package_parts %} +{# #} +{# Loop through the remote_package_parts to find the role #} +{% if clone_item['name'] == item['role_name'] | replace('os_', '') %} +{# #} +{# Check whether there is a venvwithindex setting #} +{% if clone_item['original'] | search('venvwithindex') %} +{# #} +{# Check if venvwithindex is set to boolean true #} +{% set venvwithindex=clone_item['original'] | regex_replace('(?i).*venvwithindex=(true|false).*', '\\1') %} +{% if venvwithindex | bool %} +{# #} +{# Add the extra indexes if they're defined #} +{% if repo_build_pip_default_index is defined %} +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --index-url {{ repo_build_pip_default_index }}" +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_default_index | netloc_no_port }}" +{% endif %} +{% if repo_build_pip_extra_index is defined %} +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --extra-index-url {{ repo_build_pip_extra_index }}" +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_extra_index | netloc_no_port }}" +{% endif %} +{% if repo_build_pip_extra_indexes is defined %} +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --extra-index-url {{ repo_build_pip_extra_indexes | join(' --extra-index-url ') }}" +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ') }}" +{% endif %} +{% else %} +{# If not true, then venvwithindex is set to boolean false #} +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --no-index" +{% endif %} +{% else %} +{# If not set, then assume that venvwithindex is false #} +PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --no-index" +{% endif %} +{% endif %} +{% endfor %} + +# The options used by pip when building the venvs +PIP_INSTALL_OPTIONS="{{ repo_build_venv_pip_install_options }}" + +# The command used when creating the venv +VENV_CREATE_COMMAND="{{ repo_build_venv_command_options }}" + +# The requirements list for the venv +{# #} +{# Rules for inclusion: #} +{# - The requirements are compiled from the *_pip_packages values. #} +{# - Any key containing the word 'requires' will be ignored as these #} +{# packages are destined for installation on the host, not in the venv. #} +{# - Any key containing the word 'optional' will be ignored as these #} +{# are destined for optional installation into the venv at run time #} +{# based on the user configuration. #} +{# - Any key containing the word 'proprietary' will be ignored as these #} +{# are destined for optional installation into the venv at run time #} +{# based on the user configuration and are not available on pypi. #} +{# #} +{# Input Example: #} +{# #} +{# role_name: os_neutron #} +{# role_data: neutron_optional_calico_pip_packages: #} +{# - calico #} +{# - networking-calico #} +{# - python-etcd #} +{# neutron_pip_packages: #} +{# - cliff #} +{# - configobj #} +{# - keystonemiddleware #} +{# - neutron #} +{# - neutron_dynamic_routing #} +{# - neutron_fwaas #} +{# - neutron_lbaas #} +{# - neutron_vpnaas #} +{# - pycrypto #} +{# - pymysql #} +{# - python-glanceclient #} +{# - python-keystoneclient #} +{# - python-memcached #} +{# - python-neutronclient #} +{# - python-novaclient #} +{# - repoze.lru #} +{# neutron_requires_pip_packages: #} +{# - httplib2 #} +{# - python-keystoneclient #} +{# - virtualenv #} +{# - virtualenv-tools #} +{# project_group: neutron_all #} +{# #} +{# Output Example: #} +{# #} +{# cliff #} +{# configobj #} +{# keystonemiddleware #} +{# neutron #} +{# neutron_dynamic_routing #} +{# neutron_fwaas #} +{# neutron_lbaas #} +{# neutron_vpnaas #} +{# pycrypto #} +{# pymysql #} +{# python-glanceclient #} +{# python-keystoneclient #} +{# python-memcached #} +{# python-neutronclient #} +{# python-novaclient #} +{# repoze.lru #} +{# #} +{# Here we loop through the data, and apply the rules to produce the list #} +{# of requirements. #} +{# #} +{% set requirement_list = [] %} +{% for requirement_key, requirement_data in item['role_data'].items() %} +{% if (requirement_key | match(".*_pip_packages$")) and + (not requirement_key | match(".*_requires_.*")) and + (not requirement_key | match(".*_optional_.*")) and + (not requirement_key | match(".*_proprietary_.*")) %} +{% for requirement in requirement_data %} +{% set _ = requirement_list.append(requirement) %} +{% endfor %} +{% endif %} +{% endfor %} +{# #} +{# Finally, we output the alphabetically sorted requirements. #} +{# #} +ROLE_VENV_REQUIREMENTS="{{ (requirement_list | sort) | join(' ') }}" diff --git a/templates/venv-build-script.sh.j2 b/templates/venv-build-script.sh.j2 new file mode 100644 index 0000000..34a86ee --- /dev/null +++ b/templates/venv-build-script.sh.j2 @@ -0,0 +1,93 @@ +#!/bin/bash +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +## Shell Opts ---------------------------------------------------------------- +set -e + +## Variables ----------------------------------------------------------------- + +# The options file for the venv +ROLE_VENV_REQUIREMENTS_FILE="${1}" + +## Functions ----------------------------------------------------------------- + +usage() { + cat < + +EOF +} + +## Main ---------------------------------------------------------------------- + +# Validate that an options file as been provided +if [[ -z "${ROLE_VENV_REQUIREMENTS_FILE}" ]]; then + usage + exit 1 +fi + +# Source the options file +source "${ROLE_VENV_REQUIREMENTS_FILE}" + +# Begin the venv build +pushd "{{ repo_build_venv_dir }}" + + # If the venv achive already exists, remove it + [[ -e "${ROLE_VENV_FILE}.tgz" ]] && rm -f "${ROLE_VENV_FILE}.tgz" + + # If the venv checksum file already exists, remove it + [[ -e "${ROLE_VENV_FILE}.checksum" ]] && rm -f "${ROLE_VENV_FILE}.checksum" + + # If the venv working directory already exists, remove it + [[ -d "${ROLE_VENV_PATH}" ]] && rm -rf "${ROLE_VENV_PATH}" + + # If the pip build directory already exists, remove it + [[ -d "/tmp/${ROLE_VENV_FILE}" ]] && rm -rf "/tmp/${ROLE_VENV_FILE}" + + # Create the virtualenv shell + ${VENV_CREATE_COMMAND} "${ROLE_VENV_PATH}" + + # Create the pip build directory + mkdir -p "/tmp/${ROLE_VENV_FILE}" + + # Activate the python virtual environment for good measure + source "${ROLE_VENV_PATH}/bin/activate" + + # Install the packages into the venv + ${ROLE_VENV_PATH}/bin/pip install \ + --build "/tmp/${ROLE_VENV_FILE}" \ + ${PIP_INSTALL_OPTIONS} \ + ${PIP_INDEX_OPTIONS} \ + ${ROLE_VENV_REQUIREMENTS} + + # Deactivate the venv for good measure + deactivate + + # Find and remove all of the python pyc files + find "${ROLE_VENV_PATH}" -type f -name '*.pyc' -delete + + # Create the archive + tar czf "${ROLE_VENV_FILE}.tgz" -C "${ROLE_VENV_PATH}" . + + # Create a checksum file for the archive + sha1sum "${ROLE_VENV_FILE}.tgz" | awk '{print $1}' > "${ROLE_VENV_FILE}.checksum" + + # Delete working directories + rm -rf "${ROLE_VENV_PATH}" + rm -rf "/tmp/${ROLE_VENV_FILE}" + +popd diff --git a/vars/main.yml b/vars/main.yml new file mode 100644 index 0000000..4fab8f3 --- /dev/null +++ b/vars/main.yml @@ -0,0 +1,37 @@ +--- +# Copyright 2017, Rackspace US, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The following rules are applied in order to determine the filtered list +# of roles to be processed for producing venvs. +# +# 1. The role name must be named 'os_*'. We only want the OpenStack +# services to be compiled into venvs. +# 2. If repo_build_venv_selective is disabled, then all the roles +# which pass rule 1 will have venvs built. +# 3. If the 'project_group' for the role is defined and is populated +# then the venv will be built. +# +filtered_venv_role_python_requirements: |- + {%- set filtered_role_list = [] %} + {%- for role_name, role_data in local_packages.results.0.item.role_requirements.items() %} + {%- if ("os_" in role_name) %} + {%- set project_group = role_data['project_group'] %} + {%- if (not repo_build_venv_selective | bool) or + ((groups[project_group] is defined) and (groups[project_group] | length > 0)) %} + {%- set _ = filtered_role_list.append({'role_name': role_name, 'role_data': role_data}) %} + {%- endif %} + {%- endif %} + {%- endfor %} + {{- filtered_role_list }}