Make venv build process idempotent
The venv build process currently executes on every build, destroying all existing venvs for the distribution/architecture of the repo server it executes on, then rebuilds them. It is also terribly difficult to troubleshoot due to the fact that it is executed in parallel through a single bash script using backgrounded processes. This patch breaks the build process up into two parts - the per-venv options files, and the script that uses the options to build the venvs. With this breakdown we're able to do the following: - Only execute a venv rebuild if the venv options (indexes, requirements) have changed. - Use the Ansible asynchonous execution to execute parallel venv builds. As a very welcome side-effect, this also means that the venv build execution provides individual output for success or failures, making it much easier to see what went wrong when failing. As part of the patch, the removal of the *.in, *.txt and *.html files on each wheel build is taken out. This is to protect the venv options files. The removal of those files was unnecessary anyway as they're templated and therefore replaced by Ansible if they need to be changed. Change-Id: I063c3addb6fbabb01d620be33aac2cab29a02750
This commit is contained in:
Jesse Pretorius
parent
6c3e2ea540
commit
27ec4a229d
|
|
@ -55,6 +55,12 @@ repo_build_wheel_selective: "{{ true if repo_build_venv_selective | bool else fa
|
|||
# Toggle whether venvs should be built selectively or not
|
||||
repo_build_venv_selective: yes
|
||||
|
||||
# Toggle whether a venv rebuild should be forced
|
||||
repo_build_venv_rebuild: no
|
||||
|
||||
# Timeout (in minutes) for a venv build
|
||||
repo_build_venv_timeout: 30
|
||||
|
||||
# Optionally set this to change the default index from pypi to an alternative
|
||||
#repo_build_pip_default_index: "https://pypi.python.org/simple"
|
||||
|
||||
|
|
@ -76,11 +82,11 @@ repo_build_timeout: 120
|
|||
repo_build_concurrency: "{{ (((ansible_processor_vcpus|default(1)) | int) > 4) | ternary(8, 4) }}"
|
||||
repo_build_venv_build_dir: "/tmp/openstack-venv-builder"
|
||||
repo_build_venv_dir: "{{ repo_build_base_path }}/venvs/{{ repo_build_release_tag }}/{{ repo_build_os_distro_version }}"
|
||||
repo_build_venv_pip_install_options: >
|
||||
repo_build_venv_pip_install_options: >-
|
||||
--timeout 120
|
||||
--find-links {{ repo_build_release_path }}
|
||||
--log /var/log/repo/repo_venv_builder.log
|
||||
repo_build_venv_command_options: >
|
||||
repo_build_venv_command_options: >-
|
||||
{{ virtualenv_bin }}
|
||||
--always-copy
|
||||
--extra-search-dir {{ repo_build_release_path }}
|
||||
|
|
|
|||
|
|
@ -31,7 +31,7 @@
|
|||
- include: repo_post_build.yml
|
||||
|
||||
# Venv building
|
||||
- include: repo_venv.yml
|
||||
- include: repo_venv_build.yml
|
||||
tags:
|
||||
- repo-build-venvs
|
||||
|
||||
|
|
|
|||
|
|
@ -30,9 +30,6 @@
|
|||
patterns:
|
||||
- "*{{ ansible_architecture | lower }}.whl"
|
||||
- "*none-any.whl"
|
||||
- "*.in"
|
||||
- "*.html"
|
||||
- "*.txt"
|
||||
register: os_release_files
|
||||
tags:
|
||||
- repo-clean-workspace
|
||||
|
|
|
|||
|
|
@ -1,18 +0,0 @@
|
|||
---
|
||||
# Copyright 2015, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- include: repo_venv_pre_build.yml
|
||||
- include: repo_venv_build.yml
|
||||
- include: repo_venv_post_build.yml
|
||||
|
|
@ -13,6 +13,16 @@
|
|||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Create venv directories
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: "directory"
|
||||
owner: "{{ repo_build_service_user_name }}"
|
||||
mode: 02755
|
||||
with_items:
|
||||
- "{{ repo_build_venv_build_dir }}/venvs"
|
||||
- "{{ repo_build_venv_dir }}"
|
||||
|
||||
- name: Get venv command path
|
||||
command: which virtualenv
|
||||
changed_when: false
|
||||
|
|
@ -26,21 +36,6 @@
|
|||
tags:
|
||||
- always
|
||||
|
||||
- name: Check for created venvs
|
||||
command: >
|
||||
ls -1 "{{ repo_build_venv_dir }}/"
|
||||
changed_when: false
|
||||
register: created_venvs
|
||||
tags:
|
||||
- repo-create-venv
|
||||
|
||||
- name: Set existing venv fact
|
||||
set_fact:
|
||||
existing_venvs: "{{ created_venvs.stdout_lines }}"
|
||||
tags:
|
||||
- repo-command-bin
|
||||
- repo-create-venv
|
||||
|
||||
# This is removed so that virtual env is not installing unknown
|
||||
# packages as assumed wheels, IE pip8
|
||||
- name: Ensure virtualenv_support is absent
|
||||
|
|
@ -50,28 +45,43 @@
|
|||
with_items:
|
||||
- "/usr/local/lib/python2.7/dist-packages/virtualenv_support"
|
||||
- "/usr/local/lib/python2.7/site-packages/virtualenv_support"
|
||||
tags:
|
||||
- repo-create-venv
|
||||
|
||||
- name: Create venv process script
|
||||
- name: Create venv build script
|
||||
template:
|
||||
src: "op-venv-script.sh.j2"
|
||||
dest: "/opt/op-venv-script.sh"
|
||||
tags:
|
||||
- repo-create-venv
|
||||
src: "venv-build-script.sh.j2"
|
||||
dest: "/opt/venv-build-script.sh"
|
||||
mode: "0755"
|
||||
|
||||
- name: Run venv process script
|
||||
command: "bash /opt/op-venv-script.sh"
|
||||
changed_when: false
|
||||
tags:
|
||||
- repo-create-venv
|
||||
- repo-venv-compress-archive
|
||||
- repo-create-venv-archive
|
||||
- repo-create-venv-checksum
|
||||
- name: Create venv build options files
|
||||
template:
|
||||
src: "venv-build-options.txt.j2"
|
||||
dest: "{{ repo_build_release_path }}/venv-build-options-{{ item['role_name'] }}.txt"
|
||||
with_items: "{{ filtered_venv_role_python_requirements }}"
|
||||
register: _create_venv_options_files
|
||||
|
||||
- name: Remove venv process script
|
||||
file:
|
||||
path: "/opt/op-venv-script.sh"
|
||||
state: absent
|
||||
- name: Execute the venv build scripts asynchonously
|
||||
shell: "/opt/venv-build-script.sh {{ repo_build_release_path }}/venv-build-options-{{ item['item']['role_name'] }}.txt"
|
||||
args:
|
||||
executable: "/bin/bash"
|
||||
when: (item | changed) or (repo_build_venv_rebuild | bool)
|
||||
with_items:
|
||||
- "{{ _create_venv_options_files.results }}"
|
||||
register: _build_venv
|
||||
async: "{{ repo_build_venv_timeout * 60 }}"
|
||||
poll: 0
|
||||
# This task requires the use of the shell module, so we skip lint
|
||||
# to avoid:
|
||||
# ANSIBLE0013 Use shell only when shell functionality is required
|
||||
tags:
|
||||
- repo-create-venv
|
||||
- skip_ansible_lint
|
||||
|
||||
- name: Wait for the venvs builds to complete
|
||||
async_status:
|
||||
jid: "{{ item['ansible_job_id'] }}"
|
||||
register: _venv_jobs
|
||||
until: _venv_jobs['finished'] | bool
|
||||
delay: 10
|
||||
retries: "{{ repo_build_venv_timeout * 6 }}"
|
||||
with_items: "{{ _build_venv['results'] }}"
|
||||
when:
|
||||
- item['ansible_job_id'] is defined
|
||||
|
|
|
|||
|
|
@ -1,23 +0,0 @@
|
|||
---
|
||||
# Copyright 2015, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Cleanup venv directory
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: absent
|
||||
with_items:
|
||||
- "{{ repo_build_venv_build_dir }}"
|
||||
tags:
|
||||
- repo-cleanup-venv-location
|
||||
|
|
@ -1,59 +0,0 @@
|
|||
---
|
||||
# Copyright 2015, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
- name: Make sure old venv build directories are clean
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: "absent"
|
||||
with_items:
|
||||
- "{{ repo_build_venv_build_dir }}"
|
||||
tags:
|
||||
- repo-create-venv-location
|
||||
- repo-venv-compress-archive
|
||||
- repo-create-venv-archive
|
||||
|
||||
- name: Register arch base venvs
|
||||
command: "find {{ repo_build_venv_dir }}/ -name '*{{ ansible_architecture | lower }}*'"
|
||||
args:
|
||||
removes: "{{ repo_build_venv_dir }}"
|
||||
register: base_venvs
|
||||
tags:
|
||||
- repo-create-venv-location
|
||||
- repo-venv-compress-archive
|
||||
- repo-create-venv-archive
|
||||
|
||||
- name: Destroy base venvs to rebuild them
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: "absent"
|
||||
with_items: "{{ base_venvs.stdout_lines|default([]) }}"
|
||||
tags:
|
||||
- repo-create-venv-location
|
||||
- repo-venv-compress-archive
|
||||
- repo-create-venv-archive
|
||||
|
||||
- name: Create venv directory
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: "directory"
|
||||
owner: "{{ repo_build_service_user_name }}"
|
||||
mode: 02755
|
||||
with_items:
|
||||
- "{{ repo_build_venv_build_dir }}/venvs"
|
||||
- "{{ repo_build_venv_dir }}"
|
||||
tags:
|
||||
- repo-create-venv-location
|
||||
- repo-venv-compress-archive
|
||||
- repo-create-venv-archive
|
||||
|
|
@ -1,128 +0,0 @@
|
|||
#!/usr/local/env bash
|
||||
|
||||
set -ev
|
||||
|
||||
function venv_create {
|
||||
VENV_PATH="$1"
|
||||
VENV_FILE="$2"
|
||||
ROLE_VENV_WITH_INDEX="$3"
|
||||
VENV_VALUES="$4"
|
||||
|
||||
# If the venv working directory already exists remove it
|
||||
[[ -d "/tmp/${VENV_PATH}" ]] && rm -rf "/tmp/${VENV_PATH}"
|
||||
|
||||
# If the pip build directory already exists remove it
|
||||
[[ -d "/tmp/${VENV_FILE}" ]] && rm -rf "/tmp/${VENV_FILE}"
|
||||
|
||||
# Create the virtualenv shell
|
||||
{{ repo_build_venv_command_options.split() | join(' ') }} "${VENV_PATH}"
|
||||
|
||||
# Create the pip build directory
|
||||
mkdir -p "/tmp/${VENV_FILE}"
|
||||
|
||||
# Activate the python virtual environment for good measure
|
||||
source "${VENV_PATH}/bin/activate"
|
||||
|
||||
# Run the pip install within the venv and specify a specific build directory which
|
||||
# resolves pip locking issues when run in parallel.
|
||||
{% set pip_command = [] %}
|
||||
{% set _ = pip_command.append('${VENV_PATH}/bin/pip install') %}
|
||||
{% set _ = pip_command.append('--build "/tmp/${VENV_FILE}"') %}
|
||||
{% set _ = pip_command.append(repo_build_venv_pip_install_options.strip()) %}
|
||||
|
||||
if [ "${ROLE_VENV_WITH_INDEX}" = false ]; then
|
||||
{{ pip_command | join(' ') }} --no-index ${VENV_VALUES}
|
||||
|
||||
{% if repo_build_pip_default_index is defined %}
|
||||
{% set _ = pip_command.append('--index-url ' + repo_build_pip_default_index) %}
|
||||
{% set _ = pip_command.append('--trusted-host ' + repo_build_pip_default_index | netloc_no_port) %}
|
||||
{% endif %}
|
||||
{% if repo_build_pip_extra_index is defined %}
|
||||
{% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_index) %}
|
||||
{% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_index | netloc_no_port) %}
|
||||
{% endif %}
|
||||
{% if repo_build_pip_extra_indexes is defined %}
|
||||
{% set _ = pip_command.append('--extra-index-url ' + repo_build_pip_extra_indexes | join(' --extra-index-url ')) %}
|
||||
{% set _ = pip_command.append('--trusted-host ' + repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ')) %}
|
||||
{% endif %}
|
||||
|
||||
else
|
||||
{{ pip_command | join(' ') }} ${VENV_VALUES}
|
||||
|
||||
fi
|
||||
|
||||
# Deactivate the venv for good measure
|
||||
deactivate
|
||||
|
||||
# Find and remove all of the python pyc files
|
||||
find "${VENV_PATH}" -type f -name '*.pyc' -delete
|
||||
|
||||
# Create the archive
|
||||
tar czf "${VENV_FILE}.tgz" -C "${VENV_PATH}" .
|
||||
|
||||
# Create a checksum file for the archive
|
||||
sha1sum "${VENV_FILE}.tgz" | awk '{print $1}' > "${VENV_FILE}.checksum"
|
||||
|
||||
# Remove the working directories
|
||||
rm -rf "${VENV_PATH}"
|
||||
rm -rf "/tmp/${VENV_FILE}"
|
||||
}
|
||||
|
||||
# First operation is to sort and set the known os_* roles and create a unique dict.
|
||||
# NOTE: this is a Jinja loop and will not be rendered within the script. For debugging
|
||||
# purposes the group data will be rendered as a comment.
|
||||
{% set os_group = {} %}
|
||||
{% for role_name, role_data in local_packages.results.0.item.role_requirements.items() %}
|
||||
{% set _host_group = role_data['project_group'] %}
|
||||
{% if ("os_" in role_name) and ((not repo_build_venv_selective | bool) or ((groups[_host_group] is defined) and (groups[_host_group] | length > 0))) %}
|
||||
{% set requirement_list = [] %}
|
||||
{% for requirement_key, requirement_data in role_data.items() %}
|
||||
{% if '_pip_packages' in requirement_key and 'requires' not in requirement_key and 'optional' not in requirement_key and 'proprietary' not in requirement_key %}
|
||||
{% for requirement in requirement_data %}
|
||||
{% set _ = requirement_list.append(requirement) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
# venv to build {{ role_name }}
|
||||
# * packages within the {{ role_name }} venv: {{ requirement_list }}
|
||||
{% set _ = os_group.update({role_name: requirement_list}) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
PID=()
|
||||
# Run the venv create. This will loop over all of the os_group role packages and create a python virtual env.
|
||||
# Venv creation is done parallel at a count of the known "ansible_processor_count" or using a default of 5.
|
||||
# This loop will enter the venv build directory and create tagged venvs in a distribution directory
|
||||
# If the venv archive already exists the creation process will be skipped
|
||||
pushd "{{ repo_build_venv_dir }}"
|
||||
{% for key, value in os_group.items() %}
|
||||
|
||||
{% set venvwithindex = [] %}
|
||||
{% for clone_item in local_packages.results.0.item.remote_package_parts %}
|
||||
{% if clone_item['name'] == (key | replace('os_', '')) and 'venvwithindex=True' in clone_item['original'] %}
|
||||
{% set _ = venvwithindex.append(true) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% if venvwithindex %}
|
||||
ROLE_VENV_WITH_INDEX=true
|
||||
{% else %}
|
||||
ROLE_VENV_WITH_INDEX=false
|
||||
{% endif %}
|
||||
ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ key | replace('os_', '') }}"
|
||||
ROLE_VENV_FILE="{{ key | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}"
|
||||
if [ ! -f "${ROLE_VENV_FILE}.tgz" ];then
|
||||
venv_create "${ROLE_VENV_PATH}" "${ROLE_VENV_FILE}" "${ROLE_VENV_WITH_INDEX}" "{{ value | join(' ') }}" &
|
||||
pid[{{ loop.index }}]=$!
|
||||
fi
|
||||
unset ROLE_VENV_PATH
|
||||
unset ROLE_VENV_FILE
|
||||
unset ROLE_VENV_WITH_INDEX
|
||||
{% if loop.index is divisibleby(repo_build_concurrency | int) or loop.last %}
|
||||
for job_pid in ${!pid[@]}; do
|
||||
wait ${pid[$job_pid]} || exit 99
|
||||
done
|
||||
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
popd
|
||||
|
||||
|
|
@ -0,0 +1,141 @@
|
|||
# The purpose of this file is to provide the configuration options to the venv
|
||||
# build script for each venv. These options are all set out in a file in order
|
||||
# to enable the idempotence of the venv build process.
|
||||
|
||||
# The working directory for the venv build
|
||||
ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ item['role_name'] | replace('os_', '') }}"
|
||||
|
||||
# The name used for the venv working directory, and resulting compressed venv
|
||||
ROLE_VENV_FILE="{{ item['role_name'] | replace('os_', '') }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}"
|
||||
|
||||
# The index options used by pip when building the venvs
|
||||
PIP_INDEX_OPTIONS=""
|
||||
{# #}
|
||||
{# In order to build venvs with packages not generally available in the #}
|
||||
{# repo through the wheel build, the boolean 'venvwithindex' flag provided #}
|
||||
{# in the remote_package_parts data enables the use of additional indexes. #}
|
||||
{# This is useful when building venvs which require different constraints. #}
|
||||
{# #}
|
||||
{% for clone_item in local_packages.results.0.item.remote_package_parts %}
|
||||
{# #}
|
||||
{# Loop through the remote_package_parts to find the role #}
|
||||
{% if clone_item['name'] == item['role_name'] | replace('os_', '') %}
|
||||
{# #}
|
||||
{# Check whether there is a venvwithindex setting #}
|
||||
{% if clone_item['original'] | search('venvwithindex') %}
|
||||
{# #}
|
||||
{# Check if venvwithindex is set to boolean true #}
|
||||
{% set venvwithindex=clone_item['original'] | regex_replace('(?i).*venvwithindex=(true|false).*', '\\1') %}
|
||||
{% if venvwithindex | bool %}
|
||||
{# #}
|
||||
{# Add the extra indexes if they're defined #}
|
||||
{% if repo_build_pip_default_index is defined %}
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --index-url {{ repo_build_pip_default_index }}"
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_default_index | netloc_no_port }}"
|
||||
{% endif %}
|
||||
{% if repo_build_pip_extra_index is defined %}
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --extra-index-url {{ repo_build_pip_extra_index }}"
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_extra_index | netloc_no_port }}"
|
||||
{% endif %}
|
||||
{% if repo_build_pip_extra_indexes is defined %}
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --extra-index-url {{ repo_build_pip_extra_indexes | join(' --extra-index-url ') }}"
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ') }}"
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{# If not true, then venvwithindex is set to boolean false #}
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --no-index"
|
||||
{% endif %}
|
||||
{% else %}
|
||||
{# If not set, then assume that venvwithindex is false #}
|
||||
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --no-index"
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
# The options used by pip when building the venvs
|
||||
PIP_INSTALL_OPTIONS="{{ repo_build_venv_pip_install_options }}"
|
||||
|
||||
# The command used when creating the venv
|
||||
VENV_CREATE_COMMAND="{{ repo_build_venv_command_options }}"
|
||||
|
||||
# The requirements list for the venv
|
||||
{# #}
|
||||
{# Rules for inclusion: #}
|
||||
{# - The requirements are compiled from the *_pip_packages values. #}
|
||||
{# - Any key containing the word 'requires' will be ignored as these #}
|
||||
{# packages are destined for installation on the host, not in the venv. #}
|
||||
{# - Any key containing the word 'optional' will be ignored as these #}
|
||||
{# are destined for optional installation into the venv at run time #}
|
||||
{# based on the user configuration. #}
|
||||
{# - Any key containing the word 'proprietary' will be ignored as these #}
|
||||
{# are destined for optional installation into the venv at run time #}
|
||||
{# based on the user configuration and are not available on pypi. #}
|
||||
{# #}
|
||||
{# Input Example: #}
|
||||
{# #}
|
||||
{# role_name: os_neutron #}
|
||||
{# role_data: neutron_optional_calico_pip_packages: #}
|
||||
{# - calico #}
|
||||
{# - networking-calico #}
|
||||
{# - python-etcd #}
|
||||
{# neutron_pip_packages: #}
|
||||
{# - cliff #}
|
||||
{# - configobj #}
|
||||
{# - keystonemiddleware #}
|
||||
{# - neutron #}
|
||||
{# - neutron_dynamic_routing #}
|
||||
{# - neutron_fwaas #}
|
||||
{# - neutron_lbaas #}
|
||||
{# - neutron_vpnaas #}
|
||||
{# - pycrypto #}
|
||||
{# - pymysql #}
|
||||
{# - python-glanceclient #}
|
||||
{# - python-keystoneclient #}
|
||||
{# - python-memcached #}
|
||||
{# - python-neutronclient #}
|
||||
{# - python-novaclient #}
|
||||
{# - repoze.lru #}
|
||||
{# neutron_requires_pip_packages: #}
|
||||
{# - httplib2 #}
|
||||
{# - python-keystoneclient #}
|
||||
{# - virtualenv #}
|
||||
{# - virtualenv-tools #}
|
||||
{# project_group: neutron_all #}
|
||||
{# #}
|
||||
{# Output Example: #}
|
||||
{# #}
|
||||
{# cliff #}
|
||||
{# configobj #}
|
||||
{# keystonemiddleware #}
|
||||
{# neutron #}
|
||||
{# neutron_dynamic_routing #}
|
||||
{# neutron_fwaas #}
|
||||
{# neutron_lbaas #}
|
||||
{# neutron_vpnaas #}
|
||||
{# pycrypto #}
|
||||
{# pymysql #}
|
||||
{# python-glanceclient #}
|
||||
{# python-keystoneclient #}
|
||||
{# python-memcached #}
|
||||
{# python-neutronclient #}
|
||||
{# python-novaclient #}
|
||||
{# repoze.lru #}
|
||||
{# #}
|
||||
{# Here we loop through the data, and apply the rules to produce the list #}
|
||||
{# of requirements. #}
|
||||
{# #}
|
||||
{% set requirement_list = [] %}
|
||||
{% for requirement_key, requirement_data in item['role_data'].items() %}
|
||||
{% if (requirement_key | match(".*_pip_packages$")) and
|
||||
(not requirement_key | match(".*_requires_.*")) and
|
||||
(not requirement_key | match(".*_optional_.*")) and
|
||||
(not requirement_key | match(".*_proprietary_.*")) %}
|
||||
{% for requirement in requirement_data %}
|
||||
{% set _ = requirement_list.append(requirement) %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{# #}
|
||||
{# Finally, we output the alphabetically sorted requirements. #}
|
||||
{# #}
|
||||
ROLE_VENV_REQUIREMENTS="{{ (requirement_list | sort) | join(' ') }}"
|
||||
|
|
@ -0,0 +1,93 @@
|
|||
#!/bin/bash
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## Shell Opts ----------------------------------------------------------------
|
||||
set -e
|
||||
|
||||
## Variables -----------------------------------------------------------------
|
||||
|
||||
# The options file for the venv
|
||||
ROLE_VENV_REQUIREMENTS_FILE="${1}"
|
||||
|
||||
## Functions -----------------------------------------------------------------
|
||||
|
||||
usage() {
|
||||
cat <<EOF
|
||||
|
||||
Usage:
|
||||
${0} <path to configuration options file>
|
||||
|
||||
EOF
|
||||
}
|
||||
|
||||
## Main ----------------------------------------------------------------------
|
||||
|
||||
# Validate that an options file as been provided
|
||||
if [[ -z "${ROLE_VENV_REQUIREMENTS_FILE}" ]]; then
|
||||
usage
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Source the options file
|
||||
source "${ROLE_VENV_REQUIREMENTS_FILE}"
|
||||
|
||||
# Begin the venv build
|
||||
pushd "{{ repo_build_venv_dir }}"
|
||||
|
||||
# If the venv achive already exists, remove it
|
||||
[[ -e "${ROLE_VENV_FILE}.tgz" ]] && rm -f "${ROLE_VENV_FILE}.tgz"
|
||||
|
||||
# If the venv checksum file already exists, remove it
|
||||
[[ -e "${ROLE_VENV_FILE}.checksum" ]] && rm -f "${ROLE_VENV_FILE}.checksum"
|
||||
|
||||
# If the venv working directory already exists, remove it
|
||||
[[ -d "${ROLE_VENV_PATH}" ]] && rm -rf "${ROLE_VENV_PATH}"
|
||||
|
||||
# If the pip build directory already exists, remove it
|
||||
[[ -d "/tmp/${ROLE_VENV_FILE}" ]] && rm -rf "/tmp/${ROLE_VENV_FILE}"
|
||||
|
||||
# Create the virtualenv shell
|
||||
${VENV_CREATE_COMMAND} "${ROLE_VENV_PATH}"
|
||||
|
||||
# Create the pip build directory
|
||||
mkdir -p "/tmp/${ROLE_VENV_FILE}"
|
||||
|
||||
# Activate the python virtual environment for good measure
|
||||
source "${ROLE_VENV_PATH}/bin/activate"
|
||||
|
||||
# Install the packages into the venv
|
||||
${ROLE_VENV_PATH}/bin/pip install \
|
||||
--build "/tmp/${ROLE_VENV_FILE}" \
|
||||
${PIP_INSTALL_OPTIONS} \
|
||||
${PIP_INDEX_OPTIONS} \
|
||||
${ROLE_VENV_REQUIREMENTS}
|
||||
|
||||
# Deactivate the venv for good measure
|
||||
deactivate
|
||||
|
||||
# Find and remove all of the python pyc files
|
||||
find "${ROLE_VENV_PATH}" -type f -name '*.pyc' -delete
|
||||
|
||||
# Create the archive
|
||||
tar czf "${ROLE_VENV_FILE}.tgz" -C "${ROLE_VENV_PATH}" .
|
||||
|
||||
# Create a checksum file for the archive
|
||||
sha1sum "${ROLE_VENV_FILE}.tgz" | awk '{print $1}' > "${ROLE_VENV_FILE}.checksum"
|
||||
|
||||
# Delete working directories
|
||||
rm -rf "${ROLE_VENV_PATH}"
|
||||
rm -rf "/tmp/${ROLE_VENV_FILE}"
|
||||
|
||||
popd
|
||||
|
|
@ -0,0 +1,37 @@
|
|||
---
|
||||
# Copyright 2017, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# The following rules are applied in order to determine the filtered list
|
||||
# of roles to be processed for producing venvs.
|
||||
#
|
||||
# 1. The role name must be named 'os_*'. We only want the OpenStack
|
||||
# services to be compiled into venvs.
|
||||
# 2. If repo_build_venv_selective is disabled, then all the roles
|
||||
# which pass rule 1 will have venvs built.
|
||||
# 3. If the 'project_group' for the role is defined and is populated
|
||||
# then the venv will be built.
|
||||
#
|
||||
filtered_venv_role_python_requirements: |-
|
||||
{%- set filtered_role_list = [] %}
|
||||
{%- for role_name, role_data in local_packages.results.0.item.role_requirements.items() %}
|
||||
{%- if ("os_" in role_name) %}
|
||||
{%- set project_group = role_data['project_group'] %}
|
||||
{%- if (not repo_build_venv_selective | bool) or
|
||||
((groups[project_group] is defined) and (groups[project_group] | length > 0)) %}
|
||||
{%- set _ = filtered_role_list.append({'role_name': role_name, 'role_data': role_data}) %}
|
||||
{%- endif %}
|
||||
{%- endif %}
|
||||
{%- endfor %}
|
||||
{{- filtered_role_list }}
|
||||
Loading…
Reference in New Issue