Remove venv build process

The packaged venvs are no longer used by any of the roles,
so we remove the whole process.

Change-Id: I086c7f53f90c11969f56391551a573e1bd526c50
This commit is contained in:
Jesse Pretorius 2018-09-22 18:45:16 +01:00
parent d3bcd1c951
commit b1148411cc
10 changed files with 25 additions and 423 deletions

View File

@ -50,7 +50,7 @@ repo_build_git_dir: "{{ repo_build_base_path }}/openstackgit"
repo_build_pool_dir: "{{ repo_build_base_path }}/pools/{{ repo_build_os_distro_version }}"
# Toggle whether git repositories should be cloned selectively or not
repo_build_git_selective: "{{ true if (repo_build_wheel_selective | bool and repo_build_venv_selective | bool) else false }}"
repo_build_git_selective: "{{ repo_build_wheel_selective | bool }}"
# Toggle whether a git clone should be forced
repo_build_git_reclone: no
@ -59,20 +59,11 @@ repo_build_git_reclone: no
repo_build_git_depth: 1
# Toggle whether wheels should be built selectively or not
repo_build_wheel_selective: "{{ true if repo_build_venv_selective | bool else false }}"
repo_build_wheel_selective: yes
# Toggle whether a wheel rebuild should be forced
repo_build_wheel_rebuild: no
# Toggle whether venvs should be built selectively or not
repo_build_venv_selective: yes
# Toggle whether a venv rebuild should be forced
repo_build_venv_rebuild: no
# Timeout (in minutes) for a venv build
repo_build_venv_timeout: 30
# Optionally set this to change the default index from pypi to an alternative
repo_build_pip_default_index: "https://pypi.python.org/simple"
@ -87,23 +78,6 @@ repo_build_pip_default_index: "https://pypi.python.org/simple"
# - libvirt-python
repo_build_timeout: 120
repo_build_concurrency: "{{ (((ansible_processor_vcpus|default(1)) | int) > 4) | ternary(8, 4) }}"
repo_build_venv_build_dir: "/tmp/openstack-venv-builder"
repo_build_venv_dir: "{{ repo_build_base_path }}/venvs/{{ repo_build_release_tag }}/{{ repo_build_os_distro_version }}"
# We use find-links here in order to prefer the locally
# built wheels.
repo_build_venv_pip_install_options: >-
--timeout 120
--find-links {{ repo_build_release_path }}
--pre
# NOTE(odyssey4me):
# Not using --always-copy for CentOS/SuSE due to
# https://github.com/pypa/virtualenv/issues/565
repo_build_venv_command_options: >-
{{ virtualenv_bin }}
{{ (ansible_pkg_mgr == 'apt') | ternary('--always-copy', '') }}
# Path to the repo build venv binaries
repo_build_bin: "/openstack/venvs/repo-build-{{ repo_build_release_tag }}/bin"
@ -117,7 +91,3 @@ repo_pip_packages:
- wheel
repo_build_store_pip_sources: false
# Default python packages which will be installed
# into every venv.
repo_venv_default_pip_packages: []

View File

@ -0,0 +1,21 @@
---
deprecations:
- |
The repo build process no longer builds packaged venvs. Instead, the venvs
are created on the target hosts as the install process for each service
needs to. This opens up the opportunity for roles to be capable of creating
multiple venvs, and for any role to create venvs - neither of these options
were possible in previous releases.
The following variables therefore have been removed.
* ``repo_build_venv_selective``
* ``repo_build_venv_rebuild``
* ``repo_build_venv_timeout``
* ``repo_build_concurrency``
* ``repo_build_venv_build_dir``
* ``repo_build_venv_dir``
* ``repo_build_venv_pip_install_options``
* ``repo_build_venv_command_options``
* ``repo_venv_default_pip_packages``

View File

@ -45,10 +45,6 @@
tags:
- repo-build-wheels
- include_tasks: repo_build_venvs.yml
tags:
- repo-build-venvs
- include_tasks: repo_build_index.yml
tags:
- repo-build-index

View File

@ -18,7 +18,6 @@
file_type: any
paths:
- "{{ repo_build_release_path }}"
- "{{ repo_build_venv_dir }}"
register: manifest_items
tags:
- repo-create-release-manifest

View File

@ -17,17 +17,13 @@
ini_file:
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
section: repo_build
option: "{{ item }}"
option: "need_wheel_build"
value: False
with_items:
- "need_wheel_build"
- "need_venv_build"
when:
- "(ansible_local is not defined) or
('openstack_ansible' not in ansible_local) or
('repo_build' not in ansible_local['openstack_ansible']) or
('need_wheel_build' not in ansible_local['openstack_ansible']['repo_build']) or
('need_venv_build' not in ansible_local['openstack_ansible']['repo_build'])"
('need_wheel_build' not in ansible_local['openstack_ansible']['repo_build'])"
- name: Create package directories
file:
@ -77,18 +73,6 @@
(_wheel_build_constraints | changed) or
(repo_build_wheel_rebuild | bool)
- name: Record whether a venv build is required
ini_file:
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
section: repo_build
option: "need_venv_build"
value: True
when:
- (_wheel_build_requirements | changed) or
(_wheel_build_constraints | changed) or
(repo_build_wheel_rebuild | bool) or
(repo_build_venv_rebuild | bool)
# Note(odyssey4me):
# To cater for a situation where the pip packages are changing, but the repo
# does not yet have the package built, we need to ensure that this task can

View File

@ -1,81 +0,0 @@
---
# Copyright 2015, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
- name: Create venv directories
file:
path: "{{ repo_build_venv_dir }}"
state: "directory"
owner: "{{ repo_build_service_user_name }}"
mode: 02755
- name: Get venv command path
command: which virtualenv
changed_when: false
register: virtualenv_path
- name: Set virtualenv command path
set_fact:
virtualenv_bin: "{{ virtualenv_path.stdout }}"
- name: Create venv build script
template:
src: "venv-build-script.sh.j2"
dest: "/opt/venv-build-script.sh"
mode: "0755"
register: _create_venv_build_script
- name: Create venv build options files
template:
src: "venv-build-options.txt.j2"
dest: "{{ repo_build_release_path }}/venv-build-options-{{ item['role_name'] }}.txt"
with_items: "{{ filtered_venv_role_python_requirements }}"
register: _create_venv_options_files
- name: Execute the venv build scripts asynchonously
shell: "/opt/venv-build-script.sh {{ repo_build_release_path }}/venv-build-options-{{ item['item']['role_name'] }}.txt"
args:
executable: "/bin/bash"
when:
- (item | changed) or
(_create_venv_build_script | changed) or
(ansible_local['openstack_ansible']['repo_build']['need_venv_build'] | bool)
with_items:
- "{{ _create_venv_options_files.results }}"
register: _build_venv
async: "{{ repo_build_venv_timeout * 60 }}"
poll: 0
# This task requires the use of the shell module, so we skip lint
# to avoid:
# ANSIBLE0013 Use shell only when shell functionality is required
tags:
- skip_ansible_lint
- name: Wait for the venvs builds to complete
async_status:
jid: "{{ item['ansible_job_id'] }}"
register: _venv_jobs
until: _venv_jobs['finished'] | bool
delay: 10
retries: "{{ repo_build_venv_timeout * 6 }}"
with_items: "{{ _build_venv['results'] }}"
when:
- item['ansible_job_id'] is defined
- name: Disable the venv build requirement now that it is complete
ini_file:
dest: "/etc/ansible/facts.d/openstack_ansible.fact"
section: repo_build
option: need_venv_build
value: False

View File

@ -31,8 +31,6 @@
dest: "{{ repo_build_base_path }}/{{ repo_build_release_version_path }}"
- src: "{{ repo_build_pool_dir }}"
dest: "{{ repo_build_base_path }}/pools"
- src: "{{ repo_build_venv_dir }}"
dest: "{{ repo_build_base_path }}/venvs/{{ repo_build_release_tag }}"
become: true
become_user: "{{ repo_build_service_user_name }}"
changed_when: false

View File

@ -1,149 +0,0 @@
# The purpose of this file is to provide the configuration options to the venv
# build script for each venv. These options are all set out in a file in order
# to enable the idempotence of the venv build process.
{% set venv_service_name = item['role_name'] | replace('os_', '') %}
# The working directory for the venv build
ROLE_VENV_PATH="{{ repo_build_venv_build_dir }}/venvs/{{ venv_service_name }}"
# The name used for the venv working directory, and resulting compressed venv
ROLE_VENV_FILE="{{ venv_service_name }}-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}"
# The index options used by pip when building the venvs
PIP_INDEX_OPTIONS=""
{# #}
{# In order to build venvs with packages not generally available in the #}
{# repo through the wheel build, the boolean 'venvwithindex' flag provided #}
{# in the remote_package_parts data enables the use of additional indexes. #}
{# This is useful when building venvs which require different constraints. #}
{# #}
{% set remote_package_parts = local_packages.results.0.item.remote_package_parts %}
{# #}
{# We need to verify whether there are any git sources which apply to the #}
{# role. If there are not, then we need not bother with any of this. #}
{# #}
{% set git_source_available = ((remote_package_parts | selectattr('name', 'equalto', venv_service_name)) | list) | length > 0 %}
{% if git_source_available | bool %}
{% for clone_item in remote_package_parts %}
{# #}
{# Loop through the remote_package_parts to find the role #}
{% if clone_item['name'] == item['role_name'] | replace('os_', '') %}
{# #}
{# Check whether there is a venvwithindex setting #}
{% if clone_item['original'] | search('venvwithindex') %}
{# #}
{# Check if venvwithindex is set to boolean true #}
{% set venvwithindex=clone_item['original'] | regex_replace('(?i).*venvwithindex=(true|false).*', '\\1') %}
{% if venvwithindex | bool %}
{# #}
{# Add the extra indexes if they're defined #}
{% if repo_build_pip_default_index is defined %}
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --index-url {{ repo_build_pip_default_index }}"
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_default_index | netloc_no_port }}"
{% endif %}
{% if repo_build_pip_extra_indexes is defined %}
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --extra-index-url {{ repo_build_pip_extra_indexes | join(' --extra-index-url ') }}"
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --trusted-host {{ repo_build_pip_extra_indexes | map('netloc_no_port') | join(' --trusted-host ') }}"
{% endif %}
{% else %}
{# If not true, then venvwithindex is set to boolean false #}
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --no-index"
{% endif %}
{% else %}
{# If not set, then assume that venvwithindex is false #}
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --no-index"
{% endif %}
{% endif %}
{% endfor %}
{% else %}
{# If there is no git source available, then we should not be using any #}
{# upstream indexes either. #}
PIP_INDEX_OPTIONS="${PIP_INDEX_OPTIONS} --no-index"
{% endif %}
# The options used by pip when building the venvs
PIP_INSTALL_OPTIONS="{{ repo_build_venv_pip_install_options }}"
# The command used when creating the venv
VENV_CREATE_COMMAND="{{ repo_build_venv_command_options }}"
# The requirements list for the venv
{# #}
{# Rules for inclusion: #}
{# - The requirements are compiled from the *_pip_packages values. #}
{# - Any key containing the word 'requires' will be ignored as these #}
{# packages are destined for installation on the host, not in the venv. #}
{# - Any key containing the word 'optional' will be ignored as these #}
{# are destined for optional installation into the venv at run time #}
{# based on the user configuration. #}
{# - Any key containing the word 'proprietary' will be ignored as these #}
{# are destined for optional installation into the venv at run time #}
{# based on the user configuration and are not available on pypi. #}
{# #}
{# Input Example: #}
{# #}
{# role_name: os_neutron #}
{# role_data: neutron_optional_calico_pip_packages: #}
{# - calico #}
{# - networking-calico #}
{# - python-etcd #}
{# neutron_pip_packages: #}
{# - cliff #}
{# - configobj #}
{# - keystonemiddleware #}
{# - neutron #}
{# - neutron_dynamic_routing #}
{# - neutron_fwaas #}
{# - neutron_lbaas #}
{# - neutron_vpnaas #}
{# - pycrypto #}
{# - pymysql #}
{# - python-glanceclient #}
{# - python-keystoneclient #}
{# - python-memcached #}
{# - python-neutronclient #}
{# - python-novaclient #}
{# - repoze.lru #}
{# neutron_requires_pip_packages: #}
{# - httplib2 #}
{# - python-keystoneclient #}
{# - virtualenv #}
{# project_group: neutron_all #}
{# #}
{# Output Example: #}
{# #}
{# cliff #}
{# configobj #}
{# keystonemiddleware #}
{# neutron #}
{# neutron_dynamic_routing #}
{# neutron_fwaas #}
{# neutron_lbaas #}
{# neutron_vpnaas #}
{# pycrypto #}
{# pymysql #}
{# python-glanceclient #}
{# python-keystoneclient #}
{# python-memcached #}
{# python-neutronclient #}
{# python-novaclient #}
{# repoze.lru #}
{# #}
{# Here we loop through the data, and apply the rules to produce the list #}
{# of requirements. #}
{# #}
{% set requirement_list = [] %}
{% for requirement_key, requirement_data in item['role_data'].items() %}
{% if (requirement_key | match(".*_pip_packages$")) and
(not requirement_key | match(".*_requires_.*")) and
(not requirement_key | match(".*_optional_.*")) and
(not requirement_key | match(".*_proprietary_.*")) %}
{% for requirement in requirement_data %}
{% set _ = requirement_list.append(requirement) %}
{% endfor %}
{% endif %}
{% endfor %}
{# #}
{# Finally, we output the alphabetically sorted requirements. #}
{# #}
ROLE_VENV_REQUIREMENTS="{{ (repo_venv_default_pip_packages | union(requirement_list) | sort) | join(' ') }}"

View File

@ -1,119 +0,0 @@
#!/bin/bash
# Copyright 2017, Rackspace US, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
## Shell Opts ----------------------------------------------------------------
set -e
## Variables -----------------------------------------------------------------
# The options file for the venv
ROLE_VENV_REQUIREMENTS_FILE="${1}"
## Functions -----------------------------------------------------------------
usage() {
cat <<EOF
Usage:
${0} <path to configuration options file>
EOF
}
## Main ----------------------------------------------------------------------
# Validate that an options file as been provided
if [[ -z "${ROLE_VENV_REQUIREMENTS_FILE}" ]]; then
usage
exit 1
fi
# Source the options file
source "${ROLE_VENV_REQUIREMENTS_FILE}"
# Output the beginning of the build
echo -n "Building venv ${ROLE_VENV_FILE}..."
# Set the log file path
ROLE_VENV_LOG="/var/log/repo/venv_build_${ROLE_VENV_FILE}.log"
# Begin the venv build
pushd "{{ repo_build_venv_dir }}" &>/dev/null
# If the venv achive already exists, remove it
[[ -e "${ROLE_VENV_FILE}.tgz" ]] && rm -f "${ROLE_VENV_FILE}.tgz"
# If the venv checksum file already exists, remove it
[[ -e "${ROLE_VENV_FILE}.checksum" ]] && rm -f "${ROLE_VENV_FILE}.checksum"
# If the venv working directory already exists, remove it
[[ -d "${ROLE_VENV_PATH}" ]] && rm -rf "${ROLE_VENV_PATH}"
# If the pip build directory already exists, remove it
[[ -d "/tmp/${ROLE_VENV_FILE}" ]] && rm -rf "/tmp/${ROLE_VENV_FILE}"
# Create the virtualenv shell
${VENV_CREATE_COMMAND} "${ROLE_VENV_PATH}" &>${ROLE_VENV_LOG}
# Create the pip build directory
mkdir -p "/tmp/${ROLE_VENV_FILE}"
# Activate the python virtual environment for good measure
source "${ROLE_VENV_PATH}/bin/activate"
# Upgrade pip, setuptools and wheel to the version we want
${ROLE_VENV_PATH}/bin/pip install \
--disable-pip-version-check \
--quiet --quiet \
--build "/tmp/${ROLE_VENV_FILE}" \
${PIP_INSTALL_OPTIONS} \
--constraint {{ repo_build_release_path }}/requirements_constraints.txt \
--upgrade \
--index-url {{ repo_build_pip_default_index }} \
--trusted-host {{ repo_build_pip_default_index | netloc_no_port }} \
pip setuptools wheel \
--log "${ROLE_VENV_LOG}"
# Install the packages into the venv
${ROLE_VENV_PATH}/bin/pip install \
--disable-pip-version-check \
--quiet --quiet \
--build "/tmp/${ROLE_VENV_FILE}" \
${PIP_INSTALL_OPTIONS} \
${PIP_INDEX_OPTIONS} \
${ROLE_VENV_REQUIREMENTS} \
--log "${ROLE_VENV_LOG}"
# Deactivate the venv for good measure
deactivate
# Find and remove all of the python pyc files
find "${ROLE_VENV_PATH}" -type f -name '*.pyc' -delete 2>>${ROLE_VENV_LOG}
# Create the archive
tar czf "${ROLE_VENV_FILE}.tgz" -C "${ROLE_VENV_PATH}" . 2>>${ROLE_VENV_LOG}
# Create a checksum file for the archive
sha1sum "${ROLE_VENV_FILE}.tgz" | awk '{print $1}' > "${ROLE_VENV_FILE}.checksum" 2>>${ROLE_VENV_LOG}
# Delete working directories
rm -rf "${ROLE_VENV_PATH}"
rm -rf "/tmp/${ROLE_VENV_FILE}"
popd &>/dev/null
# Output the end of the build
echo "done"

View File

@ -59,23 +59,6 @@
roles:
- "repo_build"
post_tasks:
- name: List the files in the venv folder
command: ls -1 {{ repo_build_venv_dir }}/
register: venv_folder_content
changed_when: false
- name: Ensure that the keystone venv is present
assert:
that: "'keystone-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}.tgz' in venv_folder_content.stdout"
- name: Ensure that the tempest venv is present
assert:
that: "'tempest-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}.tgz' in venv_folder_content.stdout"
- name: Ensure that the nova venv is NOT present
assert:
that: "'nova-{{ repo_build_release_tag }}-{{ ansible_architecture | lower }}.tgz' not in venv_folder_content.stdout"
- name: List the files in the git folder
command: ls -1 /var/www/repo/openstackgit/
register: git_folder_content