Fix repo constraints construction and package installation
The current repo build process has the following issues: 1. The constraints consruction doesn't properly handle two constraints which use the same name, but have different version specs. eg: ovs===2.5.0;python_version=='2.7' ovs===2.6.0.dev2;python_version=='3.4' This is a problem in the constraints construction. 2. The pip packages installed on the repo server in order to construct the repo do not respect the global pins or the upper constraint overrides. 3. The constraints are selected based on the requirements specified. This makes the process unnecessarily complex. 4. The global pins are not applied to the constructed constraints, so the wrong packages get built and installed. This patch corrects all of these issues and hopefully makes the mechanism for constructing constraints more readily apparent, and therefore easier to maintain. This backport combines selective changes implemented in the following reviews: - Add logic to NOT build "proprietary" packages https://review.openstack.org/300505 - Updated py_pkgs to index requirement files https://review.openstack.org/345448 - Fix py_pkgs lookup to not include optional packages https://review.openstack.org/347834 - Update py_pkgs to itemise global pins https://review.openstack.org/347930 - Fix repo constraints construction and package installation https://review.openstack.org/350352 - Correct global_pins depth https://review.openstack.org/350654 Change-Id: If52d073d43081468e2faf2cd063c4b211c29994f Related-Bug: #1605846 Closes-Bug: #1609056
This commit is contained in:
Jesse Pretorius
Jesse Pretorius (odyssey4me)
parent
8df2967151
commit
4c620fee1f
|
|
@ -28,6 +28,7 @@ GIT_PACKAGE_DEFAULT_PARTS = dict()
|
|||
|
||||
|
||||
ROLE_PACKAGES = dict()
|
||||
ROLE_REQUIREMENTS = dict()
|
||||
|
||||
|
||||
REQUIREMENTS_FILE_TYPES = [
|
||||
|
|
@ -162,15 +163,18 @@ class DependencyFileProcessor(object):
|
|||
# Process everything simply by calling the method
|
||||
self._process_files()
|
||||
|
||||
def _py_pkg_extend(self, packages):
|
||||
def _py_pkg_extend(self, packages, py_package=None):
|
||||
if py_package is None:
|
||||
py_package = self.pip['py_package']
|
||||
for pkg in packages:
|
||||
pkg_name = _pip_requirement_split(pkg)[0]
|
||||
for py_pkg in self.pip['py_package']:
|
||||
for py_pkg in py_package:
|
||||
py_pkg_name = _pip_requirement_split(py_pkg)[0]
|
||||
if pkg_name == py_pkg_name:
|
||||
self.pip['py_package'].remove(py_pkg)
|
||||
py_package.remove(py_pkg)
|
||||
else:
|
||||
self.pip['py_package'].extend([i.lower() for i in packages])
|
||||
py_package.extend([i.lower() for i in packages])
|
||||
return py_package
|
||||
|
||||
@staticmethod
|
||||
def _filter_files(file_names, ext):
|
||||
|
|
@ -310,7 +314,7 @@ class DependencyFileProcessor(object):
|
|||
if name not in GIT_PACKAGE_DEFAULT_PARTS:
|
||||
GIT_PACKAGE_DEFAULT_PARTS[name] = git_data.copy()
|
||||
else:
|
||||
GIT_PACKAGE_DEFAULT_PARTS[name].update()
|
||||
GIT_PACKAGE_DEFAULT_PARTS[name].update(git_data)
|
||||
|
||||
# get the repo plugin definitions, if any
|
||||
git_data['plugins'] = loaded_yaml.get(plugins_var)
|
||||
|
|
@ -321,6 +325,26 @@ class DependencyFileProcessor(object):
|
|||
git_data=git_data
|
||||
)
|
||||
|
||||
def _package_build_index(self, packages, role_name, var_name,
|
||||
pkg_index=ROLE_PACKAGES):
|
||||
self._py_pkg_extend(packages)
|
||||
if role_name:
|
||||
if role_name in pkg_index:
|
||||
role_pkgs = pkg_index[role_name]
|
||||
else:
|
||||
role_pkgs = pkg_index[role_name] = dict()
|
||||
|
||||
pkgs = role_pkgs.get(var_name, list())
|
||||
if 'optional' not in var_name:
|
||||
pkgs = self._py_pkg_extend(packages, pkgs)
|
||||
pkg_index[role_name][var_name] = pkgs
|
||||
else:
|
||||
for k, v in pkg_index.items():
|
||||
for item_name in v.keys():
|
||||
if var_name == item_name:
|
||||
pkg_index[k][item_name] = self._py_pkg_extend(packages, pkg_index[k][item_name])
|
||||
|
||||
|
||||
def _process_files(self):
|
||||
"""Process files."""
|
||||
|
||||
|
|
@ -348,25 +372,31 @@ class DependencyFileProcessor(object):
|
|||
loaded_yaml=loaded_config,
|
||||
git_item=key,
|
||||
yaml_file_name=file_name
|
||||
)
|
||||
)
|
||||
|
||||
if [i for i in BUILT_IN_PIP_PACKAGE_VARS if i in key]:
|
||||
self._py_pkg_extend(values)
|
||||
if role_name:
|
||||
if role_name in ROLE_PACKAGES:
|
||||
role_pkgs = ROLE_PACKAGES[role_name]
|
||||
else:
|
||||
role_pkgs = ROLE_PACKAGES[role_name] = dict()
|
||||
if [i for i in BUILT_IN_PIP_PACKAGE_VARS
|
||||
if i in key
|
||||
if 'proprietary' not in key]:
|
||||
self._package_build_index(
|
||||
packages=values,
|
||||
role_name=role_name,
|
||||
var_name=key
|
||||
)
|
||||
|
||||
pkgs = role_pkgs.get(key, list())
|
||||
if 'optional' not in key:
|
||||
pkgs.extend(values)
|
||||
ROLE_PACKAGES[role_name][key] = pkgs
|
||||
else:
|
||||
for k, v in ROLE_PACKAGES.items():
|
||||
for item_name in v.keys():
|
||||
if key == item_name:
|
||||
ROLE_PACKAGES[k][item_name].extend(values)
|
||||
for key, values in loaded_config.items():
|
||||
if 'proprietary' in key:
|
||||
proprietary_pkgs = [
|
||||
i for i in values
|
||||
if GIT_PACKAGE_DEFAULT_PARTS.get(i)
|
||||
]
|
||||
if proprietary_pkgs:
|
||||
self._package_build_index(
|
||||
packages=proprietary_pkgs,
|
||||
role_name=role_name,
|
||||
var_name=key
|
||||
)
|
||||
else:
|
||||
role_name = None
|
||||
|
||||
return_list = self._filter_files(self.file_names, 'txt')
|
||||
for file_name in return_list:
|
||||
|
|
@ -379,13 +409,33 @@ class DependencyFileProcessor(object):
|
|||
for file_name in return_list:
|
||||
if file_name.endswith('other-requirements.txt'):
|
||||
continue
|
||||
if 'roles' in file_name:
|
||||
_role_name = file_name.split('roles%s' % os.sep)[-1]
|
||||
role_name = _role_name.split(os.sep)[0]
|
||||
if not role_name:
|
||||
role_name = 'default'
|
||||
with open(file_name, 'r') as f:
|
||||
packages = [
|
||||
i.split()[0] for i in f.read().splitlines()
|
||||
i.split()[0].lower() for i in f.read().splitlines()
|
||||
if i
|
||||
if not i.startswith('#')
|
||||
]
|
||||
self._py_pkg_extend(packages)
|
||||
base_file_name = os.path.basename(file_name)
|
||||
if base_file_name.endswith('test-requirements.txt'):
|
||||
continue
|
||||
if base_file_name.endswith('global-requirement-pins.txt'):
|
||||
self._package_build_index(
|
||||
packages=packages,
|
||||
role_name='global_pins',
|
||||
var_name='pinned_packages',
|
||||
pkg_index=ROLE_REQUIREMENTS
|
||||
)
|
||||
self._package_build_index(
|
||||
packages=packages,
|
||||
role_name=role_name,
|
||||
var_name='txt_file_packages',
|
||||
pkg_index=ROLE_REQUIREMENTS
|
||||
)
|
||||
|
||||
|
||||
def _abs_path(path):
|
||||
|
|
@ -522,6 +572,7 @@ class LookupModule(object):
|
|||
for key, value in return_data.items():
|
||||
if isinstance(value, (list, set)):
|
||||
return_data[key] = sorted(value)
|
||||
return_data['role_requirement_files'] = ROLE_REQUIREMENTS
|
||||
return [return_data]
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -15,7 +15,6 @@
|
|||
|
||||
# Wheel building
|
||||
- include: repo_clone_git.yml
|
||||
- include: repo_build_install.yml
|
||||
- include: repo_set_facts.yml
|
||||
- include: repo_pre_build.yml
|
||||
- include: repo_build.yml
|
||||
|
|
|
|||
|
|
@ -1,29 +0,0 @@
|
|||
---
|
||||
# Copyright 2016, Rackspace US, Inc.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
|
||||
- name: Install pip packages
|
||||
pip:
|
||||
name: "{{ item }}"
|
||||
state: present
|
||||
extra_args: "--constraint {{ repo_build_git_dir }}/requirements/upper-constraints.txt {{ pip_install_options|default('') }}"
|
||||
register: install_packages
|
||||
until: install_packages|success
|
||||
retries: 5
|
||||
delay: 5
|
||||
with_items: repo_pip_packages
|
||||
tags:
|
||||
- repo-pip-packages
|
||||
|
||||
|
|
@ -56,3 +56,16 @@
|
|||
dest: "{{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt"
|
||||
tags:
|
||||
- repo-build-constraints-file
|
||||
|
||||
- name: Install pip packages
|
||||
pip:
|
||||
name: "{{ item }}"
|
||||
state: present
|
||||
extra_args: "--constraint {{ repo_build_release_path }}/{{ repo_build_release_tag }}/requirements_constraints.txt {{ pip_install_options|default('') }}"
|
||||
register: install_packages
|
||||
until: install_packages|success
|
||||
retries: 5
|
||||
delay: 5
|
||||
with_items: repo_pip_packages
|
||||
tags:
|
||||
- repo-pip-packages
|
||||
|
|
|
|||
|
|
@ -57,25 +57,8 @@
|
|||
|
||||
- name: Decode the upper constraints content
|
||||
set_fact:
|
||||
_upper_constraints: "{{ slurp_upper_constraints.content | b64decode | splitlines }}"
|
||||
upper_constraints: "{{ slurp_upper_constraints.content | b64decode | splitlines }}"
|
||||
when: slurp_upper_constraints | success
|
||||
tags:
|
||||
- repo-set-constraints
|
||||
- repo-build-constraints-file
|
||||
|
||||
- name: Normalise the upper constraints
|
||||
set_fact:
|
||||
upper_constraints: "{{ local_packages.results.0.item.packages | pip_constraint_update(_upper_constraints) }}"
|
||||
when: slurp_upper_constraints | success
|
||||
tags:
|
||||
- repo-set-constraints
|
||||
- repo-build-constraints-file
|
||||
|
||||
- name: Apply the upper constraint overrides
|
||||
set_fact:
|
||||
upper_constraints: "{{ upper_constraints | pip_constraint_update(repo_build_upper_constraints_overrides) }}"
|
||||
when: repo_build_upper_constraints_overrides | length > 0
|
||||
tags:
|
||||
- repo-set-constraints
|
||||
- repo-build-constraints-file
|
||||
|
||||
|
|
|
|||
|
|
@ -1,21 +1,47 @@
|
|||
# Computed constraints
|
||||
{% set constraint_pkgs = [] -%}
|
||||
{% for clone_item in local_packages.results.0.item.remote_package_parts -%}
|
||||
{% if 'ignorerequirements=true' not in clone_item['original'] %}
|
||||
#
|
||||
# Constraints set by SHA's in the git sources
|
||||
#
|
||||
{% set constraint_pkgs = [] %}
|
||||
{% for clone_item in local_packages.results.0.item.remote_package_parts %}
|
||||
{% if 'ignorerequirements=true' not in clone_item['original'] %}
|
||||
{{ clone_item['original'] | replace(clone_item['url'], 'file://' + repo_build_git_dir + '/' + clone_item['name'] ) }}
|
||||
{% set _ = constraint_pkgs.append(clone_item['name'] | replace('-', '_') | lower) %}
|
||||
{% endif %}
|
||||
{% set _ = constraint_pkgs.append(clone_item['name'] | replace('-', '_') | lower) %}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
# upper boundry constraints from requirements repo.
|
||||
{% for constraint_item in upper_constraints %}
|
||||
{%- set constraint_split = constraint_item.split('===') %}
|
||||
{%- set constraint_name = constraint_split[0] %}
|
||||
{%- set constraint_name_normalized = constraint_name | replace('-', '_') | lower %}
|
||||
{% if constraint_name_normalized not in constraint_pkgs %}
|
||||
{% if repo_build_use_upper_constraints | bool and (constraint_split | length) > 1 %}
|
||||
{{ constraint_split[0] | replace('-', '_') | lower }}<={{ constraint_split[1] }}
|
||||
{% elif (constraint_split | length) == 1 %}
|
||||
{{ constraint_item }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
#
|
||||
# User-provided constraints set through a variable
|
||||
#
|
||||
{% set override_packages = [] %}
|
||||
{% for constraint_override_item in repo_build_upper_constraints_overrides %}
|
||||
{% set constraint_override_name = constraint_override_item | regex_replace('(>=|<=|>|<|==|~=|!=).*$','') %}
|
||||
{% set _ = override_packages.append(constraint_override_name) %}
|
||||
{{ constraint_override_item }}
|
||||
{% endfor %}
|
||||
#
|
||||
# Global pins set through the file global-requirement-pins.txt
|
||||
#
|
||||
{% set global_pin_packages = [] %}
|
||||
{% for global_pin in local_packages.results.0.item.role_requirement_files.global_pins.pinned_packages %}
|
||||
{% set global_pin_package_name = global_pin | regex_replace('(<=|<|==).*$','') %}
|
||||
{% set _ = global_pin_packages.append(global_pin_package_name) %}
|
||||
{# we want to ensure that repo_build_upper_constraints_overrides take the highest precedence #}
|
||||
{% if global_pin_package_name not in repo_build_upper_constraints_overrides %}
|
||||
{{ global_pin }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{# we don't bother applying OpenStack upper-constraints if the deployer has opted not to #}
|
||||
{% if repo_build_use_upper_constraints | bool %}
|
||||
#
|
||||
# Upper constraints from the OpenStack requirements repo
|
||||
#
|
||||
{% for constraint_item in upper_constraints %}
|
||||
{% set constraint_name = constraint_item | regex_replace('===.*', '') %}
|
||||
{% set constraint_data = constraint_item | regex_replace('.*===', '') %}
|
||||
{# The name has to be normalised to comply with PEP standards #}
|
||||
{% set constraint_name_normalized = constraint_name | replace('-', '_') | lower %}
|
||||
{% set constraint = constraint_name_normalized + '<=' + constraint_data %}
|
||||
{% if (constraint_name_normalized not in constraint_pkgs) and (constraint_name_normalized not in override_packages) and (constraint_name_normalized not in global_pin_packages) %}
|
||||
{{ constraint }}
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
{% endif %}
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
---
|
||||
features:
|
||||
- The ``repo_build`` role now provides the ability to override the
|
||||
upper-constraints applied which are sourced from OpenStack. The
|
||||
variable ``repo_build_upper_constraints_overrides`` can be
|
||||
populated with a list of upper constraints.
|
||||
upper-constraints applied which are sourced from OpenStack and
|
||||
from the global-requirements-pins.txt file. The variable
|
||||
``repo_build_upper_constraints_overrides`` can be populated with
|
||||
a list of upper constraints. This list will take the highest
|
||||
precedence in the constraints process, with the exception of
|
||||
the pins set in the git source SHAs.
|
||||
|
|
|
|||
Loading…
Reference in New Issue