From 82671430e66a657284af629047f2c096b72fe548 Mon Sep 17 00:00:00 2001 From: Joshua Harlow Date: Thu, 9 Jul 2015 17:06:41 -0700 Subject: [PATCH] Unpin the pip version We were restricted to older versions of pip due to some import changes, and code changes the pip folks did, but this changes that so that we can now use the newer versions of pip and all the features/additions they have performed there. Change-Id: Ia4bff0229acf7823c8fe9e0e95bf3ac02f9a6ce1 --- anvil/packaging/base.py | 75 ++++++++++++++++----------- anvil/packaging/helpers/pip_helper.py | 29 ++++------- anvil/packaging/venv.py | 6 --- requirements.txt | 2 +- tools/multipip | 56 +++++++------------- tools/py2rpm | 15 ++++-- 6 files changed, 87 insertions(+), 96 deletions(-) diff --git a/anvil/packaging/base.py b/anvil/packaging/base.py index b797b5d7..4be13fab 100644 --- a/anvil/packaging/base.py +++ b/anvil/packaging/base.py @@ -18,6 +18,8 @@ # R0921: Abstract class not referenced #pylint: disable=R0902,R0921 +import functools + from anvil import colorizer from anvil import exceptions as exc from anvil import log as logging @@ -175,36 +177,9 @@ class DependencyHandler(object): sh.unlink(self.tracereader.filename()) def _scan_pip_requires(self, requires_files): + own_eggs = self._python_eggs(False) - def validate_requirement(filename, source_req): - install_egg = None - for egg_info in self._python_eggs(False): - if egg_info['name'] == source_req.key: - install_egg = egg_info - break - if not install_egg: - return - # Ensure what we are about to install/create will actually work - # with the desired version. If it is not compatible then we should - # abort and someone should update the tag/branch in the origin - # file (or fix it via some other mechanism). - if install_egg['version'] not in source_req: - msg = ("Can not satisfy '%s' with '%s', version" - " conflict found in %s") - raise exc.DependencyException(msg % (source_req, - install_egg['req'], - filename)) - - if not requires_files: - return - utils.log_iterable(sorted(requires_files), - logger=LOG, - header="Scanning %s pip 'requires' files" % (len(requires_files))) - forced_by_key = {} - for pkg in self.forced_pips: - forced_by_key[pkg.key] = pkg - mutations = 0 - for fn in sorted(requires_files): + def replace_forced_requirements(fn, forced_by_key): old_lines = sh.load_file(fn).splitlines() new_lines = [] alterations = [] @@ -232,12 +207,52 @@ class DependencyHandler(object): if alterations: contents = "# Cleaned on %s\n\n%s\n" % (utils.iso8601(), "\n".join(new_lines)) sh.write_file_and_backup(fn, contents) - mutations += len(alterations) utils.log_iterable(alterations, logger=LOG, header="Replaced %s requirements in %s" % (len(alterations), fn), color=None) + return len(alterations) + + def on_replace_done(fn, time_taken): + LOG.debug("Replacing potential forced requirements in %s" + " took %s seconds", colorizer.quote(fn), time_taken) + + def validate_requirement(filename, source_req): + install_egg = None + for egg_info in own_eggs: + if egg_info['name'] == source_req.key: + install_egg = egg_info + break + if not install_egg: + return + # Ensure what we are about to install/create will actually work + # with the desired version. If it is not compatible then we should + # abort and someone should update the tag/branch in the origin + # file (or fix it via some other mechanism). + if install_egg['version'] not in source_req: + msg = ("Can not satisfy '%s' with '%s', version" + " conflict found in %s") + raise exc.DependencyException(msg % (source_req, + install_egg['req'], + filename)) + + if not requires_files: + return + requires_files = sorted(requires_files) + utils.log_iterable(requires_files, + logger=LOG, + header="Scanning %s pip 'requires' files" % (len(requires_files))) + forced_by_key = {} + for pkg in self.forced_pips: + forced_by_key[pkg.key] = pkg + mutations = 0 + for fn in requires_files: + LOG.debug("Replacing any potential forced requirements in %s", + colorizer.quote(fn)) + mutations += utils.time_it(functools.partial(on_replace_done, fn), + replace_forced_requirements, + fn, forced_by_key) # NOTE(imelnikov): after updating requirement lists we should re-fetch # data from them again, so we drop pip helper caches here. if mutations > 0: diff --git a/anvil/packaging/helpers/pip_helper.py b/anvil/packaging/helpers/pip_helper.py index 58ffdde9..e1b6e334 100644 --- a/anvil/packaging/helpers/pip_helper.py +++ b/anvil/packaging/helpers/pip_helper.py @@ -14,7 +14,6 @@ # License for the specific language governing permissions and limitations # under the License. -from distutils import version as dist_version import glob import pkg_resources import re @@ -22,16 +21,13 @@ import sys import tempfile import threading +from pip import download as pip_download from pip import req as pip_req +from pip import utils as pip_util + import pkginfo import six -try: - from pip import util as pip_util -except ImportError: - # pip >=6 changed this location for some reason... - from pip import utils as pip_util - from anvil import log as logging from anvil import shell as sh from anvil import utils @@ -45,7 +41,6 @@ EGGS_DETAILED = {} EGGS_DETAILED_LOCK = threading.RLock() PYTHON_KEY_VERSION_RE = re.compile("^(.+)-([0-9][0-9.a-zA-Z]*)$") -PIP_VERSION = pkg_resources.get_distribution('pip').version PIP_EXECUTABLE = sh.which_first(['pip', 'pip-python']) @@ -208,6 +203,7 @@ def parse_requirements(contents): def read_requirement_files(files): pip_requirements = [] + session = pip_download.PipSession() for filename in files: if sh.isfile(filename): cache_key = "f:%s:%s" % (sh.abspth(filename), sh.getsize(filename)) @@ -215,7 +211,8 @@ def read_requirement_files(files): try: reqs = REQUIREMENT_FILE_CACHE[cache_key] except KeyError: - reqs = tuple(pip_req.parse_requirements(filename)) + reqs = tuple(pip_req.parse_requirements(filename, + session=session)) REQUIREMENT_FILE_CACHE[cache_key] = reqs pip_requirements.extend(reqs) return (pip_requirements, @@ -236,23 +233,15 @@ def download_dependencies(download_dir, pips_to_download, output_filename): if sh.isdir(build_path): sh.deldir(build_path) sh.mkdir(build_path) - # Ensure certain directories exist that we want to exist (but we don't - # want to delete them run after run). - cache_path = sh.joinpths(download_dir, ".cache") - if not sh.isdir(cache_path): - sh.mkdir(cache_path) cmdline = [ PIP_EXECUTABLE, '-v', 'install', '-I', '-U', '--download', download_dir, '--build', build_path, - '--download-cache', cache_path, + # Don't download wheels since we lack the ability to create + # rpms from them (until future when we will have it, if ever)... + "--no-use-wheel", ] - # Don't download wheels... - # - # See: https://github.com/pypa/pip/issues/1439 - if dist_version.StrictVersion(PIP_VERSION) >= dist_version.StrictVersion('1.5'): - cmdline.append("--no-use-wheel") for p in pips_to_download: for p_seg in _split(p): if p_seg: diff --git a/anvil/packaging/venv.py b/anvil/packaging/venv.py index f86294b5..43cf6c17 100644 --- a/anvil/packaging/venv.py +++ b/anvil/packaging/venv.py @@ -64,7 +64,6 @@ class VenvDependencyHandler(base.DependencyHandler): super(VenvDependencyHandler, self).__init__(distro, root_dir, instances, opts, group, prior_groups) - self.cache_dir = sh.joinpths(self.root_dir, "pip-cache") self.jobs = max(0, int(opts.get('jobs', 0))) def _venv_directory_for(self, instance): @@ -81,14 +80,9 @@ class VenvDependencyHandler(base.DependencyHandler): } if extra_env_overrides: env_overrides.update(extra_env_overrides) - sh.mkdirslist(self.cache_dir, tracewriter=self.tracewriter) cmd = list(base_pip) + ['install'] if upgrade: cmd.append("--upgrade") - cmd.extend([ - '--download-cache', - self.cache_dir, - ]) if isinstance(requirements, six.string_types): cmd.extend([ '--requirement', diff --git a/requirements.txt b/requirements.txt index 13e08191..563174ba 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # of appearance. Changing the order has an impact on the overall integration # process (and later running processes...). -pip<6 +pip>=6 pkginfo cheetah>=2.4.4 iniparse diff --git a/tools/multipip b/tools/multipip index f1b6e678..ed357b1a 100755 --- a/tools/multipip +++ b/tools/multipip @@ -5,16 +5,15 @@ from __future__ import print_function import collections import itertools import logging +import pkg_resources import re import sys import argparse import six -import pip.index -import pip.req -import pkg_resources - +from pip import download as pip_download +from pip import req as pip_req # Use this for the sorting order of operators OP_ORDER = ('!=', '==', '<', '<=', '>', '>=') @@ -84,21 +83,6 @@ def setup_logging(options): LOGGER.setLevel(level) -def install_requirement_ensure_req_field(req): - if not hasattr(req, 'req') or not req.req: - # pip 0.8 or so - link = pip.index.Link(req.url) - name = link.egg_fragment - if not name: - raise Exception("Cannot find package name from `%s'" % req.url) - req.req = pkg_resources.Requirement.parse(name) - return req - - -def install_requirement_str(req): - return req.url or str(req.req) - - def install_requirement_parse(line, comes_from): line = line.strip() if line.startswith('-e') or line.startswith('--editable'): @@ -106,11 +90,11 @@ def install_requirement_parse(line, comes_from): line = line[2:].strip() else: line = line[len('--editable'):].strip().lstrip('=') - req = pip.req.InstallRequirement.from_editable( + req = pip_req.InstallRequirement.from_editable( line, comes_from=comes_from) else: - req = pip.req.InstallRequirement.from_line(line, comes_from) - return install_requirement_ensure_req_field(req) + req = pip_req.InstallRequirement.from_line(line, comes_from) + return req def iter_combinations(elements, include_empty=False): @@ -302,7 +286,7 @@ def best_match(req_key, req_list, forced_req=None): for (op, version) in specs: spec_pieces.append("%s%s" % (op, version)) spec = "%s%s" % (req_key, ",".join(spec_pieces)) - sources.append(pip.req.InstallRequirement.from_line(spec, 'compiled')) + sources.append(pip_req.InstallRequirement.from_line(spec, 'compiled')) return sources def reform_incompatibles(incompatible_specs, versions): @@ -323,7 +307,7 @@ def best_match(req_key, req_list, forced_req=None): if not matches: spec_pieces = "%s%s" % (op, version) spec = "%s%s" % (req_key, spec_pieces) - causes.append(pip.req.InstallRequirement.from_line(spec, + causes.append(pip_req.InstallRequirement.from_line(spec, "compiled conflict")) return causes @@ -370,10 +354,10 @@ def parse_requirements(options): all_requirements.setdefault(req_key(req), []).append(req) except Exception as ex: raise RequirementException("Cannot parse `%s': %s" % (req_spec, ex)) + session = pip_download.PipSession() for filename in options.requirements: try: - for req in pip.req.parse_requirements(filename): - req = install_requirement_ensure_req_field(req) + for req in pip_req.parse_requirements(filename, session=session): if skip_match and skip_match.search(req_key(req)): continue all_requirements.setdefault(req_key(req), []).append(req) @@ -418,18 +402,18 @@ def join_requirements(requirements, ignored_requirements, forced_requirements): return (joined_requirements, incompatibles) +def reform_req(req): + if req.editable: + return "-e " + "%s#egg=%s" % (req.link.url_without_fragment, req.req) + else: + return str(req.req) + + def print_requirements(joined_requirements): formatted_requirements = [] for req_key in sorted(six.iterkeys(joined_requirements)): req = joined_requirements[req_key][0] - req_prefix = "" - if req.editable: - req_prefix = "-e " - if req.url: - req = "%s#egg=%s" % (req.url, req.req) - else: - req = str(req.req) - formatted_requirements.append("%s%s" % (req_prefix, req)) + formatted_requirements.append(reform_req(req)) for req in formatted_requirements: print(req) @@ -446,12 +430,12 @@ def print_incompatibles(incompatibles, joined_requirements): print("Choosing:", file=sys.stderr) for chosen in chosen_reqs: print("\t%s: %s" % (chosen.comes_from, - install_requirement_str(chosen)), + reform_req(chosen)), file=sys.stderr) print("Conflicting:", file=sys.stderr) for conflicting in req_incompatibles: print("\t%s: %s" % (conflicting.comes_from, - install_requirement_str(conflicting)), + reform_req(conflicting)), file=sys.stderr) diff --git a/tools/py2rpm b/tools/py2rpm index 272fdf3f..2ffa5f3a 100755 --- a/tools/py2rpm +++ b/tools/py2rpm @@ -15,9 +15,9 @@ import sys import tempfile import argparse +from pip import utils as pip_util import six -import pip.util import pkg_resources @@ -695,7 +695,7 @@ def build_rpm_spec(pkg_key, pkg_name, options, **kwargs): def build_rpm(options, filename, build_options): if os.path.isfile(filename): temp_dir = tempfile.mkdtemp('-unpack', 'py2rpm-') - pip.util.unpack_file(filename, temp_dir, None, None) + pip_util.unpack_file(filename, temp_dir, None, None) source_dir = temp_dir archive_name = filename elif os.path.isdir(filename): @@ -720,7 +720,16 @@ def build_rpm(options, filename, build_options): pkg_key = python_name_to_key(pkg_name) build_dir = options.rpm_base rpm_name = python_key_to_rpm(pkg_key) - version = one_line(info["version"]) + + # NOTE(harlowja): try not to use info["version"] to get the version, since + # currently that is getting normalized by setuptools to be a normalized + # version which can be different from the actual version... + try: + version = setup_py_one_line(source_dir, "--version") + except Exception: + version = info['version'] + logger.warning("Failed extracting version, falling back to '%s'", + version, exc_info=True) cleaned_version = version.replace('-', '_') spec_name = os.path.join(build_dir, "SPECS", "%s.spec" % rpm_name)