From 384c02d41ff292d36f24e9c82fc88a8d1f796878 Mon Sep 17 00:00:00 2001 From: Monty Taylor Date: Fri, 9 Aug 2013 19:37:57 -0300 Subject: [PATCH] Start building wheels in the mirror The mirror should build wheels, because they're awesome. However, because the wheel format does not do fine-grained enough selection on things like the high likelihood that Ubuntu precise and Ubuntu raring might have different underlying C libraries for python 2.7, we need to put our built wheels into a structure that allows them to be used safely. lsb_release -r -i is used because Centos lists lsb_release -c as "Final" which is not particularly helpful. This will wind up with identifying strings like Ubuntu-12.04 and Centos-6.4 - which should be good enough for what we need. A call to a shell is used so that the same logic can be easily duplicated in select-mirror, which is shell, to produce the appropriate pip.conf file. Change-Id: Icf360c921dbc357dbeb43386c6e8f51bd5f993ff --- jeepyb/cmd/run_mirror.py | 138 +++++++++++++++++++++++++++++---------- requirements.txt | 1 + 2 files changed, 104 insertions(+), 35 deletions(-) diff --git a/jeepyb/cmd/run_mirror.py b/jeepyb/cmd/run_mirror.py index 5e54583..d3b748e 100644 --- a/jeepyb/cmd/run_mirror.py +++ b/jeepyb/cmd/run_mirror.py @@ -158,11 +158,25 @@ class Mirror(object): def build_mirror(self, mirror): print("Building mirror: %s" % mirror['name']) - pip_format = ("%s install -U %s --exists-action=w " - "--download-cache=%s --build %s -r %s") - venv_format = ("virtualenv --clear --extra-search-dir=%s %s") - upgrade_format = ("%s install -U --exists-action=w " - "--download-cache=%s --build %s %s") + pip_format = ( + "%(pip)s install -U %(extra_args)s --exists-action=w --use-wheel" + " --download-cache=%(download_cache)s" + " --build %(build_dir)s -f %(find_links)s" + " -r %(requirements_file)s") + venv_format = ( + "virtualenv --clear --extra-search-dir=%(extra_search_dir)s" + " %(venv_dir)s") + upgrade_format = ( + "%(pip)s install -U --use-wheel --exists-action=w" + " --download-cache=%(download_cache)s --build %(build_dir)s" + " -f %(find_links)s %(requirement)s") + wheel_file_format = ( + "%(pip)s wheel --download-cache=%(download_cache)s" + " --use-wheel --wheel-dir %(wheel_dir)s -f %(find_links)s" + " -r %(requirements_file)s") + wheel_format = ( + "%(pip)s wheel --download-cache=%(download_cache)s" + " -f %(find_links)s --wheel-dir %(wheel_dir)s %(requirement)s") workdir = tempfile.mkdtemp() reqs = os.path.join(workdir, "reqs") @@ -174,11 +188,11 @@ class Mirror(object): 'projects') pip_cache_dir = os.path.join(self.config['cache-root'], 'pip', mirror['name']) + wheelhouse = os.path.join(self.config['cache-root'], "wheelhouse") if not self.args.noop: - if not os.path.exists(project_cache_dir): - os.makedirs(project_cache_dir) - if not os.path.exists(pip_cache_dir): - os.makedirs(pip_cache_dir) + for new_dir in (project_cache_dir, pip_cache_dir, wheelhouse): + if not os.path.exists(new_dir): + os.makedirs(new_dir) for project in mirror['projects']: print("Updating repository: %s" % project) @@ -218,17 +232,31 @@ class Mirror(object): if os.path.exists(requires_file): reqlist.append(requires_file) if reqlist: - out = self.run_command(venv_format % - (pip_cache_dir, venv)) - out = self.run_command(upgrade_format % - (pip, pip_cache_dir, - build, "setuptools")) - out = self.run_command(upgrade_format % - (pip, pip_cache_dir, - build, "pip")) - out = self.run_command(upgrade_format % - (pip, pip_cache_dir, - build, "virtualenv")) + out = self.run_command( + venv_format % dict( + extra_search_dir=pip_cache_dir, venv_dir=venv)) + # Need to do these separately. If you attempt to upgrade + # setuptools with something else, you can get into a + # situation where distribute has been upgraded, but pip + # attemps to install something else before installing + # the setuptools replacement. The safest thing is to + # simply upgrade setuptools by itself. + # There is a current theory that pip 1.4 may solve + # the setuptools upgrade issues, so upgrading that first + # is a good idea + for requirement in [ + "pip", "setuptools", "wheel", "virtualenv"]: + self.run_command( + upgrade_format % dict( + pip=pip, download_cache=pip_cache_dir, + build_dir=build, find_links=wheelhouse, + requirement=requirement)) + for requirement in ["pip", "setuptools", "virtualenv"]: + self.run_command( + wheel_format % dict( + pip=pip, download_cache=pip_cache_dir, + find_links=wheelhouse, wheel_dir=wheelhouse, + requirement=requirement)) if os.path.exists(build): shutil.rmtree(build) new_reqs = self.process_http_requirements(reqlist, @@ -237,9 +265,16 @@ class Mirror(object): (reqfp, reqfn) = tempfile.mkstemp() os.write(reqfp, '\n'.join(new_reqs)) os.close(reqfp) - out = self.run_command(pip_format % - (pip, "", pip_cache_dir, - build, reqfn)) + out = self.run_command( + wheel_file_format % dict( + pip=pip, download_cache=pip_cache_dir, + find_links=wheelhouse, wheel_dir=wheelhouse, + requirements_file=reqfn)) + out = self.run_command( + pip_format % dict( + pip=pip, extra_args="", + download_cache=pip_cache_dir, build_dir=build, + find_links=wheelhouse, requirements_file=reqfn)) if "\nSuccessfully installed " not in out: sys.stderr.write("Installing pip requires for %s:%s " "failed.\n%s\n" % @@ -256,13 +291,20 @@ class Mirror(object): for r in requires: reqfd.write(r + "\n") reqfd.close() - out = self.run_command(venv_format % - (pip_cache_dir, venv)) + out = self.run_command(venv_format % dict( + extra_search_dir=pip_cache_dir, venv_dir=venv)) if os.path.exists(build): shutil.rmtree(build) - out = self.run_command(pip_format % - (pip, "--no-install", - pip_cache_dir, build, reqs)) + out = self.run_command( + wheel_file_format % dict( + pip=pip, download_cache=pip_cache_dir, + find_links=wheelhouse, wheel_dir=wheelhouse, + requirements_file=reqs)) + out = self.run_command( + pip_format % dict( + pip=pip, extra_args="--no-install", + download_cache=pip_cache_dir, build_dir=build, + find_links=wheelhouse, requirements_file=reqs)) if "\nSuccessfully downloaded " not in out: sys.stderr.write("Downloading pip requires for " "%s:%s failed.\n%s\n" % @@ -273,23 +315,27 @@ class Mirror(object): print("no requirements") shutil.rmtree(workdir) + def _get_distro(self): + out = self.run_command('lsb_release -i -r -s') + return out.strip().replace('\n', '-') + def process_cache(self, mirror): if self.args.noop: return + self._write_tarball_mirror(mirror) + self._write_wheel_mirror(mirror) + + def _write_tarball_mirror(self, mirror): pip_cache_dir = os.path.join(self.config['cache-root'], 'pip', mirror['name']) destination_mirror = mirror['output'] PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]') - full_html_line = "{name}
\n" packages = {} package_count = 0 - if not os.path.exists(destination_mirror): - os.makedirs(destination_mirror) - for filename in os.listdir(pip_cache_dir): if filename.endswith('content-type'): continue @@ -304,9 +350,32 @@ class Mirror(object): package_name = name_match.group(1) version_list = packages.get(package_name, {}) - version_list[tarball] = filename + version_list[tarball] = os.path.join(pip_cache_dir, filename) packages[package_name] = version_list package_count = package_count + 1 + self._write_mirror(destination_mirror, packages, package_count) + + def _write_wheel_mirror(self, mirror): + + distro = self._get_distro() + wheelhouse = os.path.join(self.config['cache-root'], "wheelhouse") + wheel_destination_mirror = os.path.join(mirror['output'], distro) + packages = {} + package_count = 0 + + for filename in os.listdir(wheelhouse): + package_name = filename.split('-')[0].replace('_', '-') + version_list = packages.get(package_name, {}) + version_list[filename] = os.path.join(wheelhouse, filename) + packages[package_name] = version_list + package_count = package_count + 1 + self._write_mirror(wheel_destination_mirror, packages, package_count) + + def _write_mirror(self, destination_mirror, packages, package_count): + full_html_line = "{name}
\n" + + if not os.path.exists(destination_mirror): + os.makedirs(destination_mirror) full_html = open(os.path.join(destination_mirror, ".full.html"), 'w') simple_html = open(os.path.join(destination_mirror, ".index.html"), @@ -330,8 +399,7 @@ class Mirror(object): index.write(""" %s – PyPI Mirror \n""" % package_name) - for tarball, filename in versions.items(): - source_path = os.path.join(pip_cache_dir, filename) + for tarball, source_path in versions.items(): destination_path = os.path.join(destination_dir, tarball) dot_destination_path = os.path.join(destination_dir, diff --git a/requirements.txt b/requirements.txt index 17f92bb..d223638 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,3 +6,4 @@ pyyaml pkginfo PyRSS2Gen python-swiftclient +pip>=1.4