Add process_cache from openstack-ci-puppet.

2012-11-22 10:00:22 -08:00 · 2012-11-22 10:00:22 -08:00 · a140e1a13e
parent da3bada534
commit a140e1a13e
2 changed files with 95 additions and 0 deletions
--- a/gerritx/cmd/process_cache.py
+++ b/gerritx/cmd/process_cache.py
@ -0,0 +1,94 @@
+# vim: tabstop=4 shiftwidth=4 softtabstop=4
+
+# Copyright 2012  Hewlett-Packard Development Company, L.P.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+import os
+import urllib
+import datetime
+import sys
+import re
+import md5
+
+
+def main():
+    source_cache = sys.argv[1]
+    destination_mirror = sys.argv[2]
+
+    PACKAGE_VERSION_RE = re.compile(r'(.*)-[0-9]')
+    full_html_line = "<a href='{dir}/{name}'>{name}</a><br />\n"
+
+    packages = {}
+    package_count = 0
+
+    for filename in os.listdir(source_cache):
+        if filename.endswith('content-type'):
+            continue
+
+        realname = urllib.unquote(filename)
+        # The ? accounts for sourceforge downloads
+        tarball = os.path.basename(realname).split("?")[0]
+        name_match = PACKAGE_VERSION_RE.search(tarball)
+
+        if name_match is None:
+            continue
+        package_name = name_match.group(1)
+
+        version_list = packages.get(package_name, {})
+        version_list[tarball] = filename
+        packages[package_name] = version_list
+        package_count = package_count + 1
+
+    full_html = open(os.path.join(destination_mirror, "full.html"), 'w')
+    simple_html = open(os.path.join(destination_mirror, "index.html"), 'w')
+
+    header = ("<html><head><title>PyPI Mirror</title></head>"
+              "<body><h1>PyPI Mirror</h1><h2>Last update: %s</h2>\n\n"
+              % datetime.datetime.utcnow().strftime("%c UTC"))
+    full_html.write(header)
+    simple_html.write(header)
+
+    for package_name, versions in packages.items():
+        destination_dir = os.path.join(destination_mirror, package_name)
+        if not os.path.isdir(destination_dir):
+            os.makedirs(destination_dir)
+        safe_dir = urllib.quote(package_name)
+        simple_html.write("<a href='%s'>%s</a><br />\n" % (safe_dir, safe_dir))
+        with open(os.path.join(destination_dir, "index.html"), 'w') as index:
+            index.write("""<html><head>
+      <title>%s &ndash; PyPI Mirror</title>
+    </head><body>\n""" % package_name)
+            for tarball, filename in versions.items():
+                source_path = os.path.join(source_cache, filename)
+                destination_path = os.path.join(destination_dir, tarball)
+                with open(destination_path, 'w') as dest:
+                    src = open(source_path, 'r').read()
+                    md5sum = md5.md5(src).hexdigest()
+                    dest.write(src)
+
+                    safe_name = urllib.quote(tarball)
+
+                    full_html.write(full_html_line.format(dir=safe_dir,
+                                                          name=safe_name))
+                    index.write("<a href='%s#md5=%s'>%s</a>\n" % (safe_name,
+                                                                  md5sum,
+                                                                  safe_name))
+            index.write("</body></html>\n")
+    footer = """<p class='footer'>Generated by process_cache.py; %d
+    packages mirrored. </p>
+    </body></html>\n""" % package_count
+    full_html.write(footer)
+    full_html.close()
+    simple_html.write(footer)
+    simple_html.close()
--- a/setup.py
+++ b/setup.py
@ -59,6 +59,7 @@ setuptools.setup(
            'fetch-remotes=gerritx.cmd.fetch_remotes:main',
            'manage-projects=gerritx.cmd.manage_projects:main',
            'notify-impact=gerritx.cmd.notify_impact:main',
+            'process-cache=gerritx.cmd.process_cache:main',
            'run-mirror=gerritx.cmd.run_mirror:main',
            'update-blueprint=gerritx.cmd.update_blueprint:main',
            'update-bug=gerritx.cmd.update_bug:main',