zuul-swift-upload support recursive globs

Allow supplying filename and paths with '**' recursive glob matches
to zuul-swift-upload. Since bash (or shell etc) will expand on any
filenames provided to the program this needs to be used in quotes.

Usage example:
./zuul_swift_upload.py my_results.txt '**/sdist/*.zip' output.log

The hierarchy is always flattened meaning the supplied list is
placed in the topmost generated index.html. Sub-folders still keep
their hierarchy.

Change-Id: I9ba04f7e46b579dcf3f8ad0bd188f41fa5dbcad9

jenkins/scripts/zuul_swift_upload.py

@@ -21,6 +21,7 @@ credentials provided by zuul
 
 import argparse
 import logging
+import glob2
 import magic
 import os
 import Queue
@@ -264,7 +265,7 @@ def build_file_list(file_path, logserver_prefix, swift_destination_prefix,
 
                 folder_links.append(file_details)
 
-            for f in sorted(folders):
+            for f in sorted(folders, key=lambda x: x.lower()):
                 filename = os.path.basename(f) + '/'
                 full_path = os.path.join(path, filename)
                 relative_name = os.path.relpath(full_path, parent_dir)
@@ -281,7 +282,7 @@ def build_file_list(file_path, logserver_prefix, swift_destination_prefix,
 
                 folder_links.append(file_details)
 
-            for f in sorted(files):
+            for f in sorted(files, key=lambda x: x.lower()):
                 filename = os.path.basename(f)
                 full_path = os.path.join(path, filename)
                 relative_name = os.path.relpath(full_path, parent_dir)
@@ -365,6 +366,14 @@ def swift_form_post(queue, num_threads):
         t.join()
 
 
+def expand_files(paths):
+    """Expand the provided paths into a list of files/folders"""
+    results = set()
+    for p in paths:
+        results.update(glob2.glob(p))
+    return sorted(results, key=lambda x: x.lower())
+
+
 def grab_args():
     """Grab and return arguments"""
     parser = argparse.ArgumentParser(
@@ -380,7 +389,9 @@ def grab_args():
                         help='do not include links back to a parent dir')
     parser.add_argument('-n', '--name', default="logs",
                         help='The instruction-set to use')
-    parser.add_argument('files', nargs='+', help='the file(s) to upload')
+    parser.add_argument('files', nargs='+',
+                        help='the file(s) to upload with recursive glob '
+                        'matching when supplied as a string')
 
     return parser.parse_args()
 
@@ -406,7 +417,7 @@ if __name__ == '__main__':
     destination_prefix = os.path.join(logserver_prefix,
                                       swift_destination_prefix)
 
-    for file_path in args.files:
+    for file_path in expand_files(args.files):
         file_path = os.path.normpath(file_path)
         if os.path.isfile(file_path):
             filename = os.path.basename(file_path)

nodepool/elements/nodepool-base/install.d/90-venv-swift-logs

@@ -16,4 +16,5 @@
 # This is in /usr instead of /usr/local due to this bug on precise:
 # https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
 sudo -H virtualenv /usr/zuul-swift-logs-env
-sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse requests
+sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
+    requests glob2

nodepool/scripts/prepare_node.sh

@@ -177,7 +177,8 @@ sudo rm -fr /tmp/zuul
 # This is in /usr instead of /usr/local due to this bug on precise:
 # https://bugs.launchpad.net/ubuntu/+source/python2.7/+bug/839588
 sudo -H virtualenv /usr/zuul-swift-logs-env
-sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse requests
+sudo -H /usr/zuul-swift-logs-env/bin/pip install python-magic argparse \
+    requests glob2
 
 sync
 sleep 5