From 2d61e9d4fc32b85a30afc05c12e4fd3a4e40c1ed Mon Sep 17 00:00:00 2001 From: Ian Wienand Date: Fri, 17 Aug 2018 14:17:20 +1000 Subject: [PATCH] upload-logs-swift: Make indexer more generic This converts the Indexer class from something that strictly generates index.html files for folders to a more generic class that will be able to hold other types of transformations we might like to do on the total collections of uploaded files. The index.html specific arguments are moved into make_indexes() and the two helper functions that should not be called externally are renamed private. Change-Id: I388042ffb6a74c3200d92fb3a084369fcf2cf3a9 --- .../library/test_zuul_swift_upload.py | 16 +- .../library/zuul_swift_upload.py | 174 +++++++++--------- 2 files changed, 99 insertions(+), 91 deletions(-) diff --git a/roles/upload-logs-swift/library/test_zuul_swift_upload.py b/roles/upload-logs-swift/library/test_zuul_swift_upload.py index be69c78dc..39f23b40f 100644 --- a/roles/upload-logs-swift/library/test_zuul_swift_upload.py +++ b/roles/upload-logs-swift/library/test_zuul_swift_upload.py @@ -257,10 +257,10 @@ class TestFileList(testtools.TestCase): '''Test index generation creates topdir parent link''' with FileList() as fl: fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl, - create_parent_links=True, - create_topdir_parent_link=True) - ix.make_indexes() + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=True, + create_topdir_parent_link=True) self.assert_files(fl, [ ('', 'application/directory', None), @@ -312,10 +312,10 @@ class TestFileList(testtools.TestCase): '''Test index generation creates topdir parent link''' with FileList() as fl: fl.add(os.path.join(FIXTURE_DIR, 'logs/')) - ix = Indexer(fl, - create_parent_links=False, - create_topdir_parent_link=False) - ix.make_indexes() + ix = Indexer(fl) + ix.make_indexes( + create_parent_links=False, + create_topdir_parent_link=False) self.assert_files(fl, [ ('', 'application/directory', None), diff --git a/roles/upload-logs-swift/library/zuul_swift_upload.py b/roles/upload-logs-swift/library/zuul_swift_upload.py index 04d907000..ffa7398d7 100755 --- a/roles/upload-logs-swift/library/zuul_swift_upload.py +++ b/roles/upload-logs-swift/library/zuul_swift_upload.py @@ -282,87 +282,23 @@ class FileList(Sequence): class Indexer(): - """generates index.html files if requested.""" + """Index a FileList - def __init__(self, file_list, create_parent_links=True, - create_topdir_parent_link=False, - append_footer='index_footer.html'): + Functions to generate indexes and other collated data for a + FileList + - make_indexes() : make index.html in folders + """ + def __init__(self, file_list): ''' Args: - file_list (FileList): A FileList object to be updated - with index files for each directory. - create_parent_links (bool): - create_topdir_parent_link (bool): - append_footer (str): + file_list (FileList): A FileList object with all files + to be indexed. ''' assert isinstance(file_list, FileList) self.file_list = file_list - self.create_parent_links = create_parent_links - self.create_topdir_parent_link = create_topdir_parent_link - self.append_footer = append_footer - self.index_filename = 'index.html' - def make_indexes(self): - '''Make index files - - Return: - No value, the self.file_list will be updated - ''' - folders = collections.OrderedDict() - for f in self.file_list: - if f.folder: - folders[f.relative_path] = [] - folder = os.path.dirname(os.path.dirname( - f.relative_path + '/')) - if folder == '/': - folder = '' - else: - folder = os.path.dirname(f.relative_path) - folders[folder].append(f) - - indexes = {} - parent_file_detail = FileDetail(None, '..', '..') - for folder, files in folders.items(): - # Don't add the pseudo-top-directory - if files and files[0].full_path is None: - files = files[1:] - if self.create_topdir_parent_link: - files = [parent_file_detail] + files - elif self.create_parent_links: - files = [parent_file_detail] + files - - # Do generate a link to the parent directory - full_path = self.make_index_file(files, 'Index of %s' % (folder,), - self.file_list.get_tempdir()) - - if full_path: - filename = os.path.basename(full_path) - relative_name = os.path.join(folder, filename) - indexes[folder] = FileDetail(full_path, relative_name) - - # This appends the index file at the end of the group of files - # for each directory. - new_list = [] - last_dirname = None - for f in reversed(list(self.file_list)): - if f.folder: - relative_path = f.relative_path + '/' - else: - relative_path = f.relative_path - dirname = os.path.dirname(relative_path) - if dirname == '/': - dirname = '' - if dirname != last_dirname: - index = indexes.pop(dirname, None) - if index: - new_list.append(index) - last_dirname = dirname - new_list.append(f) - new_list.reverse() - self.file_list.file_list = new_list - - def make_index_file(self, folder_links, title, tempdir): + def _make_index_file(self, folder_links, title, tempdir, append_footer): """Writes an index into a file for pushing""" for file_details in folder_links: # Do not generate an index file if one exists already. @@ -370,12 +306,13 @@ class Indexer(): # content like python coverage info. if self.index_filename == file_details.filename: return - index_content = self.generate_log_index(folder_links, title) + index_content = self._generate_log_index( + folder_links, title, append_footer) fd = open(os.path.join(tempdir, self.index_filename), 'w') fd.write(index_content) return os.path.join(tempdir, self.index_filename) - def generate_log_index(self, folder_links, title): + def _generate_log_index(self, folder_links, title, append_footer): """Create an index of logfiles and links to them""" output = '%s\n' % title @@ -403,8 +340,8 @@ class Indexer(): output += '%s' % size output += '\n' - if (self.append_footer and - self.append_footer in file_details.filename): + if (append_footer and + append_footer in file_details.filename): file_details_to_append = file_details output += '' @@ -420,6 +357,78 @@ class Indexer(): output += '\n' return output + def make_indexes(self, create_parent_links=True, + create_topdir_parent_link=False, + append_footer='index_footer.html'): + '''Make index.html files + + Iterate the file list and crete index.html files for folders + + Args: + create_parent_links (bool): Create parent links + create_topdir_parent_link (bool): Create topdir parent link + append_footer (str): Filename of a footer to append to each + generated page + + Return: + No value, the self.file_list will be updated + ''' + self.index_filename = 'index.html' + + folders = collections.OrderedDict() + for f in self.file_list: + if f.folder: + folders[f.relative_path] = [] + folder = os.path.dirname(os.path.dirname( + f.relative_path + '/')) + if folder == '/': + folder = '' + else: + folder = os.path.dirname(f.relative_path) + folders[folder].append(f) + + indexes = {} + parent_file_detail = FileDetail(None, '..', '..') + for folder, files in folders.items(): + # Don't add the pseudo-top-directory + if files and files[0].full_path is None: + files = files[1:] + if create_topdir_parent_link: + files = [parent_file_detail] + files + elif create_parent_links: + files = [parent_file_detail] + files + + # Do generate a link to the parent directory + full_path = self._make_index_file(files, 'Index of %s' % (folder,), + self.file_list.get_tempdir(), + append_footer) + + if full_path: + filename = os.path.basename(full_path) + relative_name = os.path.join(folder, filename) + indexes[folder] = FileDetail(full_path, relative_name) + + # This appends the index file at the end of the group of files + # for each directory. + new_list = [] + last_dirname = None + for f in reversed(list(self.file_list)): + if f.folder: + relative_path = f.relative_path + '/' + else: + relative_path = f.relative_path + dirname = os.path.dirname(relative_path) + if dirname == '/': + dirname = '' + if dirname != last_dirname: + index = indexes.pop(dirname, None) + if index: + new_list.append(index) + last_dirname = dirname + new_list.append(f) + new_list.reverse() + self.file_list.file_list = new_list + class DeflateFilter(): chunk_size = 16384 @@ -585,18 +594,17 @@ def run(cloud, container, files, # Create the objects to make sure the arguments are sound. with FileList() as file_list: - indexer = Indexer(file_list, - create_parent_links=parent_links, - create_topdir_parent_link=topdir_parent_link, - append_footer=footer) - # Scan the files. for file_path in files: file_list.add(file_path) + indexer = Indexer(file_list) + # (Possibly) make indexes. if indexes: - indexer.make_indexes() + indexer.make_indexes(create_parent_links=parent_links, + create_topdir_parent_link=topdir_parent_link, + append_footer=footer) logging.debug("List of files prepared to upload:") for x in file_list: