diff --git a/README.md b/README.md index 20ccbac..576cd23 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,10 @@ shoebox binary data archiving library - supports uploading to object storage There are ArchiveReaders and ArchiveWriters which are managed -by RollManager. The RollManager opens and closes Archivers as +by RollManager. "Roll" comes from "roll over". When does a file roll-over +from one to the next? There is only one Archiver active at a time. + +The RollManager opens and closes Archivers as needed. "As needed" is determined by which RollChecker that was passed into the RollManager. Archive files can roll over based on file size or elapsed time (for writing). For reading, archive @@ -14,5 +17,23 @@ Roll Managers also take care of filename creation, compression of completed archives and transfer of archive files to remote storage locations. +The RollChecker's have a reference to the current Archive so +they can ask file-related questions (like "how big are you?") + +Usage: + + # Make a roll checker of whatever strategy you choose. + checker = roll_checker.NeverRollChecker() # one big file. + # Make a roll manager for reading or writing. + # Give the filename template and the checker. + # (and an optional working directory for new files) + x = roll_manager.WritingRollManager("template_%s", checker) + # Write metadata and payload ... + for index in range(10): + x.write({"index": str(index)}, "payload_%d" % index) + # WritingRollManager.write(metadata, payload) where + # metadata = string:string dict + # payload = string of data. Most likely a json structure. + TODO: How will the ReadingRollManager know which files to read from, and in which order, if the filename is templated? diff --git a/shoebox/archive.py b/shoebox/archive.py index 5967789..3b63422 100644 --- a/shoebox/archive.py +++ b/shoebox/archive.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +import disk_storage + class Archive(object): def __init__(self, filename): @@ -28,10 +30,16 @@ class ArchiveWriter(Archive): """ def __init__(self, filename): super(ArchiveWriter, self).__init__(filename) + self._open_file(filename) + + def _open_file(self, filename): + # Broken out for testing. self._handle = open(filename, "wb+") - def write(self, payload): - pass + def write(self, metadata, payload): + binary = disk_storage.pack_notification(payload, metadata) + for block in binary: + self._handle.write(block) class ArchiveReader(Archive): @@ -40,11 +48,5 @@ class ArchiveReader(Archive): def __init__(self, filename): super(ArchiveReader, self).__init__(filename) - def read_block(self): - pass - - def read_header(self): - pass - - def read_payload(self): + def read(self): pass diff --git a/shoebox/roll_manager.py b/shoebox/roll_manager.py index bd6dd95..be038ef 100644 --- a/shoebox/roll_manager.py +++ b/shoebox/roll_manager.py @@ -44,29 +44,28 @@ class RollManager(object): class ReadingRollManager(RollManager): - def __init__(self, filename_template, roll_checker, directory="."): + def __init__(self, filename_template, roll_checker, directory=".", + archive_class = archive.ArchiveReader): super(ReadingRollManager, self).__init__(filename_template, roll_checker, directory) - self.archive_class = archive.ArchiveReader + self.archive_class = archive_class - def read_block(self): - pass - - def read_header(self): - pass - - def read_payload(self): + def read(self): pass class WritingRollManager(RollManager): - def __init__(self, filename_template, roll_checker, directory="."): + def __init__(self, filename_template, roll_checker, directory=".", + archive_class = archive.ArchiveWriter): super(WritingRollManager, self).__init__(filename_template, roll_checker, directory) - self.archive_class = archive.ArchiveWriter + self.archive_class = archive_class - def write(self, payload): + def write(self, metadata, payload): + """metadata is string:string dict. + payload must be encoded as string. + """ a = self.get_active_archive() - a.write(payload) + a.write(metadata, payload) if self._should_roll_archive(): self._roll_archive()