tests work again and no temp files

This commit is contained in:
Sandy Walsh 2014-05-13 02:48:47 +00:00
parent 2dee06e43d
commit 2a2bd5e54c
3 changed files with 45 additions and 23 deletions

View File

@ -4,7 +4,10 @@ shoebox
binary data archiving library - supports uploading to object storage
There are ArchiveReaders and ArchiveWriters which are managed
by RollManager. The RollManager opens and closes Archivers as
by RollManager. "Roll" comes from "roll over". When does a file roll-over
from one to the next? There is only one Archiver active at a time.
The RollManager opens and closes Archivers as
needed. "As needed" is determined by which RollChecker that was
passed into the RollManager. Archive files can roll over based
on file size or elapsed time (for writing). For reading, archive
@ -14,5 +17,23 @@ Roll Managers also take care of filename creation, compression
of completed archives and transfer of archive files to remote
storage locations.
The RollChecker's have a reference to the current Archive so
they can ask file-related questions (like "how big are you?")
Usage:
# Make a roll checker of whatever strategy you choose.
checker = roll_checker.NeverRollChecker() # one big file.
# Make a roll manager for reading or writing.
# Give the filename template and the checker.
# (and an optional working directory for new files)
x = roll_manager.WritingRollManager("template_%s", checker)
# Write metadata and payload ...
for index in range(10):
x.write({"index": str(index)}, "payload_%d" % index)
# WritingRollManager.write(metadata, payload) where
# metadata = string:string dict
# payload = string of data. Most likely a json structure.
TODO: How will the ReadingRollManager know which files to read
from, and in which order, if the filename is templated?

View File

@ -13,6 +13,8 @@
# See the License for the specific language governing permissions and
# limitations under the License.
import disk_storage
class Archive(object):
def __init__(self, filename):
@ -28,10 +30,16 @@ class ArchiveWriter(Archive):
"""
def __init__(self, filename):
super(ArchiveWriter, self).__init__(filename)
self._open_file(filename)
def _open_file(self, filename):
# Broken out for testing.
self._handle = open(filename, "wb+")
def write(self, payload):
pass
def write(self, metadata, payload):
binary = disk_storage.pack_notification(payload, metadata)
for block in binary:
self._handle.write(block)
class ArchiveReader(Archive):
@ -40,11 +48,5 @@ class ArchiveReader(Archive):
def __init__(self, filename):
super(ArchiveReader, self).__init__(filename)
def read_block(self):
pass
def read_header(self):
pass
def read_payload(self):
def read(self):
pass

View File

@ -44,29 +44,28 @@ class RollManager(object):
class ReadingRollManager(RollManager):
def __init__(self, filename_template, roll_checker, directory="."):
def __init__(self, filename_template, roll_checker, directory=".",
archive_class = archive.ArchiveReader):
super(ReadingRollManager, self).__init__(filename_template,
roll_checker, directory)
self.archive_class = archive.ArchiveReader
self.archive_class = archive_class
def read_block(self):
pass
def read_header(self):
pass
def read_payload(self):
def read(self):
pass
class WritingRollManager(RollManager):
def __init__(self, filename_template, roll_checker, directory="."):
def __init__(self, filename_template, roll_checker, directory=".",
archive_class = archive.ArchiveWriter):
super(WritingRollManager, self).__init__(filename_template,
roll_checker, directory)
self.archive_class = archive.ArchiveWriter
self.archive_class = archive_class
def write(self, payload):
def write(self, metadata, payload):
"""metadata is string:string dict.
payload must be encoded as string.
"""
a = self.get_active_archive()
a.write(payload)
a.write(metadata, payload)
if self._should_roll_archive():
self._roll_archive()