Add manifest parser and generator
This reads the manifest. Its more based on osdk.py that I initially thought it would be. But hopefully I have cleaned it up a good bit. Add tool to generate manifest To generate a manifest with 2^24 objects (64TB for 4MB objects) run: tools/generate_manifest.py --backupsize 64000 --manifest /pathtosave/manifest Change-Id: I09a8fb64c0d4b8e4b4b436cca44b57bec87116cd
This commit is contained in:
SamYaple
parent
e6d67dca74
commit
4795802efd
|
|
@ -0,0 +1,190 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2016 Sam Yaple
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copied and licensed from https://github.com/SamYaple/osdk
|
||||
|
||||
from binascii import crc32
|
||||
from collections import namedtuple
|
||||
from datetime import datetime
|
||||
from struct import pack
|
||||
from struct import unpack
|
||||
from uuid import UUID
|
||||
|
||||
import six
|
||||
|
||||
SIGNATURE = 'd326503ab5ca49adac56c89eb0b8ef08d326503ab5ca49adac56c89eb0b8ef08'
|
||||
|
||||
|
||||
class EkkoShortReadError(Exception):
|
||||
|
||||
def __init__(self, size_read, size_requested):
|
||||
self.size_read = size_read
|
||||
self.size_requested = size_requested
|
||||
|
||||
|
||||
class EkkoManifestTooNewError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class EkkoChecksumError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class EkkoInvalidSignatureError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class Manifest(object):
|
||||
|
||||
def __init__(self, manifest):
|
||||
self.manifest = manifest
|
||||
self.metadata = {'version': 0}
|
||||
|
||||
def write_manifest(self):
|
||||
with open(self.manifest, 'wb', 1) as f:
|
||||
self.write_header(f)
|
||||
self.write_body(f)
|
||||
|
||||
def build_header(self):
|
||||
data = pack(
|
||||
'<i2IQH14s',
|
||||
utctimestamp(),
|
||||
self.metadata['info'].incremental,
|
||||
self.metadata['info'].segment_size,
|
||||
self.metadata['info'].sectors,
|
||||
len(self.metadata['bases']),
|
||||
str.encode('\0\0' * 14)
|
||||
)
|
||||
|
||||
checksum = crc32(data)
|
||||
|
||||
for i in self.metadata['bases']:
|
||||
data += i
|
||||
checksum = crc32(i, checksum)
|
||||
|
||||
return data, checksum
|
||||
|
||||
def write_body(self, f):
|
||||
checksum = 0
|
||||
|
||||
for k, v in six.iteritems(self.segments):
|
||||
data = pack(
|
||||
'<IHI2B20s',
|
||||
k,
|
||||
v.base,
|
||||
v.incremental,
|
||||
v.compression,
|
||||
v.encryption,
|
||||
self.hashes[k]
|
||||
)
|
||||
|
||||
f.write(data)
|
||||
checksum = crc32(data, checksum)
|
||||
|
||||
# Backfill the body_checksum
|
||||
f.seek(24, 0)
|
||||
f.write(pack('<i', checksum))
|
||||
|
||||
def write_header(self, f):
|
||||
data, checksum = self.build_header()
|
||||
|
||||
def read_data(self, f, size_requested):
|
||||
data = f.read(size_requested)
|
||||
size_read = len(data)
|
||||
if size_read != size_requested:
|
||||
raise EkkoShortReadError(
|
||||
'Failed to read amount of requested data',
|
||||
size_read,
|
||||
size_requested
|
||||
)
|
||||
self.checksum = crc32(data)
|
||||
return data
|
||||
|
||||
def read_signature(self, f):
|
||||
if not UUID(SIGNATURE).bytes == self.read_data(f, 32):
|
||||
raise EkkoInvalidSignatureError('File signiture is not valid')
|
||||
|
||||
def read_header(self, f):
|
||||
self.checksum = 0
|
||||
Info = namedtuple(
|
||||
'Info',
|
||||
'timestamp incremental segment_size sectors'
|
||||
)
|
||||
|
||||
self.read_signature(f)
|
||||
|
||||
version, header_checksum, body_checksum = unpack(
|
||||
'<I2i', self.read_data(f, 12)
|
||||
)
|
||||
|
||||
if self.metadata['version'] < version:
|
||||
raise EkkoManifestTooNewError(
|
||||
'The manifest version is newer than I know how to read'
|
||||
)
|
||||
|
||||
self.metadata['info'] = Info._make(
|
||||
unpack('<i2IQ', self.read_data(f, 20))
|
||||
)
|
||||
|
||||
num_of_bases, _ = unpack('<H14s', self.read_data(f, 16))
|
||||
|
||||
self.metadata['bases'] = [
|
||||
self.read_data(f, 16) for x in six.moves.range(0, num_of_bases)
|
||||
]
|
||||
|
||||
if self.checksum != header_checksum:
|
||||
raise EkkoChecksumError('Header checksum does not match')
|
||||
|
||||
return body_checksum
|
||||
|
||||
def read_body(self, f, body_checksum):
|
||||
self.checksum = 0
|
||||
self.segments = dict()
|
||||
self.hashes = dict()
|
||||
Segment = namedtuple(
|
||||
'Segment',
|
||||
'base incremental compression encryption'
|
||||
)
|
||||
|
||||
try:
|
||||
while True:
|
||||
processing_segment = True
|
||||
|
||||
segment, base = unpack('<IH', self.read_data(f, 6))
|
||||
|
||||
self.segments[segment] = Segment(
|
||||
self.metadata['bases'][base],
|
||||
unpack('<I2B', self.read_data(f, 6))
|
||||
)
|
||||
|
||||
self.hashes[segment] = unpack('<20s', self.read_data(f, 20))
|
||||
|
||||
processing_segment = False
|
||||
except EkkoShortReadError as e:
|
||||
if processing_segment or e.size_of_read != 0:
|
||||
raise
|
||||
|
||||
if self.checksum != body_checksum:
|
||||
raise EkkoChecksumError('Body checksum does not match')
|
||||
|
||||
def read_manifest(self):
|
||||
with open(self.manifest, 'rb', 1) as f:
|
||||
self.read_body(f, self.read_header(f))
|
||||
|
||||
|
||||
def utctimestamp():
|
||||
ts = datetime.utcnow() - datetime(1970, 1, 1)
|
||||
return ts.seconds + ts.days * 24 * 3600
|
||||
|
|
@ -3,3 +3,4 @@
|
|||
# process, which may cause wedges in the gate later.
|
||||
|
||||
pbr>=1.6
|
||||
six>=1.9.0
|
||||
|
|
|
|||
|
|
@ -0,0 +1,112 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2016 Sam Yaple
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copied and licensed from https://github.com/SamYaple/osdk
|
||||
|
||||
|
||||
import argparse
|
||||
from collections import namedtuple
|
||||
# from hashlib import sha1
|
||||
import os
|
||||
import sys
|
||||
from uuid import uuid4 as uuid
|
||||
|
||||
from ekko import manifest
|
||||
from six.moves import range
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Backup Block Device')
|
||||
parser.add_argument('--backupsize', required=True, type=int,
|
||||
help='Size of backup for manifest gen (size in GB)')
|
||||
parser.add_argument('--manifest', required=True,
|
||||
help='manifest file')
|
||||
parser.add_argument('--cbt', required=False,
|
||||
help='change block tracking info')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_segments(segments, size, backup):
|
||||
backup.segments = dict()
|
||||
backup.hashes = dict()
|
||||
Segment = namedtuple(
|
||||
'Segment',
|
||||
'base incremental compression encryption'
|
||||
)
|
||||
|
||||
for segment in segments:
|
||||
# Generate manifest info for each object in backup
|
||||
backup.segments[segment] = Segment(
|
||||
len(backup.metadata['bases']) - 1,
|
||||
backup.metadata['info'].incremental,
|
||||
0,
|
||||
0
|
||||
)
|
||||
# Random string simulating hash sha
|
||||
backup.hashes[segment] = os.urandom(20)
|
||||
|
||||
|
||||
def generate_mem_struct(segments, size, backup):
|
||||
b = {
|
||||
'96153320-980b-4b5e-958f-ea57812b280d': []
|
||||
}
|
||||
|
||||
for seg in segments:
|
||||
b['96153320-980b-4b5e-958f-ea57812b280d'].append({
|
||||
seg: backup.metadata['info'].incremental
|
||||
})
|
||||
|
||||
return b
|
||||
|
||||
|
||||
def check_manifest(manifest_file):
|
||||
return os.path.isfile(manifest_file)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
segment_size = 4 * 1024**2 # 4MiB
|
||||
size_of_disk = args.backupsize * 1024**3 # Convert GB to B
|
||||
num_of_sectors = int(size_of_disk / 512)
|
||||
num_of_segments = int(size_of_disk / segment_size)
|
||||
incremental = 0
|
||||
|
||||
Info = namedtuple(
|
||||
'Info',
|
||||
'timestamp incremental segment_size sectors'
|
||||
)
|
||||
|
||||
if check_manifest(args.manifest):
|
||||
print('manifest exists; exiting')
|
||||
return
|
||||
|
||||
backup = manifest.Manifest(args.manifest)
|
||||
|
||||
backup.metadata['info'] = Info(
|
||||
manifest.utctimestamp(),
|
||||
incremental,
|
||||
segment_size,
|
||||
num_of_sectors,
|
||||
)
|
||||
|
||||
backup.metadata['bases'] = [uuid().bytes]
|
||||
|
||||
# read_segments(range(0, num_of_segments - 1), segment_size, backup)
|
||||
generate_mem_struct(range(0, num_of_segments - 1), segment_size, backup)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
|
|
@ -0,0 +1,100 @@
|
|||
#!/usr/bin/python
|
||||
|
||||
# Copyright 2016 Sam Yaple
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Copied and licensed from https://github.com/SamYaple/osdk
|
||||
|
||||
|
||||
import argparse
|
||||
from collections import namedtuple
|
||||
# from hashlib import sha1
|
||||
import os
|
||||
import sys
|
||||
from uuid import uuid4 as uuid
|
||||
|
||||
sys.path.insert(0, '/root/ekko/')
|
||||
from ekko import manifest
|
||||
from six.moves import range
|
||||
|
||||
|
||||
def parse_args():
|
||||
parser = argparse.ArgumentParser(description='Backup Block Device')
|
||||
parser.add_argument('--backupsize', required=True, type=int,
|
||||
help='Size of backup for manifest gen (size in GB)')
|
||||
parser.add_argument('--manifest', required=True,
|
||||
help='manifest file')
|
||||
parser.add_argument('--cbt', required=False,
|
||||
help='change block tracking info')
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def read_segments(segments, size, backup):
|
||||
backup.segments = dict()
|
||||
backup.hashes = dict()
|
||||
Segment = namedtuple(
|
||||
'Segment',
|
||||
'base incremental compression encryption'
|
||||
)
|
||||
|
||||
for segment in segments:
|
||||
# Generate manifest info for each object in backup
|
||||
backup.segments[segment] = Segment(
|
||||
len(backup.metadata['bases']) - 1,
|
||||
backup.metadata['info'].incremental,
|
||||
0,
|
||||
0
|
||||
)
|
||||
# Random string simulating hash sha
|
||||
backup.hashes[segment] = os.urandom(20)
|
||||
|
||||
|
||||
def check_manifest(manifest_file):
|
||||
return os.path.isfile(manifest_file)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_args()
|
||||
segment_size = 4 * 1024**2 # 4MiB
|
||||
size_of_disk = args.backupsize * 1024**3 # Convert GB to B
|
||||
num_of_sectors = int(size_of_disk / 512)
|
||||
num_of_segments = int(size_of_disk / segment_size)
|
||||
incremental = 0
|
||||
|
||||
Info = namedtuple(
|
||||
'Info',
|
||||
'timestamp incremental segment_size sectors'
|
||||
)
|
||||
|
||||
if check_manifest(args.manifest):
|
||||
print('manifest exists; exiting')
|
||||
return
|
||||
|
||||
backup = manifest.Manifest(args.manifest)
|
||||
|
||||
backup.metadata['info'] = Info(
|
||||
manifest.utctimestamp(),
|
||||
incremental,
|
||||
segment_size,
|
||||
num_of_sectors,
|
||||
)
|
||||
|
||||
backup.metadata['bases'] = [uuid().bytes]
|
||||
|
||||
read_segments(range(0, num_of_segments - 1), segment_size, backup)
|
||||
|
||||
backup.write_manifest()
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
Loading…
Reference in New Issue