293 lines
10 KiB
Python
293 lines
10 KiB
Python
# Copyright (c) 2013, 2014, Kevin Greenan (kmgreen2@gmail.com)
|
|
# All rights reserved.
|
|
#
|
|
# Redistribution and use in source and binary forms, with or without
|
|
# modification, are permitted provided that the following conditions are met:
|
|
#
|
|
# Redistributions of source code must retain the above copyright notice, this
|
|
# list of conditions and the following disclaimer.
|
|
#
|
|
# Redistributions in binary form must reproduce the above copyright notice,
|
|
# this list of conditions and the following disclaimer in the documentation
|
|
# and/or other materials provided with the distribution. THIS SOFTWARE IS
|
|
# PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS
|
|
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
|
|
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN
|
|
# NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY
|
|
# DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
|
|
# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
|
|
# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
|
|
# ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
|
|
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
|
|
# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
|
|
from .ec_iface import ECDriverError
|
|
from .ec_iface import ECInsufficientFragments
|
|
from .ec_iface import PyECLib_FRAGHDRCHKSUM_Types
|
|
|
|
import math
|
|
import pyeclib_c
|
|
import sys
|
|
|
|
pyver = float('%s.%s' % sys.version_info[:2])
|
|
|
|
|
|
class ECPyECLibDriver(object):
|
|
|
|
def __init__(self, k, m, hd, ec_type,
|
|
chksum_type=PyECLib_FRAGHDRCHKSUM_Types.none,
|
|
validate=False):
|
|
self.k = k
|
|
self.m = m
|
|
self.hd = hd
|
|
self.ec_type = ec_type
|
|
self.chksum_type = chksum_type
|
|
|
|
self.inline_chksum = 0
|
|
self.algsig_chksum = 0
|
|
# crc32 is the only inline checksum type currently supported
|
|
if self.chksum_type is PyECLib_FRAGHDRCHKSUM_Types.inline_crc32:
|
|
self.inline_chksum = 1
|
|
|
|
self.handle = pyeclib_c.init(
|
|
self.k,
|
|
self.m,
|
|
ec_type.value,
|
|
self.hd,
|
|
self.inline_chksum,
|
|
self.algsig_chksum,
|
|
validate)
|
|
|
|
def __repr__(self):
|
|
return '%s(k=%r, m=%r, hd=%r, ec_type=%r, chksum_type=%r)' % (
|
|
self.__class__.__name__, self.k, self.m, self.hd, self.ec_type,
|
|
self.chksum_type)
|
|
|
|
def encode(self, data_bytes):
|
|
return pyeclib_c.encode(self.handle, data_bytes)
|
|
|
|
def _validate_and_return_fragment_size(self, fragments):
|
|
if len(fragments) == 0 or len(fragments[0]) == 0:
|
|
return -1
|
|
fragment_len = len(fragments[0])
|
|
for fragment in fragments[1:]:
|
|
if len(fragment) != fragment_len:
|
|
return -1
|
|
return fragment_len
|
|
|
|
def decode(self, fragment_payloads, ranges=None,
|
|
force_metadata_checks=False):
|
|
_fragment_payloads = list(fragment_payloads)
|
|
fragment_len = self._validate_and_return_fragment_size(
|
|
_fragment_payloads)
|
|
if fragment_len < 0:
|
|
raise ECDriverError(
|
|
"Invalid fragment payload in ECPyECLibDriver.decode")
|
|
|
|
if len(_fragment_payloads) < self.k:
|
|
raise ECInsufficientFragments(
|
|
"Not enough fragments given in ECPyECLibDriver.decode")
|
|
|
|
return pyeclib_c.decode(self.handle, _fragment_payloads,
|
|
fragment_len, ranges, force_metadata_checks)
|
|
|
|
def reconstruct(self, fragment_payloads, indexes_to_reconstruct):
|
|
_fragment_payloads = list(fragment_payloads)
|
|
fragment_len = self._validate_and_return_fragment_size(
|
|
_fragment_payloads)
|
|
if fragment_len < 0:
|
|
raise ECDriverError(
|
|
"Invalid fragment payload in ECPyECLibDriver.reconstruct")
|
|
|
|
reconstructed_data = []
|
|
|
|
# Reconstruct the data, then the parity
|
|
# The parity cannot be reconstructed until
|
|
# after all data is reconstructed
|
|
indexes_to_reconstruct.sort()
|
|
_indexes_to_reconstruct = indexes_to_reconstruct[:]
|
|
|
|
while len(_indexes_to_reconstruct) > 0:
|
|
index = _indexes_to_reconstruct.pop(0)
|
|
reconstructed = pyeclib_c.reconstruct(
|
|
self.handle, _fragment_payloads, fragment_len, index)
|
|
reconstructed_data.append(reconstructed)
|
|
_fragment_payloads.append(reconstructed)
|
|
|
|
return reconstructed_data
|
|
|
|
def fragments_needed(self, reconstruct_indexes, exclude_indexes):
|
|
required_fragments = pyeclib_c.get_required_fragments(
|
|
self.handle, reconstruct_indexes, exclude_indexes)
|
|
return required_fragments
|
|
|
|
def min_parity_fragments_needed(self):
|
|
""" FIXME - fix this to return a function of HD """
|
|
return 1
|
|
|
|
def get_metadata(self, fragment, formatted=0):
|
|
fragment_metadata = pyeclib_c.get_metadata(self.handle, fragment,
|
|
formatted)
|
|
return fragment_metadata
|
|
|
|
def verify_stripe_metadata(self, fragment_metadata_list):
|
|
success = pyeclib_c.check_metadata(self.handle, fragment_metadata_list)
|
|
return success
|
|
|
|
def get_segment_info(self, data_len, segment_size):
|
|
segment_info = pyeclib_c.get_segment_info(self.handle, data_len,
|
|
segment_size)
|
|
return segment_info
|
|
|
|
|
|
class ECNullDriver(object):
|
|
|
|
def __init__(self, k, m, hd, ec_type=None, chksum_type=None,
|
|
validate=False):
|
|
self.k = k
|
|
self.m = m
|
|
self.hd = hd
|
|
|
|
def encode(self, data_bytes):
|
|
pass
|
|
|
|
def decode(self, fragment_payloads, ranges, force_metadata_checks):
|
|
pass
|
|
|
|
def reconstruct(self, available_fragment_payloads,
|
|
missing_fragment_indexes):
|
|
pass
|
|
|
|
def fragments_needed(self, missing_fragment_indexes):
|
|
pass
|
|
|
|
def get_metadata(self, fragment, formatted=0):
|
|
pass
|
|
|
|
def min_parity_fragments_needed(self):
|
|
pass
|
|
|
|
def verify_stripe_metadata(self, fragment_metadata_list):
|
|
pass
|
|
|
|
def get_segment_info(self, data_len, segment_size):
|
|
pass
|
|
|
|
|
|
#
|
|
# A striping-only driver for EC. This is
|
|
# pretty much RAID 0.
|
|
#
|
|
class ECStripingDriver(object):
|
|
|
|
def __init__(self, k, m, hd, ec_type=None, chksum_type=None,
|
|
validate=False):
|
|
"""Stripe an arbitrary-sized string into k fragments
|
|
:param k: the number of data fragments to stripe
|
|
:param m: the number of parity fragments to stripe
|
|
:raises: ECDriverError if there is an error during encoding
|
|
"""
|
|
self.k = k
|
|
|
|
if m != 0:
|
|
raise ECDriverError("This driver only supports m=0")
|
|
|
|
self.m = m
|
|
self.hd = hd
|
|
|
|
def encode(self, data_bytes):
|
|
"""Stripe an arbitrary-sized string into k fragments
|
|
:param data_bytes: the buffer to encode
|
|
:returns: a list of k buffers (data only)
|
|
:raises: ECDriverError if there is an error during encoding
|
|
"""
|
|
# Main fragment size
|
|
fragment_size = math.ceil(len(data_bytes) / float(self.k))
|
|
|
|
# Size of last fragment
|
|
last_fragment_size = len(data_bytes) - (fragment_size * self.k - 1)
|
|
|
|
fragments = []
|
|
offset = 0
|
|
for i in range(self.k - 1):
|
|
fragments.append(data_bytes[offset:fragment_size])
|
|
offset += fragment_size
|
|
|
|
fragments.append(data_bytes[offset:last_fragment_size])
|
|
|
|
return fragments
|
|
|
|
def decode(self, fragment_payloads, ranges=None,
|
|
force_metadata_checks=False):
|
|
"""Convert a k-fragment data stripe into a string
|
|
:param fragment_payloads: fragments (in order) to convert into a string
|
|
:param ranges (unsupported): range decode
|
|
:param force_metadata_checks (unsupported): verify fragment metadata
|
|
:returns: a string containing original data
|
|
:raises: ECDriverError if there is an error during decoding
|
|
"""
|
|
if ranges is not None:
|
|
raise ECDriverError("Decode does not support range requests in the"
|
|
" striping driver.")
|
|
if force_metadata_checks is not False:
|
|
raise ECDriverError(
|
|
"Decode does not support metadata integrity checks in the "
|
|
" striping driver.")
|
|
if len(fragment_payloads) != self.k:
|
|
raise ECInsufficientFragments(
|
|
"Decode requires %d fragments, %d fragments were given" %
|
|
(len(fragment_payloads), self.k))
|
|
|
|
ret_string = ''
|
|
|
|
for fragment in fragment_payloads:
|
|
ret_string += fragment
|
|
|
|
return ret_string
|
|
|
|
def reconstruct(self, available_fragment_payloads,
|
|
missing_fragment_indexes):
|
|
"""We cannot reconstruct a fragment using other fragments. This means
|
|
that reconstruction means all fragments must be specified, otherwise we
|
|
cannot reconstruct and must raise an error.
|
|
:param available_fragment_payloads: available fragments (in order)
|
|
:param missing_fragment_indexes: indexes of missing fragments
|
|
:returns: a string containing the original data
|
|
:raises: ECDriverError if there is an error during reconstruction
|
|
"""
|
|
if len(available_fragment_payloads) != self.k:
|
|
raise ECDriverError(
|
|
"Reconstruction requires %d fragments, %d fragments given" %
|
|
(len(available_fragment_payloads), self.k))
|
|
|
|
return available_fragment_payloads
|
|
|
|
def fragments_needed(self, missing_fragment_indexes):
|
|
"""By definition, all missing fragment indexes are needed to
|
|
reconstruct, so just return the list handed to this function.
|
|
:param missing_fragment_indexes: indexes of missing fragments
|
|
:returns: missing_fragment_indexes
|
|
"""
|
|
return missing_fragment_indexes
|
|
|
|
def min_parity_fragments_needed(self):
|
|
pass
|
|
|
|
def get_metadata(self, fragment, formatted=0):
|
|
"""This driver does not include fragment metadata, so return empty
|
|
string
|
|
:param fragment: a fragment
|
|
:returns: empty string
|
|
"""
|
|
return ''
|
|
|
|
def verify_stripe_metadata(self, fragment_metadata_list):
|
|
"""This driver does not include fragment metadata, so return true
|
|
:param fragment_metadata_list: a list of fragments
|
|
:returns: True
|
|
"""
|
|
return True
|
|
|
|
def get_segment_info(self, data_len, segment_size):
|
|
pass
|