Add Phazr.IO libphazr backend support
Currently, there are several implementations of erasure codes that are available within OpenStack Swift. Most, if not all, of which are based on the Reed Solomon coding algorithm. Phazr.IO’s Erasure Coding technology uses a patented algorithm which are significantly more efficient and improves the speed of coding, decoding and reconstruction. In addition, Phazr.IO Erasure Code use a non-systematic algorithm which provides data protection at rest and in transport without the need to use encryption. Please contact support@phazr.io for more info on our technology. Change-Id: I9377fa32426a190efd0a7f0675ecf13d7e90367d
This commit is contained in:
parent
37d43c7c73
commit
69cb73f9d1
|
@ -51,6 +51,7 @@ Supported ``ec_type`` values:
|
|||
* ``isa_l_rs_vand`` => Intel Storage Acceleration Library (ISA-L) - SIMD accelerated Erasure Coding backends [4]
|
||||
* ``isa_l_rs_cauchy`` => Cauchy Reed-Solomon encoding (ISA-L variant) [4]
|
||||
* ``shss`` => NTT Lab Japan's Erasure Coding Library [5]
|
||||
* ``libphazr`` => Phazr.IO's erasure code library with built-in privacy [6]
|
||||
|
||||
|
||||
The Python API supports the following functions:
|
||||
|
@ -240,7 +241,7 @@ Quick Start
|
|||
An example for ubuntu to install dependency packages::
|
||||
|
||||
$ sudo apt-get install build-essential python-dev python-pip liberasurecode-dev
|
||||
$ sudo pip install -U bindep -r test-requirement.txt
|
||||
$ sudo pip install -U bindep -r test-requirements.txt
|
||||
|
||||
If you want to confirm all dependency packages installed successfully, try::
|
||||
|
||||
|
@ -280,3 +281,5 @@ References
|
|||
[4] Intel(R) Storage Acceleration Library (Open Source Version), https://01.org/intel%C2%AE-storage-acceleration-library-open-source-version
|
||||
|
||||
[5] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend"
|
||||
|
||||
[6] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy"
|
||||
|
|
|
@ -124,6 +124,7 @@ class PyECLib_EC_Types(PyECLibEnum):
|
|||
shss = 5
|
||||
liberasurecode_rs_vand = 6
|
||||
isa_l_rs_cauchy = 7
|
||||
libphazr = 8
|
||||
|
||||
|
||||
# Output of Erasure (en)Coding process are data "fragments". Fragment data
|
||||
|
@ -175,6 +176,8 @@ class ECDriver(object):
|
|||
elif value == "flat_xor_hd_4":
|
||||
self.hd = 4
|
||||
value = "flat_xor_hd"
|
||||
elif value == "libphazr":
|
||||
self.hd = 1
|
||||
if PyECLib_EC_Types.has_enum(value):
|
||||
self.ec_type = PyECLib_EC_Types.get_by_name(value)
|
||||
else:
|
||||
|
@ -521,6 +524,7 @@ ALL_EC_TYPES = [
|
|||
'shss',
|
||||
'liberasurecode_rs_vand',
|
||||
'isa_l_rs_cauchy',
|
||||
'libphazr',
|
||||
]
|
||||
|
||||
|
||||
|
|
|
@ -971,6 +971,9 @@ static const char* backend_id_to_str(uint8_t backend_id)
|
|||
case 7:
|
||||
backend_id_str = "isa_l_rs_cauchy\0";
|
||||
break;
|
||||
case 8:
|
||||
backend_id_str = "libphazr\0";
|
||||
break;
|
||||
default:
|
||||
backend_id_str = "unknown\0";
|
||||
}
|
||||
|
|
|
@ -134,7 +134,7 @@ class TestPyECLibDriver(unittest.TestCase):
|
|||
"Invalid Argument: m is required")
|
||||
|
||||
with self.assertRaises(ECDriverError) as err_context:
|
||||
# m is smaller than 1
|
||||
# k is smaller than 1
|
||||
ECDriver(ec_type=ec_type, k=-100, m=1)
|
||||
self.assertEqual(str(err_context.exception),
|
||||
"Invalid number of data fragments (k)")
|
||||
|
@ -151,10 +151,15 @@ class TestPyECLibDriver(unittest.TestCase):
|
|||
for _type in ALL_EC_TYPES:
|
||||
try:
|
||||
if _type is 'shss':
|
||||
_k = 10
|
||||
_m = 4
|
||||
elif _type is 'libphazr':
|
||||
_k = 4
|
||||
_m = 4
|
||||
else:
|
||||
_k = 10
|
||||
_m = 5
|
||||
ECDriver(k=10, m=_m, ec_type=_type, validate=True)
|
||||
ECDriver(k=_k, m=_m, ec_type=_type, validate=True)
|
||||
available_ec_types.append(_type)
|
||||
except Exception:
|
||||
# ignore any errors, assume backend not available
|
||||
|
@ -171,10 +176,12 @@ class TestPyECLibDriver(unittest.TestCase):
|
|||
try:
|
||||
if _type is 'shss':
|
||||
_instance = ECDriver(k=10, m=4, ec_type=_type)
|
||||
elif _type is 'libphazr':
|
||||
_instance = ECDriver(k=4, m=4, ec_type=_type)
|
||||
else:
|
||||
_instance = ECDriver(k=10, m=5, ec_type=_type)
|
||||
except ECDriverError:
|
||||
self.fail("%p: %s algorithm not supported" % _instance, _type)
|
||||
self.fail("%s algorithm not supported" % _type)
|
||||
|
||||
self.assertRaises(ECBackendNotSupported, ECDriver, k=10, m=5,
|
||||
ec_type="invalid_algo")
|
||||
|
@ -244,6 +251,11 @@ class TestPyECLibDriver(unittest.TestCase):
|
|||
chksum_type=csum))
|
||||
pyeclib_drivers.append(ECDriver(k=11, m=7, ec_type=_type6,
|
||||
chksum_type=csum))
|
||||
|
||||
_type7 = 'libphazr'
|
||||
if _type7 in VALID_EC_TYPES:
|
||||
pyeclib_drivers.append(ECDriver(k=4, m=4, ec_type=_type7,
|
||||
chksum_type=csum))
|
||||
return pyeclib_drivers
|
||||
|
||||
def test_small_encode(self):
|
||||
|
@ -740,7 +752,15 @@ class TestBackendsEnabled(unittest.TestCase):
|
|||
def dummy(self, ec_type=ec_type):
|
||||
if ec_type not in VALID_EC_TYPES:
|
||||
raise unittest.SkipTest
|
||||
k, m = 10, 4 if ec_type == 'shss' else 5
|
||||
if ec_type == 'shss':
|
||||
k = 10
|
||||
m = 4
|
||||
elif ec_type == 'libphazr':
|
||||
k = 4
|
||||
m = 4
|
||||
else:
|
||||
k = 10
|
||||
m = 5
|
||||
ECDriver(k=k, m=m, ec_type=ec_type)
|
||||
dummy.__name__ = 'test_%s_available' % ec_type
|
||||
cls_dict[dummy.__name__] = dummy
|
||||
|
|
|
@ -78,6 +78,7 @@ class TestPyECLib(unittest.TestCase):
|
|||
(PyECLib_EC_Types.shss, 10, 4),
|
||||
(PyECLib_EC_Types.shss, 20, 4),
|
||||
(PyECLib_EC_Types.shss, 11, 7)]
|
||||
self.libphazr = [(PyECLib_EC_Types.libphazr, 4, 4)]
|
||||
|
||||
# Input temp files for testing
|
||||
self.sizes = ["101-K", "202-K", "303-K"]
|
||||
|
@ -357,6 +358,36 @@ class TestPyECLib(unittest.TestCase):
|
|||
print("Reconstruct (%s): %s" %
|
||||
(size_str, self.get_throughput(avg_time, size_str)))
|
||||
|
||||
@require_backend("libphazr")
|
||||
def test_libphazr(self):
|
||||
for (ec_type, k, m) in self.libphazr:
|
||||
print(("\nRunning tests for %s k=%d, m=%d" % (ec_type, k, m)))
|
||||
|
||||
success = self._test_get_required_fragments(k, m, ec_type)
|
||||
self.assertTrue(success)
|
||||
|
||||
for size_str in self.sizes:
|
||||
avg_time = self.time_encode(k, m, ec_type.value, 0,
|
||||
size_str,
|
||||
self.iterations)
|
||||
print("Encode (%s): %s" %
|
||||
(size_str, self.get_throughput(avg_time, size_str)))
|
||||
|
||||
for size_str in self.sizes:
|
||||
success, avg_time = self.time_decode(k, m, ec_type.value, 0,
|
||||
size_str,
|
||||
self.iterations)
|
||||
self.assertTrue(success)
|
||||
print("Decode (%s): %s" %
|
||||
(size_str, self.get_throughput(avg_time, size_str)))
|
||||
|
||||
for size_str in self.sizes:
|
||||
success, avg_time = self.time_reconstruct(
|
||||
k, m, ec_type.value, 0, size_str, self.iterations)
|
||||
self.assertTrue(success)
|
||||
print("Reconstruct (%s): %s" %
|
||||
(size_str, self.get_throughput(avg_time, size_str)))
|
||||
|
||||
def _test_get_required_fragments(self, num_data, num_parity, ec_type):
|
||||
"""
|
||||
:return boolean, True if all tests passed
|
||||
|
|
Loading…
Reference in New Issue