Add Phazr.IO libphazr backend to liberasurecode

Currently, there are several implementations of erasure codes that are
available within OpenStack Swift.  Most, if not all, of which are based
on the Reed Solomon coding algorithm.

Phazr.IO’s Erasure Coding technology uses a patented algorithm which are
significantly more efficient and improves the speed of coding, decoding
and reconstruction.  In addition, Phazr.IO Erasure Code use a non-systematic
algorithm which provides data protection at rest and in transport without
the need to use encryption.

Please contact support@phazr.io for more info on our technology.

Change-Id: I4e40d02a8951e38409ad3c604c5dd6f050fa7ea0
This commit is contained in:
Jim Cheung 2017-01-23 13:55:24 -08:00 committed by jimcheung
parent 0794b31c62
commit 06e50ea86b
17 changed files with 551 additions and 32 deletions

View File

@ -17,6 +17,7 @@ Highlights
- 'ISA-L' - Intel Storage Acceleration Library - SIMD accelerated Erasure Coding backends [2]
- 'SHSS' - NTT Lab Japan's hybrid Erasure Coding backend [4]
- 'Flat XOR HD' - built-in to liberasurecode, based on [3]
- 'libphazr' - Phazr.IO's erasure code backend with built-in privacy [5]
- 'NULL' template backend implemented to help future backend writers
@ -381,6 +382,8 @@ Code organization
| | +-- isa_l_rs_vand.c --> 'isa_l_rs_vand' erasure code backend (Intel)
| | +-- shss
| | +-- shss.c --> 'shss' erasure code backend (NTT Labs)
| | +-- phazrio
| | +-- libphazr.c --> 'libphazr' erasure code backend (Phazr.IO)
| |
| |-- builtin
| | +-- xor_codes --> XOR HD code backend, built-in erasure
@ -426,3 +429,5 @@ References
[3] Greenan, Kevin M et al, "Flat XOR-based erasure codes in storage systems", http://www.kaymgee.com/Kevin_Greenan/Publications_files/greenan-msst10.pdf
[4] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend"
[5] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy"

View File

@ -49,6 +49,7 @@ typedef enum {
EC_BACKEND_SHSS = 5,
EC_BACKEND_LIBERASURECODE_RS_VAND = 6,
EC_BACKEND_ISA_L_RS_CAUCHY = 7,
EC_BACKEND_LIBPHAZR = 8,
EC_BACKENDS_MAX,
} ec_backend_id_t;

View File

@ -1,4 +1,4 @@
/*
/*
* <Copyright>
*
* Redistribution and use in source and binary forms, with or without
@ -64,6 +64,8 @@ struct ec_backend_args {
#define RECONSTRUCT reconstruct
#define ELEMENTSIZE element_size
#define ISCOMPATIBLEWITH is_compatible_with
#define GETMETADATASIZE get_backend_metadata_size
#define GETENCODEOFFSET get_encode_offset
#define FN_NAME(s) str(s)
#define str(s) #s
@ -90,7 +92,10 @@ struct ec_backend_op_stubs {
int blocksize);
int (*ELEMENTSIZE)(void *desc);
bool (*ISCOMPATIBLEWITH) (uint32_t version);
bool (*ISCOMPATIBLEWITH)(uint32_t version);
size_t (*GETMETADATASIZE)(void *desc, int blocksize);
size_t (*GETENCODEOFFSET)(void *desc, int metadata_size);
};
/* ==~=*=~==~=*=~==~=*=~= backend struct definitions =~=*=~==~=*=~==~=*==~== */
@ -109,11 +114,6 @@ struct ec_backend_common {
char soversion[MAX_LEN]; /* EC backend shared library version */
struct ec_backend_op_stubs *ops; /* EC backend stubs */
size_t backend_metadata_size;
/* EC backend custom metadata size -
* backend_metadata_size bytes are added to
* the fragment size when allocating
* data/parity fragment buffers */
uint32_t ec_backend_version; /* The revision number of this back
* end. Is used to determine whether
* a specific instance of this backend
@ -162,6 +162,21 @@ ec_backend_t liberasurecode_backend_lookup_by_name(const char *name);
*/
ec_backend_t liberasurecode_backend_instance_get_by_desc(int desc);
/* Common function for backends */
/**
* A function to return 0 for generic usage on backends for get_encode_offset
*
* Returns 0 always
*/
static inline size_t get_encode_offset_zero(void *desc, int metadata_size){ return 0; }
/**
* A function to return 0 for generic usage on backends for get_backend_metadata_size
*
* Returns 0 always
*/
static inline size_t get_backend_metadata_size_zero(void *desc, int blocksize){ return 0; }
/* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
#ifdef __cplusplus

View File

@ -26,7 +26,8 @@ liberasurecode_la_SOURCES = \
backends/isa-l/isa_l_rs_cauchy.c \
backends/rs_vand/liberasurecode_rs_vand.c \
builtin/rs_vand/rs_galois.c \
backends/shss/shss.c
backends/shss/shss.c \
backends/phazrio/libphazr.c
liberasurecode_la_CPPFLAGS = -Werror @GCOV_FLAGS@
liberasurecode_la_LIBADD = \
@ -42,4 +43,5 @@ MOSTLYCLEANFILES = *.gcda *.gcno *.gcov utils/chksum/*.gcda utils/chksum/*.gcno
backends/xor/*.gcda backends/xor/*.gcno backends/xor/*.gcov \
backends/jerasure/*.gcda backends/jerasure/*.gcno backends/jerasure/*.gcov \
backends/shss/*.gcda backends/shss/*.gcno backends/shss/*.gcov \
backends/rs_vand/*.gcda backends/rs_vand/*.gcno backends/rs_vand/*.gcov
backends/rs_vand/*.gcda backends/rs_vand/*.gcno backends/rs_vand/*.gcov \
backends/phazrio/*.gcda backends/phazrio/*.gcno backends/phazrio/*.gcov

View File

@ -71,6 +71,8 @@ struct ec_backend_op_stubs isa_l_rs_cauchy_op_stubs = {
.RECONSTRUCT = isa_l_reconstruct,
.ELEMENTSIZE = isa_l_element_size,
.ISCOMPATIBLEWITH = isa_l_rs_cauchy_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_isa_l_rs_cauchy = {
@ -79,7 +81,6 @@ struct ec_backend_common backend_isa_l_rs_cauchy = {
.soname = ISA_L_RS_CAUCHY_SO_NAME,
.soversion = ISA_L_RS_CAUCHY_LIB_VER_STR,
.ops = &isa_l_rs_cauchy_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(ISA_L_RS_CAUCHY_LIB_MAJOR,
ISA_L_RS_CAUCHY_LIB_MINOR,
ISA_L_RS_CAUCHY_LIB_REV),

View File

@ -70,6 +70,8 @@ struct ec_backend_op_stubs isa_l_rs_vand_op_stubs = {
.RECONSTRUCT = isa_l_reconstruct,
.ELEMENTSIZE = isa_l_element_size,
.ISCOMPATIBLEWITH = isa_l_rs_vand_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_isa_l_rs_vand = {
@ -78,7 +80,6 @@ struct ec_backend_common backend_isa_l_rs_vand = {
.soname = ISA_L_RS_VAND_SO_NAME,
.soversion = ISA_L_RS_VAND_LIB_VER_STR,
.ops = &isa_l_rs_vand_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(ISA_L_RS_VAND_LIB_MAJOR,
ISA_L_RS_VAND_LIB_MINOR,
ISA_L_RS_VAND_LIB_REV),

View File

@ -426,7 +426,6 @@ static bool jerasure_rs_cauchy_is_compatible_with(uint32_t version) {
return version == backend_jerasure_rs_cauchy.ec_backend_version;
}
struct ec_backend_op_stubs jerasure_rs_cauchy_op_stubs = {
.INIT = jerasure_rs_cauchy_init,
.EXIT = jerasure_rs_cauchy_exit,
@ -436,7 +435,8 @@ struct ec_backend_op_stubs jerasure_rs_cauchy_op_stubs = {
.RECONSTRUCT = jerasure_rs_cauchy_reconstruct,
.ELEMENTSIZE = jerasure_rs_cauchy_element_size,
.ISCOMPATIBLEWITH = jerasure_rs_cauchy_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_jerasure_rs_cauchy = {
@ -445,7 +445,6 @@ struct ec_backend_common backend_jerasure_rs_cauchy = {
.soname = JERASURE_RS_CAUCHY_SO_NAME,
.soversion = JERASURE_RS_CAUCHY_LIB_VER_STR,
.ops = &jerasure_rs_cauchy_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(JERASURE_RS_CAUCHY_LIB_MAJOR,
JERASURE_RS_CAUCHY_LIB_MINOR,
JERASURE_RS_CAUCHY_LIB_REV),

View File

@ -346,6 +346,8 @@ struct ec_backend_op_stubs jerasure_rs_vand_op_stubs = {
.RECONSTRUCT = jerasure_rs_vand_reconstruct,
.ELEMENTSIZE = jerasure_rs_vand_element_size,
.ISCOMPATIBLEWITH = jerasure_rs_vand_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_jerasure_rs_vand = {
@ -354,7 +356,6 @@ struct ec_backend_common backend_jerasure_rs_vand = {
.soname = JERASURE_RS_VAND_SO_NAME,
.soversion = JERASURE_RS_VAND_LIB_VER_STR,
.ops = &jerasure_rs_vand_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(JERASURE_RS_VAND_LIB_MAJOR,
JERASURE_RS_VAND_LIB_MINOR,
JERASURE_RS_VAND_LIB_REV),

View File

@ -50,7 +50,6 @@ typedef int (*null_code_encode_func)(void *, char **, char **, int);
typedef int (*null_code_decode_func)(void *, char **, char **, int *, int, int);
typedef int (*null_reconstruct_func)(char **, int, uint64_t, int, char *);
typedef int (*null_code_fragments_needed_func)(void *, int *, int *, int *);
struct null_descriptor {
/* calls required for init */
init_null_code_func init_null_code;
@ -215,6 +214,7 @@ static int null_exit(void *desc)
static bool null_is_compatible_with(uint32_t version) {
return true;
}
struct ec_backend_op_stubs null_op_stubs = {
.INIT = null_init,
.EXIT = null_exit,
@ -224,6 +224,8 @@ struct ec_backend_op_stubs null_op_stubs = {
.RECONSTRUCT = null_reconstruct,
.ELEMENTSIZE = null_element_size,
.ISCOMPATIBLEWITH = null_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_null = {
@ -232,7 +234,6 @@ struct ec_backend_common backend_null = {
.soname = NULL_SO_NAME,
.soversion = NULL_LIB_VER_STR,
.ops = &null_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(NULL_LIB_MAJOR, NULL_LIB_MINOR,
NULL_LIB_REV),
};

View File

@ -0,0 +1,391 @@
/*
* Copyright 2016 Phazr.IO Inc
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this
* list of conditions and the following disclaimer in the documentation and/or
* other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
* THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* Phazr.IO libphazr backend
*
* vi: set noai tw=79 ts=4 sw=4:
*/
#include <stdio.h>
#include <stdlib.h>
#include "erasurecode.h"
#include "erasurecode_backend.h"
#include "erasurecode_helpers.h"
#define LIBPHAZR_LIB_MAJOR 1
#define LIBPHAZR_LIB_MINOR 0
#define LIBPHAZR_LIB_REV 0
#define LIBPHAZR_LIB_VER_STR "1.0.0"
#define LIBPHAZR_LIB_NAME "libphazr"
#if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__)
#define LIBPHAZR_SO_NAME "libphazr.dylib"
#else
#define LIBPHAZR_SO_NAME "libphazr.so.1"
#endif
/* Forward declarations */
struct ec_backend libphazr;
struct ec_backend_op_stubs libphazr_ops;
struct ec_backend_common backend_libphazr;
typedef int (*pio_matrix_encode_func)(char *, char *, char **, int, int, int, int, int, int);
typedef int (*pio_matrix_decode_func)(char *, char *, char **, int *, int, int, int, int, int, int);
typedef int (*pio_matrix_reconstruct_func)(char *, char **, int *, int, int, int, int, int, int);
typedef char* (*pio_create_precoding_matrix_func)(int);
typedef char* (*pio_create_inverse_precoding_matrix_func)(int);
typedef char* (*pio_create_kmux_matrix_func)(int, int, int);
struct libphazr_descriptor {
/* calls required for init */
pio_create_precoding_matrix_func create_precoding_matrix;
pio_create_inverse_precoding_matrix_func create_inverse_precoding_matrix;
pio_create_kmux_matrix_func create_kmux_matrix;
/* calls required for encode */
pio_matrix_encode_func matrix_encode;
/* calls required for decode */
pio_matrix_decode_func matrix_decode;
/* calls required for reconstruct */
pio_matrix_reconstruct_func matrix_reconstruct;
/* fields needed to hold state */
char *matrix;
char *precoding_matrix;
char *inverse_precoding_matrix;
int k;
int m;
int w;
int hd;
};
#define DEFAULT_W 64
#define DEFAULT_HD 1
static int get_padded_blocksize(int w, int hd, int blocksize)
{
int word_size = w / 8;
return (int) ceill((double) blocksize / (word_size - hd)) * word_size;
}
static int pio_matrix_encode(void *desc, char **data, char **parity, int blocksize)
{
int i, ret = 0;
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padding_size = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize) - blocksize;
char **encoded = malloc(sizeof(char*) * (xdesc->k + xdesc->m));
if (NULL == encoded) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < xdesc->k; i++) {
encoded[i] = data[i];
}
for (i = 0; i < xdesc->m; i++) {
encoded[i + xdesc->k] = parity[i];
}
ret = xdesc->matrix_encode(xdesc->precoding_matrix, xdesc->matrix, encoded,
xdesc->k, xdesc->m, xdesc->w, xdesc->hd, blocksize, padding_size);
out:
free(encoded);
return ret;
}
static int pio_matrix_decode(void *desc, char **data, char **parity,
int *missing_idxs, int blocksize)
{
int i, ret = 0;
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padding_size = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize) - blocksize;
char **decoded = malloc(sizeof(char*) * (xdesc->k + xdesc->m));
if (NULL == decoded) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < xdesc->k; i++) {
decoded[i] = data[i];
}
for (i = 0; i < xdesc->m; i++) {
decoded[i + xdesc->k] = parity[i];
}
ret = xdesc->matrix_decode(xdesc->inverse_precoding_matrix, xdesc->matrix, decoded,
missing_idxs, xdesc->k, xdesc->m, xdesc->w, xdesc->hd, blocksize, padding_size);
out:
free(decoded);
return ret;
}
static int pio_matrix_reconstruct(void *desc, char **data, char **parity,
int *missing_idxs, int destination_idx, int blocksize)
{
int i, ret = 0;
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padding_size = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize) - blocksize;
char **encoded = malloc(sizeof(char*) * (xdesc->k + xdesc->m));
if (NULL == encoded) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < xdesc->k; i++) {
encoded[i] = data[i];
}
for (i = 0; i < xdesc->m; i++) {
encoded[i + xdesc->k] = parity[i];
}
ret = xdesc->matrix_reconstruct(xdesc->matrix, encoded, missing_idxs,
destination_idx, xdesc->k, xdesc->m, xdesc->w, blocksize, padding_size);
out:
free(encoded);
return ret;
}
static int pio_min_fragments(void *desc, int *missing_idxs,
int *fragments_to_exclude, int *fragments_needed)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (xdesc->k + xdesc->m); i++) {
if (!(missing_bm & (1 << i))) {
fragments_needed[j] = i;
j++;
}
if (j == xdesc->k) {
ret = 0;
fragments_needed[j] = -1;
break;
}
}
return ret;
}
/**
* Return the element-size, which is the number of bits stored
* on a given device, per codeword.
*/
static int pio_element_size(void *desc)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *)desc;
return xdesc->w;
}
static void * pio_init(struct ec_backend_args *args, void *backend_sohandle)
{
struct libphazr_descriptor *desc = NULL;
/* allocate and fill in libphazr_descriptor */
desc = (struct libphazr_descriptor *)malloc(sizeof(struct libphazr_descriptor));
if (NULL == desc) {
return NULL;
}
memset(desc, 0, sizeof(struct libphazr_descriptor));
desc->k = args->uargs.k;
desc->m = args->uargs.m;
desc->w = args->uargs.w;
desc->hd = args->uargs.hd;
if (desc->w <= 0)
desc->w = DEFAULT_W;
args->uargs.w = desc->w;
if (desc->hd <= 0)
desc->hd = DEFAULT_HD;
args->uargs.hd = desc->hd;
/*
* ISO C forbids casting a void* to a function pointer.
* Since dlsym return returns a void*, we use this union to
* "transform" the void* to a function pointer.
*/
union {
pio_create_precoding_matrix_func create_precoding_matrix_ptr;
pio_create_inverse_precoding_matrix_func create_inverse_precoding_matrix_ptr;
pio_create_kmux_matrix_func create_kmux_matrix_ptr;
pio_matrix_encode_func matrix_encode_ptr;
pio_matrix_decode_func matrix_decode_ptr;
pio_matrix_reconstruct_func matrix_reconstruct_ptr;
void *vptr;
} func_handle = {.vptr = NULL};
/* fill in function addresses */
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "create_precoding_matrix");
desc->create_precoding_matrix = func_handle.create_precoding_matrix_ptr;
if (NULL == desc->create_precoding_matrix) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "create_inverse_precoding_matrix");
desc->create_inverse_precoding_matrix = func_handle.create_inverse_precoding_matrix_ptr;
if (NULL == desc->create_inverse_precoding_matrix) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "create_kmux_matrix");
desc->create_kmux_matrix = func_handle.create_kmux_matrix_ptr;
if (NULL == desc->create_kmux_matrix) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "matrix_encode");
desc->matrix_encode = func_handle.matrix_encode_ptr;
if (NULL == desc->matrix_encode) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "matrix_decode");
desc->matrix_decode = func_handle.matrix_decode_ptr;
if (NULL == desc->matrix_decode) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "matrix_reconstruct");
desc->matrix_reconstruct = func_handle.matrix_reconstruct_ptr;
if (NULL == desc->matrix_reconstruct) {
goto error;
}
if (NULL == desc->precoding_matrix) {
desc->precoding_matrix = desc->create_precoding_matrix(desc->k);
if (NULL == desc->precoding_matrix) {
goto error;
}
}
if (NULL == desc->inverse_precoding_matrix) {
desc->inverse_precoding_matrix = desc->create_inverse_precoding_matrix(desc->k);
if (NULL == desc->inverse_precoding_matrix) {
goto error;
}
}
if (NULL == desc->matrix) {
desc->matrix = desc->create_kmux_matrix(desc->k, desc->m, desc->w);
if (NULL == desc->create_kmux_matrix) {
goto error;
}
}
return (void *) desc;
error:
free(desc->matrix);
free(desc->precoding_matrix);
free(desc->inverse_precoding_matrix);
free(desc);
return NULL;
}
static int pio_exit(void *desc)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor*)desc;
free(xdesc->matrix);
free(xdesc->precoding_matrix);
free(xdesc->inverse_precoding_matrix);
free(xdesc);
return 0;
}
static bool pio_is_compatible_with(uint32_t version)
{
return version == backend_libphazr.ec_backend_version;
}
static size_t pio_get_backend_metadata_size(void *desc, int blocksize)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padded_blocksize = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize);
return padded_blocksize - blocksize;
}
static size_t pio_get_encode_offset(void *desc, int metadata_size)
{
return metadata_size;
}
struct ec_backend_op_stubs libphazr_op_stubs = {
.INIT = pio_init,
.EXIT = pio_exit,
.ENCODE = pio_matrix_encode,
.DECODE = pio_matrix_decode,
.FRAGSNEEDED = pio_min_fragments,
.RECONSTRUCT = pio_matrix_reconstruct,
.ELEMENTSIZE = pio_element_size,
.ISCOMPATIBLEWITH = pio_is_compatible_with,
.GETMETADATASIZE = pio_get_backend_metadata_size,
.GETENCODEOFFSET = pio_get_encode_offset,
};
struct ec_backend_common backend_libphazr = {
.id = EC_BACKEND_LIBPHAZR,
.name = LIBPHAZR_LIB_NAME,
.soname = LIBPHAZR_SO_NAME,
.soversion = LIBPHAZR_LIB_VER_STR,
.ops = &libphazr_op_stubs,
.ec_backend_version = _VERSION(LIBPHAZR_LIB_MAJOR, LIBPHAZR_LIB_MINOR,
LIBPHAZR_LIB_REV),
};

View File

@ -298,6 +298,8 @@ struct ec_backend_op_stubs liberasurecode_rs_vand_op_stubs = {
.RECONSTRUCT = liberasurecode_rs_vand_reconstruct,
.ELEMENTSIZE = liberasurecode_rs_vand_element_size,
.ISCOMPATIBLEWITH = liberasurecode_rs_vand_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_liberasurecode_rs_vand = {
@ -306,7 +308,6 @@ struct ec_backend_common backend_liberasurecode_rs_vand = {
.soname = LIBERASURECODE_RS_VAND_SO_NAME,
.soversion = LIBERASURECODE_RS_VAND_LIB_VER_STR,
.ops = &liberasurecode_rs_vand_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(LIBERASURECODE_RS_VAND_LIB_MAJOR,
LIBERASURECODE_RS_VAND_LIB_MINOR,
LIBERASURECODE_RS_VAND_LIB_REV),

View File

@ -286,6 +286,10 @@ static bool shss_is_compatible_with(uint32_t version) {
return version == backend_shss.ec_backend_version;
}
static size_t shss_get_backend_metadata_size(void *desc, int blocksize) {
return METADATA;
}
struct ec_backend_op_stubs shss_op_stubs = {
.INIT = shss_init,
.EXIT = shss_exit,
@ -295,6 +299,8 @@ struct ec_backend_op_stubs shss_op_stubs = {
.RECONSTRUCT = shss_reconstruct,
.ELEMENTSIZE = shss_element_size,
.ISCOMPATIBLEWITH = shss_is_compatible_with,
.GETMETADATASIZE = shss_get_backend_metadata_size,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_shss = {
@ -303,5 +309,4 @@ struct ec_backend_common backend_shss = {
.soname = SHSS_SO_NAME,
.soversion = SHSS_LIB_VER_STR,
.ops = &shss_op_stubs,
.backend_metadata_size = METADATA,
};

View File

@ -178,6 +178,8 @@ struct ec_backend_op_stubs flat_xor_hd_op_stubs = {
.RECONSTRUCT = flat_xor_hd_reconstruct,
.ELEMENTSIZE = flar_xor_hd_element_size,
.ISCOMPATIBLEWITH = flat_xor_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_flat_xor_hd = {
@ -186,7 +188,6 @@ struct ec_backend_common backend_flat_xor_hd = {
.soname = FLAT_XOR_SO_NAME,
.soversion = FLAT_XOR_LIB_VER_STR,
.ops = &flat_xor_hd_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(FLAT_XOR_LIB_MAJOR,
FLAT_XOR_LIB_MINOR,
FLAT_XOR_LIB_REV),

View File

@ -50,6 +50,7 @@ extern struct ec_backend_common backend_isa_l_rs_vand;
extern struct ec_backend_common backend_shss;
extern struct ec_backend_common backend_liberasurecode_rs_vand;
extern struct ec_backend_common backend_isa_l_rs_cauchy;
extern struct ec_backend_common backend_libphazr;
ec_backend_t ec_backends_supported[] = {
(ec_backend_t) &backend_null,
@ -60,6 +61,7 @@ ec_backend_t ec_backends_supported[] = {
(ec_backend_t) &backend_shss,
(ec_backend_t) &backend_liberasurecode_rs_vand,
(ec_backend_t) &backend_isa_l_rs_cauchy,
(ec_backend_t) &backend_libphazr,
NULL,
};
@ -597,8 +599,8 @@ int liberasurecode_decode(int desc,
}
}
if (instance->common.id != EC_BACKEND_SHSS) {
/* shss (ntt_backend) must force to decode */
if (instance->common.id != EC_BACKEND_SHSS && instance->common.id != EC_BACKEND_LIBPHAZR) {
/* shss (ntt_backend) & libphazr backend must force to decode */
// TODO: Add a frag and function to handle whether the backend want to decode or not.
/*
* Try to re-assebmle the original data before attempting a decode
@ -1232,7 +1234,11 @@ int liberasurecode_get_fragment_size(int desc, int data_len)
if (NULL == instance)
return -EBACKENDNOTAVAIL;
int aligned_data_len = get_aligned_data_size(instance, data_len);
int size = (aligned_data_len / instance->args.uargs.k) + instance->common.backend_metadata_size;
int blocksize = aligned_data_len / instance->args.uargs.k;
int metadata_size = instance->common.ops->get_backend_metadata_size(
instance->desc.backend_desc,
blocksize);
int size = blocksize + metadata_size;
return size;
}

View File

@ -42,7 +42,9 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
set_fragment_payload_size(fragment, blocksize);
set_backend_id(fragment, be->common.id);
set_backend_version(fragment, be->common.ec_backend_version);
set_fragment_backend_metadata_size(fragment, be->common.backend_metadata_size);
set_fragment_backend_metadata_size(fragment, be->common.ops->get_backend_metadata_size(
be->desc.backend_desc,
blocksize));
if (add_chksum) {
set_checksum(ct, fragment, blocksize);

View File

@ -43,12 +43,19 @@ int prepare_fragments_for_encode(ec_backend_t instance,
int data_len; /* data len to write to fragment headers */
int aligned_data_len; /* EC algorithm compatible data length */
int buffer_size, payload_size = 0;
int metadata_size, data_offset = 0;
/* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
data_len = orig_data_size;
aligned_data_len = get_aligned_data_size(instance, orig_data_size);
*blocksize = payload_size = (aligned_data_len / k);
buffer_size = payload_size + instance->common.backend_metadata_size;
metadata_size = instance->common.ops->get_backend_metadata_size(
instance->desc.backend_desc,
*blocksize);
data_offset = instance->common.ops->get_encode_offset(
instance->desc.backend_desc,
metadata_size);
buffer_size = payload_size + metadata_size;
for (i = 0; i < k; i++) {
int copy_size = data_len > payload_size ? payload_size : data_len;
@ -62,7 +69,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
encoded_data[i] = get_data_ptr_from_fragment(fragment);
if (data_len > 0) {
memcpy(encoded_data[i], orig_data, copy_size);
memcpy(encoded_data[i] + data_offset, orig_data, copy_size);
}
orig_data += copy_size;
@ -357,7 +364,6 @@ int fragments_to_string(int k, int m,
char* fragment_data = get_data_ptr_from_fragment(data[i]);
int fragment_size = get_fragment_payload_size(data[i]);
int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size;
memcpy(internal_payload + string_off, fragment_data, payload_size);
orig_data_size -= payload_size;
string_off += payload_size;

View File

@ -42,6 +42,7 @@
#define ISA_L_RS_CAUCHY_BACKEND "isa_l_rs_cauchy"
#define SHSS_BACKEND "shss"
#define RS_VAND_BACKEND "liberasurecode_rs_vand"
#define LIBPHAZR_BACKEND "libphazr"
typedef void (*TEST_FUNC)();
@ -222,6 +223,13 @@ struct ec_args *liberasurecode_rs_vand_test_args[] = {
&liberasurecode_rs_vand_48_args,
NULL };
struct ec_args libphazr_args = {
.k = 4,
.m = 4,
};
struct ec_args *libphazr_test_args[] = { &libphazr_args, NULL };
struct ec_args **all_backend_tests[] = {
null_test_args,
flat_xor_test_args,
@ -230,6 +238,7 @@ struct ec_args **all_backend_tests[] = {
isa_l_test_args,
shss_test_args,
liberasurecode_rs_vand_test_args,
libphazr_test_args,
NULL};
int num_backends()
@ -288,6 +297,8 @@ char * get_name_from_backend_id(ec_backend_id_t be) {
return SHSS_BACKEND;
case EC_BACKEND_LIBERASURECODE_RS_VAND:
return RS_VAND_BACKEND;
case EC_BACKEND_LIBPHAZR:
return LIBPHAZR_BACKEND;
default:
return "UNKNOWN";
}
@ -325,6 +336,9 @@ struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int ba
case EC_BACKEND_SHSS:
backend_args_array = shss_test_args;
break;
case EC_BACKEND_LIBPHAZR:
backend_args_array = libphazr_test_args;
break;
default:
return NULL;
}
@ -932,10 +946,8 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
int num_avail_frags = 0;
char *orig_data_ptr = NULL;
int remaining = 0;
ec_backend_t be = NULL;
desc = liberasurecode_instance_create(be_id, args);
be = liberasurecode_backend_instance_get_by_desc(desc);
if (-EBACKENDNOTAVAIL == desc) {
fprintf (stderr, "Backend library not available!\n");
@ -970,14 +982,14 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
fragment_metadata_t metadata = header->meta;
assert(metadata.idx == i);
assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.backend_metadata_size);
assert(metadata.size == encoded_fragment_len - frag_header_size - metadata.frag_backend_metadata_size);
assert(metadata.orig_data_size == orig_data_size);
assert(metadata.backend_id == be_id);
assert(metadata.chksum_mismatch == 0);
data_ptr = frag + frag_header_size;
cmp_size = remaining >= metadata.size ? metadata.size : remaining;
// shss doesn't keep original data on data fragments
if (be_id != EC_BACKEND_SHSS) {
// shss & libphazr doesn't keep original data on data fragments
if (be_id != EC_BACKEND_SHSS && be_id != EC_BACKEND_LIBPHAZR) {
assert(memcmp(data_ptr, orig_data_ptr, cmp_size) == 0);
}
remaining -= cmp_size;
@ -2177,6 +2189,75 @@ struct testcase testcases[] = {
test_verify_stripe_metadata_frag_idx_invalid,
EC_BACKEND_LIBERASURECODE_RS_VAND, CHKSUM_CRC32,
.skip = false},
// libphazr backend tests
{"create_and_destroy_backend",
test_create_and_destroy_backend,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"simple_encode_libphazr",
test_simple_encode_decode,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_data_libphazr",
test_decode_with_missing_data,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_parity_libphazr",
test_decode_with_missing_parity,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_multi_data_libphazr",
test_decode_with_missing_multi_data,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_multi_parity_libphazr",
test_decode_with_missing_multi_parity,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_decode_with_missing_multi_data_parity_libphazr",
test_decode_with_missing_multi_data_parity,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"simple_reconstruct_libphazr",
test_simple_reconstruct,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_fragments_needed_libphazr",
test_fragments_needed,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_get_fragment_metadata_libphazr",
test_get_fragment_metadata,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_get_fragment_metadata_libphazr_crc32",
test_get_fragment_metadata,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata",
test_verify_stripe_metadata,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_libec_mismatch",
test_verify_stripe_metadata_libec_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_magic_mismatch",
test_verify_stripe_metadata_magic_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_id_mismatch",
test_verify_stripe_metadata_be_id_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_ver_mismatch",
test_verify_stripe_metadata_be_ver_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_frag_idx_invalid",
test_verify_stripe_metadata_frag_idx_invalid,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{ NULL, NULL, 0, 0, false },
};