Merge "Add Phazr.IO libphazr backend to liberasurecode"

This commit is contained in:
Jenkins 2017-03-01 02:21:14 +00:00 committed by Gerrit Code Review
commit 26e4742140
17 changed files with 551 additions and 32 deletions

View File

@ -17,6 +17,7 @@ Highlights
- 'ISA-L' - Intel Storage Acceleration Library - SIMD accelerated Erasure Coding backends [2]
- 'SHSS' - NTT Lab Japan's hybrid Erasure Coding backend [4]
- 'Flat XOR HD' - built-in to liberasurecode, based on [3]
- 'libphazr' - Phazr.IO's erasure code backend with built-in privacy [5]
- 'NULL' template backend implemented to help future backend writers
@ -381,6 +382,8 @@ Code organization
| | +-- isa_l_rs_vand.c --> 'isa_l_rs_vand' erasure code backend (Intel)
| | +-- shss
| | +-- shss.c --> 'shss' erasure code backend (NTT Labs)
| | +-- phazrio
| | +-- libphazr.c --> 'libphazr' erasure code backend (Phazr.IO)
| |
| |-- builtin
| | +-- xor_codes --> XOR HD code backend, built-in erasure
@ -426,3 +429,5 @@ References
[3] Greenan, Kevin M et al, "Flat XOR-based erasure codes in storage systems", http://www.kaymgee.com/Kevin_Greenan/Publications_files/greenan-msst10.pdf
[4] Kota Tsuyuzaki <tsuyuzaki.kota@lab.ntt.co.jp>, "NTT SHSS Erasure Coding backend"
[5] Jim Cheung <support@phazr.io>, "Phazr.IO libphazr erasure code backend with built-in privacy"

View File

@ -49,6 +49,7 @@ typedef enum {
EC_BACKEND_SHSS = 5,
EC_BACKEND_LIBERASURECODE_RS_VAND = 6,
EC_BACKEND_ISA_L_RS_CAUCHY = 7,
EC_BACKEND_LIBPHAZR = 8,
EC_BACKENDS_MAX,
} ec_backend_id_t;

View File

@ -1,4 +1,4 @@
/*
/*
* <Copyright>
*
* Redistribution and use in source and binary forms, with or without
@ -64,6 +64,8 @@ struct ec_backend_args {
#define RECONSTRUCT reconstruct
#define ELEMENTSIZE element_size
#define ISCOMPATIBLEWITH is_compatible_with
#define GETMETADATASIZE get_backend_metadata_size
#define GETENCODEOFFSET get_encode_offset
#define FN_NAME(s) str(s)
#define str(s) #s
@ -90,7 +92,10 @@ struct ec_backend_op_stubs {
int blocksize);
int (*ELEMENTSIZE)(void *desc);
bool (*ISCOMPATIBLEWITH) (uint32_t version);
bool (*ISCOMPATIBLEWITH)(uint32_t version);
size_t (*GETMETADATASIZE)(void *desc, int blocksize);
size_t (*GETENCODEOFFSET)(void *desc, int metadata_size);
};
/* ==~=*=~==~=*=~==~=*=~= backend struct definitions =~=*=~==~=*=~==~=*==~== */
@ -109,11 +114,6 @@ struct ec_backend_common {
char soversion[MAX_LEN]; /* EC backend shared library version */
struct ec_backend_op_stubs *ops; /* EC backend stubs */
size_t backend_metadata_size;
/* EC backend custom metadata size -
* backend_metadata_size bytes are added to
* the fragment size when allocating
* data/parity fragment buffers */
uint32_t ec_backend_version; /* The revision number of this back
* end. Is used to determine whether
* a specific instance of this backend
@ -162,6 +162,21 @@ ec_backend_t liberasurecode_backend_lookup_by_name(const char *name);
*/
ec_backend_t liberasurecode_backend_instance_get_by_desc(int desc);
/* Common function for backends */
/**
* A function to return 0 for generic usage on backends for get_encode_offset
*
* Returns 0 always
*/
static inline size_t get_encode_offset_zero(void *desc, int metadata_size){ return 0; }
/**
* A function to return 0 for generic usage on backends for get_backend_metadata_size
*
* Returns 0 always
*/
static inline size_t get_backend_metadata_size_zero(void *desc, int blocksize){ return 0; }
/* =~=*=~==~=*=~==~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~===~=*=~==~=*=~= */
#ifdef __cplusplus

View File

@ -26,7 +26,8 @@ liberasurecode_la_SOURCES = \
backends/isa-l/isa_l_rs_cauchy.c \
backends/rs_vand/liberasurecode_rs_vand.c \
builtin/rs_vand/rs_galois.c \
backends/shss/shss.c
backends/shss/shss.c \
backends/phazrio/libphazr.c
liberasurecode_la_CPPFLAGS = -Werror @GCOV_FLAGS@
liberasurecode_la_LIBADD = \
@ -42,4 +43,5 @@ MOSTLYCLEANFILES = *.gcda *.gcno *.gcov utils/chksum/*.gcda utils/chksum/*.gcno
backends/xor/*.gcda backends/xor/*.gcno backends/xor/*.gcov \
backends/jerasure/*.gcda backends/jerasure/*.gcno backends/jerasure/*.gcov \
backends/shss/*.gcda backends/shss/*.gcno backends/shss/*.gcov \
backends/rs_vand/*.gcda backends/rs_vand/*.gcno backends/rs_vand/*.gcov
backends/rs_vand/*.gcda backends/rs_vand/*.gcno backends/rs_vand/*.gcov \
backends/phazrio/*.gcda backends/phazrio/*.gcno backends/phazrio/*.gcov

View File

@ -71,6 +71,8 @@ struct ec_backend_op_stubs isa_l_rs_cauchy_op_stubs = {
.RECONSTRUCT = isa_l_reconstruct,
.ELEMENTSIZE = isa_l_element_size,
.ISCOMPATIBLEWITH = isa_l_rs_cauchy_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_isa_l_rs_cauchy = {
@ -79,7 +81,6 @@ struct ec_backend_common backend_isa_l_rs_cauchy = {
.soname = ISA_L_RS_CAUCHY_SO_NAME,
.soversion = ISA_L_RS_CAUCHY_LIB_VER_STR,
.ops = &isa_l_rs_cauchy_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(ISA_L_RS_CAUCHY_LIB_MAJOR,
ISA_L_RS_CAUCHY_LIB_MINOR,
ISA_L_RS_CAUCHY_LIB_REV),

View File

@ -70,6 +70,8 @@ struct ec_backend_op_stubs isa_l_rs_vand_op_stubs = {
.RECONSTRUCT = isa_l_reconstruct,
.ELEMENTSIZE = isa_l_element_size,
.ISCOMPATIBLEWITH = isa_l_rs_vand_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_isa_l_rs_vand = {
@ -78,7 +80,6 @@ struct ec_backend_common backend_isa_l_rs_vand = {
.soname = ISA_L_RS_VAND_SO_NAME,
.soversion = ISA_L_RS_VAND_LIB_VER_STR,
.ops = &isa_l_rs_vand_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(ISA_L_RS_VAND_LIB_MAJOR,
ISA_L_RS_VAND_LIB_MINOR,
ISA_L_RS_VAND_LIB_REV),

View File

@ -447,7 +447,6 @@ static bool jerasure_rs_cauchy_is_compatible_with(uint32_t version) {
return version == backend_jerasure_rs_cauchy.ec_backend_version;
}
struct ec_backend_op_stubs jerasure_rs_cauchy_op_stubs = {
.INIT = jerasure_rs_cauchy_init,
.EXIT = jerasure_rs_cauchy_exit,
@ -457,7 +456,8 @@ struct ec_backend_op_stubs jerasure_rs_cauchy_op_stubs = {
.RECONSTRUCT = jerasure_rs_cauchy_reconstruct,
.ELEMENTSIZE = jerasure_rs_cauchy_element_size,
.ISCOMPATIBLEWITH = jerasure_rs_cauchy_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_jerasure_rs_cauchy = {
@ -466,7 +466,6 @@ struct ec_backend_common backend_jerasure_rs_cauchy = {
.soname = JERASURE_RS_CAUCHY_SO_NAME,
.soversion = JERASURE_RS_CAUCHY_LIB_VER_STR,
.ops = &jerasure_rs_cauchy_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(JERASURE_RS_CAUCHY_LIB_MAJOR,
JERASURE_RS_CAUCHY_LIB_MINOR,
JERASURE_RS_CAUCHY_LIB_REV),

View File

@ -368,6 +368,8 @@ struct ec_backend_op_stubs jerasure_rs_vand_op_stubs = {
.RECONSTRUCT = jerasure_rs_vand_reconstruct,
.ELEMENTSIZE = jerasure_rs_vand_element_size,
.ISCOMPATIBLEWITH = jerasure_rs_vand_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_jerasure_rs_vand = {
@ -376,7 +378,6 @@ struct ec_backend_common backend_jerasure_rs_vand = {
.soname = JERASURE_RS_VAND_SO_NAME,
.soversion = JERASURE_RS_VAND_LIB_VER_STR,
.ops = &jerasure_rs_vand_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(JERASURE_RS_VAND_LIB_MAJOR,
JERASURE_RS_VAND_LIB_MINOR,
JERASURE_RS_VAND_LIB_REV),

View File

@ -50,7 +50,6 @@ typedef int (*null_code_encode_func)(void *, char **, char **, int);
typedef int (*null_code_decode_func)(void *, char **, char **, int *, int, int);
typedef int (*null_reconstruct_func)(char **, int, uint64_t, int, char *);
typedef int (*null_code_fragments_needed_func)(void *, int *, int *, int *);
struct null_descriptor {
/* calls required for init */
init_null_code_func init_null_code;
@ -215,6 +214,7 @@ static int null_exit(void *desc)
static bool null_is_compatible_with(uint32_t version) {
return true;
}
struct ec_backend_op_stubs null_op_stubs = {
.INIT = null_init,
.EXIT = null_exit,
@ -224,6 +224,8 @@ struct ec_backend_op_stubs null_op_stubs = {
.RECONSTRUCT = null_reconstruct,
.ELEMENTSIZE = null_element_size,
.ISCOMPATIBLEWITH = null_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_null = {
@ -232,7 +234,6 @@ struct ec_backend_common backend_null = {
.soname = NULL_SO_NAME,
.soversion = NULL_LIB_VER_STR,
.ops = &null_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(NULL_LIB_MAJOR, NULL_LIB_MINOR,
NULL_LIB_REV),
};

View File

@ -0,0 +1,391 @@
/*
* Copyright 2016 Phazr.IO Inc
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions are met:
*
* Redistributions of source code must retain the above copyright notice, this
* list of conditions and the following disclaimer.
*
* Redistributions in binary form must reproduce the above copyright notice, this
* list of conditions and the following disclaimer in the documentation and/or
* other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
* THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
* EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*
* Phazr.IO libphazr backend
*
* vi: set noai tw=79 ts=4 sw=4:
*/
#include <stdio.h>
#include <stdlib.h>
#include "erasurecode.h"
#include "erasurecode_backend.h"
#include "erasurecode_helpers.h"
#define LIBPHAZR_LIB_MAJOR 1
#define LIBPHAZR_LIB_MINOR 0
#define LIBPHAZR_LIB_REV 0
#define LIBPHAZR_LIB_VER_STR "1.0.0"
#define LIBPHAZR_LIB_NAME "libphazr"
#if defined(__MACOS__) || defined(__MACOSX__) || defined(__OSX__) || defined(__APPLE__)
#define LIBPHAZR_SO_NAME "libphazr.dylib"
#else
#define LIBPHAZR_SO_NAME "libphazr.so.1"
#endif
/* Forward declarations */
struct ec_backend libphazr;
struct ec_backend_op_stubs libphazr_ops;
struct ec_backend_common backend_libphazr;
typedef int (*pio_matrix_encode_func)(char *, char *, char **, int, int, int, int, int, int);
typedef int (*pio_matrix_decode_func)(char *, char *, char **, int *, int, int, int, int, int, int);
typedef int (*pio_matrix_reconstruct_func)(char *, char **, int *, int, int, int, int, int, int);
typedef char* (*pio_create_precoding_matrix_func)(int);
typedef char* (*pio_create_inverse_precoding_matrix_func)(int);
typedef char* (*pio_create_kmux_matrix_func)(int, int, int);
struct libphazr_descriptor {
/* calls required for init */
pio_create_precoding_matrix_func create_precoding_matrix;
pio_create_inverse_precoding_matrix_func create_inverse_precoding_matrix;
pio_create_kmux_matrix_func create_kmux_matrix;
/* calls required for encode */
pio_matrix_encode_func matrix_encode;
/* calls required for decode */
pio_matrix_decode_func matrix_decode;
/* calls required for reconstruct */
pio_matrix_reconstruct_func matrix_reconstruct;
/* fields needed to hold state */
char *matrix;
char *precoding_matrix;
char *inverse_precoding_matrix;
int k;
int m;
int w;
int hd;
};
#define DEFAULT_W 64
#define DEFAULT_HD 1
static int get_padded_blocksize(int w, int hd, int blocksize)
{
int word_size = w / 8;
return (int) ceill((double) blocksize / (word_size - hd)) * word_size;
}
static int pio_matrix_encode(void *desc, char **data, char **parity, int blocksize)
{
int i, ret = 0;
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padding_size = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize) - blocksize;
char **encoded = malloc(sizeof(char*) * (xdesc->k + xdesc->m));
if (NULL == encoded) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < xdesc->k; i++) {
encoded[i] = data[i];
}
for (i = 0; i < xdesc->m; i++) {
encoded[i + xdesc->k] = parity[i];
}
ret = xdesc->matrix_encode(xdesc->precoding_matrix, xdesc->matrix, encoded,
xdesc->k, xdesc->m, xdesc->w, xdesc->hd, blocksize, padding_size);
out:
free(encoded);
return ret;
}
static int pio_matrix_decode(void *desc, char **data, char **parity,
int *missing_idxs, int blocksize)
{
int i, ret = 0;
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padding_size = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize) - blocksize;
char **decoded = malloc(sizeof(char*) * (xdesc->k + xdesc->m));
if (NULL == decoded) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < xdesc->k; i++) {
decoded[i] = data[i];
}
for (i = 0; i < xdesc->m; i++) {
decoded[i + xdesc->k] = parity[i];
}
ret = xdesc->matrix_decode(xdesc->inverse_precoding_matrix, xdesc->matrix, decoded,
missing_idxs, xdesc->k, xdesc->m, xdesc->w, xdesc->hd, blocksize, padding_size);
out:
free(decoded);
return ret;
}
static int pio_matrix_reconstruct(void *desc, char **data, char **parity,
int *missing_idxs, int destination_idx, int blocksize)
{
int i, ret = 0;
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padding_size = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize) - blocksize;
char **encoded = malloc(sizeof(char*) * (xdesc->k + xdesc->m));
if (NULL == encoded) {
ret = -ENOMEM;
goto out;
}
for (i = 0; i < xdesc->k; i++) {
encoded[i] = data[i];
}
for (i = 0; i < xdesc->m; i++) {
encoded[i + xdesc->k] = parity[i];
}
ret = xdesc->matrix_reconstruct(xdesc->matrix, encoded, missing_idxs,
destination_idx, xdesc->k, xdesc->m, xdesc->w, blocksize, padding_size);
out:
free(encoded);
return ret;
}
static int pio_min_fragments(void *desc, int *missing_idxs,
int *fragments_to_exclude, int *fragments_needed)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *)desc;
uint64_t exclude_bm = convert_list_to_bitmap(fragments_to_exclude);
uint64_t missing_bm = convert_list_to_bitmap(missing_idxs) | exclude_bm;
int i;
int j = 0;
int ret = -1;
for (i = 0; i < (xdesc->k + xdesc->m); i++) {
if (!(missing_bm & (1 << i))) {
fragments_needed[j] = i;
j++;
}
if (j == xdesc->k) {
ret = 0;
fragments_needed[j] = -1;
break;
}
}
return ret;
}
/**
* Return the element-size, which is the number of bits stored
* on a given device, per codeword.
*/
static int pio_element_size(void *desc)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *)desc;
return xdesc->w;
}
static void * pio_init(struct ec_backend_args *args, void *backend_sohandle)
{
struct libphazr_descriptor *desc = NULL;
/* allocate and fill in libphazr_descriptor */
desc = (struct libphazr_descriptor *)malloc(sizeof(struct libphazr_descriptor));
if (NULL == desc) {
return NULL;
}
memset(desc, 0, sizeof(struct libphazr_descriptor));
desc->k = args->uargs.k;
desc->m = args->uargs.m;
desc->w = args->uargs.w;
desc->hd = args->uargs.hd;
if (desc->w <= 0)
desc->w = DEFAULT_W;
args->uargs.w = desc->w;
if (desc->hd <= 0)
desc->hd = DEFAULT_HD;
args->uargs.hd = desc->hd;
/*
* ISO C forbids casting a void* to a function pointer.
* Since dlsym return returns a void*, we use this union to
* "transform" the void* to a function pointer.
*/
union {
pio_create_precoding_matrix_func create_precoding_matrix_ptr;
pio_create_inverse_precoding_matrix_func create_inverse_precoding_matrix_ptr;
pio_create_kmux_matrix_func create_kmux_matrix_ptr;
pio_matrix_encode_func matrix_encode_ptr;
pio_matrix_decode_func matrix_decode_ptr;
pio_matrix_reconstruct_func matrix_reconstruct_ptr;
void *vptr;
} func_handle = {.vptr = NULL};
/* fill in function addresses */
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "create_precoding_matrix");
desc->create_precoding_matrix = func_handle.create_precoding_matrix_ptr;
if (NULL == desc->create_precoding_matrix) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "create_inverse_precoding_matrix");
desc->create_inverse_precoding_matrix = func_handle.create_inverse_precoding_matrix_ptr;
if (NULL == desc->create_inverse_precoding_matrix) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "create_kmux_matrix");
desc->create_kmux_matrix = func_handle.create_kmux_matrix_ptr;
if (NULL == desc->create_kmux_matrix) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "matrix_encode");
desc->matrix_encode = func_handle.matrix_encode_ptr;
if (NULL == desc->matrix_encode) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "matrix_decode");
desc->matrix_decode = func_handle.matrix_decode_ptr;
if (NULL == desc->matrix_decode) {
goto error;
}
func_handle.vptr = NULL;
func_handle.vptr = dlsym(backend_sohandle, "matrix_reconstruct");
desc->matrix_reconstruct = func_handle.matrix_reconstruct_ptr;
if (NULL == desc->matrix_reconstruct) {
goto error;
}
if (NULL == desc->precoding_matrix) {
desc->precoding_matrix = desc->create_precoding_matrix(desc->k);
if (NULL == desc->precoding_matrix) {
goto error;
}
}
if (NULL == desc->inverse_precoding_matrix) {
desc->inverse_precoding_matrix = desc->create_inverse_precoding_matrix(desc->k);
if (NULL == desc->inverse_precoding_matrix) {
goto error;
}
}
if (NULL == desc->matrix) {
desc->matrix = desc->create_kmux_matrix(desc->k, desc->m, desc->w);
if (NULL == desc->create_kmux_matrix) {
goto error;
}
}
return (void *) desc;
error:
free(desc->matrix);
free(desc->precoding_matrix);
free(desc->inverse_precoding_matrix);
free(desc);
return NULL;
}
static int pio_exit(void *desc)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor*)desc;
free(xdesc->matrix);
free(xdesc->precoding_matrix);
free(xdesc->inverse_precoding_matrix);
free(xdesc);
return 0;
}
static bool pio_is_compatible_with(uint32_t version)
{
return version == backend_libphazr.ec_backend_version;
}
static size_t pio_get_backend_metadata_size(void *desc, int blocksize)
{
struct libphazr_descriptor *xdesc = (struct libphazr_descriptor *) desc;
int padded_blocksize = get_padded_blocksize(xdesc->w, xdesc->hd, blocksize);
return padded_blocksize - blocksize;
}
static size_t pio_get_encode_offset(void *desc, int metadata_size)
{
return metadata_size;
}
struct ec_backend_op_stubs libphazr_op_stubs = {
.INIT = pio_init,
.EXIT = pio_exit,
.ENCODE = pio_matrix_encode,
.DECODE = pio_matrix_decode,
.FRAGSNEEDED = pio_min_fragments,
.RECONSTRUCT = pio_matrix_reconstruct,
.ELEMENTSIZE = pio_element_size,
.ISCOMPATIBLEWITH = pio_is_compatible_with,
.GETMETADATASIZE = pio_get_backend_metadata_size,
.GETENCODEOFFSET = pio_get_encode_offset,
};
struct ec_backend_common backend_libphazr = {
.id = EC_BACKEND_LIBPHAZR,
.name = LIBPHAZR_LIB_NAME,
.soname = LIBPHAZR_SO_NAME,
.soversion = LIBPHAZR_LIB_VER_STR,
.ops = &libphazr_op_stubs,
.ec_backend_version = _VERSION(LIBPHAZR_LIB_MAJOR, LIBPHAZR_LIB_MINOR,
LIBPHAZR_LIB_REV),
};

View File

@ -298,6 +298,8 @@ struct ec_backend_op_stubs liberasurecode_rs_vand_op_stubs = {
.RECONSTRUCT = liberasurecode_rs_vand_reconstruct,
.ELEMENTSIZE = liberasurecode_rs_vand_element_size,
.ISCOMPATIBLEWITH = liberasurecode_rs_vand_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_liberasurecode_rs_vand = {
@ -306,7 +308,6 @@ struct ec_backend_common backend_liberasurecode_rs_vand = {
.soname = LIBERASURECODE_RS_VAND_SO_NAME,
.soversion = LIBERASURECODE_RS_VAND_LIB_VER_STR,
.ops = &liberasurecode_rs_vand_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(LIBERASURECODE_RS_VAND_LIB_MAJOR,
LIBERASURECODE_RS_VAND_LIB_MINOR,
LIBERASURECODE_RS_VAND_LIB_REV),

View File

@ -286,6 +286,10 @@ static bool shss_is_compatible_with(uint32_t version) {
return version == backend_shss.ec_backend_version;
}
static size_t shss_get_backend_metadata_size(void *desc, int blocksize) {
return METADATA;
}
struct ec_backend_op_stubs shss_op_stubs = {
.INIT = shss_init,
.EXIT = shss_exit,
@ -295,6 +299,8 @@ struct ec_backend_op_stubs shss_op_stubs = {
.RECONSTRUCT = shss_reconstruct,
.ELEMENTSIZE = shss_element_size,
.ISCOMPATIBLEWITH = shss_is_compatible_with,
.GETMETADATASIZE = shss_get_backend_metadata_size,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_shss = {
@ -303,5 +309,4 @@ struct ec_backend_common backend_shss = {
.soname = SHSS_SO_NAME,
.soversion = SHSS_LIB_VER_STR,
.ops = &shss_op_stubs,
.backend_metadata_size = METADATA,
};

View File

@ -178,6 +178,8 @@ struct ec_backend_op_stubs flat_xor_hd_op_stubs = {
.RECONSTRUCT = flat_xor_hd_reconstruct,
.ELEMENTSIZE = flar_xor_hd_element_size,
.ISCOMPATIBLEWITH = flat_xor_is_compatible_with,
.GETMETADATASIZE = get_backend_metadata_size_zero,
.GETENCODEOFFSET = get_encode_offset_zero,
};
struct ec_backend_common backend_flat_xor_hd = {
@ -186,7 +188,6 @@ struct ec_backend_common backend_flat_xor_hd = {
.soname = FLAT_XOR_SO_NAME,
.soversion = FLAT_XOR_LIB_VER_STR,
.ops = &flat_xor_hd_op_stubs,
.backend_metadata_size = 0,
.ec_backend_version = _VERSION(FLAT_XOR_LIB_MAJOR,
FLAT_XOR_LIB_MINOR,
FLAT_XOR_LIB_REV),

View File

@ -50,6 +50,7 @@ extern struct ec_backend_common backend_isa_l_rs_vand;
extern struct ec_backend_common backend_shss;
extern struct ec_backend_common backend_liberasurecode_rs_vand;
extern struct ec_backend_common backend_isa_l_rs_cauchy;
extern struct ec_backend_common backend_libphazr;
ec_backend_t ec_backends_supported[] = {
(ec_backend_t) &backend_null,
@ -60,6 +61,7 @@ ec_backend_t ec_backends_supported[] = {
(ec_backend_t) &backend_shss,
(ec_backend_t) &backend_liberasurecode_rs_vand,
(ec_backend_t) &backend_isa_l_rs_cauchy,
(ec_backend_t) &backend_libphazr,
NULL,
};
@ -604,8 +606,8 @@ int liberasurecode_decode(int desc,
}
}
if (instance->common.id != EC_BACKEND_SHSS) {
/* shss (ntt_backend) must force to decode */
if (instance->common.id != EC_BACKEND_SHSS && instance->common.id != EC_BACKEND_LIBPHAZR) {
/* shss (ntt_backend) & libphazr backend must force to decode */
// TODO: Add a frag and function to handle whether the backend want to decode or not.
/*
* Try to re-assebmle the original data before attempting a decode
@ -1239,7 +1241,11 @@ int liberasurecode_get_fragment_size(int desc, int data_len)
if (NULL == instance)
return -EBACKENDNOTAVAIL;
int aligned_data_len = get_aligned_data_size(instance, data_len);
int size = (aligned_data_len / instance->args.uargs.k) + instance->common.backend_metadata_size;
int blocksize = aligned_data_len / instance->args.uargs.k;
int metadata_size = instance->common.ops->get_backend_metadata_size(
instance->desc.backend_desc,
blocksize);
int size = blocksize + metadata_size;
return size;
}

View File

@ -42,7 +42,9 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
set_fragment_payload_size(fragment, blocksize);
set_backend_id(fragment, be->common.id);
set_backend_version(fragment, be->common.ec_backend_version);
set_fragment_backend_metadata_size(fragment, be->common.backend_metadata_size);
set_fragment_backend_metadata_size(fragment, be->common.ops->get_backend_metadata_size(
be->desc.backend_desc,
blocksize));
if (add_chksum) {
set_checksum(ct, fragment, blocksize);

View File

@ -43,12 +43,19 @@ int prepare_fragments_for_encode(ec_backend_t instance,
int data_len; /* data len to write to fragment headers */
int aligned_data_len; /* EC algorithm compatible data length */
int buffer_size, payload_size = 0;
int metadata_size, data_offset = 0;
/* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
data_len = orig_data_size;
aligned_data_len = get_aligned_data_size(instance, orig_data_size);
*blocksize = payload_size = (aligned_data_len / k);
buffer_size = payload_size + instance->common.backend_metadata_size;
metadata_size = instance->common.ops->get_backend_metadata_size(
instance->desc.backend_desc,
*blocksize);
data_offset = instance->common.ops->get_encode_offset(
instance->desc.backend_desc,
metadata_size);
buffer_size = payload_size + metadata_size;
for (i = 0; i < k; i++) {
int copy_size = data_len > payload_size ? payload_size : data_len;
@ -62,7 +69,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
encoded_data[i] = get_data_ptr_from_fragment(fragment);
if (data_len > 0) {
memcpy(encoded_data[i], orig_data, copy_size);
memcpy(encoded_data[i] + data_offset, orig_data, copy_size);
}
orig_data += copy_size;
@ -357,7 +364,6 @@ int fragments_to_string(int k, int m,
char* fragment_data = get_data_ptr_from_fragment(data[i]);
int fragment_size = get_fragment_payload_size(data[i]);
int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size;
memcpy(internal_payload + string_off, fragment_data, payload_size);
orig_data_size -= payload_size;
string_off += payload_size;

View File

@ -42,6 +42,7 @@
#define ISA_L_RS_CAUCHY_BACKEND "isa_l_rs_cauchy"
#define SHSS_BACKEND "shss"
#define RS_VAND_BACKEND "liberasurecode_rs_vand"
#define LIBPHAZR_BACKEND "libphazr"
typedef void (*TEST_FUNC)();
@ -222,6 +223,13 @@ struct ec_args *liberasurecode_rs_vand_test_args[] = {
&liberasurecode_rs_vand_48_args,
NULL };
struct ec_args libphazr_args = {
.k = 4,
.m = 4,
};
struct ec_args *libphazr_test_args[] = { &libphazr_args, NULL };
struct ec_args **all_backend_tests[] = {
null_test_args,
flat_xor_test_args,
@ -230,6 +238,7 @@ struct ec_args **all_backend_tests[] = {
isa_l_test_args,
shss_test_args,
liberasurecode_rs_vand_test_args,
libphazr_test_args,
NULL};
int num_backends()
@ -288,6 +297,8 @@ char * get_name_from_backend_id(ec_backend_id_t be) {
return SHSS_BACKEND;
case EC_BACKEND_LIBERASURECODE_RS_VAND:
return RS_VAND_BACKEND;
case EC_BACKEND_LIBPHAZR:
return LIBPHAZR_BACKEND;
default:
return "UNKNOWN";
}
@ -325,6 +336,9 @@ struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int ba
case EC_BACKEND_SHSS:
backend_args_array = shss_test_args;
break;
case EC_BACKEND_LIBPHAZR:
backend_args_array = libphazr_test_args;
break;
default:
return NULL;
}
@ -991,10 +1005,8 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
int num_avail_frags = 0;
char *orig_data_ptr = NULL;
int remaining = 0;
ec_backend_t be = NULL;
desc = liberasurecode_instance_create(be_id, args);
be = liberasurecode_backend_instance_get_by_desc(desc);
if (-EBACKENDNOTAVAIL == desc) {
fprintf (stderr, "Backend library not available!\n");
@ -1029,14 +1041,14 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
fragment_metadata_t metadata = header->meta;
assert(metadata.idx == i);
assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.backend_metadata_size);
assert(metadata.size == encoded_fragment_len - frag_header_size - metadata.frag_backend_metadata_size);
assert(metadata.orig_data_size == orig_data_size);
assert(metadata.backend_id == be_id);
assert(metadata.chksum_mismatch == 0);
data_ptr = frag + frag_header_size;
cmp_size = remaining >= metadata.size ? metadata.size : remaining;
// shss doesn't keep original data on data fragments
if (be_id != EC_BACKEND_SHSS) {
// shss & libphazr doesn't keep original data on data fragments
if (be_id != EC_BACKEND_SHSS && be_id != EC_BACKEND_LIBPHAZR) {
assert(memcmp(data_ptr, orig_data_ptr, cmp_size) == 0);
}
remaining -= cmp_size;
@ -2243,6 +2255,75 @@ struct testcase testcases[] = {
test_verify_stripe_metadata_frag_idx_invalid,
EC_BACKEND_LIBERASURECODE_RS_VAND, CHKSUM_CRC32,
.skip = false},
// libphazr backend tests
{"create_and_destroy_backend",
test_create_and_destroy_backend,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"simple_encode_libphazr",
test_simple_encode_decode,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_data_libphazr",
test_decode_with_missing_data,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_parity_libphazr",
test_decode_with_missing_parity,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_multi_data_libphazr",
test_decode_with_missing_multi_data,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"decode_with_missing_multi_parity_libphazr",
test_decode_with_missing_multi_parity,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_decode_with_missing_multi_data_parity_libphazr",
test_decode_with_missing_multi_data_parity,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"simple_reconstruct_libphazr",
test_simple_reconstruct,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_fragments_needed_libphazr",
test_fragments_needed,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_get_fragment_metadata_libphazr",
test_get_fragment_metadata,
EC_BACKEND_LIBPHAZR, CHKSUM_NONE,
.skip = false},
{"test_get_fragment_metadata_libphazr_crc32",
test_get_fragment_metadata,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata",
test_verify_stripe_metadata,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_libec_mismatch",
test_verify_stripe_metadata_libec_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_magic_mismatch",
test_verify_stripe_metadata_magic_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_id_mismatch",
test_verify_stripe_metadata_be_id_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_be_ver_mismatch",
test_verify_stripe_metadata_be_ver_mismatch,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{"test_verify_stripe_metadata_frag_idx_invalid",
test_verify_stripe_metadata_frag_idx_invalid,
EC_BACKEND_LIBPHAZR, CHKSUM_CRC32,
.skip = false},
{ NULL, NULL, 0, 0, false },
};