openstack
/
liberasurecode
Code Issues Proposed changes

This is the fix for Issue #13: 

https://bitbucket.org/tsg-/liberasurecode/issue/13/decode-fails-for-many-cases-when-m-k

This fix includes:

1.) Proper buffer allocation for the 'missing_idxs' structure, which was not allocating enough
    space when k > m.
2.) Checks to use header fields of parity fragments during decode when *no* data fragments
    are available.
3.) Fixed the unit tests to properly handle the case where k <= m.
4.) Extended the unit test framework to support multiple tests per backend
5.) Added tests for all RS implementations: (4,8), (4,4), (10,10)
This commit is contained in:
Kevin Greenan 2015-04-26 16:59:04 -07:00
parent a380246762
commit a01b1818c8
3 changed files with 186 additions and 36 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
src/erasurecode.c
View File

@ -587,7 +587,7 @@ int liberasurecode_decode(int desc,
goto out;
}
missing_idxs = alloc_and_set_buffer(sizeof(char*) * k, -1);
missing_idxs = alloc_and_set_buffer(sizeof(char*) * (k + m), -1);
if (NULL == missing_idxs) {
log_error("Could not allocate missing_idxs buffer!");
goto out;
@ -771,7 +771,7 @@ int liberasurecode_reconstruct_fragment(int desc,
goto out;
}
missing_idxs = alloc_and_set_buffer(sizeof(int*) * k, -1);
missing_idxs = alloc_and_set_buffer(sizeof(int*) * (k + m), -1);
if (NULL == missing_idxs) {
log_error("Could not allocate missing_idxs buffer!");
goto out;

16
src/erasurecode_preprocessing.c
View File

@ -119,7 +119,7 @@ int prepare_fragments_for_decode(
unsigned long long missing_bm; /* bitmap form of missing indexes list */
int orig_data_size = -1;
int payload_size = -1;
missing_bm = convert_list_to_bitmap(missing_idxs);
/*
@ -192,6 +192,20 @@ int prepare_fragments_for_decode(
*realloc_bm = *realloc_bm | (1 << (k + i));
}
/* Need to determine the size of the original data */
if (((missing_bm & (1 << (k + i))) == 0) && orig_data_size < 0) {
orig_data_size = get_orig_data_size(parity[i]);
if (orig_data_size < 0) {
log_error("Invalid orig_data_size in fragment header!");
return -EBADHEADER;
}
payload_size = get_fragment_payload_size(parity[i]);
if (orig_data_size < 0) {
log_error("Invalid fragment_size in fragment header!");
return -EBADHEADER;
}
}
}
*orig_size = orig_data_size;

202
test/liberasurecode_test.c
View File

@ -57,6 +57,8 @@ struct ec_args null_args = {
.ct = CHKSUM_NONE,
};
struct ec_args *null_test_args[] = { &null_args, NULL };
struct ec_args flat_xor_hd_args = {
.k = 3,
.m = 3,
@ -64,6 +66,8 @@ struct ec_args flat_xor_hd_args = {
.ct = CHKSUM_NONE,
};
struct ec_args *flat_xor_test_args[] = { &flat_xor_hd_args, NULL };
struct ec_args jerasure_rs_vand_args = {
.k = 10,
.m = 4,
@ -72,6 +76,35 @@ struct ec_args jerasure_rs_vand_args = {
.ct = CHKSUM_NONE,
};
struct ec_args jerasure_rs_vand_44_args = {
.k = 4,
.m = 4,
.w = 16,
.hd = 5,
.ct = CHKSUM_NONE,
};
struct ec_args jerasure_rs_vand_48_args = {
.k = 4,
.m = 8,
.w = 16,
.hd = 9,
.ct = CHKSUM_NONE,
};
struct ec_args jerasure_rs_vand_1010_args = {
.k = 10,
.m = 10,
.w = 16,
.hd = 11,
.ct = CHKSUM_NONE,
};
struct ec_args *jerasure_rs_vand_test_args[] = { &jerasure_rs_vand_args,
&jerasure_rs_vand_44_args,
&jerasure_rs_vand_1010_args,
&jerasure_rs_vand_48_args,
NULL };
struct ec_args jerasure_rs_cauchy_args = {
.k = 10,
.m = 4,
@ -80,6 +113,37 @@ struct ec_args jerasure_rs_cauchy_args = {
.ct = CHKSUM_NONE,
};
struct ec_args jerasure_rs_cauchy_44_args = {
.k = 4,
.m = 4,
.w = 4,
.hd = 5,
.ct = CHKSUM_NONE,
};
struct ec_args jerasure_rs_cauchy_48_args = {
.k = 4,
.m = 8,
.w = 8,
.hd = 9,
.ct = CHKSUM_NONE,
};
struct ec_args jerasure_rs_cauchy_1010_args = {
.k = 10,
.m = 10,
.w = 8,
.hd = 11,
.ct = CHKSUM_NONE,
};
struct ec_args *jerasure_rs_cauchy_test_args[] = { &jerasure_rs_cauchy_args,
&jerasure_rs_cauchy_44_args,
&jerasure_rs_cauchy_48_args,
&jerasure_rs_cauchy_1010_args,
NULL };
struct ec_args isa_l_args = {
.k = 10,
.m = 4,
@ -87,6 +151,25 @@ struct ec_args isa_l_args = {
.hd = 5,
};
struct ec_args isa_l_44_args = {
.k = 4,
.m = 4,
.w = 8,
.hd = 5,
};
struct ec_args isa_l_1010_args = {
.k = 10,
.m = 10,
.w = 8,
.hd = 11,
};
struct ec_args *isa_l_test_args[] = { &isa_l_args,
&isa_l_44_args,
&isa_l_1010_args,
NULL };
int priv = 128;
struct ec_args shss_args = {
.k = 6,
@ -95,6 +178,43 @@ struct ec_args shss_args = {
.priv_args2 = &priv,
};
struct ec_args *shss_test_args[] = { &shss_args, NULL };
struct ec_args **all_backend_tests[] = { null_test_args,
flat_xor_test_args,
jerasure_rs_vand_test_args,
jerasure_rs_cauchy_test_args,
isa_l_test_args,
shss_test_args , NULL};
int num_backends()
{
int i = 0;
while (NULL != all_backend_tests[i]) {
i++;
}
return i;
}
int max_tests_for_backends()
{
int n_backends = num_backends();
int i = 0;
int max = 0;
for (i = 0; i < n_backends; i++) {
int j = 0;
while (NULL != all_backend_tests[i][j]) {
j++;
}
if (j > max) {
max = j;
}
}
return max;
}
typedef enum {
LIBEC_VERSION_MISMATCH,
@ -122,32 +242,48 @@ char * get_name_from_backend_id(ec_backend_id_t be) {
}
}
struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct)
struct ec_args *create_ec_args(ec_backend_id_t be, ec_checksum_type_t ct, int backend_test_idx)
{
size_t ec_args_size = sizeof(struct ec_args);
struct ec_args *template = NULL;
struct ec_args** backend_args_array = NULL;
int i = 0;
switch(be) {
case EC_BACKEND_NULL:
template = &null_args;
backend_args_array = null_test_args;
break;
case EC_BACKEND_JERASURE_RS_VAND:
template = &jerasure_rs_vand_args;
backend_args_array = jerasure_rs_vand_test_args;
break;
case EC_BACKEND_JERASURE_RS_CAUCHY:
template = &jerasure_rs_cauchy_args;
backend_args_array = jerasure_rs_cauchy_test_args;
break;
case EC_BACKEND_FLAT_XOR_HD:
template = &flat_xor_hd_args;
backend_args_array = flat_xor_test_args;
break;
case EC_BACKEND_ISA_L_RS_VAND:
template = &isa_l_args;
backend_args_array = isa_l_test_args;
break;
case EC_BACKEND_SHSS:
template = &shss_args;
backend_args_array = shss_test_args;
break;
default:
return NULL;
}
while (NULL != backend_args_array && NULL != backend_args_array[i]) {
if (i == backend_test_idx) {
template = backend_args_array[i];
break;
}
i++;
}
if (NULL == template) {
return NULL;
}
struct ec_args *args = malloc(ec_args_size);
assert(args);
memcpy(args, template, ec_args_size);
@ -1077,13 +1213,13 @@ static void test_decode_with_missing_parity(const ec_backend_id_t be_id,
static void test_decode_with_missing_multi_data(const ec_backend_id_t be_id,
struct ec_args *args)
{
int max_num_missing = args->hd - 1;
int max_num_missing = args->k <= (args->hd - 1) ? args->k : args->hd - 1;
int i,j;
for (i = 0; i < args->k - max_num_missing + 1; i++) {
int *skip = create_skips_array(args,-1);
assert(skip != NULL);
for (j = i; j < i + max_num_missing; j++) {
skip[j]=1;
skip[j % args->k]=1;
}
encode_decode_test_impl(be_id, args, skip);
free(skip);
@ -1111,8 +1247,8 @@ static void test_decode_with_missing_multi_data_parity(
{
int i,j;
int max_num_missing = args->hd - 1;
int start = args->k - max_num_missing + 1;
for (i = start; i < start + max_num_missing -1; i++) {
int end = (args->k + args->m) - max_num_missing + 1;
for (i = 0; i < end; i++) {
int *skip = create_skips_array(args,-1);
assert(skip != NULL);
for (j = i; j < i + max_num_missing; j++) {
@ -1651,30 +1787,30 @@ struct testcase testcases[] = {
int main(int argc, char **argv)
{
int ii = 0, num_cases = 0;
int ii = 0, num_cases = 0, i = 0;
int max_backend_tests = max_tests_for_backends();
for (num_cases = 0; testcases[num_cases].description; num_cases++) {
/* Just counting */
}
printf("1..%d\n", num_cases);
for (ii = 0; testcases[ii].description != NULL; ++ii) {
const char *testname = get_name_from_backend_id(testcases[ii].be_id);
fflush(stdout);
if (testcases[ii].skip) {
fprintf(stdout, "ok # SKIP %d - %s: %s\n", ii + 1,
testcases[ii].description,
(testname) ? testname : "");
continue;
for (i = 0; i < max_backend_tests; i++) {
for (ii = 0; testcases[ii].description != NULL; ++ii) {
const char *testname = get_name_from_backend_id(testcases[ii].be_id);
fflush(stdout);
if (testcases[ii].skip) {
fprintf(stdout, "ok # SKIP %d - %s: %s (idx=%d)\n", num_cases,
testcases[ii].description,
(testname) ? testname : "", i);
continue;
}
struct ec_args *args = create_ec_args(testcases[ii].be_id, testcases[ii].ct, i);
if (NULL != args) {
testcases[ii].function(testcases[ii].be_id, args);
fprintf(stdout, "ok %d - %s: %s (idx=%d)\n", num_cases,
testcases[ii].description,
(testname) ? testname : "", i);
fflush(stdout);
free(args);
num_cases++;
}
}
struct ec_args *args = create_ec_args(testcases[ii].be_id, testcases[ii].ct);
testcases[ii].function(testcases[ii].be_id, args);
fprintf(stdout, "ok %d - %s: %s\n", ii + 1,
testcases[ii].description,
(testname) ? testname : "");
fflush(stdout);
free(args);
}
return 0;
}