490 lines
15 KiB
C
490 lines
15 KiB
C
/*
|
|
* Copyright 2014 Eric Lambert
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* Redistributions of source code must retain the above copyright notice, this
|
|
* list of conditions and the following disclaimer.
|
|
*
|
|
* Redistributions in binary form must reproduce the above copyright notice, this
|
|
* list of conditions and the following disclaimer in the documentation and/or
|
|
* other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
|
|
* THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
|
|
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
* EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
|
|
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
|
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*/
|
|
|
|
/*
|
|
* EDL 8/26/2014: This test is based on the test/test_xor_hd_code. It runs
|
|
* through a similar set of conditions but instead uses the liberasurecode
|
|
* API as opposed to directly talking to xor implementation. In the original
|
|
* test_xor_hd_code, we measured the performance of a series of encode/decode
|
|
* ops. For the time being, I have "disabled" the performance measurement in
|
|
* this test ... the main reason for doing so was that we need to some more
|
|
* memory management when using the API and I did not want those management
|
|
* ops polluting the resutls. When I have some time I will renable address
|
|
* this (figure out how to make sure memory management does not affect perf
|
|
* numbers).
|
|
*/
|
|
|
|
#include <stdio.h>
|
|
#include <stdlib.h>
|
|
#include <string.h>
|
|
#include <time.h>
|
|
#include <assert.h>
|
|
#include "erasurecode.h"
|
|
#include "erasurecode_helpers.h"
|
|
#include "erasurecode_helpers_ext.h"
|
|
#include "builtin/xor_codes/test_xor_hd_code.h"
|
|
|
|
struct frag_array_set {
|
|
unsigned int num_fragments;
|
|
char **array;
|
|
};
|
|
|
|
void print_mask(unsigned long mask)
|
|
{
|
|
unsigned int i = 0;
|
|
long pos = 1;
|
|
|
|
if (mask == 0) {
|
|
fprintf(stderr," No Missing fragments");
|
|
return;
|
|
}
|
|
fprintf(stderr," Missing fragments = ");
|
|
for (i = 0; i < (sizeof(size_t) * 8) - 1; i++) {
|
|
if ((mask & (pos << i)) != 0) {
|
|
fprintf(stderr,"%d ",i);
|
|
}
|
|
}
|
|
}
|
|
|
|
void missing_mask_to_array(long mask, int *missing)
|
|
{
|
|
unsigned int i = 0;
|
|
long pos = 1;
|
|
|
|
for (i = 0; i < (sizeof(size_t) * 8) - 1; i++) {
|
|
if ((mask & (pos << i)) != 0) {
|
|
*missing = i;
|
|
}
|
|
}
|
|
}
|
|
|
|
size_t add_item_to_missing_mask(unsigned long mask, long pos)
|
|
{
|
|
if (pos < 0) {
|
|
return mask;
|
|
}
|
|
unsigned long f = 1L << pos;
|
|
mask |= f;
|
|
return mask;
|
|
}
|
|
|
|
static int create_frags_array_set(struct frag_array_set *set,
|
|
char **data,
|
|
unsigned int num_data_frags,
|
|
char **parity,
|
|
unsigned int num_parity_frags,
|
|
unsigned long missing_mask)
|
|
{
|
|
int rc =0;
|
|
unsigned int num_frags = 0;
|
|
unsigned long i = 0;
|
|
fragment_header_t *header = NULL;
|
|
size_t size = (num_data_frags + num_parity_frags) * sizeof(char *);
|
|
char **array = malloc(size);
|
|
|
|
if (array == NULL) {
|
|
rc = -1;
|
|
goto out;
|
|
}
|
|
|
|
//add data frags
|
|
memset(array, 0, size);
|
|
for (i = 0; i < num_data_frags; i++) {
|
|
if ( (missing_mask | 1L << i) == 1) {
|
|
continue;
|
|
}
|
|
header = (fragment_header_t*)data[i];
|
|
if (header == NULL ||
|
|
header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
|
|
continue;
|
|
}
|
|
array[num_frags++] = data[i];
|
|
}
|
|
|
|
//add parity frags
|
|
for (i = 0; i < num_parity_frags; i++) {
|
|
if ( (missing_mask | 1L << (i + num_data_frags)) == 1) {
|
|
continue;
|
|
}
|
|
header = (fragment_header_t*)parity[i];
|
|
if (header == NULL ||
|
|
header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
|
|
continue;
|
|
}
|
|
array[num_frags++] = parity[i];
|
|
}
|
|
|
|
set->num_fragments = num_frags;
|
|
set->array = array;
|
|
out:
|
|
return rc;
|
|
}
|
|
|
|
static void fill_buffer(char *buf, size_t size, int seed)
|
|
{
|
|
size_t i;
|
|
buf[0] = seed;
|
|
|
|
for (i=1; i < size; i++) {
|
|
buf[i] = ((buf[i-1] + i) % 256);
|
|
}
|
|
}
|
|
|
|
static int test_hd_code(struct ec_args *args,
|
|
int num_failure_combs,
|
|
int failure_combs[][4])
|
|
{
|
|
int i, j, err;
|
|
unsigned int num_iter = 1000;
|
|
size_t blocksize = 32768;
|
|
int missing_idxs[4] = { -1, -1, -1, -1 };
|
|
int excluded_idxs[4] = { -1, -1, -1, -1 };
|
|
int ret = 0;
|
|
char *data, **parity;
|
|
int *fragments_needed;
|
|
char **encoded_data = NULL;
|
|
char **encoded_parity = NULL;
|
|
uint64_t encoded_fragment_len = 0;
|
|
int rc = 0;
|
|
char *out_data = NULL;
|
|
uint64_t out_data_len = 0;
|
|
unsigned long mask = 0;
|
|
int desc = -1;
|
|
struct frag_array_set frags; //MOVE ME
|
|
|
|
srand(time(NULL));
|
|
|
|
/*
|
|
* Set up data and parity fragments.
|
|
*/
|
|
|
|
fragments_needed = (int*)malloc(args->k*args->m*sizeof(int));
|
|
if (!fragments_needed) {
|
|
fprintf(stderr, "Could not allocate memory for fragments\n");
|
|
exit(2);
|
|
}
|
|
memset(fragments_needed, 0, args->k*args->m*sizeof(int));
|
|
|
|
err = posix_memalign((void **) &data, 16, blocksize * args->k);
|
|
if (err != 0 || !data) {
|
|
fprintf(stderr, "Could not allocate memory for data\n");
|
|
exit(1);
|
|
}
|
|
fill_buffer(data, blocksize * args->k, 0);
|
|
|
|
parity = (char**)malloc(args->m * sizeof(char*));
|
|
for (i=0; i < args->m; i++) {
|
|
err = posix_memalign((void **) &parity[i], 16, blocksize);
|
|
if (err != 0 || !parity[i]) {
|
|
fprintf(stderr, "Could not allocate memory for parity %d\n", i);
|
|
exit(1);
|
|
}
|
|
memset(parity[i], 0, blocksize);
|
|
}
|
|
|
|
/*
|
|
* Get handle
|
|
*/
|
|
desc = liberasurecode_instance_create(EC_BACKEND_FLAT_XOR_HD, args);
|
|
if (desc <= 0) {
|
|
fprintf(stderr, "Could not create libec descriptor\n");
|
|
exit(1);
|
|
}
|
|
|
|
/*
|
|
* Run Encode test
|
|
*/
|
|
for (i=0; i < num_iter-1; i++) {
|
|
rc = liberasurecode_encode(desc, data, blocksize * args->k,
|
|
&encoded_data, &encoded_parity,
|
|
&encoded_fragment_len);
|
|
//FIXME: this and the following free's taint the perf test
|
|
assert(0 == rc);
|
|
for (j = 0; j < args->k; j++) {
|
|
free(encoded_data[j]);
|
|
}
|
|
free(encoded_data);
|
|
for (j = 0; j < args->m; j++) {
|
|
free(encoded_parity[j]);
|
|
}
|
|
free(encoded_parity);
|
|
}
|
|
fprintf(stderr, " Encode: OK\n");
|
|
|
|
for (i=0; i < args->m; i++) {
|
|
memset(parity[i], 0, blocksize);
|
|
}
|
|
rc = liberasurecode_encode(desc, data, blocksize * args->k, &encoded_data,
|
|
&encoded_parity, &encoded_fragment_len);
|
|
assert(0 == rc);
|
|
|
|
/*
|
|
* Run Decode Test
|
|
*/
|
|
for (i=0; i < num_failure_combs; i++) {
|
|
mask = 0;
|
|
for (j = 0; j < 3; j++) {
|
|
int idx = failure_combs[i][j];
|
|
if (idx == -1) {
|
|
continue;
|
|
}
|
|
mask = add_item_to_missing_mask(mask, idx);
|
|
}
|
|
|
|
/*
|
|
* Spot check to ensure missing elements are not included in
|
|
* list of fragments needed and that decode is 'doable'
|
|
*/
|
|
missing_mask_to_array(mask, missing_idxs);
|
|
ret = liberasurecode_fragments_needed(desc, missing_idxs, excluded_idxs,
|
|
fragments_needed); //known leak
|
|
if (ret < 0) {
|
|
fprintf(stderr,"xor_hd_fragments_needed thinks reconstruction not possible, when it is!\n");
|
|
exit(2);
|
|
}
|
|
|
|
/*
|
|
* Make sure that none of the missig fragments are in the set of
|
|
* fragments needed to reconstruct the object.
|
|
*/
|
|
j = 0;
|
|
while (fragments_needed[j] > -1) {
|
|
if (fragments_needed[j] == missing_idxs[0] ||
|
|
fragments_needed[j] == missing_idxs[1] ||
|
|
fragments_needed[j] == missing_idxs[2]) {
|
|
fprintf(stderr,
|
|
"fragments_needed[%d]=%d in missing index list: (%d %d %d)!\n",
|
|
j, fragments_needed[j], missing_idxs[0],
|
|
missing_idxs[1], missing_idxs[2]);
|
|
exit(2);
|
|
}
|
|
j++;
|
|
}
|
|
create_frags_array_set(&frags,encoded_data, args->k, encoded_parity,
|
|
args->m, mask);
|
|
rc = liberasurecode_decode(desc, frags.array, frags.num_fragments,
|
|
encoded_fragment_len, 1,
|
|
&out_data, &out_data_len);
|
|
assert(rc == 0);
|
|
assert(out_data_len == blocksize * args->k);
|
|
if (memcmp(data, out_data, out_data_len) != 0) {
|
|
fprintf(stderr, "Decode did not work: (%d %d %d)!\n",
|
|
missing_idxs[0], missing_idxs[1], missing_idxs[2]);
|
|
exit(2);
|
|
}
|
|
free(frags.array);
|
|
free(out_data);
|
|
}
|
|
|
|
for (i=0; i < num_iter; i++) {
|
|
mask = 0;
|
|
int mi = rand() % (args->k + args->m);
|
|
|
|
mask = add_item_to_missing_mask(mask, mi);
|
|
for (j=1; j < args->hd-1;j++) {
|
|
mi = mi + 1 % (args->k + args->m);
|
|
mask = add_item_to_missing_mask(mask, mi);
|
|
}
|
|
create_frags_array_set(&frags,encoded_data, args->k, encoded_parity,
|
|
args->m, mask);
|
|
rc = liberasurecode_decode(desc, frags.array, frags.num_fragments,
|
|
encoded_fragment_len, 1,
|
|
&out_data, &out_data_len);
|
|
free(frags.array);
|
|
free(out_data);
|
|
|
|
assert(rc == 0);
|
|
//print_mask(mask);
|
|
}
|
|
for (j = 0; j < args->k; j++) {
|
|
free(encoded_data[j]);
|
|
}
|
|
free(encoded_data);
|
|
for (j = 0; j < args->m; j++) {
|
|
free(encoded_parity[j]);
|
|
}
|
|
free(encoded_parity);
|
|
free(fragments_needed);
|
|
free(data);
|
|
for (i = 0; i < args->m; i++) {
|
|
free(parity[i]);
|
|
}
|
|
free(parity);
|
|
|
|
liberasurecode_instance_destroy(desc);
|
|
return 0;
|
|
}
|
|
|
|
static int run_test(int k, int m, int hd)
|
|
{
|
|
int ret = -1;
|
|
struct ec_args args = {
|
|
.k = k,
|
|
.m = m,
|
|
.hd = hd,
|
|
};
|
|
|
|
fprintf(stderr, "Running (%d, %d, %d):\n", k, m, hd);
|
|
|
|
switch(k+m)
|
|
{
|
|
case 10:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_10_3_COMBS, failure_combs_10_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_10_4_COMBS, failure_combs_10_4);
|
|
}
|
|
break;
|
|
case 11:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_11_3_COMBS, failure_combs_11_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_11_4_COMBS, failure_combs_11_4);
|
|
}
|
|
break;
|
|
case 12:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_12_3_COMBS, failure_combs_12_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_12_4_COMBS, failure_combs_12_4);
|
|
}
|
|
break;
|
|
case 13:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_13_3_COMBS, failure_combs_13_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_13_4_COMBS, failure_combs_13_4);
|
|
}
|
|
break;
|
|
case 14:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_14_3_COMBS, failure_combs_14_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_14_4_COMBS, failure_combs_14_4);
|
|
}
|
|
break;
|
|
case 15:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_15_3_COMBS, failure_combs_15_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_15_4_COMBS, failure_combs_15_4);
|
|
}
|
|
break;
|
|
case 16:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_16_3_COMBS, failure_combs_16_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_16_4_COMBS, failure_combs_16_4);
|
|
}
|
|
break;
|
|
case 17:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_17_3_COMBS, failure_combs_17_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_17_4_COMBS, failure_combs_17_4);
|
|
}
|
|
break;
|
|
case 18:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_18_3_COMBS, failure_combs_18_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_18_4_COMBS, failure_combs_18_4);
|
|
}
|
|
break;
|
|
case 19:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_19_3_COMBS, failure_combs_19_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_19_4_COMBS, failure_combs_19_4);
|
|
}
|
|
break;
|
|
case 20:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_20_3_COMBS, failure_combs_20_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_20_4_COMBS, failure_combs_20_4);
|
|
}
|
|
break;
|
|
case 21:
|
|
if (hd == 3) {
|
|
ret = test_hd_code(&args, NUM_21_3_COMBS, failure_combs_21_3);
|
|
} else {
|
|
ret = test_hd_code(&args, NUM_21_4_COMBS, failure_combs_21_4);
|
|
}
|
|
break;
|
|
case 22:
|
|
ret = test_hd_code(&args, NUM_22_4_COMBS, failure_combs_22_4);
|
|
break;
|
|
case 23:
|
|
ret = test_hd_code(&args, NUM_23_4_COMBS, failure_combs_23_4);
|
|
break;
|
|
case 24:
|
|
ret = test_hd_code(&args, NUM_24_4_COMBS, failure_combs_24_4);
|
|
break;
|
|
case 25:
|
|
ret = test_hd_code(&args, NUM_25_4_COMBS, failure_combs_25_4);
|
|
break;
|
|
case 26:
|
|
ret = test_hd_code(&args, NUM_26_4_COMBS, failure_combs_26_4);
|
|
break;
|
|
default:
|
|
ret = -1;
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
int main()
|
|
{
|
|
int ret = 0;
|
|
int i;
|
|
for (i=6; i < 16; i++) {
|
|
ret = run_test(i, 6, 3);
|
|
if (ret != 0) {
|
|
return ret;
|
|
}
|
|
}
|
|
|
|
for (i=5; i < 11; i++) {
|
|
ret = run_test(i, 5, 3);
|
|
if (ret != 0) {
|
|
return ret;
|
|
}
|
|
}
|
|
|
|
for (i=6; i < 21; i++) {
|
|
ret = run_test(i, 6, 4);
|
|
if (ret != 0) {
|
|
return ret;
|
|
}
|
|
}
|
|
for (i=5; i < 11; i++) {
|
|
ret = run_test(i, 5, 4);
|
|
if (ret != 0) {
|
|
return ret;
|
|
}
|
|
}
|
|
exit(ret);
|
|
}
|
|
|