385 lines
13 KiB
C
385 lines
13 KiB
C
/*
|
|
* Copyright 2014 Tushar Gohad, Kevin M Greenan, Eric Lambert, Mark Storer
|
|
*
|
|
* Redistribution and use in source and binary forms, with or without
|
|
* modification, are permitted provided that the following conditions are met:
|
|
*
|
|
* Redistributions of source code must retain the above copyright notice, this
|
|
* list of conditions and the following disclaimer.
|
|
*
|
|
* Redistributions in binary form must reproduce the above copyright notice, this
|
|
* list of conditions and the following disclaimer in the documentation and/or
|
|
* other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY
|
|
* THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED
|
|
* WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
|
|
* MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO
|
|
* EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
|
|
* INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
|
|
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
|
|
* DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
|
|
* LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE
|
|
* OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
|
|
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
*
|
|
* liberasurecode proprocssing helpers implementation
|
|
*
|
|
* vi: set noai tw=79 ts=4 sw=4:
|
|
*/
|
|
|
|
#include "erasurecode_backend.h"
|
|
#include "erasurecode_helpers.h"
|
|
#include "erasurecode_helpers_ext.h"
|
|
#include "erasurecode_log.h"
|
|
#include "erasurecode_preprocessing.h"
|
|
#include "erasurecode_stdinc.h"
|
|
|
|
int prepare_fragments_for_encode(ec_backend_t instance,
|
|
int k, int m,
|
|
const char *orig_data, uint64_t orig_data_size, /* input */
|
|
char **encoded_data, char **encoded_parity, /* output */
|
|
int *blocksize)
|
|
{
|
|
int i, ret = 0;
|
|
int data_len; /* data len to write to fragment headers */
|
|
int aligned_data_len; /* EC algorithm compatible data length */
|
|
int buffer_size, payload_size = 0;
|
|
int metadata_size, data_offset = 0;
|
|
|
|
/* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
|
|
data_len = orig_data_size;
|
|
aligned_data_len = get_aligned_data_size(instance, orig_data_size);
|
|
*blocksize = payload_size = (aligned_data_len / k);
|
|
metadata_size = instance->common.ops->get_backend_metadata_size(
|
|
instance->desc.backend_desc,
|
|
*blocksize);
|
|
data_offset = instance->common.ops->get_encode_offset(
|
|
instance->desc.backend_desc,
|
|
metadata_size);
|
|
buffer_size = payload_size + metadata_size;
|
|
|
|
for (i = 0; i < k; i++) {
|
|
int copy_size = data_len > payload_size ? payload_size : data_len;
|
|
char *fragment = (char *) alloc_fragment_buffer(buffer_size);
|
|
if (NULL == fragment) {
|
|
ret = -ENOMEM;
|
|
goto out_error;
|
|
}
|
|
|
|
/* Copy existing data into clean, zero'd out buffer */
|
|
encoded_data[i] = get_data_ptr_from_fragment(fragment);
|
|
|
|
if (data_len > 0) {
|
|
memcpy(encoded_data[i] + data_offset, orig_data, copy_size);
|
|
}
|
|
|
|
orig_data += copy_size;
|
|
data_len -= copy_size;
|
|
}
|
|
|
|
for (i = 0; i < m; i++) {
|
|
char *fragment = (char *) alloc_fragment_buffer(buffer_size);
|
|
if (NULL == fragment) {
|
|
ret = -ENOMEM;
|
|
goto out_error;
|
|
}
|
|
|
|
encoded_parity[i] = get_data_ptr_from_fragment(fragment);
|
|
}
|
|
|
|
out:
|
|
return ret;
|
|
|
|
out_error:
|
|
printf ("ERROR in encode\n");
|
|
if (encoded_data) {
|
|
for (i = 0; i < k; i++) {
|
|
if (encoded_data[i])
|
|
free_fragment_buffer(encoded_data[i]);
|
|
}
|
|
check_and_free_buffer(encoded_data);
|
|
}
|
|
|
|
if (encoded_parity) {
|
|
for (i = 0; i < m; i++) {
|
|
if (encoded_parity[i])
|
|
free_fragment_buffer(encoded_parity[i]);
|
|
}
|
|
check_and_free_buffer(encoded_parity);
|
|
}
|
|
|
|
goto out;
|
|
}
|
|
|
|
/*
|
|
* Note that the caller should always check realloc_bm during success or
|
|
* failure to free buffers allocated here. We could free up in this function,
|
|
* but it is internal to this library and only used in a few places. In any
|
|
* case, the caller has to free up in the success case, so it may as well do
|
|
* so in the failure case.
|
|
*/
|
|
int prepare_fragments_for_decode(
|
|
int k, int m,
|
|
char **data, char **parity,
|
|
int *missing_idxs,
|
|
int *orig_size, int *fragment_payload_size, int fragment_size,
|
|
uint64_t *realloc_bm)
|
|
{
|
|
int i; /* a counter */
|
|
unsigned long long missing_bm; /* bitmap form of missing indexes list */
|
|
int orig_data_size = -1;
|
|
int payload_size = -1;
|
|
|
|
missing_bm = convert_list_to_bitmap(missing_idxs);
|
|
|
|
/*
|
|
* Determine if each data fragment is:
|
|
* 1.) Alloc'd: if not, alloc new buffer (for missing fragments)
|
|
* 2.) Aligned to 16-byte boundaries: if not, alloc a new buffer
|
|
* memcpy the contents and free the old buffer
|
|
*/
|
|
for (i = 0; i < k; i++) {
|
|
/*
|
|
* Allocate or replace with aligned buffer if the buffer was not
|
|
* aligned.
|
|
* DO NOT FREE: the python GC should free the original when cleaning up
|
|
* 'data_list'
|
|
*/
|
|
if (NULL == data[i]) {
|
|
data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
|
|
if (NULL == data[i]) {
|
|
log_error("Could not allocate data buffer!");
|
|
return -ENOMEM;
|
|
}
|
|
*realloc_bm = *realloc_bm | (1 << i);
|
|
} else if (!is_addr_aligned((unsigned long)data[i], 16)) {
|
|
char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
|
|
if (NULL == tmp_buf) {
|
|
log_error("Could not allocate temp buffer!");
|
|
return -ENOMEM;
|
|
}
|
|
memcpy(tmp_buf, data[i], fragment_size);
|
|
data[i] = tmp_buf;
|
|
*realloc_bm = *realloc_bm | (1 << i);
|
|
}
|
|
|
|
/* Need to determine the size of the original data */
|
|
if (((missing_bm & (1 << i)) == 0) && orig_data_size < 0) {
|
|
orig_data_size = get_orig_data_size(data[i]);
|
|
if (orig_data_size < 0) {
|
|
log_error("Invalid orig_data_size in fragment header!");
|
|
return -EBADHEADER;
|
|
}
|
|
payload_size = get_fragment_payload_size(data[i]);
|
|
if (orig_data_size < 0) {
|
|
log_error("Invalid fragment_size in fragment header!");
|
|
return -EBADHEADER;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* Perform the same allocation, alignment checks on the parity fragments */
|
|
for (i = 0; i < m; i++) {
|
|
/*
|
|
* Allocate or replace with aligned buffer, if the buffer was not aligned.
|
|
* DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
|
|
*/
|
|
if (NULL == parity[i]) {
|
|
parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
|
|
if (NULL == parity[i]) {
|
|
log_error("Could not allocate parity buffer!");
|
|
return -ENOMEM;
|
|
}
|
|
*realloc_bm = *realloc_bm | (1 << (k + i));
|
|
} else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
|
|
char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
|
|
if (NULL == tmp_buf) {
|
|
log_error("Could not allocate temp buffer!");
|
|
return -ENOMEM;
|
|
}
|
|
memcpy(tmp_buf, parity[i], fragment_size);
|
|
parity[i] = tmp_buf;
|
|
*realloc_bm = *realloc_bm | (1 << (k + i));
|
|
}
|
|
|
|
/* Need to determine the size of the original data */
|
|
if (((missing_bm & (1 << (k + i))) == 0) && orig_data_size < 0) {
|
|
orig_data_size = get_orig_data_size(parity[i]);
|
|
if (orig_data_size < 0) {
|
|
log_error("Invalid orig_data_size in fragment header!");
|
|
return -EBADHEADER;
|
|
}
|
|
payload_size = get_fragment_payload_size(parity[i]);
|
|
if (orig_data_size < 0) {
|
|
log_error("Invalid fragment_size in fragment header!");
|
|
return -EBADHEADER;
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
*orig_size = orig_data_size;
|
|
*fragment_payload_size = payload_size;
|
|
|
|
return 0;
|
|
}
|
|
|
|
int get_fragment_partition(
|
|
int k, int m,
|
|
char **fragments, int num_fragments,
|
|
char **data, char **parity, int *missing)
|
|
{
|
|
int i = 0;
|
|
int num_missing = 0;
|
|
int index;
|
|
|
|
/*
|
|
* Set all data and parity entries to NULL
|
|
*/
|
|
for (i = 0; i < k; i++) {
|
|
data[i] = NULL;
|
|
}
|
|
for (i = 0; i < m; i++) {
|
|
parity[i] = NULL;
|
|
}
|
|
|
|
/*
|
|
* Fill in data and parity with available fragments
|
|
*/
|
|
for (i = 0; i < num_fragments; i++) {
|
|
index = get_fragment_idx(fragments[i]);
|
|
if (index < 0 || index > (k + m)) {
|
|
return -EBADHEADER;
|
|
}
|
|
if (index < k) {
|
|
data[index] = fragments[i];
|
|
} else {
|
|
parity[index - k] = fragments[i];
|
|
}
|
|
}
|
|
|
|
/*
|
|
* Fill in missing array with missing indexes
|
|
*/
|
|
for (i = 0; i < k; i++) {
|
|
if (NULL == data[i]) {
|
|
missing[num_missing] = i;
|
|
num_missing++;
|
|
}
|
|
}
|
|
for (i = 0; i < m; i++) {
|
|
if (NULL == parity[i]) {
|
|
missing[num_missing] = i + k;
|
|
num_missing++;
|
|
}
|
|
}
|
|
// TODO: In general, it is possible to reconstruct one or more fragments
|
|
// when more than m fragments are missing (e.g. flat XOR codes)
|
|
return (num_missing > m) ? -EINSUFFFRAGS : 0;
|
|
}
|
|
|
|
int fragments_to_string(int k, int m,
|
|
char **fragments, int num_fragments,
|
|
char **orig_payload, uint64_t *payload_len)
|
|
{
|
|
char *internal_payload = NULL;
|
|
char **data = NULL;
|
|
int orig_data_size = -1;
|
|
int i;
|
|
int index;
|
|
int data_size;
|
|
int num_data = 0;
|
|
int string_off = 0;
|
|
int ret = -1;
|
|
|
|
if (num_fragments < k) {
|
|
/*
|
|
* This is not necessarily an error condition, so *do not log here*
|
|
* We can maybe debug log, if necessary.
|
|
*/
|
|
goto out;
|
|
}
|
|
|
|
data = (char **) get_aligned_buffer16(sizeof(char *) * k);
|
|
|
|
if (NULL == data) {
|
|
log_error("Could not allocate buffer for data!!");
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
for (i = 0; i < num_fragments; i++) {
|
|
index = get_fragment_idx(fragments[i]);
|
|
data_size = get_fragment_payload_size(fragments[i]);
|
|
if ((index < 0) || (data_size < 0)) {
|
|
log_error("Invalid fragment header information!");
|
|
ret = -EBADHEADER;
|
|
goto out;
|
|
}
|
|
|
|
/* Validate the original data size */
|
|
if (orig_data_size < 0) {
|
|
orig_data_size = get_orig_data_size(fragments[i]);
|
|
} else {
|
|
if (get_orig_data_size(fragments[i]) != orig_data_size) {
|
|
log_error("Inconsistent orig_data_size in fragment header!");
|
|
ret = -EBADHEADER;
|
|
goto out;
|
|
}
|
|
}
|
|
|
|
/* Skip parity fragments, put data fragments in index order */
|
|
if (index >= k) {
|
|
continue;
|
|
} else {
|
|
/* Make sure we account for duplicates */
|
|
if (NULL == data[index]) {
|
|
data[index] = fragments[i];
|
|
num_data++;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* We do not have enough data fragments to do this! */
|
|
if (num_data != k) {
|
|
/*
|
|
* This is not necessarily an error condition, so *do not log here*
|
|
* We can maybe debug log, if necessary.
|
|
*/
|
|
goto out;
|
|
}
|
|
|
|
/* Create the string to return */
|
|
internal_payload = (char *) get_aligned_buffer16(orig_data_size);
|
|
if (NULL == internal_payload) {
|
|
log_error("Could not allocate buffer for decoded string!");
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
/* Pass the original data length back */
|
|
*payload_len = orig_data_size;
|
|
|
|
/* Copy fragment data into cstring (fragments should be in index order) */
|
|
for (i = 0; i < num_data && orig_data_size > 0; i++) {
|
|
char* fragment_data = get_data_ptr_from_fragment(data[i]);
|
|
int fragment_size = get_fragment_payload_size(data[i]);
|
|
int payload_size = orig_data_size > fragment_size ? fragment_size : orig_data_size;
|
|
memcpy(internal_payload + string_off, fragment_data, payload_size);
|
|
orig_data_size -= payload_size;
|
|
string_off += payload_size;
|
|
}
|
|
|
|
/* Everything worked just fine */
|
|
ret = 0;
|
|
|
|
out:
|
|
if (NULL != data) {
|
|
free(data);
|
|
}
|
|
|
|
*orig_payload = internal_payload;
|
|
return ret;
|
|
}
|
|
|
|
|