openstack
/
liberasurecode
Code Issues Proposed changes

Move backend metadata adding to fragment allocation 

On the first consideration[1], metadata_adder is defined as a extra byte
size for "each" fragment. However, current implementation is an element
affects to data_len. (i.e. aligned_data_size for original segment data)

We should make metadata_adder to be a fixed value against to each fragment,
otherwise the extra bytes for the fragments will have flexible length depends
on "K". It will be quite complex for backend implementation to know "How large
bytes the raw data size is", "How large bytes the backend could use as extra
bytes for each fragment".

1: 032b57d9b1?at=master
This commit is contained in:
Kota Tsuyuzaki 2015-01-05 03:06:47 -08:00
parent 712bc80ca7
commit 7cf824af56
5 changed files with 22 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/erasurecode/erasurecode_helpers.h
View File

@ -119,7 +119,7 @@ void init_fragment_header(char *buf)
void *alloc_zeroed_buffer(int size);
void *alloc_and_set_buffer(int size, int value);
void *check_and_free_buffer(void *buf);
char *alloc_fragment_buffer(int size);
char *alloc_fragment_buffer(ec_backend_t instance, int size);
int free_fragment_buffer(char *buf);
void *get_aligned_buffer16(int size);
int get_aligned_data_size(ec_backend_t instance, int data_len);

1
include/erasurecode/erasurecode_preprocessing.h
View File

@ -37,6 +37,7 @@ int prepare_fragments_for_encode(
int *blocksize);
int prepare_fragments_for_decode(
ec_backend_t instance,
int k, int m,
char **data, char **parity,
int *missing_idxs,

6
src/erasurecode.c
View File

@ -634,8 +634,8 @@ int liberasurecode_decode(int desc,
* (realloc_bm).
*
*/
ret = prepare_fragments_for_decode(k, m,
data, parity, missing_idxs,
ret = prepare_fragments_for_decode(instance, k, m,
data, parity, missing_idxs,
&orig_data_size, &blocksize,
fragment_len, &realloc_bm);
if (ret < 0) {
@ -802,7 +802,7 @@ int liberasurecode_reconstruct_fragment(int desc,
* It passes back a bitmap telling us which buffers need to be freed by
* us (realloc_bm).
*/
ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
ret = prepare_fragments_for_decode(instance, k, m, data, parity, missing_idxs,
&orig_data_size, &blocksize,
fragment_len, &realloc_bm);
if (ret < 0) {

10
src/erasurecode_helpers.c
View File

@ -116,11 +116,16 @@ void * check_and_free_buffer(void * buf)
return NULL;
}
char *alloc_fragment_buffer(int size)
char *alloc_fragment_buffer(ec_backend_t instance, int size)
{
char *buf;
fragment_header_t *header = NULL;
if (NULL != instance){
/* Account for any custom metadata the backend wants to add in data_len */
size += instance->common.metadata_adder;
}
size += sizeof(fragment_header_t);
buf = get_aligned_buffer16(size);
@ -188,9 +193,6 @@ int get_aligned_data_size(ec_backend_t instance, int data_len)
int alignment_multiple;
int aligned_size = 0;
/* Account for any custom metadata the backend wants to add in data_len */
data_len += instance->common.metadata_adder;
/*
* For Cauchy reed-solomon align to k*word_size*packet_size
* For Vandermonde reed-solomon and flat-XOR, align to k*word_size

17
src/erasurecode_preprocessing.c
View File

@ -49,7 +49,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
for (i = 0; i < k; i++) {
int payload_size = data_len > bsize ? bsize : data_len;
char *fragment = (char *) alloc_fragment_buffer(bsize);
char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
@ -67,7 +67,7 @@ int prepare_fragments_for_encode(ec_backend_t instance,
}
for (i = 0; i < m; i++) {
char *fragment = (char *) alloc_fragment_buffer(bsize);
char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
@ -108,6 +108,7 @@ out_error:
* so in the failure case.
*/
int prepare_fragments_for_decode(
ec_backend_t instance,
int k, int m,
char **data, char **parity,
int *missing_idxs,
@ -135,14 +136,16 @@ int prepare_fragments_for_decode(
* 'data_list'
*/
if (NULL == data[i]) {
data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
data[i] = alloc_fragment_buffer(
instance, fragment_size - sizeof(fragment_header_t));
if (NULL == data[i]) {
log_error("Could not allocate data buffer!");
return -1;
}
*realloc_bm = *realloc_bm | (1 << i);
} else if (!is_addr_aligned((unsigned long)data[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
char *tmp_buf = alloc_fragment_buffer(
instance, fragment_size - sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -1;
@ -174,14 +177,16 @@ int prepare_fragments_for_decode(
* DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
*/
if (NULL == parity[i]) {
parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
parity[i] = alloc_fragment_buffer(
instance, fragment_size-sizeof(fragment_header_t));
if (NULL == parity[i]) {
log_error("Could not allocate parity buffer!");
return -1;
}
*realloc_bm = *realloc_bm | (1 << (k + i));
} else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
char *tmp_buf = alloc_fragment_buffer(
instance, fragment_size-sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -1;