openstack
/
liberasurecode
Code Issues Proposed changes

Enable to get fragment_len includes metadata_adder 

This patch allows to get correct fragment size includes metadata_adder.

Current implementaion automatically allocates extra bytes for the metadata_adder
in alloc_buffer, and then, no information about the extra bytes will be returned
to the api caller side. It's too confusable because callers couldn't know how size they
assumes as the fragment size.

To be easy to find out the size infomation, this patch adds "frag_adder_size"
variable into fragment metadata and also some functions to get fragment size.

The definitions of these size infomation are here,

fragment_meta:
- size-> raw data size used to encode/fragment_to_string
- frag_adder_size-> metadata_adder of backend specification

And the definitions of functions are here,

- get_fragment_size:
  -> return sizeof(fragument_header) + size + frag_adder_size

- get_fragment_buffer_size:
  -> return size + frag_adder_size

- get_fragment_payload_size:
  -> return size

By using these function above, users could get the size information
directly from fragments. It results in enabling to return fragment_len
to the caller side easily.
This commit is contained in:
Kota Tsuyuzaki 2015-02-05 21:54:38 -08:00
parent 1b08cb4376
commit 0addebdbf6
8 changed files with 71 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
include/erasurecode/erasurecode.h
View File

@ -247,6 +247,7 @@ fragment_metadata
{
uint32_t idx; /* 4 */
uint32_t size; /* 4 */
uint32_t frag_adder_size; /* 4 */
uint64_t orig_data_size; /* 8 */
uint8_t chksum_type; /* 1 */
uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */

9
include/erasurecode/erasurecode_helpers.h
View File

@ -45,12 +45,12 @@
typedef struct __attribute__((__packed__)) fragment_header_s
{
fragment_metadata_t meta; /* 55 bytes */
fragment_metadata_t meta; /* 59 bytes */
uint32_t magic; /* 4 bytes */
uint32_t libec_version; /* 4 bytes */
// We must be aligned to 16-byte boundaries
// So, size this array accordingly
uint8_t aligned_padding[1];
uint8_t aligned_padding[13];
} fragment_header_t;
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
@ -119,7 +119,7 @@ void init_fragment_header(char *buf)
void *alloc_zeroed_buffer(int size);
void *alloc_and_set_buffer(int size, int value);
void *check_and_free_buffer(void *buf);
char *alloc_fragment_buffer(ec_backend_t instance, int size);
char *alloc_fragment_buffer(int size);
int free_fragment_buffer(char *buf);
void *get_aligned_buffer16(int size);
int get_aligned_data_size(ec_backend_t instance, int data_len);
@ -135,6 +135,9 @@ int set_fragment_idx(char *buf, int idx);
int get_fragment_idx(char *buf);
int set_fragment_payload_size(char *buf, int size);
int get_fragment_payload_size(char *buf);
int set_fragment_adder_size(char *buf, int size);
int get_fragment_adder_size(char *buf);
int get_fragment_buffer_size(char *buf);
int set_orig_data_size(char *buf, int orig_data_size);
int get_orig_data_size(char *buf);
int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize);

1
include/erasurecode/erasurecode_preprocessing.h
View File

@ -37,7 +37,6 @@ int prepare_fragments_for_encode(
int *blocksize);
int prepare_fragments_for_decode(
ec_backend_t instance,
int k, int m,
char **data, char **parity,
int *missing_idxs,

4
src/erasurecode.c
View File

@ -631,7 +631,7 @@ int liberasurecode_decode(int desc,
* (realloc_bm).
*
*/
ret = prepare_fragments_for_decode(instance, k, m,
ret = prepare_fragments_for_decode(k, m,
data, parity, missing_idxs,
&orig_data_size, &blocksize,
fragment_len, &realloc_bm);
@ -799,7 +799,7 @@ int liberasurecode_reconstruct_fragment(int desc,
* It passes back a bitmap telling us which buffers need to be freed by
* us (realloc_bm).
*/
ret = prepare_fragments_for_decode(instance, k, m, data, parity, missing_idxs,
ret = prepare_fragments_for_decode(k, m, data, parity, missing_idxs,
&orig_data_size, &blocksize,
fragment_len, &realloc_bm);
if (ret < 0) {

52
src/erasurecode_helpers.c
View File

@ -116,16 +116,11 @@ void * check_and_free_buffer(void * buf)
return NULL;
}
char *alloc_fragment_buffer(ec_backend_t instance, int size)
char *alloc_fragment_buffer(int size)
{
char *buf;
fragment_header_t *header = NULL;
if (NULL != instance){
/* Account for any custom metadata the backend wants to add in data_len */
size += instance->common.metadata_adder;
}
size += sizeof(fragment_header_t);
buf = get_aligned_buffer16(size);
@ -168,13 +163,11 @@ int free_fragment_buffer(char *buf)
*/
uint64_t get_fragment_size(char *buf)
{
fragment_header_t *header = NULL;
if (NULL == buf)
return -1;
header = (fragment_header_t *) buf;
return (header->meta.size + sizeof(fragment_header_t));
return get_fragment_buffer_size(buf) + sizeof(fragment_header_t);
}
/**
@ -331,6 +324,47 @@ int get_fragment_payload_size(char *buf)
return header->meta.size;
}
int set_fragment_adder_size(char *buf, int size)
{
fragment_header_t *header = (fragment_header_t *) buf;
assert(NULL != header);
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (set adder size)!");
return -1;
}
header->meta.frag_adder_size = size;
return 0;
}
int get_fragment_adder_size(char *buf)
{
fragment_header_t *header = (fragment_header_t *) buf;
assert(NULL != header);
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (get adder size)!");
return -1;
}
return header->meta.frag_adder_size;
}
int get_fragment_buffer_size(char *buf)
{
fragment_header_t *header = (fragment_header_t *) buf;
assert(NULL != header);
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (get size)!");
return -1;
}
return header->meta.size + header->meta.frag_adder_size;
}
int set_orig_data_size(char *buf, int orig_data_size)
{
fragment_header_t *header = (fragment_header_t *) buf;

3
src/erasurecode_postprocessing.c
View File

@ -41,7 +41,8 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
set_fragment_payload_size(fragment, blocksize);
set_backend_id(fragment, be->common.id);
set_backend_version(fragment, be->common.ec_backend_version);
set_fragment_adder_size(fragment, be->common.metadata_adder);
if (add_chksum) {
set_checksum(ct, fragment, blocksize);
}

30
src/erasurecode_preprocessing.c
View File

@ -40,16 +40,17 @@ int prepare_fragments_for_encode(ec_backend_t instance,
int i, ret = 0;
int data_len; /* data len to write to fragment headers */
int aligned_data_len; /* EC algorithm compatible data length */
int bsize = 0;
int buffer_size, payload_size = 0;
/* Calculate data sizes, aligned_data_len guaranteed to be divisible by k*/
data_len = orig_data_size;
aligned_data_len = get_aligned_data_size(instance, orig_data_size);
*blocksize = bsize = (aligned_data_len / k);
*blocksize = payload_size = (aligned_data_len / k);
buffer_size = payload_size + instance->common.metadata_adder;
for (i = 0; i < k; i++) {
int payload_size = data_len > bsize ? bsize : data_len;
char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
int copy_size = data_len > payload_size ? payload_size : data_len;
char *fragment = (char *) alloc_fragment_buffer(buffer_size);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
@ -59,15 +60,15 @@ int prepare_fragments_for_encode(ec_backend_t instance,
encoded_data[i] = get_data_ptr_from_fragment(fragment);
if (data_len > 0) {
memcpy(encoded_data[i], orig_data, payload_size);
memcpy(encoded_data[i], orig_data, copy_size);
}
orig_data += payload_size;
data_len -= payload_size;
orig_data += copy_size;
data_len -= copy_size;
}
for (i = 0; i < m; i++) {
char *fragment = (char *) alloc_fragment_buffer(instance, bsize);
char *fragment = (char *) alloc_fragment_buffer(buffer_size);
if (NULL == fragment) {
ret = -ENOMEM;
goto out_error;
@ -108,7 +109,6 @@ out_error:
* so in the failure case.
*/
int prepare_fragments_for_decode(
ec_backend_t instance,
int k, int m,
char **data, char **parity,
int *missing_idxs,
@ -136,16 +136,14 @@ int prepare_fragments_for_decode(
* 'data_list'
*/
if (NULL == data[i]) {
data[i] = alloc_fragment_buffer(
instance, fragment_size - sizeof(fragment_header_t));
data[i] = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == data[i]) {
log_error("Could not allocate data buffer!");
return -1;
}
*realloc_bm = *realloc_bm | (1 << i);
} else if (!is_addr_aligned((unsigned long)data[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(
instance, fragment_size - sizeof(fragment_header_t));
char *tmp_buf = alloc_fragment_buffer(fragment_size - sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -1;
@ -177,16 +175,14 @@ int prepare_fragments_for_decode(
* DO NOT FREE: the python GC should free the original when cleaning up 'data_list'
*/
if (NULL == parity[i]) {
parity[i] = alloc_fragment_buffer(
instance, fragment_size-sizeof(fragment_header_t));
parity[i] = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
if (NULL == parity[i]) {
log_error("Could not allocate parity buffer!");
return -1;
}
*realloc_bm = *realloc_bm | (1 << (k + i));
} else if (!is_addr_aligned((unsigned long)parity[i], 16)) {
char *tmp_buf = alloc_fragment_buffer(
instance, fragment_size-sizeof(fragment_header_t));
char *tmp_buf = alloc_fragment_buffer(fragment_size-sizeof(fragment_header_t));
if (NULL == tmp_buf) {
log_error("Could not allocate temp buffer!");
return -1;

5
test/liberasurecode_test.c
View File

@ -592,8 +592,11 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
int num_avail_frags = 0;
char *orig_data_ptr = NULL;
int remaining = 0;
ec_backend_t be = NULL;
desc = liberasurecode_instance_create(be_id, args);
be = liberasurecode_backend_instance_get_by_desc(desc);
if (-EBACKENDNOTAVAIL == desc) {
fprintf (stderr, "Backend library not available!\n");
return;
@ -614,7 +617,7 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
assert(header != NULL);
fragment_metadata_t metadata = header->meta;
assert(metadata.idx == i);
assert(metadata.size == encoded_fragment_len - frag_header_size);
assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.metadata_adder);
assert(metadata.orig_data_size == orig_data_size);
char *data_ptr = frag + frag_header_size;
int cmp_size = remaining >= metadata.size ? metadata.size : remaining;