Merged in frag_metadata_checksum (pull request #20)
Fragment metadata checksumming support
This commit is contained in:
commit
35000cb0c3
|
@ -256,15 +256,15 @@ int liberasurecode_fragments_needed(int desc,
|
|||
typedef struct __attribute__((__packed__))
|
||||
fragment_metadata
|
||||
{
|
||||
uint32_t idx; /* 4 */
|
||||
uint32_t size; /* 4 */
|
||||
uint32_t frag_backend_metadata_size; /* 4 */
|
||||
uint64_t orig_data_size; /* 8 */
|
||||
uint8_t chksum_type; /* 1 */
|
||||
uint32_t idx; /* 4 */
|
||||
uint32_t size; /* 4 */
|
||||
uint32_t frag_backend_metadata_size; /* 4 */
|
||||
uint64_t orig_data_size; /* 8 */
|
||||
uint8_t chksum_type; /* 1 */
|
||||
uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */
|
||||
uint8_t chksum_mismatch; /* 1 */
|
||||
uint8_t backend_id; /* 1 */
|
||||
uint32_t backend_version; /* 4 */
|
||||
uint8_t chksum_mismatch; /* 1 */
|
||||
uint8_t backend_id; /* 1 */
|
||||
uint32_t backend_version; /* 4 */
|
||||
} fragment_metadata_t;
|
||||
|
||||
/**
|
||||
|
@ -321,12 +321,13 @@ int liberasurecode_verify_stripe_metadata(int desc,
|
|||
|
||||
typedef struct __attribute__((__packed__)) fragment_header_s
|
||||
{
|
||||
fragment_metadata_t meta; /* 59 bytes */
|
||||
uint32_t magic; /* 4 bytes */
|
||||
uint32_t libec_version; /* 4 bytes */
|
||||
fragment_metadata_t meta; /* 59 bytes */
|
||||
uint32_t magic; /* 4 bytes */
|
||||
uint32_t libec_version; /* 4 bytes */
|
||||
uint32_t metadata_chksum; /* 4 bytes */
|
||||
// We must be aligned to 16-byte boundaries
|
||||
// So, size this array accordingly
|
||||
uint8_t aligned_padding[13];
|
||||
uint8_t aligned_padding[9];
|
||||
} fragment_header_t;
|
||||
|
||||
#define FRAGSIZE_2_BLOCKSIZE(fragment_size) \
|
||||
|
|
|
@ -65,13 +65,16 @@ int get_fragment_buffer_size(char *buf);
|
|||
int set_orig_data_size(char *buf, int orig_data_size);
|
||||
int get_orig_data_size(char *buf);
|
||||
int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize);
|
||||
int get_checksum(char *buf); //TODO implement this
|
||||
int get_checksum(char *buf);
|
||||
int set_libec_version(char *fragment);
|
||||
int get_libec_version(char *fragment, uint32_t *ver);
|
||||
int set_backend_id(char *buf, ec_backend_id_t id);
|
||||
int get_backend_id(char *buf, ec_backend_id_t *id);
|
||||
int set_backend_version(char *buf, uint32_t version);
|
||||
int get_backend_version(char *buf, uint32_t *version);
|
||||
int set_metadata_chksum(char *buf);
|
||||
uint32_t *get_metadata_chksum(char *buf);
|
||||
int is_invalid_fragment_header(fragment_header_t *header);
|
||||
|
||||
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
|
||||
|
||||
|
|
|
@ -26,8 +26,8 @@
|
|||
#define _ERASURECODE_VERSION_H_
|
||||
|
||||
#define _MAJOR 1
|
||||
#define _MINOR 1
|
||||
#define _REV 2
|
||||
#define _MINOR 2
|
||||
#define _REV 0
|
||||
#define _VERSION(x, y, z) ((x << 16) | (y << 8) | (z))
|
||||
|
||||
#define LIBERASURECODE_VERSION _VERSION(_MAJOR, _MINOR, _REV)
|
||||
|
|
|
@ -26,6 +26,7 @@
|
|||
* vi: set noai tw=79 ts=4 sw=4:
|
||||
*/
|
||||
|
||||
#include "assert.h"
|
||||
#include "list.h"
|
||||
#include "erasurecode.h"
|
||||
#include "erasurecode_backend.h"
|
||||
|
@ -578,10 +579,22 @@ int liberasurecode_decode(int desc,
|
|||
m = instance->args.uargs.m;
|
||||
|
||||
if (num_fragments < k) {
|
||||
log_error("Not enough fragments to decode, got %d, need %d!",
|
||||
num_fragments, k);
|
||||
ret = -EINSUFFFRAGS;
|
||||
goto out;
|
||||
}
|
||||
|
||||
for (i = 0; i < num_fragments; ++i) {
|
||||
/* Verify metadata checksum */
|
||||
if (is_invalid_fragment_header(
|
||||
(fragment_header_t *) available_fragments[i])) {
|
||||
log_error("Invalid fragment header information!");
|
||||
ret = -EBADHEADER;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
if (instance->common.id != EC_BACKEND_SHSS) {
|
||||
/* shss (ntt_backend) must force to decode */
|
||||
// TODO: Add a frag and function to handle whether the backend want to decode or not.
|
||||
|
@ -783,6 +796,16 @@ int liberasurecode_reconstruct_fragment(int desc,
|
|||
k = instance->args.uargs.k;
|
||||
m = instance->args.uargs.m;
|
||||
|
||||
for (i = 0; i < num_fragments; i++) {
|
||||
/* Verify metadata checksum */
|
||||
if (is_invalid_fragment_header(
|
||||
(fragment_header_t *) available_fragments[i])) {
|
||||
log_error("Invalid fragment header information!");
|
||||
ret = -EBADHEADER;
|
||||
goto out;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Allocate arrays for data, parity and missing_idxs
|
||||
*/
|
||||
|
@ -1004,6 +1027,14 @@ int liberasurecode_get_fragment_metadata(char *fragment,
|
|||
goto out;
|
||||
}
|
||||
|
||||
/* Verify metadata checksum */
|
||||
if (is_invalid_fragment_header(
|
||||
(fragment_header_t *) fragment)) {
|
||||
log_error("Invalid fragment header information!");
|
||||
ret = -EBADHEADER;
|
||||
goto out;
|
||||
}
|
||||
|
||||
memcpy(fragment_metadata, fragment, sizeof(struct fragment_metadata));
|
||||
fragment_hdr = (fragment_header_t *) fragment;
|
||||
if (LIBERASURECODE_FRAG_HEADER_MAGIC != fragment_hdr->magic) {
|
||||
|
@ -1037,6 +1068,20 @@ out:
|
|||
return ret;
|
||||
}
|
||||
|
||||
int is_invalid_fragment_header(fragment_header_t *header)
|
||||
{
|
||||
uint32_t *stored_csum = NULL, csum = 0;
|
||||
assert (NULL != header);
|
||||
if (header->libec_version < _VERSION(1,2,0))
|
||||
/* no metadata checksum support */
|
||||
return 0;
|
||||
stored_csum = get_metadata_chksum((char *) header);
|
||||
if (NULL == stored_csum)
|
||||
return 1; /* can't verify, possibly crc32 call error */
|
||||
csum = crc32(0, &header->meta, sizeof(fragment_metadata_t));
|
||||
return (*stored_csum != csum);
|
||||
}
|
||||
|
||||
int liberasurecode_verify_fragment_metadata(ec_backend_t be,
|
||||
fragment_metadata_t *md)
|
||||
{
|
||||
|
@ -1054,6 +1099,27 @@ int liberasurecode_verify_fragment_metadata(ec_backend_t be,
|
|||
return 0;
|
||||
}
|
||||
|
||||
int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
|
||||
{
|
||||
ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
|
||||
if (!be) {
|
||||
log_error("Unable to verify fragment metadata: invalid backend id %d.",
|
||||
desc);
|
||||
return -EINVALIDPARAMS;
|
||||
}
|
||||
if (liberasurecode_verify_fragment_metadata(be,
|
||||
fragment_metadata) != 0) {
|
||||
return -EBADHEADER;
|
||||
}
|
||||
if (!be->common.ops->is_compatible_with(fragment_metadata->backend_version)) {
|
||||
return -EBADHEADER;
|
||||
}
|
||||
if (fragment_metadata->chksum_mismatch == 1) {
|
||||
return -EBADCHKSUM;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int is_invalid_fragment(int desc, char *fragment)
|
||||
{
|
||||
uint32_t ver = 0;
|
||||
|
@ -1069,41 +1135,18 @@ int is_invalid_fragment(int desc, char *fragment)
|
|||
return 1;
|
||||
}
|
||||
if (get_libec_version(fragment, &ver) != 0 ||
|
||||
ver != LIBERASURECODE_VERSION) {
|
||||
ver > LIBERASURECODE_VERSION) {
|
||||
return 1;
|
||||
}
|
||||
if (liberasurecode_get_fragment_metadata(fragment,
|
||||
&fragment_metadata) != 0) {
|
||||
if (liberasurecode_get_fragment_metadata(fragment, &fragment_metadata) != 0) {
|
||||
return 1;
|
||||
}
|
||||
if (liberasurecode_verify_fragment_metadata(be,
|
||||
&fragment_metadata) != 0) {
|
||||
if (is_invalid_fragment_metadata(desc, &fragment_metadata) != 0) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int is_valid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
|
||||
{
|
||||
ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
|
||||
if (!be) {
|
||||
log_error("Unable to verify stripe metadata: invalid backend id %d.",
|
||||
desc);
|
||||
return -EINVALIDPARAMS;
|
||||
}
|
||||
if (liberasurecode_verify_fragment_metadata(be,
|
||||
fragment_metadata) != 0) {
|
||||
return -EBADHEADER;
|
||||
}
|
||||
if (!be->common.ops->is_compatible_with(fragment_metadata->backend_version)) {
|
||||
return -EBADHEADER;
|
||||
}
|
||||
if (fragment_metadata->chksum_mismatch == 1) {
|
||||
return -EBADCHKSUM;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int liberasurecode_verify_stripe_metadata(int desc,
|
||||
char **fragments, int num_fragments)
|
||||
{
|
||||
|
@ -1120,7 +1163,7 @@ int liberasurecode_verify_stripe_metadata(int desc,
|
|||
|
||||
for (i = 0; i < num_fragments; i++) {
|
||||
fragment_metadata_t *fragment_metadata = (fragment_metadata_t*)fragments[i];
|
||||
int ret = is_valid_fragment_metadata(desc, fragment_metadata);
|
||||
int ret = is_invalid_fragment_metadata(desc, fragment_metadata);
|
||||
if (ret < 0) {
|
||||
return ret;
|
||||
}
|
||||
|
|
|
@ -501,3 +501,45 @@ inline uint32_t* get_chksum(char *buf)
|
|||
|
||||
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
|
||||
|
||||
#if LIBERASURECODE_VERSION >= _VERSION(1,2,0)
|
||||
inline int set_metadata_chksum(char *buf)
|
||||
{
|
||||
fragment_header_t* header = (fragment_header_t*) buf;
|
||||
|
||||
assert(NULL != header);
|
||||
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
|
||||
log_error("Invalid fragment header (set meta chksum)!\n");
|
||||
return -1;
|
||||
}
|
||||
|
||||
header->metadata_chksum = crc32(0, &header->meta,
|
||||
sizeof(fragment_metadata_t));
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline uint32_t* get_metadata_chksum(char *buf)
|
||||
{
|
||||
fragment_header_t* header = (fragment_header_t*) buf;
|
||||
|
||||
assert(NULL != header);
|
||||
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
|
||||
log_error("Invalid fragment header (get meta chksum)!");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
return (uint32_t *) &header->metadata_chksum;
|
||||
}
|
||||
#else
|
||||
inline int set_metadata_chksum(char *buf)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
|
||||
inline uint32_t* get_metadata_chksum(char *buf)
|
||||
{
|
||||
return (uint32_t *) 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
|
||||
|
||||
|
|
|
@ -47,6 +47,7 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
|
|||
if (add_chksum) {
|
||||
set_checksum(ct, fragment, blocksize);
|
||||
}
|
||||
set_metadata_chksum(fragment);
|
||||
}
|
||||
|
||||
int finalize_fragments_after_encode(ec_backend_t instance,
|
||||
|
|
|
@ -940,17 +940,29 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
|
|||
assert(0 == rc);
|
||||
orig_data_ptr = orig_data;
|
||||
remaining = orig_data_size;
|
||||
for (i = 0; i < args->k; i++)
|
||||
for (i = 0; i < args->k + args->m; i++)
|
||||
{
|
||||
char *frag = encoded_data[i];
|
||||
int cmp_size = -1;
|
||||
char *data_ptr = NULL;
|
||||
char *frag = NULL;
|
||||
uint32_t *mcksum = NULL;
|
||||
|
||||
frag = (i < args->k) ? encoded_data[i] : encoded_parity[i - args->k];
|
||||
assert(frag != NULL);
|
||||
fragment_header_t *header = (fragment_header_t*)frag;
|
||||
assert(header != NULL);
|
||||
mcksum = get_metadata_chksum(frag);
|
||||
assert(mcksum != NULL);
|
||||
assert(header->metadata_chksum == *mcksum);
|
||||
|
||||
fragment_metadata_t metadata = header->meta;
|
||||
assert(metadata.idx == i);
|
||||
assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.backend_metadata_size);
|
||||
assert(metadata.orig_data_size == orig_data_size);
|
||||
char *data_ptr = frag + frag_header_size;
|
||||
int cmp_size = remaining >= metadata.size ? metadata.size : remaining;
|
||||
assert(metadata.backend_id == be_id);
|
||||
assert(metadata.chksum_mismatch == 0);
|
||||
data_ptr = frag + frag_header_size;
|
||||
cmp_size = remaining >= metadata.size ? metadata.size : remaining;
|
||||
// shss doesn't keep original data on data fragments
|
||||
if (be_id != EC_BACKEND_SHSS) {
|
||||
assert(memcmp(data_ptr, orig_data_ptr, cmp_size) == 0);
|
||||
|
|
Loading…
Reference in New Issue