Merged in frag_metadata_checksum (pull request #20)

Fragment metadata checksumming support
This commit is contained in:
Kevin Greenan 2016-03-10 22:20:43 -08:00
commit 35000cb0c3
7 changed files with 148 additions and 46 deletions

View File

@ -256,15 +256,15 @@ int liberasurecode_fragments_needed(int desc,
typedef struct __attribute__((__packed__))
fragment_metadata
{
uint32_t idx; /* 4 */
uint32_t size; /* 4 */
uint32_t frag_backend_metadata_size; /* 4 */
uint64_t orig_data_size; /* 8 */
uint8_t chksum_type; /* 1 */
uint32_t idx; /* 4 */
uint32_t size; /* 4 */
uint32_t frag_backend_metadata_size; /* 4 */
uint64_t orig_data_size; /* 8 */
uint8_t chksum_type; /* 1 */
uint32_t chksum[LIBERASURECODE_MAX_CHECKSUM_LEN]; /* 32 */
uint8_t chksum_mismatch; /* 1 */
uint8_t backend_id; /* 1 */
uint32_t backend_version; /* 4 */
uint8_t chksum_mismatch; /* 1 */
uint8_t backend_id; /* 1 */
uint32_t backend_version; /* 4 */
} fragment_metadata_t;
/**
@ -321,12 +321,13 @@ int liberasurecode_verify_stripe_metadata(int desc,
typedef struct __attribute__((__packed__)) fragment_header_s
{
fragment_metadata_t meta; /* 59 bytes */
uint32_t magic; /* 4 bytes */
uint32_t libec_version; /* 4 bytes */
fragment_metadata_t meta; /* 59 bytes */
uint32_t magic; /* 4 bytes */
uint32_t libec_version; /* 4 bytes */
uint32_t metadata_chksum; /* 4 bytes */
// We must be aligned to 16-byte boundaries
// So, size this array accordingly
uint8_t aligned_padding[13];
uint8_t aligned_padding[9];
} fragment_header_t;
#define FRAGSIZE_2_BLOCKSIZE(fragment_size) \

View File

@ -65,13 +65,16 @@ int get_fragment_buffer_size(char *buf);
int set_orig_data_size(char *buf, int orig_data_size);
int get_orig_data_size(char *buf);
int set_checksum(ec_checksum_type_t ct, char *buf, int blocksize);
int get_checksum(char *buf); //TODO implement this
int get_checksum(char *buf);
int set_libec_version(char *fragment);
int get_libec_version(char *fragment, uint32_t *ver);
int set_backend_id(char *buf, ec_backend_id_t id);
int get_backend_id(char *buf, ec_backend_id_t *id);
int set_backend_version(char *buf, uint32_t version);
int get_backend_version(char *buf, uint32_t *version);
int set_metadata_chksum(char *buf);
uint32_t *get_metadata_chksum(char *buf);
int is_invalid_fragment_header(fragment_header_t *header);
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */

View File

@ -26,8 +26,8 @@
#define _ERASURECODE_VERSION_H_
#define _MAJOR 1
#define _MINOR 1
#define _REV 2
#define _MINOR 2
#define _REV 0
#define _VERSION(x, y, z) ((x << 16) | (y << 8) | (z))
#define LIBERASURECODE_VERSION _VERSION(_MAJOR, _MINOR, _REV)

View File

@ -26,6 +26,7 @@
* vi: set noai tw=79 ts=4 sw=4:
*/
#include "assert.h"
#include "list.h"
#include "erasurecode.h"
#include "erasurecode_backend.h"
@ -578,10 +579,22 @@ int liberasurecode_decode(int desc,
m = instance->args.uargs.m;
if (num_fragments < k) {
log_error("Not enough fragments to decode, got %d, need %d!",
num_fragments, k);
ret = -EINSUFFFRAGS;
goto out;
}
for (i = 0; i < num_fragments; ++i) {
/* Verify metadata checksum */
if (is_invalid_fragment_header(
(fragment_header_t *) available_fragments[i])) {
log_error("Invalid fragment header information!");
ret = -EBADHEADER;
goto out;
}
}
if (instance->common.id != EC_BACKEND_SHSS) {
/* shss (ntt_backend) must force to decode */
// TODO: Add a frag and function to handle whether the backend want to decode or not.
@ -783,6 +796,16 @@ int liberasurecode_reconstruct_fragment(int desc,
k = instance->args.uargs.k;
m = instance->args.uargs.m;
for (i = 0; i < num_fragments; i++) {
/* Verify metadata checksum */
if (is_invalid_fragment_header(
(fragment_header_t *) available_fragments[i])) {
log_error("Invalid fragment header information!");
ret = -EBADHEADER;
goto out;
}
}
/*
* Allocate arrays for data, parity and missing_idxs
*/
@ -1004,6 +1027,14 @@ int liberasurecode_get_fragment_metadata(char *fragment,
goto out;
}
/* Verify metadata checksum */
if (is_invalid_fragment_header(
(fragment_header_t *) fragment)) {
log_error("Invalid fragment header information!");
ret = -EBADHEADER;
goto out;
}
memcpy(fragment_metadata, fragment, sizeof(struct fragment_metadata));
fragment_hdr = (fragment_header_t *) fragment;
if (LIBERASURECODE_FRAG_HEADER_MAGIC != fragment_hdr->magic) {
@ -1037,6 +1068,20 @@ out:
return ret;
}
int is_invalid_fragment_header(fragment_header_t *header)
{
uint32_t *stored_csum = NULL, csum = 0;
assert (NULL != header);
if (header->libec_version < _VERSION(1,2,0))
/* no metadata checksum support */
return 0;
stored_csum = get_metadata_chksum((char *) header);
if (NULL == stored_csum)
return 1; /* can't verify, possibly crc32 call error */
csum = crc32(0, &header->meta, sizeof(fragment_metadata_t));
return (*stored_csum != csum);
}
int liberasurecode_verify_fragment_metadata(ec_backend_t be,
fragment_metadata_t *md)
{
@ -1054,6 +1099,27 @@ int liberasurecode_verify_fragment_metadata(ec_backend_t be,
return 0;
}
int is_invalid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
{
ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
if (!be) {
log_error("Unable to verify fragment metadata: invalid backend id %d.",
desc);
return -EINVALIDPARAMS;
}
if (liberasurecode_verify_fragment_metadata(be,
fragment_metadata) != 0) {
return -EBADHEADER;
}
if (!be->common.ops->is_compatible_with(fragment_metadata->backend_version)) {
return -EBADHEADER;
}
if (fragment_metadata->chksum_mismatch == 1) {
return -EBADCHKSUM;
}
return 0;
}
int is_invalid_fragment(int desc, char *fragment)
{
uint32_t ver = 0;
@ -1069,41 +1135,18 @@ int is_invalid_fragment(int desc, char *fragment)
return 1;
}
if (get_libec_version(fragment, &ver) != 0 ||
ver != LIBERASURECODE_VERSION) {
ver > LIBERASURECODE_VERSION) {
return 1;
}
if (liberasurecode_get_fragment_metadata(fragment,
&fragment_metadata) != 0) {
if (liberasurecode_get_fragment_metadata(fragment, &fragment_metadata) != 0) {
return 1;
}
if (liberasurecode_verify_fragment_metadata(be,
&fragment_metadata) != 0) {
if (is_invalid_fragment_metadata(desc, &fragment_metadata) != 0) {
return 1;
}
return 0;
}
int is_valid_fragment_metadata(int desc, fragment_metadata_t *fragment_metadata)
{
ec_backend_t be = liberasurecode_backend_instance_get_by_desc(desc);
if (!be) {
log_error("Unable to verify stripe metadata: invalid backend id %d.",
desc);
return -EINVALIDPARAMS;
}
if (liberasurecode_verify_fragment_metadata(be,
fragment_metadata) != 0) {
return -EBADHEADER;
}
if (!be->common.ops->is_compatible_with(fragment_metadata->backend_version)) {
return -EBADHEADER;
}
if (fragment_metadata->chksum_mismatch == 1) {
return -EBADCHKSUM;
}
return 0;
}
int liberasurecode_verify_stripe_metadata(int desc,
char **fragments, int num_fragments)
{
@ -1120,7 +1163,7 @@ int liberasurecode_verify_stripe_metadata(int desc,
for (i = 0; i < num_fragments; i++) {
fragment_metadata_t *fragment_metadata = (fragment_metadata_t*)fragments[i];
int ret = is_valid_fragment_metadata(desc, fragment_metadata);
int ret = is_invalid_fragment_metadata(desc, fragment_metadata);
if (ret < 0) {
return ret;
}

View File

@ -501,3 +501,45 @@ inline uint32_t* get_chksum(char *buf)
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
#if LIBERASURECODE_VERSION >= _VERSION(1,2,0)
inline int set_metadata_chksum(char *buf)
{
fragment_header_t* header = (fragment_header_t*) buf;
assert(NULL != header);
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (set meta chksum)!\n");
return -1;
}
header->metadata_chksum = crc32(0, &header->meta,
sizeof(fragment_metadata_t));
return 0;
}
inline uint32_t* get_metadata_chksum(char *buf)
{
fragment_header_t* header = (fragment_header_t*) buf;
assert(NULL != header);
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (get meta chksum)!");
return NULL;
}
return (uint32_t *) &header->metadata_chksum;
}
#else
inline int set_metadata_chksum(char *buf)
{
return 0;
}
inline uint32_t* get_metadata_chksum(char *buf)
{
return (uint32_t *) 0;
}
#endif
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */

View File

@ -47,6 +47,7 @@ void add_fragment_metadata(ec_backend_t be, char *fragment,
if (add_chksum) {
set_checksum(ct, fragment, blocksize);
}
set_metadata_chksum(fragment);
}
int finalize_fragments_after_encode(ec_backend_t instance,

View File

@ -940,17 +940,29 @@ static void encode_decode_test_impl(const ec_backend_id_t be_id,
assert(0 == rc);
orig_data_ptr = orig_data;
remaining = orig_data_size;
for (i = 0; i < args->k; i++)
for (i = 0; i < args->k + args->m; i++)
{
char *frag = encoded_data[i];
int cmp_size = -1;
char *data_ptr = NULL;
char *frag = NULL;
uint32_t *mcksum = NULL;
frag = (i < args->k) ? encoded_data[i] : encoded_parity[i - args->k];
assert(frag != NULL);
fragment_header_t *header = (fragment_header_t*)frag;
assert(header != NULL);
mcksum = get_metadata_chksum(frag);
assert(mcksum != NULL);
assert(header->metadata_chksum == *mcksum);
fragment_metadata_t metadata = header->meta;
assert(metadata.idx == i);
assert(metadata.size == encoded_fragment_len - frag_header_size - be->common.backend_metadata_size);
assert(metadata.orig_data_size == orig_data_size);
char *data_ptr = frag + frag_header_size;
int cmp_size = remaining >= metadata.size ? metadata.size : remaining;
assert(metadata.backend_id == be_id);
assert(metadata.chksum_mismatch == 0);
data_ptr = frag + frag_header_size;
cmp_size = remaining >= metadata.size ? metadata.size : remaining;
// shss doesn't keep original data on data fragments
if (be_id != EC_BACKEND_SHSS) {
assert(memcmp(data_ptr, orig_data_ptr, cmp_size) == 0);