openstack
/
liberasurecode
Code Issues Proposed changes

Allow reading of little-endian frags on big-endian 

... and vice-versa. We'll fix up frag header values for our output
parameter from liberasurecode_get_fragment_metadata but otherwise
avoid manipulating the in-memory fragment much.

Change-Id: Idd6833bdea60e27c9a0148ee28b4a2c1070be148
This commit is contained in:
Tim Burke 2018-09-11 19:36:10 +00:00
parent 72af842b28
commit 7b547e0e46
3 changed files with 117 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
include/erasurecode/erasurecode_helpers.h
View File

@ -93,5 +93,21 @@ void *get_aligned_buffer16(int size);
/* ==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~==~=*=~== */
#ifndef bswap_32
static __inline uint32_t __libec_bswap_32(uint32_t __x)
{
return __x>>24 | (__x>>8&0xff00) | (__x<<8&0xff0000) | __x<<24;
}
#define bswap_32(x) __libec_bswap_32(x)
#endif
#ifndef bswap_64
static __inline uint64_t __libec_bswap_64(uint64_t __x)
{
return (__libec_bswap_32(__x)+0ULL)<<32 | __libec_bswap_32(__x>>32);
}
#define bswap_64(x) __libec_bswap_64(x)
#endif
#endif // _ERASURECODE_HELPERS_H_

60
src/erasurecode.c
View File

@ -1055,17 +1055,36 @@ int liberasurecode_get_fragment_metadata(char *fragment,
memcpy(fragment_metadata, fragment, sizeof(struct fragment_metadata));
fragment_hdr = (fragment_header_t *) fragment;
if (LIBERASURECODE_FRAG_HEADER_MAGIC != fragment_hdr->magic) {
log_error("Invalid fragment, illegal magic value");
ret = -EINVALIDPARAMS;
goto out;
if (LIBERASURECODE_FRAG_HEADER_MAGIC != bswap_32(fragment_hdr->magic)) {
log_error("Invalid fragment, illegal magic value");
ret = -EINVALIDPARAMS;
goto out;
} else {
// Must've written this on an opposite-endian architecture.
// Fix it in fragment_metadata
fragment_metadata->idx = bswap_32(fragment_metadata->idx);
fragment_metadata->size = bswap_32(fragment_metadata->size);
fragment_metadata->frag_backend_metadata_size =
bswap_32(fragment_metadata->frag_backend_metadata_size);
fragment_metadata->orig_data_size =
bswap_64(fragment_metadata->orig_data_size);
fragment_metadata->chksum_type =
bswap_32(fragment_metadata->chksum_type);
for (int i = 0; i < LIBERASURECODE_MAX_CHECKSUM_LEN; i++) {
fragment_metadata->chksum[i] =
bswap_32(fragment_metadata->chksum[i]);
}
fragment_metadata->backend_version =
bswap_32(fragment_metadata->backend_version);
}
}
switch(fragment_hdr->meta.chksum_type) {
switch(fragment_metadata->chksum_type) {
case CHKSUM_CRC32: {
uint32_t computed_chksum = 0;
uint32_t stored_chksum = fragment_hdr->meta.chksum[0];
uint32_t stored_chksum = fragment_metadata->chksum[0];
char *fragment_data = get_data_ptr_from_fragment(fragment);
uint64_t fragment_size = fragment_hdr->meta.size;
uint64_t fragment_size = fragment_metadata->size;
computed_chksum = crc32(0, (unsigned char *) fragment_data, fragment_size);
if (stored_chksum != computed_chksum) {
// Try again with our "alternative" crc32; see
@ -1095,28 +1114,39 @@ out:
int is_invalid_fragment_header(fragment_header_t *header)
{
uint32_t csum = 0;
uint32_t csum = 0, metadata_chksum = 0, libec_version = 0;
assert (NULL != header);
if (header->libec_version == 0)
/* libec_version must be bigger than 0 */
return 1;
if (header->libec_version < _VERSION(1,2,0))
metadata_chksum = header->metadata_chksum;
libec_version = header->libec_version;
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
if (bswap_32(header->magic) != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (get meta chksum)!");
return 1;
} else {
// Must've written this on an opposite-endian architecture.
// Fix our reference checksum and version, but *don't touch
// the header data yet*. Leave that for when we're extracting
// in liberasurecode_get_fragment_metadata
metadata_chksum = bswap_32(metadata_chksum);
libec_version = bswap_32(libec_version);
}
}
if (libec_version < _VERSION(1,2,0))
/* no metadata checksum support */
return 0;
if (header->magic != LIBERASURECODE_FRAG_HEADER_MAGIC) {
log_error("Invalid fragment header (get meta chksum)!");
return 1;
}
csum = crc32(0, (unsigned char *) &header->meta, sizeof(fragment_metadata_t));
if (header->metadata_chksum == csum) {
if (metadata_chksum == csum) {
return 0;
}
// Else, try again with our "alternative" crc32; see
// https://bugs.launchpad.net/liberasurecode/+bug/1666320
csum = liberasurecode_crc32_alt(0, &header->meta, sizeof(fragment_metadata_t));
return (header->metadata_chksum != csum);
return (metadata_chksum != csum);
}
int liberasurecode_verify_fragment_metadata(ec_backend_t be,

63
test/liberasurecode_test.c
View File

@ -1762,29 +1762,77 @@ static void test_verify_stripe_metadata_frag_idx_invalid(
verify_fragment_metadata_mismatch_impl(be_id, args, FRAGIDX_OUT_OF_RANGE);
}
static void test_metadata_crcs()
static void test_metadata_crcs_le()
{
// We've observed headers like this in the wild, using our busted crc32
char header[] =
char orig_header[] =
"\x03\x00\x00\x00\x00\x00\x04\x00\x00\x00\x00\x00\x00\x00\x10\x00"
"\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x07\x01\x0e\x02\x00\xcc\x5e\x0c\x0b\x00"
"\x04\x01\x00\x22\xee\x45\xb9\x00\x00\x00\x00\x00\x00\x00\x00\x00";
char header[sizeof(orig_header)];
memcpy(header, orig_header, sizeof(orig_header));
fragment_metadata_t res;
assert(liberasurecode_get_fragment_metadata(header, &res) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
assert(res.backend_version == _VERSION(2, 14, 1));
assert(is_invalid_fragment_header((fragment_header_t *) header) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
// Switch it to zlib's implementation
header[70] = '\x18';
header[69] = '\x73';
header[68] = '\xf8';
header[67] = '\xec';
orig_header[70] = '\x18';
orig_header[69] = '\x73';
orig_header[68] = '\xf8';
orig_header[67] = '\xec';
memcpy(header, orig_header, sizeof(orig_header));
assert(liberasurecode_get_fragment_metadata(header, &res) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
assert(res.backend_version == _VERSION(2, 14, 1));
assert(is_invalid_fragment_header((fragment_header_t *) header) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
// Write down the wrong thing
header[70] = '\xff';
assert(liberasurecode_get_fragment_metadata(header, &res) == -EBADHEADER);
assert(is_invalid_fragment_header((fragment_header_t *) header) == 1);
}
static void test_metadata_crcs_be()
{
// Like above, but big-endian
char orig_header[] =
"\x00\x00\x00\x03\x00\x04\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x10\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00"
"\x00\x00\x00\x00\x00\x00\x07\x00\x02\x0e\x01\x0b\x0c\x5e\xcc\x00"
"\x01\x04\x00\xfa\x85\x40\x70\x00\x00\x00\x00\x00\x00\x00\x00\x00";
char header[sizeof(orig_header)];
memcpy(header, orig_header, sizeof(orig_header));
fragment_metadata_t res;
assert(liberasurecode_get_fragment_metadata(header, &res) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
assert(res.backend_version == _VERSION(2, 14, 1));
assert(is_invalid_fragment_header((fragment_header_t *) header) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
// Switch it to zlib's implementation
orig_header[67] = '\xe3';
orig_header[68] = '\x73';
orig_header[69] = '\x88';
orig_header[70] = '\xa0';
memcpy(header, orig_header, sizeof(orig_header));
assert(liberasurecode_get_fragment_metadata(header, &res) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
assert(res.backend_version == _VERSION(2, 14, 1));
assert(is_invalid_fragment_header((fragment_header_t *) header) == 0);
assert(memcmp(header, orig_header, sizeof(orig_header)) == 0);
// Write down the wrong thing
header[70] = '\xff';
@ -1832,7 +1880,8 @@ struct testcase testcases[] = {
TEST(test_fragments_needed_invalid_args, EC_BACKENDS_MAX, CHKSUM_TYPES_MAX),
TEST(test_get_fragment_partition, EC_BACKENDS_MAX, CHKSUM_TYPES_MAX),
TEST(test_liberasurecode_get_version, EC_BACKENDS_MAX, CHKSUM_TYPES_MAX),
TEST(test_metadata_crcs, EC_BACKENDS_MAX, 0),
TEST(test_metadata_crcs_le, EC_BACKENDS_MAX, 0),
TEST(test_metadata_crcs_be, EC_BACKENDS_MAX, 0),
// NULL backend test
TEST(test_create_and_destroy_backend, EC_BACKEND_NULL, CHKSUM_NONE),
TEST(test_simple_encode_decode, EC_BACKEND_NULL, CHKSUM_NONE),