openstack
/
liberasurecode
Code Issues Proposed changes

Ripping out galois.[ch] and dynamically pulling the multiplication functon from jerasure

This commit is contained in:
Kevin Greenan 2014-07-22 13:32:23 -07:00
parent bde7428542
commit 59bec942cb
11 changed files with 70 additions and 1281 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Makefile.am
View File

@ -15,7 +15,6 @@ AM_CFLAGS = -fPIC $(AM_CPPFLAGS) -L/usr/local/lib
include_HEADERS = \
include/erasurecode/alg_sig.h \
include/erasurecode/galois.h \
include/erasurecode/erasurecode.h \
include/erasurecode/erasurecode_backend.h \
include/erasurecode/erasurecode_helpers.h \

17
configure.ac
View File

@ -32,23 +32,6 @@ AC_CHECK_HEADERS(sys/types.h stdio.h stdlib.h stddef.h stdarg.h malloc.h memory.
signal.h dlfcn.h pthread.h unistd.h limits.h errno.h syslog.h)
AC_CHECK_FUNCS(malloc calloc realloc free openlog)
# Check for gf_complete headers
AC_CHECK_HEADERS(galois.h gf_complete.h)
# Enable this check when gf_complete is external
AC_CHECK_LIB([gf_complete], [gf_init_easy],
[
AM_CONDITIONAL([GF_COMPLETE_INSTALLED], [true])
AC_DEFINE([HAVE_LIBGF_COMPLETE], [1], ["Defined if gf-complete is installed"])
],
[
echo "Warning gf_complete is required for *high performance* algorithmic signature support."
echo "gf_complete is available from https://bitbucket.org/jimplank/gf-complete.git"
echo "On Debian/Ubuntu systems, you can use \'apt-get install gf-complete\'"
echo "This will install with a slower Galois field backend!"
AM_CONDITIONAL([GF_COMPLETE_INSTALLED], [false])
])
AC_ARG_ENABLE([debug],
[ --enable-debug Turn on debugging],
[case "${enableval}" in

8
include/erasurecode/alg_sig.h
View File

@ -27,10 +27,18 @@
#include "galois.h"
struct jerasure_mult_routines {
int (*galois_single_multiply)(int, int, int);
};
#define JERASURE_SONAME "libJerasure.dylib"
typedef struct alg_sig_s
{
int gf_w;
int sig_len;
struct jerasure_mult_routines mult_routines;
void *jerasure_sohandle;
int *tbl1_l;
int *tbl1_r;
int *tbl2_l;

2
include/erasurecode/erasurecode.h
View File

@ -159,7 +159,7 @@ int liberasurecode_encode(int desc,
int liberasurecode_decode(int desc,
char **available_fragments, /* input */
int num_fragments, uint64_t fragment_len, /* input */
char *out_data, uint64_t *out_data_len); /* output */
char **out_data, uint64_t *out_data_len); /* output */
/**
* Reconstruct a missing fragment from a subset of available fragments

101
include/erasurecode/galois.h
View File

@ -1,101 +0,0 @@
/* *
* Copyright (c) 2013, James S. Plank and Kevin Greenan
* All rights reserved.
*
* Jerasure - A C/C++ Library for a Variety of Reed-Solomon and RAID-6 Erasure
* Coding Techniques
*
* Revision 2.0: Galois Field backend now links to GF-Complete
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
*
* - Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
*
* - Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
* the documentation and/or other materials provided with the
* distribution.
*
* - Neither the name of the University of Tennessee nor the names of its
* contributors may be used to endorse or promote products derived
* from this software without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
* "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
* LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
* A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
* HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
* INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
* BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
* OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
* AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY
* WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
* POSSIBILITY OF SUCH DAMAGE.
*/
#ifndef _GALOIS_H
#define _GALOIS_H
#include <config.h>
#include <stdio.h>
#include <stdlib.h>
#ifdef HAVE_LIBGF_COMPLETE
#include <gf_complete.h>
extern void galois_change_technique(gf_t *gf, int w);
gf_t* galois_init_field(int w,
int mult_type,
int region_type,
int divide_type,
uint64_t prim_poly,
int arg1,
int arg2);
gf_t* galois_init_composite_field(int w,
int region_type,
int divide_type,
int degree,
gf_t* base_gf);
gf_t * galois_get_field_ptr(int w);
#endif
extern int galois_single_multiply(int a, int b, int w);
extern int galois_single_divide(int a, int b, int w);
extern int galois_inverse(int x, int w);
void galois_region_xor( char *src, /* Source Region */
char *dest, /* Dest Region (holds result) */
int nbytes); /* Number of bytes in region */
/* These multiply regions in w=8, w=16 and w=32. They are much faster
than calling galois_single_multiply. The regions must be long word aligned. */
void galois_w08_region_multiply(char *region, /* Region to multiply */
int multby, /* Number to multiply by */
int nbytes, /* Number of bytes in region */
char *r2, /* If r2 != NULL, products go here.
Otherwise region is overwritten */
int add); /* If (r2 != NULL && add) the produce is XOR'd with r2 */
void galois_w16_region_multiply(char *region, /* Region to multiply */
int multby, /* Number to multiply by */
int nbytes, /* Number of bytes in region */
char *r2, /* If r2 != NULL, products go here.
Otherwise region is overwritten */
int add); /* If (r2 != NULL && add) the produce is XOR'd with r2 */
void galois_w32_region_multiply(char *region, /* Region to multiply */
int multby, /* Number to multiply by */
int nbytes, /* Number of bytes in region */
char *r2, /* If r2 != NULL, products go here.
Otherwise region is overwritten */
int add); /* If (r2 != NULL && add) the produce is XOR'd with r2 */
#endif

5
src/Makefile.am
View File

@ -14,7 +14,6 @@ liberasurecode_la_SOURCES = \
erasurecode_postprocessing.c \
utils/chksum/crc32.c \
utils/chksum/alg_sig.c \
utils/chksum/galois.c \
backends/xor/flat_xor_hd.c \
backends/jerasure/jerasure_rs_vand.c
@ -22,10 +21,6 @@ liberasurecode_la_CPPFLAGS = -Werror
liberasurecode_la_LIBADD = \
builtin/xor_codes/libXorcode.la -lpthread -lm
if GF_COMPLETE_INSTALLED
liberasurecode_la_LIBADD += -lgf_complete
endif
# Version format (C - A).(A).(R) for C:R:A input
liberasurecode_la_LDFLAGS = -rpath '$(libdir)' -version-info 9:10:9

6
src/erasurecode.c
View File

@ -386,7 +386,7 @@ out:
int liberasurecode_decode(int desc,
char **available_fragments, /* input */
int num_fragments, uint64_t fragment_len, /* input */
char *out_data, uint64_t *out_data_len) /* output */
char **out_data, uint64_t *out_data_len) /* output */
{
int i, j;
int ret = 0;
@ -415,7 +415,7 @@ int liberasurecode_decode(int desc,
*/
ret = fragments_to_string(k, m,
available_fragments, num_fragments,
&out_data, out_data_len);
out_data, out_data_len);
if (ret == 0) {
/* We were able to get the original data without decoding! */
@ -507,7 +507,7 @@ int liberasurecode_decode(int desc,
}
/* Try to generate the original string */
ret = fragments_to_string(k, m, data, k, &out_data, out_data_len);
ret = fragments_to_string(k, m, data, k, out_data, out_data_len);
if (ret < 0) {
log_error("Could not prepare convert decoded fragments to a string!");

71
src/utils/chksum/alg_sig.c
View File

@ -22,6 +22,7 @@
* ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
*/
#include <dlfcn.h>
#include <alg_sig.h>
#include <stdlib.h>
#include <string.h>
@ -29,6 +30,19 @@
int valid_gf_w[] = { 8, 16, -1 };
int valid_pairs[][2] = { { 8, 32}, {16, 32}, {16, 64}, {-1, -1} };
void *get_jerasure_sohandle()
{
return dlopen(JERASURE_SONAME, RTLD_LAZY | RTLD_LOCAL);
}
int load_gf_functions(void *sohandle, struct jerasure_mult_routines *routines)
{
routines->galois_single_multiply = dlsym(sohandle, "galois_single_multiply");
if (NULL == routines->galois_single_multiply) {
return -1;
}
return 0;
}
static
alg_sig_t *init_alg_sig_w8(int sig_len)
@ -39,9 +53,23 @@ alg_sig_t *init_alg_sig_w8(int sig_len)
int w = 8;
int alpha = 2, beta = 4, gamma = 8;
int num_components = sig_len / w;
struct jerasure_mult_routines g_jerasure_mult_routines;
void *jerasure_sohandle = get_jerasure_sohandle();
if (NULL == jerasure_sohandle) {
return NULL;
}
alg_sig_handle = (alg_sig_t *)malloc(sizeof(alg_sig_t));
if (alg_sig_handle == NULL) {
if (NULL == alg_sig_handle) {
return NULL;
}
alg_sig_handle->jerasure_sohandle = jerasure_sohandle;
if (load_gf_functions(alg_sig_handle->jerasure_sohandle, &(alg_sig_handle->mult_routines)) < 0) {
free(alg_sig_handle);
return NULL;
}
@ -66,14 +94,14 @@ alg_sig_t *init_alg_sig_w8(int sig_len)
*/
for (i = 0; i < 16; i++) {
if (num_components >= 4) {
alg_sig_handle->tbl1_l[i] = galois_single_multiply((unsigned char) (i << 4) & 0xf0, alpha, w);
alg_sig_handle->tbl1_r[i] = galois_single_multiply((unsigned char) i, alpha, w);
alg_sig_handle->tbl1_l[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned char)(i << 4) & 0xf0, alpha, w);
alg_sig_handle->tbl1_r[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned char) i, alpha, w);
alg_sig_handle->tbl2_l[i] = galois_single_multiply((unsigned char) (i << 4) & 0xf0, beta, w);
alg_sig_handle->tbl2_r[i] = galois_single_multiply((unsigned char) i, beta, w);
alg_sig_handle->tbl2_l[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned char) (i << 4) & 0xf0, beta, w);
alg_sig_handle->tbl2_r[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned char) i, beta, w);
alg_sig_handle->tbl3_l[i] = galois_single_multiply((unsigned char) (i << 4) & 0xf0, gamma, w);
alg_sig_handle->tbl3_r[i] = galois_single_multiply((unsigned char) i, gamma, w);
alg_sig_handle->tbl3_l[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned char) (i << 4) & 0xf0, gamma, w);
alg_sig_handle->tbl3_r[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned char) i, gamma, w);
}
}
@ -89,12 +117,24 @@ alg_sig_t *init_alg_sig_w16(int sig_len)
int w = 16;
int alpha = 2, beta = 4, gamma = 8;
int num_components = sig_len / w;
void *jerasure_sohandle = get_jerasure_sohandle();
if (jerasure_sohandle == NULL) {
return NULL;
}
alg_sig_handle = (alg_sig_t *)malloc(sizeof(alg_sig_t));
if (alg_sig_handle == NULL) {
return NULL;
}
alg_sig_handle->jerasure_sohandle = jerasure_sohandle;
if (load_gf_functions(alg_sig_handle->jerasure_sohandle, &(alg_sig_handle->mult_routines)) < 0) {
free(alg_sig_handle);
return NULL;
}
alg_sig_handle->sig_len = sig_len;
alg_sig_handle->gf_w = w;
@ -118,15 +158,15 @@ alg_sig_t *init_alg_sig_w16(int sig_len)
* Note that \gamme = 8 (\alpha ^ 3 MOD 2^16)
*/
for (i = 0; i < 256; i++) {
alg_sig_handle->tbl1_l[i] = galois_single_multiply((unsigned short) (i << 8), alpha, w);
alg_sig_handle->tbl1_r[i] = galois_single_multiply((unsigned short) i, alpha, w);
alg_sig_handle->tbl1_l[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned short) (i << 8), alpha, w);
alg_sig_handle->tbl1_r[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned short) i, alpha, w);
if (num_components >= 4) {
alg_sig_handle->tbl2_l[i] = galois_single_multiply((unsigned short) (i << 8), beta, w);
alg_sig_handle->tbl2_r[i] = galois_single_multiply((unsigned short) i, beta, w);
alg_sig_handle->tbl2_l[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned short) (i << 8), beta, w);
alg_sig_handle->tbl2_r[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned short) i, beta, w);
alg_sig_handle->tbl3_l[i] = galois_single_multiply((unsigned short) (i << 8), gamma, w);
alg_sig_handle->tbl3_r[i] = galois_single_multiply((unsigned short) i, gamma, w);
alg_sig_handle->tbl3_l[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned short) (i << 8), gamma, w);
alg_sig_handle->tbl3_r[i] = alg_sig_handle->mult_routines.galois_single_multiply((unsigned short) i, gamma, w);
}
}
@ -165,6 +205,9 @@ void destroy_alg_sig(alg_sig_t* alg_sig_handle)
free(alg_sig_handle);
return;
}
dlclose(alg_sig_handle->jerasure_sohandle);
int num_components = alg_sig_handle->sig_len / alg_sig_handle->gf_w;
free(alg_sig_handle->tbl1_l);

1131
src/utils/chksum/galois.c
View File

File diff suppressed because it is too large Load Diff

7
test/Makefile.am
View File

@ -17,10 +17,3 @@ liberasurecode_test_SOURCES = liberasurecode_test.c
liberasurecode_test_CPPFLAGS = -I$(top_srcdir)/include -I$(top_srcdir)/include/erasurecode
liberasurecode_test_LDFLAGS = $(top_srcdir)/src/liberasurecode.la -ldl -lpthread
check_PROGRAMS += liberasurecode_test
if GF_COMPLETE_INSTALLED
alg_sig_test_LDFLAGS += -lgf_complete
liberasurecode_test_LDFLAGS += -lgf_complete
test_xor_hd_code_LDFLAGS += -lgf_complete
endif

2
test/liberasurecode_test.c
View File

@ -96,7 +96,7 @@ static int test_simple_encode_decode(
assert(0 == rc);
rc = liberasurecode_decode(desc, encoded_data,
10, encoded_fragment_len, decoded_data, &decoded_data_len);
10, encoded_fragment_len, &decoded_data, &decoded_data_len);
assert(0 == rc);
assert(decoded_data_len == orig_data_size);
assert(memcmp(decoded_data, orig_data, orig_data_size) == 0);