Skip to content

Commit

Permalink
Add support for Good General Cauchy Matrices (#2)
Browse files Browse the repository at this point in the history 
add _gc versions of encode/decode that use the 'good cauchy' matrix for the particular k, m and w. If w is not supplied it is calculated as `ceil(math:log2(K+M))`. These should perform better in all cases than the Vandermonde matrixes used in the regular encode/decode.

Note, however, that sometimes w should be higher than the default for ideal performance. This is not done automatically.
  • Loading branch information
@vihu @Vagabond
vihu authored and Vagabond committed Jan 7, 2019
1 parent ab0595a commit 1697137
Show file tree
Hide file tree
Showing 5 changed files with 464 additions and 35 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ clean:
$(REBAR) clean

test:
$(REBAR) eqc -n 100
$(REBAR) ct --verbose; $(REBAR) eqc -n 100

typecheck:
$(REBAR) dialyzer
Expand Down
332 changes: 323 additions & 9 deletions c_src/erasure.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,9 @@
#include <stdbool.h>
#include <string.h>
#include <jerasure.h>
#include <cauchy.h>
#include <reed_sol.h>
#include <math.h>

static ERL_NIF_TERM ATOM_TRUE;
static ERL_NIF_TERM ATOM_FALSE;
Expand Down Expand Up @@ -93,7 +95,7 @@ encode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])

ERL_NIF_TERM list = enif_make_list(env, 0);

for (int i = k+m - 1; i >= 0; i--)
for (int i = k+m - 1; i >= 0; i--)
{
ERL_NIF_TERM binary;
unsigned char *bindata = enif_make_new_binary(env, blocksize, &binary);
Expand All @@ -110,6 +112,120 @@ encode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
return enif_make_tuple2(env, enif_make_atom(env, "ok"), list);
}


// ==================================================================
// Encode using the Good general cauchy matrix

static ERL_NIF_TERM
encode_gc(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
{
int k;
if (!enif_get_int(env, argv[0], &k))
{
return enif_make_badarg(env);
}

int m;
if (!enif_get_int(env, argv[1], &m))
{
return enif_make_badarg(env);
}

int w;
if (!enif_get_int(env, argv[2], &w))
{
return enif_make_badarg(env);
}


ErlNifBinary input;
if (!enif_is_binary(env, argv[3]) || !enif_inspect_binary(env, argv[3], &input))
{
return enif_make_badarg(env);
}

int blocksize = input.size / k;
int padding = 0;
int remainder = input.size % k;

if (remainder != 0) {
// payload is not cleanly divible by K, we need to pad
padding = (k - (input.size % k));
blocksize = (input.size + padding) / k;
}

while (blocksize % sizeof(long) != 0 || blocksize % w != 0) {
blocksize++;
padding++;
}

// block spacing has to be a multiple of 16
int blockspacing = blocksize + (16 - (blocksize % 16));

int bytes_per_shard = input.size / k;
int extra_bytes = input.size % k;

char *shards = calloc(k+m, blockspacing);
char *data_ptrs[k];
char *coding_ptrs[m];

unsigned char *p = input.data;
for (int i = 0; i < k+m; i++) {
memset(shards+(blockspacing*i), 0, blockspacing);
if (i < k) {
data_ptrs[i] = shards+(blockspacing*i);
memcpy(shards+(blockspacing*i), p, bytes_per_shard);
p += bytes_per_shard;
if (extra_bytes > 0) {
memcpy(shards+(blockspacing*i)+bytes_per_shard, p, 1);
p++;
extra_bytes--;
}
} else {
coding_ptrs[i-k] = shards+(blockspacing*i);
}
}

int *matrix, *bitmatrix, **schedule;

matrix = cauchy_good_general_coding_matrix(k, m, w);
if (matrix == NULL) {
free(shards);
return enif_make_badarg(env);
}
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
if (bitmatrix == NULL) {
free(shards);
free(matrix);
return enif_make_badarg(env);
}
schedule = jerasure_smart_bitmatrix_to_schedule(k, m, w, bitmatrix);
if (schedule == NULL) {
return enif_make_badarg(env);
}
jerasure_schedule_encode(k, m, w, schedule, data_ptrs, coding_ptrs, blocksize, blocksize/w);

ERL_NIF_TERM list = enif_make_list(env, 0);

for (int i = k+m - 1; i >= 0; i--)
{
ERL_NIF_TERM binary;
unsigned char *bindata = enif_make_new_binary(env, blocksize, &binary);
memcpy(bindata, shards+(blockspacing*i), blocksize);
list = enif_make_list_cell(env,
enif_make_tuple3(env,
enif_make_int(env, i),
enif_make_int(env, input.size),
binary
), list);
}
free(shards);
free(matrix);
return enif_make_tuple2(env, enif_make_atom(env, "ok"), list);
}

// ==================================================================

static ERL_NIF_TERM
decode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
{
Expand Down Expand Up @@ -265,22 +381,220 @@ decode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])

cleanup:

if (matrix != NULL) {
free(matrix);
if (matrix != NULL) {
free(matrix);
}
if (shards != NULL) {
free(shards);
}
if (erasures != NULL) {
free(erasures);
}

return result;
}

// ==================================================================
// Decode using the Good general cauchy matrix

static ERL_NIF_TERM
decode_gc(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
{
int k;
ERL_NIF_TERM result;
if (!enif_get_int(env, argv[0], &k))
{
return enif_make_badarg(env);
}
if (shards != NULL) {
free(shards);

int m;
if (!enif_get_int(env, argv[1], &m))
{
return enif_make_badarg(env);
}

int w;
if (!enif_get_int(env, argv[2], &w))
{
return enif_make_badarg(env);
}
if (erasures != NULL) {
free(erasures);

unsigned len;
if (!enif_is_list(env, argv[3]) || !enif_get_list_length(env, argv[3], &len) || len < k)
{
// need at least K shards, sorry
return enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "insufficent_shards"));
}

return result;
int *matrix = NULL;
int *bitmatrix = NULL;
char *shards = NULL;
char *data_ptrs[k];
char *coding_ptrs[m];

int *erasures = NULL;

for (int i = 0; i < k; i++) {
data_ptrs[i] = NULL;
}

for (int i = 0; i < m; i++) {
coding_ptrs[i] = NULL;
}

// all the shards must be the same size
// and all the indices need to be in-bounds
ERL_NIF_TERM head, tail;
const ERL_NIF_TERM *tuple;
int arity, id, totalsize, lasttotalsize = 0;
int padding, blocksize = 0, remainder=0, blockspacing=0;
tail = argv[3];
while (enif_get_list_cell(env, tail, &head, &tail))
{
if (!enif_get_tuple(env, head, &arity, &tuple) || arity != 3) {
result = enif_make_badarg(env);
goto cleanup;
}
if (!enif_get_int(env, tuple[0], &id) || id < 0 || id >= k+m) {
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "invalid_shard_id"));
goto cleanup;
}
if ((id < k && data_ptrs[id] != NULL) || ( id >=k && coding_ptrs[id-k] != NULL)) {
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "duplicate_shard_id"));
goto cleanup;
}
if (!enif_get_int(env, tuple[1], &totalsize) || totalsize <= 0) {
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "invalid_total_size"));
goto cleanup;
}
if (lasttotalsize != 0 && totalsize != lasttotalsize) {
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "inconsistent_total_size"));
goto cleanup;
}
lasttotalsize=totalsize;

if (blocksize == 0) {
blocksize = totalsize / k;
padding = 0;

remainder = totalsize % k;
if (remainder != 0) {
// payload is not cleanly divible by K, we need to pad
padding = (k - (totalsize % k));
blocksize = (totalsize + padding) / k;
}
while (blocksize % sizeof(long) != 0 || blocksize % w != 0) {
blocksize++;
padding++;
}
// block spacing has to be a multiple of 16
blockspacing = blocksize + (16 - (blocksize % 16));
// Note more random padding
shards = calloc(k+m+1, blockspacing);
}

ErlNifBinary input;
if (!enif_is_binary(env, tuple[2]) || !enif_inspect_binary(env, tuple[2], &input) || input.size != blocksize)
{
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "invalid_shard"));
goto cleanup;
}

if (id < k) {
data_ptrs[id] = shards+(blockspacing*id);
} else {
coding_ptrs[id-k] = shards+(blockspacing*id);
}

memset(shards+(blockspacing*id), 0, blockspacing);
memcpy(shards+(blockspacing*id), input.data, blocksize);
}

erasures = calloc(k+m, sizeof(int));
int j = 0;

int bytes_per_shard = totalsize / k;
int extra_bytes = totalsize % k;

// calculate the missing shards and track them in the erasures array
for (int i = 0; i < k+m; i++) {
if ((i < k && data_ptrs[i] == NULL) || (i >= k && coding_ptrs[i-k] == NULL)) {
// set up any missing data or coding pointers
if (i < k && data_ptrs[i] == NULL) {
data_ptrs[i] = shards+(blockspacing*i);
} else if (i >= k && coding_ptrs[i-k] == NULL) {
coding_ptrs[i-k] = shards+(blockspacing*i);
}
erasures[j] = i;
j++;
}
}
erasures[j] = -1;

matrix = cauchy_good_general_coding_matrix(k, m, w);
if (matrix == NULL) {
result = enif_make_badarg(env);
goto cleanup;
}
bitmatrix = jerasure_matrix_to_bitmatrix(k, m, w, matrix);
if (bitmatrix == NULL) {
result = enif_make_badarg(env);
goto cleanup;
}

int res = -1;

if (len == k+m) {
// we have all the shards (why are we decoding at all?) smart lazy decoding segfaults if you have all the shards
// so we use the simpler form here
res = jerasure_bitmatrix_decode(k, m, w, bitmatrix, 0, erasures, data_ptrs, coding_ptrs, blocksize, blocksize/w);
} else {
// we're actually missing something here, so we can use the smart lazy mode
res = jerasure_schedule_decode_lazy(k, m, w, bitmatrix, erasures, data_ptrs, coding_ptrs, blocksize, blocksize/w, 1);
}

if (res == -1) {
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "decode_failed"));
goto cleanup;
}

ERL_NIF_TERM decoded;
unsigned char* decoded_data = enif_make_new_binary(env, totalsize, &decoded);
memset(decoded_data, 0, totalsize);
unsigned char *p = decoded_data;

for (int i = 0; i < k; i++) {
memcpy(p, shards+(blockspacing*i), bytes_per_shard);
p += bytes_per_shard;
if (extra_bytes > 0) {
memcpy(p, shards+(blockspacing*i)+bytes_per_shard, 1);
extra_bytes--;
p++;
}
}

result = enif_make_tuple2(env, enif_make_atom(env, "ok"), decoded);

cleanup:

if (matrix != NULL) {
free(matrix);
}
if (shards != NULL) {
free(shards);
}
if (erasures != NULL) {
free(erasures);
}

return result;
}

static ErlNifFunc nif_funcs[] =
{{"encode", 3, encode, 0},
{"decode", 3, decode, 0}};
{"encode_gc", 4, encode_gc, 0},
{"decode", 3, decode, 0},
{"decode_gc", 4, decode_gc, 0}};

#define ATOM(Id, Value) \
{ \
Expand Down
Loading

0 comments on commit 1697137

Please sign in to comment.