Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement proper block padding for 32 bit systems #1

Merged
merged 2 commits into from
Aug 29, 2018
Merged

Implement proper block padding for 32 bit systems #1

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
706bdcf
Implement proper block padding for 32 bit systems
Vagabond Aug 28, 2018
9f86bca
Use variable length arrays for data and coding pointer arrays
Vagabond Aug 29, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Next Next commit
Implement proper block padding for 32 bit systems 
On ARM (not ARM64) where the sizeof(long) is 4 (vs 8 on 64 bit systems)
the GF-Complete requirement that data blocks be aligned on a multiple of
16 bytes can be violated. This patch reworks how we allocate and pad the
data shards such that:

* All shards are allocated in contiguous memory
* The spacing between shards is always a multiple of 16 bytes
  • Loading branch information
@Vagabond
Vagabond committed Aug 28, 2018
commit 706bdcfdc71deb7e83130cf4438de5a87974777d
89 changes: 57 additions & 32 deletions c_src/erasure.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,47 +59,55 @@ encode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
}
}

// block spacing has to be a multiple of 16
int blockspacing = blocksize + (16 - (blocksize % 16));

int bytes_per_shard = input.size / k;
int extra_bytes = input.size % k;


char *shards[k+m];
char *shards = calloc(k+m, blockspacing);
char **data_ptrs = calloc(k, sizeof(char*));
char **coding_ptrs = calloc(m, sizeof(char*));

unsigned char *p = input.data;
for (int i = 0; i < k+m; i++) {
shards[i] = (char *)malloc(sizeof(char)*blocksize);
memset(shards[i], 0, blocksize);
memset(shards+(blockspacing*i), 0, blockspacing);
if (i < k) {
memcpy(shards[i], p, bytes_per_shard);
data_ptrs[i] = shards+(blockspacing*i);
memcpy(shards+(blockspacing*i), p, bytes_per_shard);
p += bytes_per_shard;
if (extra_bytes > 0) {
memcpy(shards[i]+bytes_per_shard, p, 1);
memcpy(shards+(blockspacing*i)+bytes_per_shard, p, 1);
p++;
extra_bytes--;
}
} else {
coding_ptrs[i-k] = shards+(blockspacing*i);
}
}


int w = 8;
int *matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
jerasure_matrix_encode(k, m, w, matrix, shards, shards+k, blocksize);
jerasure_matrix_encode(k, m, w, matrix, data_ptrs, coding_ptrs, blocksize);

ERL_NIF_TERM list = enif_make_list(env, 0);

for (int i = k+m - 1; i >= 0; i--)
{
ERL_NIF_TERM binary;
unsigned char *bindata = enif_make_new_binary(env, blocksize, &binary);
memcpy(bindata, shards[i], blocksize);
free(shards[i]);
memcpy(bindata, shards+(blockspacing*i), blocksize);
list = enif_make_list_cell(env,
enif_make_tuple3(env,
enif_make_int(env, i),
enif_make_int(env, input.size),
binary
), list);
}
free(shards);
free(data_ptrs);
free(coding_ptrs);
return enif_make_tuple2(env, enif_make_atom(env, "ok"), list);
}

Expand All @@ -126,19 +134,26 @@ decode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
return enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "insufficent_shards"));
}

char **shards = malloc(sizeof(char*)*(k+m));
char *shards = NULL;
char **data_ptrs = calloc(k, sizeof(char*));
Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

why not allocate these on the stack?

char **coding_ptrs = calloc(m, sizeof(char*));

int *erasures = NULL;

for (int i = 0; i < k+m; i++) {
shards[i] = NULL;
for (int i = 0; i < k; i++) {
data_ptrs[i] = NULL;
}

for (int i = 0; i < m; i++) {
coding_ptrs[i] = NULL;
}

// all the shards must be the same size
// and all the indices need to be in-bounds
ERL_NIF_TERM head, tail;
const ERL_NIF_TERM *tuple;
int arity, id, totalsize, lasttotalsize = 0;
int padding, blocksize = 0, remainder=0;
int padding, blocksize = 0, remainder=0, blockspacing=0;
tail = argv[2];
while (enif_get_list_cell(env, tail, &head, &tail))
{
Expand All @@ -150,7 +165,7 @@ decode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "invalid_shard_id"));
goto cleanup;
}
if (shards[id] != NULL) {
if ((id < k && data_ptrs[id] != NULL) || ( id >=k && coding_ptrs[id-k] != NULL)) {
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "duplicate_shard_id"));
goto cleanup;
}
Expand Down Expand Up @@ -178,6 +193,9 @@ decode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
padding++;
}
}
// block spacing has to be a multiple of 16
blockspacing = blocksize + (16 - (blocksize % 16));
shards = calloc(k+m, blockspacing);
}

ErlNifBinary input;
Expand All @@ -187,48 +205,57 @@ decode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])
goto cleanup;
}

shards[id] = (char *)malloc(sizeof(char)*blocksize);
memset(shards[id], 0, blocksize);
memcpy(shards[id], input.data, blocksize);
if (id < k) {
data_ptrs[id] = shards+(blockspacing*id);
} else {
coding_ptrs[id-k] = shards+(blockspacing*id);
}

memset(shards+(blockspacing*id), 0, blockspacing);
memcpy(shards+(blockspacing*id), input.data, blocksize);
}

erasures = malloc(sizeof(int)*(k+m));
erasures = calloc(k+m, sizeof(int));
int j = 0;

int bytes_per_shard = totalsize / k;
int extra_bytes = totalsize % k;

// calculate the missing shards and fill them in with 0s
// calculate the missing shards and track them in the erasures array
for (int i = 0; i < k+m; i++) {
if (shards[i] == NULL) {
if ((i < k && data_ptrs[i] == NULL) || (i >= k && coding_ptrs[i-k] == NULL)) {
// set up any missing data or coding pointers
if (i < k && data_ptrs[i] == NULL) {
data_ptrs[i] = shards+(blockspacing*i);
} else if (i >= k && coding_ptrs[i-k] == NULL) {
coding_ptrs[i-k] = shards+(blockspacing*i);
}
erasures[j] = i;
j++;
shards[i] = (char *)malloc(sizeof(char)*blocksize);
memset(shards[i], 0, blocksize);
}
}
erasures[j] = -1;

int w = 8;
int *matrix = reed_sol_vandermonde_coding_matrix(k, m, w);
int res = jerasure_matrix_decode(k, m, w, matrix, 1, erasures, shards, shards+k, blocksize);
int res = jerasure_matrix_decode(k, m, w, matrix, 1, erasures, data_ptrs, coding_ptrs, blocksize);
//abort();

if (res == -1) {
result = enif_make_tuple2(env, enif_make_atom(env, "error"), enif_make_atom(env, "decode_failed"));
goto cleanup;
}


ERL_NIF_TERM decoded;
unsigned char* decoded_data = enif_make_new_binary(env, totalsize, &decoded);
memset(decoded_data, 0, totalsize);
unsigned char *p = decoded_data;

for (int i = 0; i < k; i++) {
memcpy(p, shards[i], bytes_per_shard);
memcpy(p, shards+(blockspacing*i), bytes_per_shard);
p += bytes_per_shard;
if (extra_bytes > 0) {
memcpy(p, shards[i]+bytes_per_shard, 1);
memcpy(p, shards+(blockspacing*i)+bytes_per_shard, 1);
extra_bytes--;
p++;
}
Expand All @@ -238,13 +265,11 @@ decode(ErlNifEnv * env, int argc, const ERL_NIF_TERM argv[])

cleanup:

for (int i = 0; i < k+m; i++) {
if (shards[i] != NULL) {
free(shards[i]);
}
if (shards != NULL) {
free(shards);
}

free(shards);
free(coding_ptrs);
free(data_ptrs);
if (erasures != NULL) {
free(erasures);
}
Expand Down
14 changes: 8 additions & 6 deletions src/erasure.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,13 +41,15 @@ not_loaded(Line) ->
-include_lib("eunit/include/eunit.hrl").

simple_test() ->
K = 7,
M = 3,
Data = <<"Lorem ipsum dolor sit amet, consectetur adipiscing elit. Morbi nec nisi interdum, ultricies mauris eget, congue ante. Fusce erat diam, lacinia eu volutpat ut, gravida quis justo. Maecenas sagittis, ligula.">>,
{ok, Shards} = encode(5, 2, Data),
?assertEqual({ok, Data}, decode(5, 2, Shards)),
?assertEqual({ok, Data}, decode(5, 2, lists:sublist(Shards, 5))),
?assertEqual({ok, Data}, decode(5, 2, lists:reverse(lists:sublist(Shards, 5)))),
?assertMatch({error, _}, decode(5, 2, lists:sublist(Shards, 4))),
?assertMatch({error, _}, decode(5, 2, lists:sublist(Shards, 4) ++ [hd(Shards)])),
{ok, Shards} = encode(K, M, Data),
?assertEqual({ok, Data}, decode(K, M, Shards)),
?assertEqual({ok, Data}, decode(K, M, lists:sublist(Shards, K))),
?assertEqual({ok, Data}, decode(K, M, lists:reverse(lists:sublist(Shards, K)))),
?assertMatch({error, _}, decode(K, M, lists:sublist(Shards, K - 1))),
?assertMatch({error, _}, decode(K, M, lists:sublist(Shards, K - 1) ++ [hd(Shards)])),
ok.

-endif.