Skip to content

Commit

Permalink
Add support for simple mutations which operate over chunks in the fil…
Browse files Browse the repository at this point in the history
…es (not using decision seeds)
RafaelTupynamba committed Mar 25, 2022
1 parent c8aad8f commit 417687a
Showing 5 changed files with 192 additions and 60 deletions.
12 changes: 8 additions & 4 deletions bt.h
Original file line number Diff line number Diff line change
@@ -251,6 +251,8 @@ void end_generation() {
chunk_name = back.name;
if (is_delete) {
is_following = true;
delete_start = back.min;
delete_end = back.max;
}
}

@@ -268,12 +270,14 @@ void end_generation() {

if (get_all_chunks) {
if (back.rand_start != back.rand_start_real) {
optional_chunks.emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name);
insertion_points[file_index].emplace_back(back.rand_start, variable_types[back.name].c_str(), back.name);
optional_chunks.emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name, back.min, back.max);
insertion_points[file_index].emplace_back(back.rand_start, variable_types[back.name].c_str(), back.name, back.min);
is_following = true;
chunk_name = back.name;
rand_start = back.rand_start;
rand_end = file_acc.rand_pos - 1;
delete_start = back.min;
delete_end = back.max;
} else if (file_acc.rand_pos > back.rand_start) {
int size = non_optional_index[file_index].size();
int i;
@@ -286,10 +290,10 @@ void end_generation() {
if (i == size) {
non_optional_index[file_index].emplace_back(variable_types[back.name].c_str(), non_optional_chunks[variable_types[back.name]].size(), 1);
}
non_optional_chunks[variable_types[back.name]].emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name);
non_optional_chunks[variable_types[back.name]].emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name, back.min, back.max);
}
if (file_acc.rand_last != UINT_MAX) {
insertion_points[file_index].emplace_back(file_acc.rand_last, variable_types[back.name].c_str(), back.name);
insertion_points[file_index].emplace_back(file_acc.rand_last, variable_types[back.name].c_str(), back.name, back.max + 1);
}
}

4 changes: 2 additions & 2 deletions checkers/wav.sh
Original file line number Diff line number Diff line change
@@ -1,2 +1,2 @@
ffmpeg -y -i - output.wav <out.wav 2>/dev/null
#wavpack -y - -o output.wav <out.wav 2>/dev/null
wavpack -y - -o output.wav <out.wav 2>/dev/null
#ffmpeg -y -i - output.wav <out.wav 2>/dev/null
7 changes: 5 additions & 2 deletions file_accessor.h
Original file line number Diff line number Diff line change
@@ -33,6 +33,8 @@ unsigned rand_start;
unsigned rand_end;
unsigned rand_start2;
unsigned rand_end2;
unsigned delete_start;
unsigned delete_end;
bool is_optional = false;
bool is_delete = false;

@@ -44,6 +46,7 @@ std::vector<int> optional_index = { 0 };
std::unordered_map<std::string, std::vector<Chunk>> non_optional_chunks;
std::vector<std::vector<NonOptional>> non_optional_index;
std::vector<std::string> rand_names;
std::vector<std::string> file_names;

void swap_bytes(void* b, unsigned size) {
if (is_big_endian) {
@@ -313,7 +316,7 @@ class file_accessor {
following_is_optional = lookahead;
is_following = false;
if (get_all_chunks && lookahead) {
deletable_chunks[file_index].emplace_back(file_index, rand_start, rand_end, variable_types[chunk_name].c_str(), chunk_name);
deletable_chunks[file_index].emplace_back(file_index, rand_start, rand_end, variable_types[chunk_name].c_str(), chunk_name, delete_start, delete_end);
}
}
}
@@ -387,7 +390,7 @@ class file_accessor {
chunk_name = "file";
}
if (get_parse_tree && get_all_chunks && rand_last != UINT_MAX) {
insertion_points[file_index].emplace_back(rand_last, "File", "file");
insertion_points[file_index].emplace_back(rand_last, "File", "file", file_pos);
}
}
}
21 changes: 19 additions & 2 deletions formatfuzzer.h
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#define MAX_RAND_SIZE 131072
#define MAX_FILE_SIZE 65536
#define SIMPLE_MUTATIONS 0

#include <vector>

@@ -8,7 +9,14 @@ struct InsertionPoint {
unsigned pos;
const char* type;
const char* name;
InsertionPoint(unsigned pos, const char* type, const char* name) : pos(pos), type(type), name(name) {}
#ifdef SIMPLE_MUTATIONS
unsigned pos_file;
#endif
InsertionPoint(unsigned pos, const char* type, const char* name, unsigned pos_file) : pos(pos), type(type), name(name)
#ifdef SIMPLE_MUTATIONS
, pos_file(pos_file)
#endif
{}
};

struct Chunk {
@@ -17,7 +25,15 @@ struct Chunk {
unsigned end;
const char* type;
const char* name;
Chunk(int file_index, unsigned start, unsigned end, const char* type, const char* name) : file_index(file_index), start(start), end(end), type(type), name(name) {}
#ifdef SIMPLE_MUTATIONS
unsigned start_file;
unsigned end_file;
#endif
Chunk(int file_index, unsigned start, unsigned end, const char* type, const char* name, unsigned start_file, unsigned end_file) : file_index(file_index), start(start), end(end), type(type), name(name)
#ifdef SIMPLE_MUTATIONS
, start_file(start_file), end_file(end_file)
#endif
{}
};

struct NonOptional {
@@ -35,6 +51,7 @@ extern std::vector<int> optional_index;
extern std::unordered_map<std::string, std::vector<Chunk>> non_optional_chunks;
extern std::vector<std::vector<NonOptional>> non_optional_index;
extern std::vector<std::string> rand_names;
extern std::vector<std::string> file_names;

void set_parser();

208 changes: 158 additions & 50 deletions fuzzer.cpp
Original file line number Diff line number Diff line change
@@ -1224,6 +1224,7 @@ smaller number of decision bytes.

extern "C" int process_file(const char *file_name, const char *rand_name) {
rand_names.push_back(rand_name);
file_names.push_back(file_name);
insertion_points.push_back({});
deletable_chunks.push_back({});
non_optional_index.push_back({});
@@ -1260,15 +1261,15 @@ extern "C" int process_file(const char *file_name, const char *rand_name) {

}

unsigned read_rand_file(const char* file_name, unsigned char* rand_buffer) {
unsigned read_file(const char* file_name, unsigned char* file_buffer) {
int file_fd = open(file_name, O_RDONLY);
if (file_fd == -1) {
perror(file_name);
exit(1);
}
ssize_t size = read(file_fd, rand_buffer, MAX_RAND_SIZE);
ssize_t size = read(file_fd, file_buffer, MAX_RAND_SIZE);
if (size < 0) {
perror("Failed to read seed file");
perror("Failed to read file");
exit(1);
}
close(file_fd);
@@ -1284,6 +1285,12 @@ void reset_info() {
buf_size = 1024;
}

void print_info() {
*print_pos = '\0';
if (debug_print)
printf("%s", mutation_info);
}

void log_info(const char * fmt, ...) {
va_list args;
va_start(args,fmt);
@@ -1308,8 +1315,17 @@ extern "C" void generate_random_file(unsigned char** file, unsigned* file_size)
}


enum SMART_MUTATION {
SMART_MUTATION_RANDOM,
SMART_REPLACE_NONOPT,
SMART_REPLACE_OPT,
SMART_INSERT,
SMART_DELETE,
SMART_ABSTRACT
};

extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, unsigned* file_size) {

extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, unsigned* file_size, SMART_MUTATION mut = SMART_MUTATION_RANDOM, unsigned char** file_simple = NULL, unsigned* file_size_simple = NULL) {
static unsigned char *original_rand_t = NULL;
static unsigned char *rand_t = NULL;
static unsigned char *rand_s = NULL;
@@ -1323,17 +1339,47 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
printf("Read only %ld bytes from /dev/urandom\n", r);
close(rand_fd);
}
if (file_simple && file_size_simple) {
*file_simple = NULL;
*file_size_simple = 0;
}
static int previous_file_index = -1;
static unsigned len_t = 0;
if (target_file_index != previous_file_index) {
len_t = read_rand_file(rand_names[target_file_index].c_str(), original_rand_t);
len_t = read_file(rand_names[target_file_index].c_str(), original_rand_t);
previous_file_index = target_file_index;
}

if (mut == SMART_MUTATION_RANDOM) {
switch (rand() % (deletable_chunks[target_file_index].size() ? 10 : 9)) {
case 0:
mut = SMART_REPLACE_NONOPT;
break;
case 1:
case 2:
mut = SMART_REPLACE_OPT;
break;
case 3:
case 4:
mut = SMART_INSERT;
break;
case 5:
case 6:
case 7:
case 8:
mut = SMART_ABSTRACT;
break;
case 9:
mut = SMART_DELETE;
break;
}
}

reset_info();
bool old_debug_print = debug_print;
switch (rand() % (deletable_chunks[target_file_index].size() ? 10 : 9)) {
case 0:

switch (mut) {
case SMART_REPLACE_NONOPT:
{
if (non_optional_index[target_file_index].size() == 0)
goto fail;
@@ -1345,9 +1391,15 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
if (non_optional_chunks[no.type].size() == 0)
goto fail;
Chunk& s = non_optional_chunks[no.type][rand() % non_optional_chunks[no.type].size()];
if (&t == &s)
goto fail;
#ifdef SIMPLE_MUTATIONS
log_info("Replacing: source non-optional chunk from file %d position %u %u %s %s\ninto target file %d non-optional chunk position %u %u %s %s\n", s.file_index, s.start_file, s.end_file, s.type, s.name, t.file_index, t.start_file, t.end_file, t.type, t.name);
#else
log_info("Replacing: source non-optional chunk from file %d position %u %u %s %s\ninto target file %d non-optional chunk position %u %u %s %s\n", s.file_index, s.start, s.end, s.type, s.name, t.file_index, t.start, t.end, t.type, t.name);
#endif
memcpy(rand_t, original_rand_t, len_t);
read_rand_file(rand_names[s.file_index].c_str(), rand_s);
read_file(rand_names[s.file_index].c_str(), rand_s);

unsigned rand_size = len_t + (s.end - s.start) - (t.end - t.start);
if (rand_size > MAX_RAND_SIZE) {
@@ -1376,20 +1428,32 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
debug_print = old_debug_print;
if (!(*file) || !(*file_size)) {
log_info("Failed to generate mutated file!\n");
if (debug_print)
printf("%s", mutation_info);
print_info();
return -2;
}
if (rand_end0 < rand_end)
log_info("Warning: Consumed %u more decision bytes than expected while generating chunk.\n", rand_end - rand_end0);
if (rand_end0 > rand_end)
log_info("Warning: Consumed %u less decision bytes than expected while generating chunk.\n", rand_end0 - rand_end);
if (debug_print)
printf("%s", mutation_info);
print_info();

#ifdef SIMPLE_MUTATIONS
if (file_simple && file_size_simple) {
unsigned len = read_file(file_names[target_file_index].c_str(), rand_t);
read_file(file_names[s.file_index].c_str(), rand_s);
if (s.end_file - s.start_file == t.end_file - t.start_file && !memcmp(rand_t + t.start_file, rand_s + s.start_file, s.end_file + 1 - s.start_file))
goto fail;
memmove(rand_t + t.start_file + s.end_file + 1 - s.start_file, rand_t + t.end_file + 1, len - (t.end_file + 1));
memcpy(rand_t + t.start_file, rand_s + s.start_file, s.end_file + 1 - s.start_file);
*file_simple = rand_t;
*file_size_simple = len + (s.end_file - s.start_file) - (t.end_file - t.start_file);
}
#endif

return (rand_end > rand_end0) - (rand_end < rand_end0);
}
case 1:
case 2:

case SMART_REPLACE_OPT:
{
if ((optional_index[target_file_index+1] - optional_index[target_file_index]) == 0)
goto fail;
@@ -1398,9 +1462,13 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
if (optional_chunks.size() == 0)
goto fail;
Chunk& s = optional_chunks[rand() % optional_chunks.size()];
#ifdef SIMPLE_MUTATIONS
log_info("Replacing: source optional chunk from file %d position %u %u %s %s\ninto target file %d optional chunk position %u %u %s %s\n", s.file_index, s.start_file, s.end_file, s.type, s.name, t.file_index, t.start_file, t.end_file, t.type, t.name);
#else
log_info("Replacing: source optional chunk from file %d position %u %u %s %s\ninto target file %d optional chunk position %u %u %s %s\n", s.file_index, s.start, s.end, s.type, s.name, t.file_index, t.start, t.end, t.type, t.name);
#endif
memcpy(rand_t, original_rand_t, len_t);
read_rand_file(rand_names[s.file_index].c_str(), rand_s);
read_file(rand_names[s.file_index].c_str(), rand_s);

unsigned rand_size = len_t + (s.end - s.start) - (t.end - t.start);
if (rand_size > MAX_RAND_SIZE) {
@@ -1429,31 +1497,45 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
debug_print = old_debug_print;
if (!(*file) || !(*file_size)) {
log_info("Failed to generate mutated file!\n");
if (debug_print)
printf("%s", mutation_info);
print_info();
return -2;
}
if (rand_end0 < rand_end)
log_info("Warning: Consumed %u more decision bytes than expected while generating chunk.\n", rand_end - rand_end0);
if (rand_end0 > rand_end)
log_info("Warning: Consumed %u less decision bytes than expected while generating chunk.\n", rand_end0 - rand_end);

if (debug_print)
printf("%s", mutation_info);
print_info();

#ifdef SIMPLE_MUTATIONS
if (file_simple && file_size_simple) {
unsigned len = read_file(file_names[target_file_index].c_str(), rand_t);
read_file(file_names[s.file_index].c_str(), rand_s);
memmove(rand_t + t.start_file + s.end_file + 1 - s.start_file, rand_t + t.end_file + 1, len - (t.end_file + 1));
memcpy(rand_t + t.start_file, rand_s + s.start_file, s.end_file + 1 - s.start_file);
*file_simple = rand_t;
*file_size_simple = len + (s.end_file - s.start_file) - (t.end_file - t.start_file);
}
#endif

return (rand_end > rand_end0) - (rand_end < rand_end0);
}
case 3:
case 4:

case SMART_INSERT:
{
if (insertion_points[target_file_index].size() == 0)
goto fail;
InsertionPoint& ip = insertion_points[target_file_index][rand() % insertion_points[target_file_index].size()];
if (optional_chunks.size() == 0)
goto fail;
Chunk& s = optional_chunks[rand() % optional_chunks.size()];
#ifdef SIMPLE_MUTATIONS
log_info("Inserting: source chunk from file %d position %u %u %s %s\ninto target file %d position %u %s %s\n", s.file_index, s.start_file, s.end_file, s.type, s.name, target_file_index, ip.pos_file, ip.type, ip.name);
#else
log_info("Inserting: source chunk from file %d position %u %u %s %s\ninto target file %d position %u %s %s\n", s.file_index, s.start, s.end, s.type, s.name, target_file_index, ip.pos, ip.type, ip.name);
#endif
memcpy(rand_t, original_rand_t, len_t);
read_rand_file(rand_names[s.file_index].c_str(), rand_s);
read_file(rand_names[s.file_index].c_str(), rand_s);

unsigned rand_size = len_t + (s.end + 1 - s.start);
if (rand_size > MAX_RAND_SIZE) {
@@ -1482,23 +1564,31 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
debug_print = old_debug_print;
if (!(*file) || !(*file_size)) {
log_info("Failed to generate mutated file!\n");
if (debug_print)
printf("%s", mutation_info);
print_info();
return -2;
}
if (rand_end0 < rand_end)
log_info("Warning: Consumed %u more decision bytes than expected while generating chunk.\n", rand_end - rand_end0);
if (rand_end0 > rand_end)
log_info("Warning: Consumed %u less decision bytes than expected while generating chunk.\n", rand_end0 - rand_end);

if (debug_print)
printf("%s", mutation_info);
print_info();

#ifdef SIMPLE_MUTATIONS
if (file_simple && file_size_simple) {
unsigned len = read_file(file_names[target_file_index].c_str(), rand_t);
read_file(file_names[s.file_index].c_str(), rand_s);
memmove(rand_t + ip.pos_file + s.end_file + 1 - s.start_file, rand_t + ip.pos_file, len - ip.pos_file);
memcpy(rand_t + ip.pos_file, rand_s + s.start_file, s.end_file + 1 - s.start_file);
*file_simple = rand_t;
*file_size_simple = len + s.end_file + 1 - s.start_file;
}
#endif

return (rand_end > rand_end0) - (rand_end < rand_end0);
}
case 5:
case 6:
case 7:
case 8:

case SMART_ABSTRACT:
{
int start_t = -1;
int end_t = -1;
@@ -1548,31 +1638,33 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
*file_size = ff_generate(rand_t, MAX_RAND_SIZE, file);
get_parse_tree = false;
debug_print = old_debug_print;
if (!(*file) || !(*file_size)) {
log_info("Failed to generate mutated file!\n");
print_info();
return -2;
}
if (smart_abstraction) {
smart_abstraction = false;
log_info("Abstracted chunk was not created!\n");
if (debug_print)
printf("%s", mutation_info);
print_info();
return -1;
}
if (!(*file) || !(*file_size)) {
log_info("Failed to generate mutated file!\n");
if (debug_print)
printf("%s", mutation_info);
return -2;
}

if (debug_print)
printf("%s", mutation_info);
print_info();
return 0;
}
case 9:

case SMART_DELETE:
{
if (deletable_chunks[target_file_index].size() == 0)
goto fail;
int index = rand() % deletable_chunks[target_file_index].size();
Chunk& t = deletable_chunks[target_file_index][index];
Chunk t = deletable_chunks[target_file_index][index];
#ifdef SIMPLE_MUTATIONS
log_info("Deleting from file %d chunk %u %u %s %s\n", t.file_index, t.start_file, t.end_file, t.type, t.name);
#else
log_info("Deleting from file %d chunk %u %u %s %s\n", t.file_index, t.start, t.end, t.type, t.name);
#endif
memcpy(rand_t, original_rand_t, len_t);

memmove(rand_t + t.start, rand_t + t.end + 1, len_t - (t.end + 1));
@@ -1587,15 +1679,26 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u
debug_print = old_debug_print;
if (!(*file) || !(*file_size)) {
log_info("Failed to generate mutated file!\n");
if (debug_print)
printf("%s", mutation_info);
print_info();
return -2;
}

if (debug_print)
printf("%s", mutation_info);
print_info();

#ifdef SIMPLE_MUTATIONS
if (file_simple && file_size_simple) {
unsigned len = read_file(file_names[target_file_index].c_str(), rand_t);
memmove(rand_t + t.start_file, rand_t + t.end_file + 1, len - (t.end_file + 1));
*file_simple = rand_t;
*file_size_simple = len - (t.end_file + 1 - t.start_file);
}
#endif

return 0;
}

default:
return -2;
}
return -2;
fail:
@@ -1616,12 +1719,17 @@ int mutations(int argc, char **argv)
}
unsigned char* file;
unsigned size;
unsigned char* file_simple;
unsigned file_size_simple;
debug_print = true;
print_errors = true;
for (int i = 0; i < 10000; ++i) {
int result = one_smart_mutation(i % rand_names.size(), &file, &size);
if (debug_print)
file = file_simple = NULL;
size = file_size_simple = 0;
int result = one_smart_mutation(i % rand_names.size(), &file, &size, SMART_MUTATION_RANDOM, &file_simple, &file_size_simple);
if (debug_print) {
printf("%d\n", result);
}
}
return 0;
}
@@ -1702,7 +1810,7 @@ int benchmark(int argc, char *argv[])
unsigned char* new_data = NULL;
int generated = 0;
int valid = 0;
uint64_t total_bytes = 0;
unsigned long long total_bytes = 0;
int i;
int iterations = 10000;
std::unordered_map<int,int> status;
@@ -1740,7 +1848,7 @@ int benchmark(int argc, char *argv[])
if (argc > 1)
printf("Valid %d/%d = %f\n", valid, generated, (double)valid/(double)generated);
if (generated)
printf("Average file size %lu bytes.\n", total_bytes / generated);
printf("Average file size %llu bytes.\n", total_bytes / generated);
printf("Speed %f / s.\n", generated / time);
delete[] data;
return 0;

0 comments on commit 417687a

Please sign in to comment.