diff --git a/bt.h b/bt.h index 10cf409..2ac3df1 100644 --- a/bt.h +++ b/bt.h @@ -251,6 +251,8 @@ void end_generation() { chunk_name = back.name; if (is_delete) { is_following = true; + delete_start = back.min; + delete_end = back.max; } } @@ -268,12 +270,14 @@ void end_generation() { if (get_all_chunks) { if (back.rand_start != back.rand_start_real) { - optional_chunks.emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name); - insertion_points[file_index].emplace_back(back.rand_start, variable_types[back.name].c_str(), back.name); + optional_chunks.emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name, back.min, back.max); + insertion_points[file_index].emplace_back(back.rand_start, variable_types[back.name].c_str(), back.name, back.min); is_following = true; chunk_name = back.name; rand_start = back.rand_start; rand_end = file_acc.rand_pos - 1; + delete_start = back.min; + delete_end = back.max; } else if (file_acc.rand_pos > back.rand_start) { int size = non_optional_index[file_index].size(); int i; @@ -286,10 +290,10 @@ void end_generation() { if (i == size) { non_optional_index[file_index].emplace_back(variable_types[back.name].c_str(), non_optional_chunks[variable_types[back.name]].size(), 1); } - non_optional_chunks[variable_types[back.name]].emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name); + non_optional_chunks[variable_types[back.name]].emplace_back(file_index, back.rand_start, file_acc.rand_pos - 1, variable_types[back.name].c_str(), back.name, back.min, back.max); } if (file_acc.rand_last != UINT_MAX) { - insertion_points[file_index].emplace_back(file_acc.rand_last, variable_types[back.name].c_str(), back.name); + insertion_points[file_index].emplace_back(file_acc.rand_last, variable_types[back.name].c_str(), back.name, back.max + 1); } } diff --git a/checkers/wav.sh b/checkers/wav.sh index 793e727..4457c85 100755 --- a/checkers/wav.sh +++ b/checkers/wav.sh @@ -1,2 +1,2 @@ -ffmpeg -y -i - output.wav /dev/null -#wavpack -y - -o output.wav /dev/null +wavpack -y - -o output.wav /dev/null +#ffmpeg -y -i - output.wav /dev/null diff --git a/file_accessor.h b/file_accessor.h index 928da1e..99ec1a6 100644 --- a/file_accessor.h +++ b/file_accessor.h @@ -33,6 +33,8 @@ unsigned rand_start; unsigned rand_end; unsigned rand_start2; unsigned rand_end2; +unsigned delete_start; +unsigned delete_end; bool is_optional = false; bool is_delete = false; @@ -44,6 +46,7 @@ std::vector optional_index = { 0 }; std::unordered_map> non_optional_chunks; std::vector> non_optional_index; std::vector rand_names; +std::vector file_names; void swap_bytes(void* b, unsigned size) { if (is_big_endian) { @@ -313,7 +316,7 @@ class file_accessor { following_is_optional = lookahead; is_following = false; if (get_all_chunks && lookahead) { - deletable_chunks[file_index].emplace_back(file_index, rand_start, rand_end, variable_types[chunk_name].c_str(), chunk_name); + deletable_chunks[file_index].emplace_back(file_index, rand_start, rand_end, variable_types[chunk_name].c_str(), chunk_name, delete_start, delete_end); } } } @@ -387,7 +390,7 @@ class file_accessor { chunk_name = "file"; } if (get_parse_tree && get_all_chunks && rand_last != UINT_MAX) { - insertion_points[file_index].emplace_back(rand_last, "File", "file"); + insertion_points[file_index].emplace_back(rand_last, "File", "file", file_pos); } } } diff --git a/formatfuzzer.h b/formatfuzzer.h index 7e4f117..e2d4532 100644 --- a/formatfuzzer.h +++ b/formatfuzzer.h @@ -1,5 +1,6 @@ #define MAX_RAND_SIZE 131072 #define MAX_FILE_SIZE 65536 +#define SIMPLE_MUTATIONS 0 #include @@ -8,7 +9,14 @@ struct InsertionPoint { unsigned pos; const char* type; const char* name; - InsertionPoint(unsigned pos, const char* type, const char* name) : pos(pos), type(type), name(name) {} +#ifdef SIMPLE_MUTATIONS + unsigned pos_file; +#endif + InsertionPoint(unsigned pos, const char* type, const char* name, unsigned pos_file) : pos(pos), type(type), name(name) +#ifdef SIMPLE_MUTATIONS + , pos_file(pos_file) +#endif + {} }; struct Chunk { @@ -17,7 +25,15 @@ struct Chunk { unsigned end; const char* type; const char* name; - Chunk(int file_index, unsigned start, unsigned end, const char* type, const char* name) : file_index(file_index), start(start), end(end), type(type), name(name) {} +#ifdef SIMPLE_MUTATIONS + unsigned start_file; + unsigned end_file; +#endif + Chunk(int file_index, unsigned start, unsigned end, const char* type, const char* name, unsigned start_file, unsigned end_file) : file_index(file_index), start(start), end(end), type(type), name(name) +#ifdef SIMPLE_MUTATIONS + , start_file(start_file), end_file(end_file) +#endif + {} }; struct NonOptional { @@ -35,6 +51,7 @@ extern std::vector optional_index; extern std::unordered_map> non_optional_chunks; extern std::vector> non_optional_index; extern std::vector rand_names; +extern std::vector file_names; void set_parser(); diff --git a/fuzzer.cpp b/fuzzer.cpp index 795aca2..03b858c 100644 --- a/fuzzer.cpp +++ b/fuzzer.cpp @@ -1224,6 +1224,7 @@ smaller number of decision bytes. extern "C" int process_file(const char *file_name, const char *rand_name) { rand_names.push_back(rand_name); + file_names.push_back(file_name); insertion_points.push_back({}); deletable_chunks.push_back({}); non_optional_index.push_back({}); @@ -1260,15 +1261,15 @@ extern "C" int process_file(const char *file_name, const char *rand_name) { } -unsigned read_rand_file(const char* file_name, unsigned char* rand_buffer) { +unsigned read_file(const char* file_name, unsigned char* file_buffer) { int file_fd = open(file_name, O_RDONLY); if (file_fd == -1) { perror(file_name); exit(1); } - ssize_t size = read(file_fd, rand_buffer, MAX_RAND_SIZE); + ssize_t size = read(file_fd, file_buffer, MAX_RAND_SIZE); if (size < 0) { - perror("Failed to read seed file"); + perror("Failed to read file"); exit(1); } close(file_fd); @@ -1284,6 +1285,12 @@ void reset_info() { buf_size = 1024; } +void print_info() { + *print_pos = '\0'; + if (debug_print) + printf("%s", mutation_info); +} + void log_info(const char * fmt, ...) { va_list args; va_start(args,fmt); @@ -1308,8 +1315,17 @@ extern "C" void generate_random_file(unsigned char** file, unsigned* file_size) } +enum SMART_MUTATION { + SMART_MUTATION_RANDOM, + SMART_REPLACE_NONOPT, + SMART_REPLACE_OPT, + SMART_INSERT, + SMART_DELETE, + SMART_ABSTRACT +}; -extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, unsigned* file_size) { + +extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, unsigned* file_size, SMART_MUTATION mut = SMART_MUTATION_RANDOM, unsigned char** file_simple = NULL, unsigned* file_size_simple = NULL) { static unsigned char *original_rand_t = NULL; static unsigned char *rand_t = NULL; static unsigned char *rand_s = NULL; @@ -1323,17 +1339,47 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u printf("Read only %ld bytes from /dev/urandom\n", r); close(rand_fd); } + if (file_simple && file_size_simple) { + *file_simple = NULL; + *file_size_simple = 0; + } static int previous_file_index = -1; static unsigned len_t = 0; if (target_file_index != previous_file_index) { - len_t = read_rand_file(rand_names[target_file_index].c_str(), original_rand_t); + len_t = read_file(rand_names[target_file_index].c_str(), original_rand_t); previous_file_index = target_file_index; } + if (mut == SMART_MUTATION_RANDOM) { + switch (rand() % (deletable_chunks[target_file_index].size() ? 10 : 9)) { + case 0: + mut = SMART_REPLACE_NONOPT; + break; + case 1: + case 2: + mut = SMART_REPLACE_OPT; + break; + case 3: + case 4: + mut = SMART_INSERT; + break; + case 5: + case 6: + case 7: + case 8: + mut = SMART_ABSTRACT; + break; + case 9: + mut = SMART_DELETE; + break; + } + } + reset_info(); bool old_debug_print = debug_print; - switch (rand() % (deletable_chunks[target_file_index].size() ? 10 : 9)) { - case 0: + + switch (mut) { + case SMART_REPLACE_NONOPT: { if (non_optional_index[target_file_index].size() == 0) goto fail; @@ -1345,9 +1391,15 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u if (non_optional_chunks[no.type].size() == 0) goto fail; Chunk& s = non_optional_chunks[no.type][rand() % non_optional_chunks[no.type].size()]; + if (&t == &s) + goto fail; +#ifdef SIMPLE_MUTATIONS + log_info("Replacing: source non-optional chunk from file %d position %u %u %s %s\ninto target file %d non-optional chunk position %u %u %s %s\n", s.file_index, s.start_file, s.end_file, s.type, s.name, t.file_index, t.start_file, t.end_file, t.type, t.name); +#else log_info("Replacing: source non-optional chunk from file %d position %u %u %s %s\ninto target file %d non-optional chunk position %u %u %s %s\n", s.file_index, s.start, s.end, s.type, s.name, t.file_index, t.start, t.end, t.type, t.name); +#endif memcpy(rand_t, original_rand_t, len_t); - read_rand_file(rand_names[s.file_index].c_str(), rand_s); + read_file(rand_names[s.file_index].c_str(), rand_s); unsigned rand_size = len_t + (s.end - s.start) - (t.end - t.start); if (rand_size > MAX_RAND_SIZE) { @@ -1376,20 +1428,32 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u debug_print = old_debug_print; if (!(*file) || !(*file_size)) { log_info("Failed to generate mutated file!\n"); - if (debug_print) - printf("%s", mutation_info); + print_info(); return -2; } if (rand_end0 < rand_end) log_info("Warning: Consumed %u more decision bytes than expected while generating chunk.\n", rand_end - rand_end0); if (rand_end0 > rand_end) log_info("Warning: Consumed %u less decision bytes than expected while generating chunk.\n", rand_end0 - rand_end); - if (debug_print) - printf("%s", mutation_info); + print_info(); + +#ifdef SIMPLE_MUTATIONS + if (file_simple && file_size_simple) { + unsigned len = read_file(file_names[target_file_index].c_str(), rand_t); + read_file(file_names[s.file_index].c_str(), rand_s); + if (s.end_file - s.start_file == t.end_file - t.start_file && !memcmp(rand_t + t.start_file, rand_s + s.start_file, s.end_file + 1 - s.start_file)) + goto fail; + memmove(rand_t + t.start_file + s.end_file + 1 - s.start_file, rand_t + t.end_file + 1, len - (t.end_file + 1)); + memcpy(rand_t + t.start_file, rand_s + s.start_file, s.end_file + 1 - s.start_file); + *file_simple = rand_t; + *file_size_simple = len + (s.end_file - s.start_file) - (t.end_file - t.start_file); + } +#endif + return (rand_end > rand_end0) - (rand_end < rand_end0); } - case 1: - case 2: + + case SMART_REPLACE_OPT: { if ((optional_index[target_file_index+1] - optional_index[target_file_index]) == 0) goto fail; @@ -1398,9 +1462,13 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u if (optional_chunks.size() == 0) goto fail; Chunk& s = optional_chunks[rand() % optional_chunks.size()]; +#ifdef SIMPLE_MUTATIONS + log_info("Replacing: source optional chunk from file %d position %u %u %s %s\ninto target file %d optional chunk position %u %u %s %s\n", s.file_index, s.start_file, s.end_file, s.type, s.name, t.file_index, t.start_file, t.end_file, t.type, t.name); +#else log_info("Replacing: source optional chunk from file %d position %u %u %s %s\ninto target file %d optional chunk position %u %u %s %s\n", s.file_index, s.start, s.end, s.type, s.name, t.file_index, t.start, t.end, t.type, t.name); +#endif memcpy(rand_t, original_rand_t, len_t); - read_rand_file(rand_names[s.file_index].c_str(), rand_s); + read_file(rand_names[s.file_index].c_str(), rand_s); unsigned rand_size = len_t + (s.end - s.start) - (t.end - t.start); if (rand_size > MAX_RAND_SIZE) { @@ -1429,8 +1497,7 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u debug_print = old_debug_print; if (!(*file) || !(*file_size)) { log_info("Failed to generate mutated file!\n"); - if (debug_print) - printf("%s", mutation_info); + print_info(); return -2; } if (rand_end0 < rand_end) @@ -1438,12 +1505,23 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u if (rand_end0 > rand_end) log_info("Warning: Consumed %u less decision bytes than expected while generating chunk.\n", rand_end0 - rand_end); - if (debug_print) - printf("%s", mutation_info); + print_info(); + +#ifdef SIMPLE_MUTATIONS + if (file_simple && file_size_simple) { + unsigned len = read_file(file_names[target_file_index].c_str(), rand_t); + read_file(file_names[s.file_index].c_str(), rand_s); + memmove(rand_t + t.start_file + s.end_file + 1 - s.start_file, rand_t + t.end_file + 1, len - (t.end_file + 1)); + memcpy(rand_t + t.start_file, rand_s + s.start_file, s.end_file + 1 - s.start_file); + *file_simple = rand_t; + *file_size_simple = len + (s.end_file - s.start_file) - (t.end_file - t.start_file); + } +#endif + return (rand_end > rand_end0) - (rand_end < rand_end0); } - case 3: - case 4: + + case SMART_INSERT: { if (insertion_points[target_file_index].size() == 0) goto fail; @@ -1451,9 +1529,13 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u if (optional_chunks.size() == 0) goto fail; Chunk& s = optional_chunks[rand() % optional_chunks.size()]; +#ifdef SIMPLE_MUTATIONS + log_info("Inserting: source chunk from file %d position %u %u %s %s\ninto target file %d position %u %s %s\n", s.file_index, s.start_file, s.end_file, s.type, s.name, target_file_index, ip.pos_file, ip.type, ip.name); +#else log_info("Inserting: source chunk from file %d position %u %u %s %s\ninto target file %d position %u %s %s\n", s.file_index, s.start, s.end, s.type, s.name, target_file_index, ip.pos, ip.type, ip.name); +#endif memcpy(rand_t, original_rand_t, len_t); - read_rand_file(rand_names[s.file_index].c_str(), rand_s); + read_file(rand_names[s.file_index].c_str(), rand_s); unsigned rand_size = len_t + (s.end + 1 - s.start); if (rand_size > MAX_RAND_SIZE) { @@ -1482,8 +1564,7 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u debug_print = old_debug_print; if (!(*file) || !(*file_size)) { log_info("Failed to generate mutated file!\n"); - if (debug_print) - printf("%s", mutation_info); + print_info(); return -2; } if (rand_end0 < rand_end) @@ -1491,14 +1572,23 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u if (rand_end0 > rand_end) log_info("Warning: Consumed %u less decision bytes than expected while generating chunk.\n", rand_end0 - rand_end); - if (debug_print) - printf("%s", mutation_info); + print_info(); + +#ifdef SIMPLE_MUTATIONS + if (file_simple && file_size_simple) { + unsigned len = read_file(file_names[target_file_index].c_str(), rand_t); + read_file(file_names[s.file_index].c_str(), rand_s); + memmove(rand_t + ip.pos_file + s.end_file + 1 - s.start_file, rand_t + ip.pos_file, len - ip.pos_file); + memcpy(rand_t + ip.pos_file, rand_s + s.start_file, s.end_file + 1 - s.start_file); + *file_simple = rand_t; + *file_size_simple = len + s.end_file + 1 - s.start_file; + } +#endif + return (rand_end > rand_end0) - (rand_end < rand_end0); } - case 5: - case 6: - case 7: - case 8: + + case SMART_ABSTRACT: { int start_t = -1; int end_t = -1; @@ -1548,31 +1638,33 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u *file_size = ff_generate(rand_t, MAX_RAND_SIZE, file); get_parse_tree = false; debug_print = old_debug_print; + if (!(*file) || !(*file_size)) { + log_info("Failed to generate mutated file!\n"); + print_info(); + return -2; + } if (smart_abstraction) { smart_abstraction = false; log_info("Abstracted chunk was not created!\n"); - if (debug_print) - printf("%s", mutation_info); + print_info(); return -1; } - if (!(*file) || !(*file_size)) { - log_info("Failed to generate mutated file!\n"); - if (debug_print) - printf("%s", mutation_info); - return -2; - } - if (debug_print) - printf("%s", mutation_info); + print_info(); return 0; } - case 9: + + case SMART_DELETE: { if (deletable_chunks[target_file_index].size() == 0) goto fail; int index = rand() % deletable_chunks[target_file_index].size(); - Chunk& t = deletable_chunks[target_file_index][index]; + Chunk t = deletable_chunks[target_file_index][index]; +#ifdef SIMPLE_MUTATIONS + log_info("Deleting from file %d chunk %u %u %s %s\n", t.file_index, t.start_file, t.end_file, t.type, t.name); +#else log_info("Deleting from file %d chunk %u %u %s %s\n", t.file_index, t.start, t.end, t.type, t.name); +#endif memcpy(rand_t, original_rand_t, len_t); memmove(rand_t + t.start, rand_t + t.end + 1, len_t - (t.end + 1)); @@ -1587,15 +1679,26 @@ extern "C" int one_smart_mutation(int target_file_index, unsigned char** file, u debug_print = old_debug_print; if (!(*file) || !(*file_size)) { log_info("Failed to generate mutated file!\n"); - if (debug_print) - printf("%s", mutation_info); + print_info(); return -2; } - if (debug_print) - printf("%s", mutation_info); + print_info(); + +#ifdef SIMPLE_MUTATIONS + if (file_simple && file_size_simple) { + unsigned len = read_file(file_names[target_file_index].c_str(), rand_t); + memmove(rand_t + t.start_file, rand_t + t.end_file + 1, len - (t.end_file + 1)); + *file_simple = rand_t; + *file_size_simple = len - (t.end_file + 1 - t.start_file); + } +#endif + return 0; } + + default: + return -2; } return -2; fail: @@ -1616,12 +1719,17 @@ int mutations(int argc, char **argv) } unsigned char* file; unsigned size; + unsigned char* file_simple; + unsigned file_size_simple; debug_print = true; print_errors = true; for (int i = 0; i < 10000; ++i) { - int result = one_smart_mutation(i % rand_names.size(), &file, &size); - if (debug_print) + file = file_simple = NULL; + size = file_size_simple = 0; + int result = one_smart_mutation(i % rand_names.size(), &file, &size, SMART_MUTATION_RANDOM, &file_simple, &file_size_simple); + if (debug_print) { printf("%d\n", result); + } } return 0; } @@ -1702,7 +1810,7 @@ int benchmark(int argc, char *argv[]) unsigned char* new_data = NULL; int generated = 0; int valid = 0; - uint64_t total_bytes = 0; + unsigned long long total_bytes = 0; int i; int iterations = 10000; std::unordered_map status; @@ -1740,7 +1848,7 @@ int benchmark(int argc, char *argv[]) if (argc > 1) printf("Valid %d/%d = %f\n", valid, generated, (double)valid/(double)generated); if (generated) - printf("Average file size %lu bytes.\n", total_bytes / generated); + printf("Average file size %llu bytes.\n", total_bytes / generated); printf("Speed %f / s.\n", generated / time); delete[] data; return 0;