From a061e1304da4f300699b059fe200602457661e5e Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 30 Mar 2023 10:18:45 +0200 Subject: [PATCH 001/149] further magma adaption --- utils/aflpp_driver/aflpp_driver.c | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index b92e68e9..05e23642 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -59,10 +59,10 @@ $AFL_HOME/afl-fuzz -i IN -o OUT ./a.out #endif #ifdef MAGMA_PATCH -int __afl_sharedmem_fuzzing = 0; +int __afl_sharedmem_fuzzing = 0; #else // AFL++ shared memory fuzz cases -int __afl_sharedmem_fuzzing = 1; +int __afl_sharedmem_fuzzing = 1; #endif extern unsigned int *__afl_fuzz_len; extern unsigned char *__afl_fuzz_ptr; @@ -75,8 +75,8 @@ extern unsigned int __afl_map_size; __attribute__((weak)) int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); __attribute__((weak)) int LLVMFuzzerInitialize(int *argc, char ***argv); -int LLVMFuzzerRunDriver(int *argc, char ***argv, - int (*callback)(const uint8_t *data, size_t size)); +__attribute__((weak)) int LLVMFuzzerRunDriver( + int *argc, char ***argv, int (*callback)(const uint8_t *data, size_t size)); // Default nop ASan hooks for manual poisoning when not linking the ASan // runtime @@ -99,13 +99,15 @@ __attribute__((weak)) void __asan_unpoison_memory_region( __attribute__((weak)) void *__asan_region_is_poisoned(void *beg, size_t size); +#ifndef MAGMA_PATCH // Notify AFL about persistent mode. static volatile char AFL_PERSISTENT[] = "##SIG_AFL_PERSISTENT##"; // Notify AFL about deferred forkserver. static volatile char AFL_DEFER_FORKSVR[] = "##SIG_AFL_DEFER_FORKSRV##"; +#endif -int __afl_persistent_loop(unsigned int); -void __afl_manual_init(); +int __afl_persistent_loop(unsigned int); +void __afl_manual_init(); // Use this optionally defined function to output sanitizer messages even if // user asks to close stderr. @@ -200,7 +202,8 @@ static void maybe_close_fd_mask() { // Define LLVMFuzzerMutate to avoid link failures for targets that use it // with libFuzzer's LLVMFuzzerCustomMutator. -size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, size_t MaxSize) { +__attribute__((weak)) size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size, + size_t MaxSize) { // assert(false && "LLVMFuzzerMutate should not be called from afl_driver"); return 0; @@ -284,8 +287,9 @@ __attribute__((weak)) int main(int argc, char **argv) { } -int LLVMFuzzerRunDriver(int *argcp, char ***argvp, - int (*callback)(const uint8_t *data, size_t size)) { +__attribute__((weak)) int LLVMFuzzerRunDriver( + int *argcp, char ***argvp, + int (*callback)(const uint8_t *data, size_t size)) { int argc = *argcp; char **argv = *argvp; From 2d0d1e267eac4723c24997116c959472aae7027f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 30 Mar 2023 14:07:33 +0200 Subject: [PATCH 002/149] better aflpp driver --- utils/aflpp_driver/aflpp_driver.c | 39 +++++++++++++------------------ 1 file changed, 16 insertions(+), 23 deletions(-) diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 05e23642..5fef4730 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -1,12 +1,14 @@ -//===- afl_driver.cpp - a glue between AFL++ and libFuzzer ------*- C++ -* ===// -//===----------------------------------------------------------------------===// +// +// afl_driver.cpp - a glue between AFL++ and libFuzzer +// /* This file allows to fuzz libFuzzer-style target functions (LLVMFuzzerTestOneInput) with AFL++ using persistent in-memory fuzzing. Usage: -################################################################################ -cat << EOF > test_fuzzer.cc + +# Example target: +$ cat << EOF > test_fuzzer.cc #include #include extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { @@ -20,16 +22,13 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { } EOF -# Build your target with -fsanitize-coverage=trace-pc-guard using fresh clang. -clang -c aflpp_driver.c -# Build afl-compiler-rt.o.c from the AFL distribution. -clang -c $AFL_HOME/instrumentation/afl-compiler-rt.o.c -# Build this file, link it with afl-compiler-rt.o.o and the target code. -afl-clang-fast -o test_fuzzer test_fuzzer.cc afl-compiler-rt.o aflpp_driver.o + +# Build your target with afl-cc -fsanitize=fuzzer +$ afl-c++ -fsanitize=fuzzer -o test_fuzzer test_fuzzer.cc # Run AFL: -rm -rf IN OUT; mkdir IN OUT; echo z > IN/z; -$AFL_HOME/afl-fuzz -i IN -o OUT ./a.out -################################################################################ +$ rm -rf in out; mkdir in out; echo z > in/foo; +$ afl-fuzz -i in -o out -- ./test_fuzzer + */ #include @@ -58,12 +57,8 @@ $AFL_HOME/afl-fuzz -i IN -o OUT ./a.out #include "hash.h" #endif -#ifdef MAGMA_PATCH -int __afl_sharedmem_fuzzing = 0; -#else // AFL++ shared memory fuzz cases -int __afl_sharedmem_fuzzing = 1; -#endif +int __afl_sharedmem_fuzzing = 1; extern unsigned int *__afl_fuzz_len; extern unsigned char *__afl_fuzz_ptr; @@ -99,15 +94,13 @@ __attribute__((weak)) void __asan_unpoison_memory_region( __attribute__((weak)) void *__asan_region_is_poisoned(void *beg, size_t size); -#ifndef MAGMA_PATCH // Notify AFL about persistent mode. static volatile char AFL_PERSISTENT[] = "##SIG_AFL_PERSISTENT##"; +int __afl_persistent_loop(unsigned int); + // Notify AFL about deferred forkserver. static volatile char AFL_DEFER_FORKSVR[] = "##SIG_AFL_DEFER_FORKSRV##"; -#endif - -int __afl_persistent_loop(unsigned int); -void __afl_manual_init(); +void __afl_manual_init(); // Use this optionally defined function to output sanitizer messages even if // user asks to close stderr. From be96253f5293a0446aae8b2f1795119456c0d5d5 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 31 Mar 2023 11:02:27 +0200 Subject: [PATCH 003/149] nits --- instrumentation/SanitizerCoverageLTO.so.cc | 2 ++ instrumentation/afl-compiler-rt.o.c | 11 ++++++----- src/afl-fuzz-bitmap.c | 10 +++++++--- src/afl-fuzz-one.c | 6 +++--- 4 files changed, 18 insertions(+), 11 deletions(-) diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 85f13c19..9691f17f 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -431,6 +431,8 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( if ((afl_global_id = atoi(ptr)) < 0) FATAL("AFL_LLVM_LTO_STARTID value of \"%s\" is negative\n", ptr); + if (afl_global_id < 4) { afl_global_id = 4; } + if ((ptr = getenv("AFL_LLVM_DOCUMENT_IDS")) != NULL) { dFile.open(ptr, std::ofstream::out | std::ofstream::app); diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index a88396d4..26fc7256 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -1536,7 +1536,7 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } - if (start == stop || *start) return; + if (start == stop || *start) { return; } x = getenv("AFL_INST_RATIO"); if (x) { @@ -1563,12 +1563,13 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { "[-] FATAL: forkserver is already up, but an instrumented dlopen() " "library loaded afterwards. You must AFL_PRELOAD such libraries to " "be able to fuzz them or LD_PRELOAD to run outside of afl-fuzz.\n" - "To ignore this set AFL_IGNORE_PROBLEMS=1.\n"); + "To ignore this set AFL_IGNORE_PROBLEMS=1 but this will be bad for " + "coverage.\n"); abort(); } else { - static u32 offset = 4; + static u32 offset = 5; while (start < stop) { @@ -1582,7 +1583,7 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } - if (unlikely(++offset >= __afl_final_loc)) { offset = 4; } + if (unlikely(++offset >= __afl_final_loc)) { offset = 5; } } @@ -1596,7 +1597,7 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { to avoid duplicate calls (which can happen as an artifact of the underlying implementation in LLVM). */ - if (__afl_final_loc < 3) __afl_final_loc = 3; // we skip the first 4 entries + if (__afl_final_loc < 5) __afl_final_loc = 5; // we skip the first 5 entries *(start++) = ++__afl_final_loc; diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index d9c792d1..556bb5d1 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -465,7 +465,8 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { u8 fn[PATH_MAX]; u8 *queue_fn = ""; - u8 new_bits = 0, keeping = 0, res, classified = 0, is_timeout = 0; + u8 new_bits = 0, keeping = 0, res, classified = 0, is_timeout = 0, + need_hash = 1; s32 fd; u64 cksum = 0; @@ -477,6 +478,7 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { classify_counts(&afl->fsrv); classified = 1; + need_hash = 0; cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); @@ -499,6 +501,8 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { new_bits = has_new_bits_unclassified(afl, afl->virgin_bits); + if (unlikely(new_bits)) { classified = 1; } + } if (likely(!new_bits)) { @@ -565,12 +569,12 @@ save_if_interesting(afl_state_t *afl, void *mem, u32 len, u8 fault) { } - if (unlikely(!classified && new_bits)) { + if (unlikely(need_hash && new_bits)) { /* due to classify counts we have to recalculate the checksum */ afl->queue_top->exec_cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, HASH_CONST); - classified = 1; + need_hash = 0; } diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 070669c5..eec5e4b5 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -2071,7 +2071,7 @@ havoc_stage: afl->stage_short = "havoc"; afl->stage_max = ((doing_det ? HAVOC_CYCLES_INIT : HAVOC_CYCLES) * perf_score / afl->havoc_div) >> - 7; + 8; } else { @@ -2080,7 +2080,7 @@ havoc_stage: snprintf(afl->stage_name_buf, STAGE_BUF_SIZE, "splice %u", splice_cycle); afl->stage_name = afl->stage_name_buf; afl->stage_short = "splice"; - afl->stage_max = (SPLICE_HAVOC * perf_score / afl->havoc_div) >> 7; + afl->stage_max = (SPLICE_HAVOC * perf_score / afl->havoc_div) >> 8; } @@ -4640,7 +4640,7 @@ pacemaker_fuzzing: MOpt_globals.splice_stageformat, splice_cycle); afl->stage_name = afl->stage_name_buf; afl->stage_short = MOpt_globals.splice_stagenameshort; - afl->stage_max = (SPLICE_HAVOC * perf_score / afl->havoc_div) >> 7; + afl->stage_max = (SPLICE_HAVOC * perf_score / afl->havoc_div) >> 8; } From 4d02d8e43dd9b413ed819e27bb012e16022d85f6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 2 Apr 2023 11:52:36 +0200 Subject: [PATCH 004/149] fix buffer overflow in pizza mode :( --- src/afl-fuzz-stats.c | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 25ebe987..88844322 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -753,20 +753,20 @@ void show_stats_normal(afl_state_t *afl) { #ifdef __linux__ if (afl->fsrv.nyx_mode) { - sprintf(banner + banner_pad, - "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s] - Nyx", - afl->crash_mode ? cPIN "peruvian were-rabbit" - : cYEL "american fuzzy lop", - si, afl->use_banner, afl->power_name); + snprintf(banner + banner_pad, sizeof(banner) - banner_pad, + "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s] - Nyx", + afl->crash_mode ? cPIN "peruvian were-rabbit" + : cYEL "american fuzzy lop", + si, afl->use_banner, afl->power_name); } else { #endif - sprintf(banner + banner_pad, - "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", - afl->crash_mode ? cPIN "peruvian were-rabbit" - : cYEL "american fuzzy lop", - si, afl->use_banner, afl->power_name); + snprintf(banner + banner_pad, sizeof(banner) - banner_pad, + "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", + afl->crash_mode ? cPIN "peruvian were-rabbit" + : cYEL "american fuzzy lop", + si, afl->use_banner, afl->power_name); #ifdef __linux__ @@ -1557,20 +1557,22 @@ void show_stats_pizza(afl_state_t *afl) { #ifdef __linux__ if (afl->fsrv.nyx_mode) { - sprintf(banner + banner_pad, - "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s] - Nyx", - afl->crash_mode ? cPIN "Mozzarbella Pizzeria table booking system" - : cYEL "Mozzarbella Pizzeria management system", - si, afl->use_banner, afl->power_name); + snprintf(banner + banner_pad, sizeof(banner) - banner_pad, + "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s] - Nyx", + afl->crash_mode ? cPIN + "Mozzarbella Pizzeria table booking system" + : cYEL "Mozzarbella Pizzeria management system", + si, afl->use_banner, afl->power_name); } else { #endif - sprintf(banner + banner_pad, - "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", - afl->crash_mode ? cPIN "Mozzarbella Pizzeria table booking system" - : cYEL "Mozzarbella Pizzeria management system", - si, afl->use_banner, afl->power_name); + snprintf(banner + banner_pad, sizeof(banner) - banner_pad, + "%s " cLCY VERSION cLBL " {%s} " cLGN "(%s) " cPIN "[%s]", + afl->crash_mode ? cPIN + "Mozzarbella Pizzeria table booking system" + : cYEL "Mozzarbella Pizzeria management system", + si, afl->use_banner, afl->power_name); #ifdef __linux__ From e46e0bce44f0799731f5e7724ba3dfacafd4c41a Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 2 Apr 2023 12:03:45 +0200 Subject: [PATCH 005/149] allow pizza mode to be disabled --- docs/Changelog.md | 3 +++ docs/env_variables.md | 2 +- include/afl-fuzz.h | 6 ++++-- src/afl-fuzz-state.c | 10 +++++++++- 4 files changed, 17 insertions(+), 4 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 3c06a785..fbf50137 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,6 +9,9 @@ - added `AFL_NO_WARN_INSTABILITY` - added `AFL_FRIDA_STATS_INTERVAL` - added time_wo_finds to fuzzer_stats + - fixed a crash in pizza (1st april easter egg) mode. Sorry for + everyone who was affected! + - allow pizza mode to be disabled when AFL_PIZZA_MODE is set to -1 - afl-cc: - add CFI sanitizer variant to gcc targets - llvm 16 support (thanks to @devnexen!) diff --git a/docs/env_variables.md b/docs/env_variables.md index c9dc1bbd..a6a0ae44 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -581,7 +581,7 @@ checks or alter some of the more exotic semantics of the tool: constructors in your target, you can set `AFL_EARLY_FORKSERVER`. Note that this is not a compile time option but a runtime option :-) - - Set `AFL_PIZZA_MODE` to 1 to enable the April 1st stats menu, set to 0 + - Set `AFL_PIZZA_MODE` to 1 to enable the April 1st stats menu, set to -1 to disable although it is 1st of April. - If you need a specific interval to update fuzzer_stats file, you can diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 58d02af5..5fd393dd 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -398,8 +398,8 @@ typedef struct afl_env_vars { afl_bench_until_crash, afl_debug_child, afl_autoresume, afl_cal_fast, afl_cycle_schedules, afl_expand_havoc, afl_statsd, afl_cmplog_only_new, afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, - afl_keep_timeouts, afl_pizza_mode, afl_no_crash_readme, - afl_ignore_timeouts, afl_no_startup_calibration, afl_no_warn_instability; + afl_keep_timeouts, afl_no_crash_readme, afl_ignore_timeouts, + afl_no_startup_calibration, afl_no_warn_instability; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, @@ -408,6 +408,8 @@ typedef struct afl_env_vars { *afl_testcache_entries, *afl_child_kill_signal, *afl_fsrv_kill_signal, *afl_target_env, *afl_persistent_record, *afl_exit_on_time; + s32 afl_pizza_mode; + } afl_env_vars_t; struct afl_pass_stat { diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index f9aa5cfe..46b67def 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -648,7 +648,15 @@ void read_afl_environment(afl_state_t *afl, char **envp) { } - if (afl->afl_env.afl_pizza_mode) { afl->pizza_is_served = 1; } + if (afl->afl_env.afl_pizza_mode > 0) { + + afl->pizza_is_served = 1; + + } else if (afl->afl_env.afl_pizza_mode < 0) { + + OKF("Pizza easter egg mode is now disabled."); + + } if (issue_detected) { sleep(2); } From 319b2e8e6f64bd35c5395c8a9e5053f25875a86d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 2 Apr 2023 12:08:54 +0200 Subject: [PATCH 006/149] fix ui layout with slow targets in pizza mode --- src/afl-fuzz-stats.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 88844322..07157bf7 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -1803,10 +1803,10 @@ void show_stats_pizza(afl_state_t *afl) { /* Show a warning about slow execution. */ - if (afl->stats_avg_exec < 100) { + if (afl->stats_avg_exec < 20) { sprintf(tmp, "%s/sec (%s)", u_stringify_float(IB(0), afl->stats_avg_exec), - afl->stats_avg_exec < 20 ? "zzzz..." : "Gennarino is at it again!"); + "zzzz..."); SAYF(bV bSTOP " pizza making speed : " cLRD "%-22s ", From a01138e1c822b97cf187bfcad9938597bf3edc1e Mon Sep 17 00:00:00 2001 From: nj00001 <42004790+nj00001@users.noreply.github.com> Date: Mon, 3 Apr 2023 15:32:52 +0800 Subject: [PATCH 007/149] fix getOperand out of range This commit fixes a sloppy function call that should normally check the number of insn's Operand before calling insn's getOperand method. The fix is that if it is 0 it should continue the loop. I solved problem https://github.com/AFLplusplus/AFLplusplus/issues/1688#issue-1648543691 using this modification --- instrumentation/afl-llvm-common.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index dc34d191..de334fb7 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -289,6 +289,9 @@ void scanForDangerousFunctions(llvm::Module *M) { StringRef ifunc_name = IF.getName(); Constant *r = IF.getResolver(); + if(r->getNumOperands() == 0){ + continue ; + } StringRef r_name = cast(r->getOperand(0))->getName(); if (!be_quiet) fprintf(stderr, From 36127fb1970746f53fec44f9394061f57a4e94c3 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 5 Apr 2023 12:59:20 +0200 Subject: [PATCH 008/149] add -z switch --- docs/Changelog.md | 1 + include/afl-fuzz.h | 3 ++- src/afl-fuzz-queue.c | 7 ++++++- src/afl-fuzz.c | 5 +++++ 4 files changed, 14 insertions(+), 2 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index fbf50137..8127e594 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -12,6 +12,7 @@ - fixed a crash in pizza (1st april easter egg) mode. Sorry for everyone who was affected! - allow pizza mode to be disabled when AFL_PIZZA_MODE is set to -1 + - add -z switch to prefer new coverage findings in seed selection - afl-cc: - add CFI sanitizer variant to gcc targets - llvm 16 support (thanks to @devnexen!) diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 5fd393dd..7ff3315b 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -501,7 +501,8 @@ typedef struct afl_state { custom_splice_optout, /* Custom mutator no splice buffer */ is_main_node, /* if this is the main node */ is_secondary_node, /* if this is a secondary instance */ - pizza_is_served; /* pizza mode */ + pizza_is_served, /* pizza mode */ + prefer_new; /* prefer new queue entries */ u32 stats_update_freq; /* Stats update frequency (execs) */ diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 40184645..1cdc8b54 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -74,9 +74,14 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, if (likely(afl->schedule < RARE)) { weight *= (avg_exec_us / q->exec_us); } weight *= (log(q->bitmap_size) / avg_bitmap_size); weight *= (1 + (q->tc_ref / avg_top_size)); - if (unlikely(weight < 1.0)) { weight = 1.0; } + if (unlikely(weight < 0.1)) { weight = 0.1; } if (unlikely(q->favored)) { weight *= 5; } if (unlikely(!q->was_fuzzed)) { weight *= 2; } + if (unlikely(afl->prefer_new)) { + + weight *= (2.0 * (q->id / (afl->queued_items - 1))); + + } return weight; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 3380fd90..0f01360e 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -132,6 +132,7 @@ static void usage(u8 *argv0, int more_help) { " fast(default), explore, exploit, seek, rare, mmopt, " "coe, lin\n" " quad -- see docs/FAQ.md for more information\n" + " -z - prefer new coverage findings when fuzzing\n" " -f file - location read by the fuzzed program (default: stdin " "or @@)\n" " -t msec - timeout for each run (auto-scaled, default %u ms). " @@ -569,6 +570,10 @@ int main(int argc, char **argv_orig, char **envp) { afl->max_length = atoi(optarg); break; + case 'z': + afl->prefer_new = 1; + break; + case 'Z': afl->old_seed_selection = 1; break; From dba93705a79fb859ce0ede606f79dd4ed203725b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 5 Apr 2023 13:07:22 +0200 Subject: [PATCH 009/149] better new weighting --- src/afl-fuzz-queue.c | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 1cdc8b54..3335e869 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -74,14 +74,16 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, if (likely(afl->schedule < RARE)) { weight *= (avg_exec_us / q->exec_us); } weight *= (log(q->bitmap_size) / avg_bitmap_size); weight *= (1 + (q->tc_ref / avg_top_size)); + + if (unlikely(afl->prefer_new)) { + + weight *= (2.0 * ((1 + q->id) / afl->queued_items)); + + } + if (unlikely(weight < 0.1)) { weight = 0.1; } if (unlikely(q->favored)) { weight *= 5; } if (unlikely(!q->was_fuzzed)) { weight *= 2; } - if (unlikely(afl->prefer_new)) { - - weight *= (2.0 * (q->id / (afl->queued_items - 1))); - - } return weight; From d67ee1777859b55b1660cef15fc09219fb165140 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 5 Apr 2023 13:30:06 +0200 Subject: [PATCH 010/149] fix --- src/afl-fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 0f01360e..c7eb985c 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -557,7 +557,7 @@ int main(int argc, char **argv_orig, char **envp) { while ( (opt = getopt( argc, argv, - "+Ab:B:c:CdDe:E:hi:I:f:F:g:G:l:L:m:M:nNOo:p:RQs:S:t:T:UV:WXx:YZ")) > + "+Ab:B:c:CdDe:E:hi:I:f:F:g:G:l:L:m:M:nNOo:p:RQs:S:t:T:UV:WXx:YzZ")) > 0) { switch (opt) { From f9851dbfbbfbc013172f500099b5f4dc572c508b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 8 Apr 2023 13:00:57 +0200 Subject: [PATCH 011/149] hopefully better -z algorithm --- src/afl-fuzz-queue.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 3335e869..6fc3c743 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -75,12 +75,6 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, weight *= (log(q->bitmap_size) / avg_bitmap_size); weight *= (1 + (q->tc_ref / avg_top_size)); - if (unlikely(afl->prefer_new)) { - - weight *= (2.0 * ((1 + q->id) / afl->queued_items)); - - } - if (unlikely(weight < 0.1)) { weight = 0.1; } if (unlikely(q->favored)) { weight *= 5; } if (unlikely(!q->was_fuzzed)) { weight *= 2; } @@ -155,6 +149,26 @@ void create_alias_table(afl_state_t *afl) { } + if (unlikely(afl->prefer_new) && afl->queued_discovered) { + + double avg_weight = sum / active; + + for (i = n - afl->queued_discovered; i < n; i++) { + + struct queue_entry *q = afl->queue_buf[i]; + + if (likely(!q->disabled) && q->weight > avg_weight) { + + double prev_weight = q->weight; + q->weight *= (2.0 * (i / n)); + sum += (q->weight - prev_weight); + + } + + } + + } + for (i = 0; i < n; i++) { // weight is always 0 for disabled entries From 75d7a094691550afe86519a1d669def0d698b5ce Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 8 Apr 2023 13:48:07 +0200 Subject: [PATCH 012/149] show custom mutator name in UI --- docs/Changelog.md | 1 + src/afl-fuzz-mutators.c | 16 ++++++++++++---- src/afl-fuzz-one.c | 1 + src/afl-fuzz-python.c | 13 ++++++++++++- 4 files changed, 26 insertions(+), 5 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 8127e594..40c328ec 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -13,6 +13,7 @@ everyone who was affected! - allow pizza mode to be disabled when AFL_PIZZA_MODE is set to -1 - add -z switch to prefer new coverage findings in seed selection + - print name of custom mutator in UI - afl-cc: - add CFI sanitizer variant to gcc targets - llvm 16 support (thanks to @devnexen!) diff --git a/src/afl-fuzz-mutators.c b/src/afl-fuzz-mutators.c index 9ea46e7a..64dbe7c6 100644 --- a/src/afl-fuzz-mutators.c +++ b/src/afl-fuzz-mutators.c @@ -179,11 +179,19 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) { void *dh; struct custom_mutator *mutator = ck_alloc(sizeof(struct custom_mutator)); - mutator->name = fn; - if (memchr(fn, '/', strlen(fn))) - mutator->name_short = strrchr(fn, '/') + 1; - else + if (memchr(fn, '/', strlen(fn))) { + + mutator->name_short = strdup(strrchr(fn, '/') + 1); + + } else { + mutator->name_short = strdup(fn); + + } + + if (strlen(mutator->name_short) > 22) { mutator->name_short[21] = 0; } + + mutator->name = fn; ACTF("Loading custom mutator library from '%s'...", fn); dh = dlopen(fn, RTLD_NOW); diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index eec5e4b5..ee562f96 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1931,6 +1931,7 @@ custom_mutator_stage: if (el->afl_custom_fuzz) { afl->current_custom_fuzz = el; + afl->stage_name = el->name_short; if (el->afl_custom_fuzz_count) { diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 2799268b..673e5a6c 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -414,10 +414,21 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl, struct custom_mutator *mutator; mutator = ck_alloc(sizeof(struct custom_mutator)); - mutator->name = module_name; ACTF("Loading Python mutator library from '%s'...", module_name); + if (memchr(module_name, '/', strlen(module_name))) { + + mutator->name_short = strdup(strrchr(module_name, '/') + 1); + + } else { + + mutator->name_short = strdup(module_name); + + } + + if (strlen(mutator->name_short) > 22) { mutator->name_short[21] = 0; } + py_mutator_t *py_mutator; py_mutator = init_py_module(afl, module_name); mutator->data = py_mutator; From 35151cefe8e96451da9a6a25cae2181178ef53d6 Mon Sep 17 00:00:00 2001 From: Hardik Shah Date: Sat, 8 Apr 2023 17:57:08 +0530 Subject: [PATCH 013/149] display instance name --- afl-whatsup | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/afl-whatsup b/afl-whatsup index 5546523a..2f5c9675 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -141,7 +141,8 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do sed 's/^command_line.*$/_skip:1/;s/[ ]*:[ ]*/="/;s/$/"/' "$i" >"$TMP" . "$TMP" - + DIR=$(dirname "$i") + DIR=${DIR##*/} RUN_UNIX=$run_time RUN_DAYS=$((RUN_UNIX / 60 / 60 / 24)) RUN_HRS=$(((RUN_UNIX / 60 / 60) % 24)) @@ -154,7 +155,7 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do if [ "$SUMMARY_ONLY" = "" ]; then - echo ">>> $afl_banner ($RUN_DAYS days, $RUN_HRS hrs) fuzzer PID: $fuzzer_pid <<<" + echo ">>> $afl_banner instance: $DIR ($RUN_DAYS days, $RUN_HRS hrs) fuzzer PID: $fuzzer_pid <<<" echo fi From c1af004451e2732b855aafb567fccbd615b5214e Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 8 Apr 2023 15:29:43 +0200 Subject: [PATCH 014/149] afl-whatsup tput --- afl-whatsup | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/afl-whatsup b/afl-whatsup index 2f5c9675..cec1ae28 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -70,10 +70,10 @@ if [ -d queue ]; then fi -RED=`tput setaf 9 1 1` -GREEN=`tput setaf 2 1 1` -BLUE=`tput setaf 4 1 1` -YELLOW=`tput setaf 11 1 1` +RED=`tput setaf 9 1 1 2>/dev/null` +GREEN=`tput setaf 2 1 1 2>/dev/null` +BLUE=`tput setaf 4 1 1 2>/dev/null` +YELLOW=`tput setaf 11 1 1 2>/dev/null` NC=`tput sgr0` RESET="$NC" From 0eace0212e35e9ed6ba5d084f787402406fa440c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Neusch=C3=A4fer?= Date: Sat, 8 Apr 2023 17:19:09 +0200 Subject: [PATCH 015/149] afl-cc: Avoid casts of string literals to char*, in definition of __AFL_INIT() etc. With the right -W options, compilers may complain about the cast of string literals (for PERSIST_SIG and DEFER_SIG) to (char*), and they're right to do so, because string literals are constant. Since some projects enable -Werror, this can lead to a broken build with afl-cc. Let's simply cast to (const char *), which preserves the constness of the string literal. --- src/afl-cc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/afl-cc.c b/src/afl-cc.c index 7b059d40..35b90a5c 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -1171,8 +1171,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { cc_params[cc_par_cnt++] = "-D__AFL_LOOP(_A)=" - "({ static volatile char *_B __attribute__((used,unused)); " - " _B = (char*)\"" PERSIST_SIG + "({ static volatile const char *_B __attribute__((used,unused)); " + " _B = (const char*)\"" PERSIST_SIG "\"; " "extern int __afl_connected;" #ifdef __APPLE__ @@ -1187,8 +1187,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { cc_params[cc_par_cnt++] = "-D__AFL_INIT()=" - "do { static volatile char *_A __attribute__((used,unused)); " - " _A = (char*)\"" DEFER_SIG + "do { static volatile const char *_A __attribute__((used,unused)); " + " _A = (const char*)\"" DEFER_SIG "\"; " #ifdef __APPLE__ "__attribute__((visibility(\"default\"))) " From 0782ed38414bed37168feafc971fd102b8294510 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 9 Apr 2023 10:33:39 +0200 Subject: [PATCH 016/149] remove pointer to removed doc --- src/afl-fuzz.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c7eb985c..a0c322da 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -166,7 +166,6 @@ static void usage(u8 *argv0, int more_help) { " pacemaker mode (minutes of no new finds). 0 = " "immediately,\n" " -1 = immediately and together with normal mutation.\n" - " See docs/README.MOpt.md\n" " -c program - enable CmpLog by specifying a binary compiled for " "it.\n" " if using QEMU/FRIDA or the fuzzing target is " From a0818c4fceff67e2cff986432c98e3ac1211bdd8 Mon Sep 17 00:00:00 2001 From: David CARLIER Date: Mon, 10 Apr 2023 12:38:26 +0100 Subject: [PATCH 017/149] LLVM instrumentation disable build warning. Since clang 16 is the version for Ubuntu 23 04/Fedora 38 and is easy enough to fix.. --- instrumentation/SanitizerCoverageLTO.so.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 9691f17f..b024179a 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1781,6 +1781,7 @@ INITIALIZE_PASS_END(ModuleSanitizerCoverageLTOLegacyPass, "sancov-lto", "Pass for instrumenting coverage on functions", false, false) +#if LLVM_VERSION_MAJOR < 16 static void registerLTOPass(const PassManagerBuilder &, legacy::PassManagerBase &PM) { @@ -1789,7 +1790,6 @@ static void registerLTOPass(const PassManagerBuilder &, } -#if LLVM_VERSION_MAJOR < 16 static RegisterStandardPasses RegisterCompTransPass( PassManagerBuilder::EP_OptimizerLast, registerLTOPass); From 9e3e1a5512acd44020430f24c9ec1260aeeda285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20Neusch=C3=A4fer?= Date: Mon, 10 Apr 2023 12:22:42 +0200 Subject: [PATCH 018/149] afl-cc: Don't offer __AFL_INIT() etc. in GCC/CLANG modes instrumentation/README.persistent_mode.md documents in the section about deferred forkserver initialization: > With the location selected, add this code in the appropriate spot: > > ```c > #ifdef __AFL_HAVE_MANUAL_CONTROL > __AFL_INIT(); > #endif > ``` > > You don't need the #ifdef guards, but including them ensures that the program > will keep working normally when compiled with a tool other than afl-clang-fast/ > afl-clang-lto/afl-gcc-fast. > > Finally, recompile the program with afl-clang-fast/afl-clang-lto/afl-gcc-fast > (afl-gcc or afl-clang will *not* generate a deferred-initialization binary) - > and you should be all set! This strongly implies that you can compile a program that uses __AFL_INIT() under an `#ifdef __AFL_HAVE_MANUAL_CONTROL` guard with afl-gcc/-clang. However, this currently fails: $ cat example.c #include int main(void) { #ifdef __AFL_HAVE_MANUAL_CONTROL __AFL_INIT(); #endif puts("Hello"); } $ afl-gcc example.c -o example afl-cc++4.06a by Michal Zalewski, Laszlo Szekeres, Marc Heuse - mode: GCC-GCC [!] WARNING: You are using outdated instrumentation, install LLVM and/or gcc-plugin and use afl-clang-fast/afl-clang-lto/afl-gcc-fast instead! afl-as++4.06a by Michal Zalewski [+] Instrumented 1 locations (64-bit, non-hardened mode, ratio 100%). /usr/bin/ld: /tmp/ccuJHcpt.o: in function `main': /home/jn/dev/fuzz/AFLplusplus/example.c:5: undefined reference to `__afl_manual_init' collect2: error: ld returned 1 exit status The issue here is an inconsistency in afl-gcc (i.e. afl-cc operating in GCC mode): - afl-cc defines __AFL_HAVE_MANUAL_CONTROL and __AFL_INIT unconditionally - __AFL_INIT relies on __afl_manual_init, which is defined in afl-compiler-rt.o - afl-cc doesn't link afl-compiler-rt in GCC or CLANG mode Since afl-gcc/-clang is documented as not supporting deferred forkserver initialization, this patch omits the definitions of __AFL_HAVE_MANUAL_CONTROL and related macros in GCC/CLANG mode. This restores the ability to compile a deferred-forkserver program under afl-gcc, if it can also be compiled under gcc. [ In case someone reads this an feels adventurous enough (as I did) to think about enabling deferred forkserver under afl-gcc: Whether the deferred forkserver actually works can be verified by placing a usleep(100000) or similar at the start of main (before __AFL_INIT()), and watching the execution speed. It doesn't work. ] --- src/afl-cc.c | 108 ++++++++++++++++++++++++++++----------------------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/src/afl-cc.c b/src/afl-cc.c index 35b90a5c..7f15ad76 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -1101,37 +1101,45 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (!have_c) cc_params[cc_par_cnt++] = "-lrt"; #endif - cc_params[cc_par_cnt++] = "-D__AFL_HAVE_MANUAL_CONTROL=1"; cc_params[cc_par_cnt++] = "-D__AFL_COMPILER=1"; cc_params[cc_par_cnt++] = "-DFUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION=1"; - /* When the user tries to use persistent or deferred forkserver modes by - appending a single line to the program, we want to reliably inject a - signature into the binary (to be picked up by afl-fuzz) and we want - to call a function from the runtime .o file. This is unnecessarily - painful for three reasons: + /* As documented in instrumentation/README.persistent_mode.md, deferred + forkserver initialization and persistent mode are not available in afl-gcc + and afl-clang. */ + if (compiler_mode != GCC && compiler_mode != CLANG) { - 1) We need to convince the compiler not to optimize out the signature. - This is done with __attribute__((used)). + cc_params[cc_par_cnt++] = "-D__AFL_HAVE_MANUAL_CONTROL=1"; - 2) We need to convince the linker, when called with -Wl,--gc-sections, - not to do the same. This is done by forcing an assignment to a - 'volatile' pointer. + /* When the user tries to use persistent or deferred forkserver modes by + appending a single line to the program, we want to reliably inject a + signature into the binary (to be picked up by afl-fuzz) and we want + to call a function from the runtime .o file. This is unnecessarily + painful for three reasons: - 3) We need to declare __afl_persistent_loop() in the global namespace, - but doing this within a method in a class is hard - :: and extern "C" - are forbidden and __attribute__((alias(...))) doesn't work. Hence the - __asm__ aliasing trick. + 1) We need to convince the compiler not to optimize out the signature. + This is done with __attribute__((used)). - */ + 2) We need to convince the linker, when called with -Wl,--gc-sections, + not to do the same. This is done by forcing an assignment to a + 'volatile' pointer. - cc_params[cc_par_cnt++] = - "-D__AFL_FUZZ_INIT()=" - "int __afl_sharedmem_fuzzing = 1;" - "extern unsigned int *__afl_fuzz_len;" - "extern unsigned char *__afl_fuzz_ptr;" - "unsigned char __afl_fuzz_alt[1048576];" - "unsigned char *__afl_fuzz_alt_ptr = __afl_fuzz_alt;"; + 3) We need to declare __afl_persistent_loop() in the global namespace, + but doing this within a method in a class is hard - :: and extern "C" + are forbidden and __attribute__((alias(...))) doesn't work. Hence the + __asm__ aliasing trick. + + */ + + cc_params[cc_par_cnt++] = + "-D__AFL_FUZZ_INIT()=" + "int __afl_sharedmem_fuzzing = 1;" + "extern unsigned int *__afl_fuzz_len;" + "extern unsigned char *__afl_fuzz_ptr;" + "unsigned char __afl_fuzz_alt[1048576];" + "unsigned char *__afl_fuzz_alt_ptr = __afl_fuzz_alt;"; + + } if (plusplus_mode) { @@ -1169,35 +1177,39 @@ static void edit_params(u32 argc, char **argv, char **envp) { "(*__afl_fuzz_len = read(0, __afl_fuzz_alt_ptr, 1048576)) == 0xffffffff " "? 0 : *__afl_fuzz_len)"; - cc_params[cc_par_cnt++] = - "-D__AFL_LOOP(_A)=" - "({ static volatile const char *_B __attribute__((used,unused)); " - " _B = (const char*)\"" PERSIST_SIG - "\"; " - "extern int __afl_connected;" -#ifdef __APPLE__ - "__attribute__((visibility(\"default\"))) " - "int _L(unsigned int) __asm__(\"___afl_persistent_loop\"); " -#else - "__attribute__((visibility(\"default\"))) " - "int _L(unsigned int) __asm__(\"__afl_persistent_loop\"); " -#endif /* ^__APPLE__ */ - // if afl is connected, we run _A times, else once. - "_L(__afl_connected ? _A : 1); })"; + if (compiler_mode != GCC && compiler_mode != CLANG) { - cc_params[cc_par_cnt++] = - "-D__AFL_INIT()=" - "do { static volatile const char *_A __attribute__((used,unused)); " - " _A = (const char*)\"" DEFER_SIG - "\"; " + cc_params[cc_par_cnt++] = + "-D__AFL_LOOP(_A)=" + "({ static volatile const char *_B __attribute__((used,unused)); " + " _B = (const char*)\"" PERSIST_SIG + "\"; " + "extern int __afl_connected;" #ifdef __APPLE__ - "__attribute__((visibility(\"default\"))) " - "void _I(void) __asm__(\"___afl_manual_init\"); " + "__attribute__((visibility(\"default\"))) " + "int _L(unsigned int) __asm__(\"___afl_persistent_loop\"); " #else - "__attribute__((visibility(\"default\"))) " - "void _I(void) __asm__(\"__afl_manual_init\"); " + "__attribute__((visibility(\"default\"))) " + "int _L(unsigned int) __asm__(\"__afl_persistent_loop\"); " #endif /* ^__APPLE__ */ - "_I(); } while (0)"; + // if afl is connected, we run _A times, else once. + "_L(__afl_connected ? _A : 1); })"; + + cc_params[cc_par_cnt++] = + "-D__AFL_INIT()=" + "do { static volatile const char *_A __attribute__((used,unused)); " + " _A = (const char*)\"" DEFER_SIG + "\"; " +#ifdef __APPLE__ + "__attribute__((visibility(\"default\"))) " + "void _I(void) __asm__(\"___afl_manual_init\"); " +#else + "__attribute__((visibility(\"default\"))) " + "void _I(void) __asm__(\"__afl_manual_init\"); " +#endif /* ^__APPLE__ */ + "_I(); } while (0)"; + + } if (x_set) { From 743ae507756db0707a213cd8272b219b0de0f514 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 12 Apr 2023 10:53:23 +0200 Subject: [PATCH 019/149] nits --- GNUmakefile | 2 +- instrumentation/afl-llvm-common.cc | 4 +--- utils/aflpp_driver/GNUmakefile | 4 ---- utils/aflpp_driver/aflpp_driver.c | 8 +++++--- 4 files changed, 7 insertions(+), 11 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 72da14ea..208e965b 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -196,7 +196,7 @@ ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python3-config 2>/dev/null)" "" PYTHON_INCLUDE ?= $(shell python3-config --includes) PYTHON_VERSION ?= $(strip $(shell python3 --version 2>&1)) - # Starting with python3.8, we need to pass the `embed` flag. Earier versions didn't know this flag. + # Starting with python3.8, we need to pass the `embed` flag. Earlier versions didn't know this flag. ifeq "$(shell python3-config --embed --libs 2>/dev/null | grep -q lpython && echo 1 )" "1" PYTHON_LIB ?= $(shell python3-config --libs --embed --ldflags) else diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index de334fb7..5d82aa25 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -289,9 +289,7 @@ void scanForDangerousFunctions(llvm::Module *M) { StringRef ifunc_name = IF.getName(); Constant *r = IF.getResolver(); - if(r->getNumOperands() == 0){ - continue ; - } + if (r->getNumOperands() == 0) { continue; } StringRef r_name = cast(r->getOperand(0))->getName(); if (!be_quiet) fprintf(stderr, diff --git a/utils/aflpp_driver/GNUmakefile b/utils/aflpp_driver/GNUmakefile index 409789cb..b973f96a 100644 --- a/utils/aflpp_driver/GNUmakefile +++ b/utils/aflpp_driver/GNUmakefile @@ -20,10 +20,6 @@ endif CFLAGS := -O3 -funroll-loops -g -fPIC -ifdef MAGMA_PATCH - CFLAGS += -DMAGMA_PATCH -endif - all: libAFLDriver.a libAFLQemuDriver.a aflpp_qemu_driver_hook.so aflpp_driver.o: aflpp_driver.c diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 5fef4730..ec304c8c 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -1,8 +1,10 @@ // -// afl_driver.cpp - a glue between AFL++ and libFuzzer +// afl_driver.cpp - a glue between AFL++ and LLVMFuzzerTestOneInput harnesses // -/* This file allows to fuzz libFuzzer-style target functions +/* + + This file allows to fuzz libFuzzer-style target functions (LLVMFuzzerTestOneInput) with AFL++ using persistent in-memory fuzzing. Usage: @@ -26,7 +28,7 @@ EOF # Build your target with afl-cc -fsanitize=fuzzer $ afl-c++ -fsanitize=fuzzer -o test_fuzzer test_fuzzer.cc # Run AFL: -$ rm -rf in out; mkdir in out; echo z > in/foo; +$ mkdir -p in ; echo z > in/foo; $ afl-fuzz -i in -o out -- ./test_fuzzer */ From d304f4e4f190409e9b1cc6993355128673eb0c3b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 12 Apr 2023 11:24:56 +0200 Subject: [PATCH 020/149] update lto doc --- instrumentation/README.lto.md | 126 ++++++++++++---------------------- 1 file changed, 43 insertions(+), 83 deletions(-) diff --git a/instrumentation/README.lto.md b/instrumentation/README.lto.md index a1c2f44a..df59cc2a 100644 --- a/instrumentation/README.lto.md +++ b/instrumentation/README.lto.md @@ -2,36 +2,37 @@ ## TL;DR: -This version requires a current llvm 11+ compiled from the GitHub master. +This version requires a LLVM 11 or newer. -1. Use afl-clang-lto/afl-clang-lto++ because it is faster and gives better - coverage than anything else that is out there in the AFL world. +1. Use afl-clang-lto/afl-clang-lto++ because the resulting binaries run + slightly faster and give better coverage. -2. You can use it together with llvm_mode: laf-intel and the instrument file - listing features and can be combined with cmplog/Redqueen. +2. You can use it together with COMPCOV, COMPLOG and the instrument file + listing features. -3. It only works with llvm 11+. +3. It only works with LLVM 11 or newer. -4. AUTODICTIONARY feature (see below)! +4. AUTODICTIONARY feature (see below) -5. If any problems arise, be sure to set `AR=llvm-ar RANLIB=llvm-ranlib`. Some - targets might need `LD=afl-clang-lto` and others `LD=afl-ld-lto`. +5. If any problems arise, be sure to set `AR=llvm-ar RANLIB=llvm-ranlib AS=llvm-as`. + Some targets might need `LD=afl-clang-lto` and others `LD=afl-ld-lto`. ## Introduction and problem description -A big issue with how AFL++ works is that the basic block IDs that are set during -compilation are random - and hence naturally the larger the number of -instrumented locations, the higher the number of edge collisions are in the map. -This can result in not discovering new paths and therefore degrade the +A big issue with how vanilla AFL worked was that the basic block IDs that are +set during compilation are random - and hence naturally the larger the number +of instrumented locations, the higher the number of edge collisions are in the +map. This can result in not discovering new paths and therefore degrade the efficiency of the fuzzing process. -*This issue is underestimated in the fuzzing community!* With a 2^16 = 64kb +*This issue is underestimated in the fuzzing community* With a 2^16 = 64kb standard map at already 256 instrumented blocks, there is on average one collision. On average, a target has 10.000 to 50.000 instrumented blocks, hence the real collisions are between 750-18.000! -To reach a solution that prevents any collisions took several approaches and -many dead ends until we got to this: +Note that PCGUARD (our own modified implementation and the SANCOV PCGUARD +implementation from libfuzzer) also provides collision free coverage. +It is a bit slower though and can a few targets with very early constructors. * We instrument at link time when we have all files pre-compiled. * To instrument at link time, we compile in LTO (link time optimization) mode. @@ -45,9 +46,9 @@ many dead ends until we got to this: The result: * 10-25% speed gain compared to llvm_mode -* guaranteed non-colliding edge coverage :-) +* guaranteed non-colliding edge coverage * The compile time, especially for binaries to an instrumented library, can be - much longer. + much (and sometimes much much) longer. Example build output from a libtiff build: @@ -59,71 +60,30 @@ AUTODICTIONARY: 11 strings found [+] Instrumented 12071 locations with no collisions (on average 1046 collisions would be in afl-gcc/afl-clang-fast) (non-hardened mode). ``` -## Getting llvm 11+ +## Getting LLVM 11+ -### Installing llvm version 11 or 12 +### Installing llvm -llvm 11 or even 12 should be available in all current Linux repositories. If you -use an outdated Linux distribution, read the next section. - -### Installing llvm from the llvm repository (version 12+) - -Installing the llvm snapshot builds is easy and mostly painless: - -In the following line, change `NAME` for your Debian or Ubuntu release name -(e.g., buster, focal, eon, etc.): +The best way to install LLVM is to follow [https://apt.llvm.org/](https://apt.llvm.org/) +e.g. for LLVM 15: ``` -echo deb http://apt.llvm.org/NAME/ llvm-toolchain-NAME NAME >> /etc/apt/sources.list +wget https://apt.llvm.org/llvm.sh +chmod +x llvm.sh +sudo ./llvm.sh 15 all ``` -Then add the pgp key of llvm and install the packages: +LLVM 11 to 16 should be available in all current Linux repositories. + +## How to build afl-clang-lto + +That part is easy. +Just set `LLVM_CONFIG` to the llvm-config-VERSION and build AFL++, e.g. for +LLVM 15: ``` -wget -O - https://apt.llvm.org/llvm-snapshot.gpg.key | apt-key add - -apt-get update && apt-get upgrade -y -apt-get install -y clang-12 clang-tools-12 libc++1-12 libc++-12-dev \ - libc++abi1-12 libc++abi-12-dev libclang1-12 libclang-12-dev \ - libclang-common-12-dev libclang-cpp12 libclang-cpp12-dev liblld-12 \ - liblld-12-dev liblldb-12 liblldb-12-dev libllvm12 libomp-12-dev \ - libomp5-12 lld-12 lldb-12 llvm-12 llvm-12-dev llvm-12-runtime llvm-12-tools -``` - -### Building llvm yourself (version 12+) - -Building llvm from GitHub takes quite some time and is not painless: - -```sh -sudo apt install binutils-dev # this is *essential*! -git clone --depth=1 https://github.com/llvm/llvm-project -cd llvm-project -mkdir build -cd build - -# Add -G Ninja if ninja-build installed -# "Building with ninja significantly improves your build time, especially with -# incremental builds, and improves your memory usage." -cmake \ - -DCLANG_INCLUDE_DOCS="OFF" \ - -DCMAKE_BUILD_TYPE=Release \ - -DLLVM_BINUTILS_INCDIR=/usr/include/ \ - -DLLVM_BUILD_LLVM_DYLIB="ON" \ - -DLLVM_ENABLE_BINDINGS="OFF" \ - -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;libcxx;libcxxabi;libunwind;lld' \ - -DLLVM_ENABLE_WARNINGS="OFF" \ - -DLLVM_INCLUDE_BENCHMARKS="OFF" \ - -DLLVM_INCLUDE_DOCS="OFF" \ - -DLLVM_INCLUDE_EXAMPLES="OFF" \ - -DLLVM_INCLUDE_TESTS="OFF" \ - -DLLVM_LINK_LLVM_DYLIB="ON" \ - -DLLVM_TARGETS_TO_BUILD="host" \ - ../llvm/ -# NOTE: for llvm 16 this needs to be changed to: -# -DLLVM_ENABLE_PROJECTS='clang;compiler-rt;lld' \ -# -DLLVM_ENABLE_RUNTIMES='libcxx;libcxxabi' \ -cmake --build . -j4 -export LLVM_CONFIG="$(pwd)/bin/llvm-config" -cd /path/to/AFLplusplus/ +cd ~/AFLplusplus +export LLVM_CONFIG=llvm-config-15 make sudo make install ``` @@ -136,10 +96,10 @@ Also, the instrument file listing (AFL_LLVM_ALLOWLIST/AFL_LLVM_DENYLIST -> [README.instrument_list.md](README.instrument_list.md)) and laf-intel/compcov (AFL_LLVM_LAF_* -> [README.laf-intel.md](README.laf-intel.md)) work. -Example: +Example (note that you might need to add the version, e.g. `llvm-ar-15`: ``` -CC=afl-clang-lto CXX=afl-clang-lto++ RANLIB=llvm-ranlib AR=llvm-ar ./configure +CC=afl-clang-lto CXX=afl-clang-lto++ RANLIB=llvm-ranlib AR=llvm-ar AS=llvm-as ./configure make ``` @@ -317,13 +277,13 @@ AS=llvm-as ... afl-clang-lto is still work in progress. Known issues: -* Anything that llvm 11+ cannot compile, afl-clang-lto cannot compile either - +* Anything that LLVM 11+ cannot compile, afl-clang-lto cannot compile either - obviously. * Anything that does not compile with LTO, afl-clang-lto cannot compile either - obviously. Hence, if building a target with afl-clang-lto fails, try to build it with -llvm12 and LTO enabled (`CC=clang-12`, `CXX=clang++-12`, `CFLAGS=-flto=full`, +LLVM 12 and LTO enabled (`CC=clang-12`, `CXX=clang++-12`, `CFLAGS=-flto=full`, and `CXXFLAGS=-flto=full`). If this succeeds, then there is an issue with afl-clang-lto. Please report at @@ -341,7 +301,7 @@ knows what this is doing. And the developer who implemented this didn't respond to emails.) In December then came the idea to implement this as a pass that is run via the -llvm "opt" program, which is performed via an own linker that afterwards calls +LLVM "opt" program, which is performed via an own linker that afterwards calls the real linker. This was first implemented in January and work ... kinda. The LTO time instrumentation worked, however, "how" the basic blocks were instrumented was a problem, as reducing duplicates turned out to be very, very @@ -353,13 +313,13 @@ dead-end too. The final idea to solve this came from domenukk who proposed to insert a block into an edge and then just use incremental counters ... and this worked! After some trials and errors to implement this vanhauser-thc found out that there is -actually an llvm function for this: SplitEdge() :-) +actually an LLVM function for this: SplitEdge() :-) -Still more problems came up though as this only works without bugs from llvm 9 +Still more problems came up though as this only works without bugs from LLVM 9 onwards, and with high optimization the link optimization ruins the instrumented control flow graph. -This is all now fixed with llvm 11+. The llvm's own linker is now able to load +This is all now fixed with LLVM 11+. The llvm's own linker is now able to load passes and this bypasses all problems we had. Happy end :) From 824385f52ce3133ecd033e587aa1a3b324adf76c Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 12 Apr 2023 14:03:29 +0200 Subject: [PATCH 021/149] make llvm 17 work --- docs/Changelog.md | 2 +- instrumentation/SanitizerCoverageLTO.so.cc | 2 ++ instrumentation/SanitizerCoveragePCGUARD.so.cc | 2 ++ 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 40c328ec..736deb30 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -16,7 +16,7 @@ - print name of custom mutator in UI - afl-cc: - add CFI sanitizer variant to gcc targets - - llvm 16 support (thanks to @devnexen!) + - llvm 16 + 17 support (thanks to @devnexen!) - support llvm 15 native pcguard changes - support for LLVMFuzzerTestOneInput -1 return - qemu_mode: diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index b024179a..5603c455 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -17,7 +17,9 @@ #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" +#if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" +#endif #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ValueTracking.h" diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index c4a564f7..5f23698b 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -13,7 +13,9 @@ #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" +#if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" +#endif #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/IR/CFG.h" From 6cc8d607fb24e060591ece4b42d83fc06de68fc6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 13 Apr 2023 11:44:39 +0200 Subject: [PATCH 022/149] remove -z option, use -p mmopt instead --- GNUmakefile | 2 +- docs/Changelog.md | 2 +- include/afl-fuzz.h | 3 +-- instrumentation/SanitizerCoverageLTO.so.cc | 2 +- instrumentation/SanitizerCoveragePCGUARD.so.cc | 2 +- src/afl-fuzz-queue.c | 14 ++++---------- src/afl-fuzz.c | 7 +------ 7 files changed, 10 insertions(+), 22 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 208e965b..85f164f5 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -546,7 +546,7 @@ ifndef AFL_NO_X86 test_build: afl-cc afl-gcc afl-as afl-showmap @echo "[*] Testing the CC wrapper afl-cc and its instrumentation output..." @unset AFL_MAP_SIZE AFL_USE_UBSAN AFL_USE_CFISAN AFL_USE_LSAN AFL_USE_ASAN AFL_USE_MSAN; ASAN_OPTIONS=detect_leaks=0 AFL_INST_RATIO=100 AFL_PATH=. ./afl-cc test-instr.c $(LDFLAGS) -o test-instr 2>&1 || (echo "Oops, afl-cc failed"; exit 1 ) - - ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -o .test-instr0 ./test-instr < /dev/null + -ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -q -m none -o .test-instr0 ./test-instr < /dev/null -echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr @rm -f test-instr @cmp -s .test-instr0 .test-instr1; DR="$$?"; rm -f .test-instr0 .test-instr1; if [ "$$DR" = "0" ]; then echo; echo "Oops, the instrumentation of afl-cc does not seem to be behaving correctly!"; echo; echo "Please post to https://github.com/AFLplusplus/AFLplusplus/issues to troubleshoot the issue."; echo; exit 1; fi diff --git a/docs/Changelog.md b/docs/Changelog.md index 736deb30..501300b1 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -12,7 +12,7 @@ - fixed a crash in pizza (1st april easter egg) mode. Sorry for everyone who was affected! - allow pizza mode to be disabled when AFL_PIZZA_MODE is set to -1 - - add -z switch to prefer new coverage findings in seed selection + - option `-p mmopt` now also selects new queue items more often - print name of custom mutator in UI - afl-cc: - add CFI sanitizer variant to gcc targets diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 7ff3315b..5fd393dd 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -501,8 +501,7 @@ typedef struct afl_state { custom_splice_optout, /* Custom mutator no splice buffer */ is_main_node, /* if this is the main node */ is_secondary_node, /* if this is a secondary instance */ - pizza_is_served, /* pizza mode */ - prefer_new; /* prefer new queue entries */ + pizza_is_served; /* pizza mode */ u32 stats_update_freq; /* Stats update frequency (execs) */ diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 5603c455..e41f19b6 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -18,7 +18,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #if LLVM_VERSION_MAJOR < 17 -#include "llvm/ADT/Triple.h" + #include "llvm/ADT/Triple.h" #endif #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 5f23698b..85b1ddd5 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -14,7 +14,7 @@ #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #if LLVM_VERSION_MAJOR < 17 -#include "llvm/ADT/Triple.h" + #include "llvm/ADT/Triple.h" #endif #include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 6fc3c743..8ad7cd97 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -149,21 +149,15 @@ void create_alias_table(afl_state_t *afl) { } - if (unlikely(afl->prefer_new) && afl->queued_discovered) { + if (unlikely(afl->schedule == MMOPT) && afl->queued_discovered) { - double avg_weight = sum / active; + u32 cnt = afl->queued_discovered >= 5 ? 5 : afl->queued_discovered; - for (i = n - afl->queued_discovered; i < n; i++) { + for (i = n - cnt; i < n; i++) { struct queue_entry *q = afl->queue_buf[i]; - if (likely(!q->disabled) && q->weight > avg_weight) { - - double prev_weight = q->weight; - q->weight *= (2.0 * (i / n)); - sum += (q->weight - prev_weight); - - } + if (likely(!q->disabled)) { q->weight *= 2.0; } } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index a0c322da..5ba54d0b 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -132,7 +132,6 @@ static void usage(u8 *argv0, int more_help) { " fast(default), explore, exploit, seek, rare, mmopt, " "coe, lin\n" " quad -- see docs/FAQ.md for more information\n" - " -z - prefer new coverage findings when fuzzing\n" " -f file - location read by the fuzzed program (default: stdin " "or @@)\n" " -t msec - timeout for each run (auto-scaled, default %u ms). " @@ -556,7 +555,7 @@ int main(int argc, char **argv_orig, char **envp) { while ( (opt = getopt( argc, argv, - "+Ab:B:c:CdDe:E:hi:I:f:F:g:G:l:L:m:M:nNOo:p:RQs:S:t:T:UV:WXx:YzZ")) > + "+Ab:B:c:CdDe:E:hi:I:f:F:g:G:l:L:m:M:nNOo:p:RQs:S:t:T:UV:WXx:YZ")) > 0) { switch (opt) { @@ -569,10 +568,6 @@ int main(int argc, char **argv_orig, char **envp) { afl->max_length = atoi(optarg); break; - case 'z': - afl->prefer_new = 1; - break; - case 'Z': afl->old_seed_selection = 1; break; From f756734ad2782c3ed56feadb4b7b23fc82a7a968 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 13 Apr 2023 12:07:27 +0200 Subject: [PATCH 023/149] fix attempt at post_process implementation --- docs/Changelog.md | 1 + docs/custom_mutators.md | 9 ++++++-- include/afl-fuzz.h | 9 ++++++-- src/afl-fuzz-python.c | 49 ++++++++--------------------------------- src/afl-fuzz-run.c | 7 +++++- 5 files changed, 30 insertions(+), 45 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 501300b1..9436fc9f 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -13,6 +13,7 @@ everyone who was affected! - allow pizza mode to be disabled when AFL_PIZZA_MODE is set to -1 - option `-p mmopt` now also selects new queue items more often + - fix bug in post_process custom mutator implementation - print name of custom mutator in UI - afl-cc: - add CFI sanitizer variant to gcc targets diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index 82131c92..a1de479e 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -118,7 +118,7 @@ def deinit(): # optional for Python ### Custom Mutation -- `init`: +- `init` (optional in Python): This method is called when AFL++ starts up and is used to seed RNG and set up buffers and state. @@ -184,6 +184,11 @@ def deinit(): # optional for Python to the target, e.g. if it is too short, too corrupted, etc. If so, return a NULL buffer and zero length (or a 0 length string in Python). + NOTE: Do not make any random changes to the data in this function! + + PERFORMANCE for C/C++: If possible make the changes in-place (so modify + the `*data` directly, and return it as `*outbuf = data`. + - `fuzz_send` (optional): This method can be used if you want to send data to the target yourself, @@ -202,7 +207,7 @@ def deinit(): # optional for Python discovered if compiled with INTROSPECTION. The custom mutator can then return a string (const char *) that reports the exact mutations used. -- `deinit`: +- `deinit` (optional in Python): The last method to be called, deinitializing the state. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 5fd393dd..8b6502b4 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -885,14 +885,19 @@ struct custom_mutator { * A post-processing function to use right before AFL writes the test case to * disk in order to execute the target. * - * (Optional) If this functionality is not needed, simply don't define this + * NOTE: Do not do any random changes to the data in this function! + * + * PERFORMANCE: If you can modify the data in-place you will have a better + * performance. Modify *data and set `*out_buf = data`. + * + * (Optional) If this functionality is not needed, simply do not define this * function. * * @param[in] data pointer returned in afl_custom_init by this custom mutator * @param[in] buf Buffer containing the test case to be executed * @param[in] buf_size Size of the test case * @param[out] out_buf Pointer to the buffer storing the test case after - * processing. External library should allocate memory for out_buf. + * processing. The external library should allocate memory for out_buf. * It can chose to alter buf in-place, if the space is large enough. * @return Size of the output buffer. */ diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index 673e5a6c..7dad0770 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -219,11 +219,14 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { if (py_module != NULL) { - u8 py_notrim = 0, py_idx; - /* init, required */ + u8 py_notrim = 0; py_functions[PY_FUNC_INIT] = PyObject_GetAttrString(py_module, "init"); - if (!py_functions[PY_FUNC_INIT]) - FATAL("init function not found in python module"); + if (!py_functions[PY_FUNC_INIT]) { + + WARNF("init function not found in python module"); + + } + py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "fuzz"); if (!py_functions[PY_FUNC_FUZZ]) py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "mutate"); @@ -231,12 +234,6 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { PyObject_GetAttrString(py_module, "describe"); py_functions[PY_FUNC_FUZZ_COUNT] = PyObject_GetAttrString(py_module, "fuzz_count"); - if (!py_functions[PY_FUNC_FUZZ]) { - - WARNF("fuzz function not found in python module"); - - } - py_functions[PY_FUNC_POST_PROCESS] = PyObject_GetAttrString(py_module, "post_process"); py_functions[PY_FUNC_INIT_TRIM] = @@ -263,36 +260,6 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) { if (!py_functions[PY_FUNC_DEINIT]) WARNF("deinit function not found in python module"); - for (py_idx = 0; py_idx < PY_FUNC_COUNT; ++py_idx) { - - if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { - - if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { - - // Implementing the trim API is optional for now - if (PyErr_Occurred()) { PyErr_Print(); } - py_notrim = 1; - - } else if (py_idx >= PY_OPTIONAL) { - - // Only _init and _deinit are not optional currently - - if (PyErr_Occurred()) { PyErr_Print(); } - - } else { - - fprintf(stderr, - "Cannot find/call function with index %d in external " - "Python module.\n", - py_idx); - return NULL; - - } - - } - - } - if (py_notrim) { py_functions[PY_FUNC_INIT_TRIM] = NULL; @@ -345,6 +312,8 @@ static void init_py(afl_state_t *afl, py_mutator_t *py_mutator, (void)afl; + if (py_mutator->py_functions[PY_FUNC_INIT] == NULL) { return; } + PyObject *py_args, *py_value; /* Provide the init function a seed for the Python RNG */ diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index f5425011..26e8549d 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -133,7 +133,12 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - if (new_mem != *mem) { *mem = new_mem; } + if (new_mem != *mem && new_mem != NULL && new_size > 0) { + + *mem = afl_realloc((void **)mem, new_size); + memmove(*mem, new_mem, new_size); + + } if (unlikely(afl->custom_mutators_count)) { From 2adf5aac0fcf9b9263bfe3dd93e4d860e3668ae4 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 13 Apr 2023 15:32:06 +0200 Subject: [PATCH 024/149] libfuzzer driver nits --- dynamic_list.txt | 1 + utils/aflpp_driver/aflpp_driver.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+) diff --git a/dynamic_list.txt b/dynamic_list.txt index d06eae4e..50c0c6b8 100644 --- a/dynamic_list.txt +++ b/dynamic_list.txt @@ -54,4 +54,5 @@ "__sanitizer_cov_trace_pc_guard"; "__sanitizer_cov_trace_pc_guard_init"; "__sanitizer_cov_trace_switch"; + "LLVMFuzzerTestOneInput"; }; diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index ec304c8c..19f0d9da 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -33,6 +33,11 @@ $ afl-fuzz -i in -o out -- ./test_fuzzer */ +#ifdef __cplusplus +extern "C" { + +#endif + #include #include #include @@ -260,6 +265,13 @@ static int ExecuteFilesOnyByOne(int argc, char **argv, __attribute__((weak)) int main(int argc, char **argv) { + if (!LLVMFuzzerTestOneInput) { + + fprintf(stderr, "Error: function LLVMFuzzerTestOneInput() not found!\n"); + abort(); + + } + if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) printf( "============================== INFO ================================\n" @@ -409,3 +421,9 @@ __attribute__((weak)) int LLVMFuzzerRunDriver( } +#ifdef __cplusplus + +} + +#endif + From eefd98f3741b5feca32c75b34a8d7b33e34044d0 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Fri, 14 Apr 2023 02:25:33 +0200 Subject: [PATCH 025/149] add Nyx support in various tools (like afl-cmin) --- afl-cmin | 14 +++++-- include/common.h | 6 +++ include/forkserver.h | 3 ++ src/afl-analyze.c | 64 ++++++++++++++++++++++++++-- src/afl-common.c | 31 ++++++++++++++ src/afl-forkserver.c | 65 +++++++++++++++++++++++++++++ src/afl-fuzz.c | 63 ---------------------------- src/afl-showmap.c | 99 ++++++++++++++++++++++++++++++++++++++++++-- src/afl-tmin.c | 59 +++++++++++++++++++++++++- 9 files changed, 330 insertions(+), 74 deletions(-) diff --git a/afl-cmin b/afl-cmin index 15b61f89..12791584 100755 --- a/afl-cmin +++ b/afl-cmin @@ -109,6 +109,7 @@ function usage() { " -O - use binary-only instrumentation (FRIDA mode)\n" \ " -Q - use binary-only instrumentation (QEMU mode)\n" \ " -U - use unicorn-based instrumentation (unicorn mode)\n" \ +" -X - use Nyx mode\n" \ "\n" \ "Minimization settings:\n" \ " -A - allow crashes and timeouts (not recommended)\n" \ @@ -156,7 +157,7 @@ BEGIN { # process options Opterr = 1 # default is to diagnose Optind = 1 # skip ARGV[0] - while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQU?")) != -1) { + while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUX?")) != -1) { if (_go_c == "i") { if (!Optarg) usage() if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} @@ -217,6 +218,12 @@ BEGIN { extra_par = extra_par " -U" unicorn_mode = 1 continue + } else + if (_go_c == "X") { + if (nyx_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -X" + nyx_mode = 1 + continue } else if (_go_c == "?") { exit 1 @@ -291,7 +298,8 @@ BEGIN { exit 1 } - if (target_bin && !exists_and_is_executable(target_bin)) { + + if (!nyx_mode && target_bin && !exists_and_is_executable(target_bin)) { "command -v "target_bin" 2>/dev/null" | getline tnew if (!tnew || !exists_and_is_executable(tnew)) { @@ -311,7 +319,7 @@ BEGIN { } } - if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !frida_mode && !unicorn_mode) { + if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !frida_mode && !unicorn_mode && !nyx_mode) { if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" exit 1 diff --git a/include/common.h b/include/common.h index 0958b035..279a5f47 100644 --- a/include/common.h +++ b/include/common.h @@ -147,5 +147,11 @@ s32 create_file(u8 *fn); void *afl_memmem(const void *haystack, size_t haystacklen, const void *needle, size_t needlelen); +#ifdef __linux__ +/* Nyx helper functions to create and remove tmp workdirs */ +char* create_nyx_tmp_workdir(void); +void remove_nyx_tmp_workdir(char* nyx_out_dir_path); +#endif + #endif diff --git a/include/forkserver.h b/include/forkserver.h index 50898a08..273a9255 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -75,6 +75,9 @@ typedef struct { } nyx_plugin_handler_t; +/* Imports helper functions to enable Nyx mode (Linux only )*/ +nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary); + #endif typedef struct afl_forkserver { diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 548956d8..0bdadfdc 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -121,9 +121,9 @@ static void kill_child() { } -static void classify_counts(u8 *mem) { +static void classify_counts(u8 *mem, u32 mem_size) { - u32 i = map_size; + u32 i = mem_size; if (edges_only) { @@ -222,7 +222,7 @@ static u64 analyze_run_target(u8 *mem, u32 len, u8 first_run) { } - classify_counts(fsrv.trace_bits); + classify_counts(fsrv.trace_bits, fsrv.map_size); total_execs++; if (stop_soon) { @@ -768,6 +768,7 @@ static void usage(u8 *argv0) { " -U - use unicorn-based instrumentation (Unicorn mode)\n" " -W - use qemu-based instrumentation with Wine (Wine " "mode)\n" + " -X - use Nyx mode\n" #endif "\n" @@ -814,7 +815,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_fsrv_init(&fsrv); - while ((opt = getopt(argc, argv, "+i:f:m:t:eAOQUWh")) > 0) { + while ((opt = getopt(argc, argv, "+i:f:m:t:eAOQUWXh")) > 0) { switch (opt) { @@ -965,6 +966,22 @@ int main(int argc, char **argv_orig, char **envp) { fsrv.mem_limit = mem_limit; break; + + #ifdef __linux__ + case 'X': /* NYX mode */ + + if (fsrv.nyx_mode) { FATAL("Multiple -X options not supported"); } + + fsrv.nyx_mode = 1; + fsrv.nyx_parent = true; + fsrv.nyx_standalone = true; + + break; + #else + case 'X': + FATAL("Nyx mode is only availabe on linux..."); + break; + #endif case 'h': usage(argv[0]); @@ -997,7 +1014,17 @@ int main(int argc, char **argv_orig, char **envp) { set_up_environment(argv); +#ifdef __linux__ + if(!fsrv.nyx_mode){ + fsrv.target_path = find_binary(argv[optind]); + } + else{ + fsrv.target_path = ck_strdup(argv[optind]); + } +#else fsrv.target_path = find_binary(argv[optind]); +#endif + fsrv.trace_bits = afl_shm_init(&shm, map_size, 0); detect_file_args(argv + optind, fsrv.out_file, &use_stdin); signal(SIGALRM, kill_child); @@ -1020,6 +1047,23 @@ int main(int argc, char **argv_orig, char **envp) { use_argv = get_cs_argv(argv[0], &target_path, argc - optind, argv + optind); +#ifdef __linux__ + } else if (fsrv.nyx_mode) { + + fsrv.nyx_id = 0; + + u8 *libnyx_binary = find_afl_binary(argv[0], "libnyx.so"); + fsrv.nyx_handlers = afl_load_libnyx_plugin(libnyx_binary); + if (fsrv.nyx_handlers == NULL) { + FATAL("failed to initialize libnyx.so..."); + } + + fsrv.out_dir_path = create_nyx_tmp_workdir(); + fsrv.nyx_bind_cpu_id = 0; + + use_argv = argv + optind; +#endif + } else { use_argv = argv + optind; @@ -1045,7 +1089,13 @@ int main(int argc, char **argv_orig, char **envp) { &fsrv, NULL, NULL, (fsrv.qemu_mode || unicorn_mode) ? SIGKILL : SIGTERM); read_initial_file(); +#ifdef __linux__ + if(!fsrv.nyx_mode){ + (void)check_binary_signatures(fsrv.target_path); + } +#else (void)check_binary_signatures(fsrv.target_path); +#endif ACTF("Performing dry run (mem limit = %llu MB, timeout = %u ms%s)...", mem_limit, exec_tmout, edges_only ? ", edges only" : ""); @@ -1069,6 +1119,12 @@ int main(int argc, char **argv_orig, char **envp) { OKF("We're done here. Have a nice day!\n"); +#ifdef __linux__ + if (fsrv.nyx_mode) { + remove_nyx_tmp_workdir(fsrv.out_dir_path); + } +#endif + afl_shm_deinit(&shm); afl_fsrv_deinit(&fsrv); if (fsrv.target_path) { ck_free(fsrv.target_path); } diff --git a/src/afl-common.c b/src/afl-common.c index 86226c9f..7dbf7129 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -1359,3 +1359,34 @@ s32 create_file(u8 *fn) { } +#ifdef __linux__ + +/* Nyx requires a tmp workdir to access specific files (such as mmapped files, + * etc.). This helper function basically creates both a path to a tmp workdir + * and the workdir itself. If the environment variable TMPDIR is set, we use + * that as the base directory, otherwise we use /tmp. */ +char* create_nyx_tmp_workdir(void) { + + char *tmpdir = getenv("TMPDIR"); + + if (!tmpdir) { tmpdir = "/tmp"; } + + char* nyx_out_dir_path = alloc_printf("%s/.nyx_tmp_%d/", tmpdir, (u32)getpid()); + + if (mkdir(nyx_out_dir_path, 0700)) { + PFATAL("Unable to create nyx workdir"); + } + + return nyx_out_dir_path; +} + +/* Vice versa, we remove the tmp workdir for nyx with this helper function. */ +void remove_nyx_tmp_workdir(char* nyx_out_dir_path) { + /* Fix me: This is not recursive, so it will always fail. Use a libnyx helper function instead + * to remove the workdir safely (and not risking to wipe the whole filesystem accidentally). */ + //if (rmdir(nyx_out_dir_path)) { + // PFATAL("Unable to remove nyx workdir"); + //} + free(nyx_out_dir_path); +} +#endif diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 5aa4c2ff..95328aa2 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -49,6 +49,71 @@ #include #include +#ifdef __linux__ +#include + +/* function to load nyx_helper function from libnyx.so */ + +nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { + + void *handle; + nyx_plugin_handler_t *plugin = calloc(1, sizeof(nyx_plugin_handler_t)); + + ACTF("Trying to load libnyx.so plugin..."); + handle = dlopen((char *)libnyx_binary, RTLD_NOW); + if (!handle) { goto fail; } + + plugin->nyx_new = dlsym(handle, "nyx_new"); + if (plugin->nyx_new == NULL) { goto fail; } + + plugin->nyx_new_parent = dlsym(handle, "nyx_new_parent"); + if (plugin->nyx_new_parent == NULL) { goto fail; } + + plugin->nyx_new_child = dlsym(handle, "nyx_new_child"); + if (plugin->nyx_new_child == NULL) { goto fail; } + + plugin->nyx_shutdown = dlsym(handle, "nyx_shutdown"); + if (plugin->nyx_shutdown == NULL) { goto fail; } + + plugin->nyx_option_set_reload_mode = + dlsym(handle, "nyx_option_set_reload_mode"); + if (plugin->nyx_option_set_reload_mode == NULL) { goto fail; } + + plugin->nyx_option_set_timeout = dlsym(handle, "nyx_option_set_timeout"); + if (plugin->nyx_option_set_timeout == NULL) { goto fail; } + + plugin->nyx_option_apply = dlsym(handle, "nyx_option_apply"); + if (plugin->nyx_option_apply == NULL) { goto fail; } + + plugin->nyx_set_afl_input = dlsym(handle, "nyx_set_afl_input"); + if (plugin->nyx_set_afl_input == NULL) { goto fail; } + + plugin->nyx_exec = dlsym(handle, "nyx_exec"); + if (plugin->nyx_exec == NULL) { goto fail; } + + plugin->nyx_get_bitmap_buffer = dlsym(handle, "nyx_get_bitmap_buffer"); + if (plugin->nyx_get_bitmap_buffer == NULL) { goto fail; } + + plugin->nyx_get_bitmap_buffer_size = + dlsym(handle, "nyx_get_bitmap_buffer_size"); + if (plugin->nyx_get_bitmap_buffer_size == NULL) { goto fail; } + + plugin->nyx_get_aux_string = dlsym(handle, "nyx_get_aux_string"); + if (plugin->nyx_get_aux_string == NULL) { goto fail; } + + OKF("libnyx plugin is ready!"); + return plugin; + +fail: + + FATAL("failed to load libnyx: %s\n", dlerror()); + free(plugin); + return NULL; + +} + +#endif + /** * The correct fds for reading and writing pipes */ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index a0c322da..8b4fe1e5 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -436,69 +436,6 @@ static void fasan_check_afl_preload(char *afl_preload) { } - #ifdef __linux__ - #include - -nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { - - void *handle; - nyx_plugin_handler_t *plugin = calloc(1, sizeof(nyx_plugin_handler_t)); - - ACTF("Trying to load libnyx.so plugin..."); - handle = dlopen((char *)libnyx_binary, RTLD_NOW); - if (!handle) { goto fail; } - - plugin->nyx_new = dlsym(handle, "nyx_new"); - if (plugin->nyx_new == NULL) { goto fail; } - - plugin->nyx_new_parent = dlsym(handle, "nyx_new_parent"); - if (plugin->nyx_new_parent == NULL) { goto fail; } - - plugin->nyx_new_child = dlsym(handle, "nyx_new_child"); - if (plugin->nyx_new_child == NULL) { goto fail; } - - plugin->nyx_shutdown = dlsym(handle, "nyx_shutdown"); - if (plugin->nyx_shutdown == NULL) { goto fail; } - - plugin->nyx_option_set_reload_mode = - dlsym(handle, "nyx_option_set_reload_mode"); - if (plugin->nyx_option_set_reload_mode == NULL) { goto fail; } - - plugin->nyx_option_set_timeout = dlsym(handle, "nyx_option_set_timeout"); - if (plugin->nyx_option_set_timeout == NULL) { goto fail; } - - plugin->nyx_option_apply = dlsym(handle, "nyx_option_apply"); - if (plugin->nyx_option_apply == NULL) { goto fail; } - - plugin->nyx_set_afl_input = dlsym(handle, "nyx_set_afl_input"); - if (plugin->nyx_set_afl_input == NULL) { goto fail; } - - plugin->nyx_exec = dlsym(handle, "nyx_exec"); - if (plugin->nyx_exec == NULL) { goto fail; } - - plugin->nyx_get_bitmap_buffer = dlsym(handle, "nyx_get_bitmap_buffer"); - if (plugin->nyx_get_bitmap_buffer == NULL) { goto fail; } - - plugin->nyx_get_bitmap_buffer_size = - dlsym(handle, "nyx_get_bitmap_buffer_size"); - if (plugin->nyx_get_bitmap_buffer_size == NULL) { goto fail; } - - plugin->nyx_get_aux_string = dlsym(handle, "nyx_get_aux_string"); - if (plugin->nyx_get_aux_string == NULL) { goto fail; } - - OKF("libnyx plugin is ready!"); - return plugin; - -fail: - - FATAL("failed to load libnyx: %s\n", dlerror()); - free(plugin); - return NULL; - -} - - #endif - /* Main entry point */ int main(int argc, char **argv_orig, char **envp) { diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 29abeb13..3ddebaad 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -434,6 +434,20 @@ static u32 read_file(u8 *in_file) { } +#ifdef __linux__ +/* Execute the target application with an empty input (in Nyx mode). */ +static void showmap_run_target_nyx_mode(afl_forkserver_t *fsrv) { + + afl_fsrv_write_to_testcase(fsrv, NULL, 0); + + if (afl_fsrv_run_target(fsrv, fsrv->exec_tmout, &stop_soon) == + FSRV_RUN_ERROR) { + + FATAL("Error running target in Nyx mode"); + } +} +#endif + /* Execute target application. */ static void showmap_run_target(afl_forkserver_t *fsrv, char **argv) { @@ -797,6 +811,7 @@ static void usage(u8 *argv0) { " -W - use qemu-based instrumentation with Wine (Wine mode)\n" " (Not necessary, here for consistency with other afl-* " "tools)\n" + " -X - use Nyx mode\n" #endif "\n" "Other settings:\n" @@ -875,7 +890,7 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_QUIET") != NULL) { be_quiet = true; } - while ((opt = getopt(argc, argv, "+i:o:f:m:t:AeqCZOH:QUWbcrsh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:AeqCZOH:QUWbcrshX")) > 0) { switch (opt) { @@ -1063,6 +1078,22 @@ int main(int argc, char **argv_orig, char **envp) { break; + #ifdef __linux__ + case 'X': /* NYX mode */ + + if (fsrv->nyx_mode) { FATAL("Multiple -X options not supported"); } + + fsrv->nyx_mode = 1; + fsrv->nyx_parent = true; + fsrv->nyx_standalone = true; + + break; + #else + case 'X': + FATAL("Nyx mode is only availabe on linux..."); + break; + #endif + case 'b': /* Secret undocumented mode. Writes output in raw binary format @@ -1134,7 +1165,17 @@ int main(int argc, char **argv_orig, char **envp) { set_up_environment(fsrv, argv); +#ifdef __linux__ + if(!fsrv->nyx_mode){ + fsrv->target_path = find_binary(argv[optind]); + } + else{ + fsrv->target_path = ck_strdup(argv[optind]); + } +#else fsrv->target_path = find_binary(argv[optind]); +#endif + fsrv->trace_bits = afl_shm_init(&shm, map_size, 0); if (!quiet_mode) { @@ -1190,6 +1231,26 @@ int main(int argc, char **argv_orig, char **envp) { use_argv = get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); +#ifdef __linux__ + } else if (fsrv->nyx_mode) { + + use_argv = ck_alloc(sizeof(char *) * (1)); + use_argv[0] = argv[0]; + + fsrv->nyx_id = 0; + + u8 *libnyx_binary = find_afl_binary(use_argv[0], "libnyx.so"); + fsrv->nyx_handlers = afl_load_libnyx_plugin(libnyx_binary); + if (fsrv->nyx_handlers == NULL) { + + FATAL("failed to initialize libnyx.so..."); + + } + + fsrv->out_dir_path = create_nyx_tmp_workdir(); + fsrv->nyx_bind_cpu_id = 0; +#endif + } else { use_argv = argv + optind; @@ -1226,7 +1287,13 @@ int main(int argc, char **argv_orig, char **envp) { } +#ifdef __linux__ + if(!fsrv->nyx_mode && in_dir){ + (void)check_binary_signatures(fsrv->target_path); + } +#else if (in_dir) { (void)check_binary_signatures(fsrv->target_path); } +#endif shm_fuzz = ck_alloc(sizeof(sharedmem_t)); @@ -1247,7 +1314,13 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->shmem_fuzz = map + sizeof(u32); configure_afl_kill_signals( - fsrv, NULL, NULL, (fsrv->qemu_mode || unicorn_mode) ? SIGKILL : SIGTERM); + fsrv, NULL, NULL, (fsrv->qemu_mode || unicorn_mode + #ifdef __linux__ + || fsrv->nyx_mode + #endif + ) + ? SIGKILL + : SIGTERM); if (!fsrv->cs_mode && !fsrv->qemu_mode && !unicorn_mode) { @@ -1370,6 +1443,12 @@ int main(int argc, char **argv_orig, char **envp) { if (execute_testcases(in_dir) == 0) { +#ifdef __linux__ + if (fsrv->nyx_mode) { + remove_nyx_tmp_workdir(fsrv->out_dir_path); + fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); + } +#endif FATAL("could not read input testcases from %s", in_dir); } @@ -1390,7 +1469,15 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->support_shmem_fuzz && !fsrv->use_shmem_fuzz) shm_fuzz = deinit_shmem(fsrv, shm_fuzz); - showmap_run_target(fsrv, use_argv); +#ifdef __linux__ + if(!fsrv->nyx_mode){ +#endif + showmap_run_target(fsrv, use_argv); +#ifdef __linux__ + } else { + showmap_run_target_nyx_mode(fsrv); + } +#endif tcnt = write_results_to_file(fsrv, out_file); if (!quiet_mode) { @@ -1441,6 +1528,12 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->target_path) { ck_free(fsrv->target_path); } +#ifdef __linux__ + if (fsrv->nyx_mode) { + remove_nyx_tmp_workdir(fsrv->out_dir_path); + } +#endif + afl_fsrv_deinit(fsrv); if (stdin_file) { ck_free(stdin_file); } diff --git a/src/afl-tmin.c b/src/afl-tmin.c index c0087f5f..942525d4 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -789,6 +789,7 @@ static void usage(u8 *argv0) { "mode)\n" " (Not necessary, here for consistency with other afl-* " "tools)\n" + " -X - use Nyx mode\n" #endif "\n" @@ -845,7 +846,7 @@ int main(int argc, char **argv_orig, char **envp) { SAYF(cCYA "afl-tmin" VERSION cRST " by Michal Zalewski\n"); - while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeAOQUWHh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeAOQUWXHh")) > 0) { switch (opt) { @@ -1003,6 +1004,22 @@ int main(int argc, char **argv_orig, char **envp) { break; + #ifdef __linux__ + case 'X': /* NYX mode */ + + if (fsrv->nyx_mode) { FATAL("Multiple -X options not supported"); } + + fsrv->nyx_mode = 1; + fsrv->nyx_parent = true; + fsrv->nyx_standalone = true; + + break; + #else + case 'X': + FATAL("Nyx mode is only availabe on linux..."); + break; + #endif + case 'H': /* Hang Mode */ /* Minimizes a testcase to the minimum that still times out */ @@ -1068,7 +1085,17 @@ int main(int argc, char **argv_orig, char **envp) { set_up_environment(fsrv, argv); +#ifdef __linux__ + if(!fsrv->nyx_mode){ + fsrv->target_path = find_binary(argv[optind]); + } + else{ + fsrv->target_path = ck_strdup(argv[optind]); + } +#else fsrv->target_path = find_binary(argv[optind]); +#endif + fsrv->trace_bits = afl_shm_init(&shm, map_size, 0); detect_file_args(argv + optind, out_file, &fsrv->use_stdin); signal(SIGALRM, kill_child); @@ -1092,6 +1119,23 @@ int main(int argc, char **argv_orig, char **envp) { use_argv = get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); +#ifdef __linux__ + } else if (fsrv->nyx_mode) { + + fsrv->nyx_id = 0; + + u8 *libnyx_binary = find_afl_binary(argv[0], "libnyx.so"); + fsrv->nyx_handlers = afl_load_libnyx_plugin(libnyx_binary); + if (fsrv->nyx_handlers == NULL) { + FATAL("failed to initialize libnyx.so..."); + } + + fsrv->out_dir_path = create_nyx_tmp_workdir(); + fsrv->nyx_bind_cpu_id = 0; + + use_argv = argv + optind; +#endif + } else { use_argv = argv + optind; @@ -1161,7 +1205,14 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->shmem_fuzz = map + sizeof(u32); read_initial_file(); + +#ifdef __linux__ + if(!fsrv->nyx_mode){ + (void)check_binary_signatures(fsrv->target_path); + } +#else (void)check_binary_signatures(fsrv->target_path); +#endif if (!fsrv->qemu_mode && !unicorn_mode) { @@ -1265,6 +1316,12 @@ int main(int argc, char **argv_orig, char **envp) { OKF("We're done here. Have a nice day!\n"); +#ifdef __linux__ + if (fsrv->nyx_mode) { + remove_nyx_tmp_workdir(fsrv->out_dir_path); + } +#endif + remove_shm = 0; afl_shm_deinit(&shm); if (fsrv->use_shmem_fuzz) shm_fuzz = deinit_shmem(fsrv, shm_fuzz); From afc47868ee06cf8e466fd88881b36d5a7c71f29a Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Fri, 14 Apr 2023 04:39:15 +0200 Subject: [PATCH 026/149] bump QEMU-Nyx version The QEMU-Nyx compile script does not set "--enable-gtk" anymore. So it is no longer necessary to patch the compile_qemu_nyx.sh script manually. --- nyx_mode/QEMU-Nyx | 2 +- nyx_mode/QEMU_NYX_VERSION | 2 +- nyx_mode/build_nyx_support.sh | 5 ----- 3 files changed, 2 insertions(+), 7 deletions(-) diff --git a/nyx_mode/QEMU-Nyx b/nyx_mode/QEMU-Nyx index 5c8cf793..60c216bc 160000 --- a/nyx_mode/QEMU-Nyx +++ b/nyx_mode/QEMU-Nyx @@ -1 +1 @@ -Subproject commit 5c8cf793ec615b0df5fa722878c8f6906ad7936f +Subproject commit 60c216bc9e4c79834716d4099993d8397a3a8fd9 diff --git a/nyx_mode/QEMU_NYX_VERSION b/nyx_mode/QEMU_NYX_VERSION index f5888136..98cb134f 100644 --- a/nyx_mode/QEMU_NYX_VERSION +++ b/nyx_mode/QEMU_NYX_VERSION @@ -1 +1 @@ -5c8cf793ec +60c216bc9e diff --git a/nyx_mode/build_nyx_support.sh b/nyx_mode/build_nyx_support.sh index e7fca64f..581a8292 100755 --- a/nyx_mode/build_nyx_support.sh +++ b/nyx_mode/build_nyx_support.sh @@ -60,11 +60,6 @@ fi echo "[*] Checking QEMU-Nyx ..." if [ ! -f "QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64" ]; then - - if ! dpkg -s gtk3-devel > /dev/null 2>&1; then - echo "[-] Disabling GTK because gtk3-devel is not installed." - sed -i 's/--enable-gtk//g' QEMU-Nyx/compile_qemu_nyx.sh - fi (cd QEMU-Nyx && ./compile_qemu_nyx.sh static) fi From e2fedce6ecfa690fa7037328b6432b80a72d5acf Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Fri, 14 Apr 2023 04:40:26 +0200 Subject: [PATCH 027/149] bump libnyx version --- nyx_mode/LIBNYX_VERSION | 2 +- nyx_mode/libnyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nyx_mode/LIBNYX_VERSION b/nyx_mode/LIBNYX_VERSION index 00165a63..461499ec 100644 --- a/nyx_mode/LIBNYX_VERSION +++ b/nyx_mode/LIBNYX_VERSION @@ -1 +1 @@ -acaf7f6 +2822aa1 diff --git a/nyx_mode/libnyx b/nyx_mode/libnyx index acaf7f63..2822aa1b 160000 --- a/nyx_mode/libnyx +++ b/nyx_mode/libnyx @@ -1 +1 @@ -Subproject commit acaf7f6346eeb5f1e2cf043543316909fca43650 +Subproject commit 2822aa1b14c5e7e43343abf4c988c4b50f90faf9 From a96cdc649fbdf2ae6f40d4e966812c46083032a2 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Fri, 14 Apr 2023 05:59:12 +0200 Subject: [PATCH 028/149] switch to latest libnyx API --- include/forkserver.h | 25 +++++++++++++------- src/afl-forkserver.c | 56 ++++++++++++++++++++++++++++---------------- 2 files changed, 53 insertions(+), 28 deletions(-) diff --git a/include/forkserver.h b/include/forkserver.h index 273a9255..7cbad8c8 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -51,16 +51,23 @@ typedef enum NyxReturnValue { } NyxReturnValue; +typedef enum NyxProcessRole { + StandAlone, + Parent, + Child, +} NyxProcessRole; + typedef struct { - void *(*nyx_new)(const char *sharedir, const char *workdir, uint32_t cpu_id, - uint32_t input_buffer_size, - bool input_buffer_write_protection); - void *(*nyx_new_parent)(const char *sharedir, const char *workdir, - uint32_t cpu_id, uint32_t input_buffer_size, - bool input_buffer_write_protection); - void *(*nyx_new_child)(const char *sharedir, const char *workdir, - uint32_t cpu_id, uint32_t worker_id); + void *(*nyx_config_load)(const char *sharedir); + void (*nyx_config_set_workdir_path)(void *config, const char *workdir); + void (*nyx_config_set_input_buffer_size)(void *config, uint32_t input_buffer_size); + void (*nyx_config_set_input_buffer_write_protection)(void *config, bool input_buffer_write_protection); + void (*nyx_config_set_hprintf_fd)(void *config, int32_t hprintf_fd); + void (*nyx_config_set_process_role)(void *config, enum NyxProcessRole role); + void (*nyx_config_set_reuse_snapshot_path)(void *config, const char *reuse_snapshot_path); + + void *(*nyx_new)(void *config, uint32_t worker_id); void (*nyx_shutdown)(void *qemu_process); void (*nyx_option_set_reload_mode)(void *qemu_process, bool enable); void (*nyx_option_set_timeout)(void *qemu_process, uint8_t timeout_sec, @@ -73,6 +80,8 @@ typedef struct { uint32_t (*nyx_get_aux_string)(void *nyx_process, uint8_t *buffer, uint32_t size); + bool (*nyx_remove_work_dir)(const char *workdir); + } nyx_plugin_handler_t; /* Imports helper functions to enable Nyx mode (Linux only )*/ diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 95328aa2..33c46b8c 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -63,15 +63,30 @@ nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { handle = dlopen((char *)libnyx_binary, RTLD_NOW); if (!handle) { goto fail; } + plugin->nyx_config_load = dlsym(handle, "nyx_config_load"); + if (plugin->nyx_config_load == NULL) { goto fail; } + + plugin->nyx_config_set_workdir_path = dlsym(handle, "nyx_config_set_workdir_path"); + if (plugin->nyx_config_set_workdir_path == NULL) { goto fail; } + + plugin->nyx_config_set_input_buffer_size = dlsym(handle, "nyx_config_set_input_buffer_size"); + if (plugin->nyx_config_set_input_buffer_size == NULL) { goto fail; } + + plugin->nyx_config_set_input_buffer_write_protection = dlsym(handle, "nyx_config_set_input_buffer_write_protection"); + if (plugin->nyx_config_set_input_buffer_write_protection == NULL) { goto fail; } + + plugin->nyx_config_set_hprintf_fd = dlsym(handle, "nyx_config_set_hprintf_fd"); + if (plugin->nyx_config_set_hprintf_fd == NULL) { goto fail; } + + plugin->nyx_config_set_process_role = dlsym(handle, "nyx_config_set_process_role"); + if (plugin->nyx_config_set_process_role == NULL) { goto fail; } + + plugin->nyx_config_set_reuse_snapshot_path = dlsym(handle, "nyx_config_set_reuse_snapshot_path"); + if (plugin->nyx_config_set_reuse_snapshot_path == NULL) { goto fail; } + plugin->nyx_new = dlsym(handle, "nyx_new"); if (plugin->nyx_new == NULL) { goto fail; } - plugin->nyx_new_parent = dlsym(handle, "nyx_new_parent"); - if (plugin->nyx_new_parent == NULL) { goto fail; } - - plugin->nyx_new_child = dlsym(handle, "nyx_new_child"); - if (plugin->nyx_new_child == NULL) { goto fail; } - plugin->nyx_shutdown = dlsym(handle, "nyx_shutdown"); if (plugin->nyx_shutdown == NULL) { goto fail; } @@ -101,6 +116,10 @@ nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { plugin->nyx_get_aux_string = dlsym(handle, "nyx_get_aux_string"); if (plugin->nyx_get_aux_string == NULL) { goto fail; } + plugin->nyx_remove_work_dir = dlsym(handle, "nyx_remove_work_dir"); + if (plugin->nyx_remove_work_dir == NULL) { goto fail; } + + OKF("libnyx plugin is ready!"); return plugin; @@ -474,27 +493,24 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, } + void* nyx_config = fsrv->nyx_handlers->nyx_config_load(fsrv->target_path); + + fsrv->nyx_handlers->nyx_config_set_workdir_path(nyx_config, x); + fsrv->nyx_handlers->nyx_config_set_input_buffer_size(nyx_config, MAX_FILE); + fsrv->nyx_handlers->nyx_config_set_input_buffer_write_protection(nyx_config, true); + if (fsrv->nyx_standalone) { - - fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new( - fsrv->target_path, x, fsrv->nyx_bind_cpu_id, MAX_FILE, true); - + fsrv->nyx_handlers->nyx_config_set_process_role(nyx_config, StandAlone); } else { - if (fsrv->nyx_parent) { - - fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new_parent( - fsrv->target_path, x, fsrv->nyx_bind_cpu_id, MAX_FILE, true); - + fsrv->nyx_handlers->nyx_config_set_process_role(nyx_config, Parent); } else { - - fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new_child( - fsrv->target_path, x, fsrv->nyx_bind_cpu_id, fsrv->nyx_id); - + fsrv->nyx_handlers->nyx_config_set_process_role(nyx_config, Child); } - } + fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_bind_cpu_id); + ck_free(x); if (fsrv->nyx_runner == NULL) { FATAL("Something went wrong ..."); } From 4f6ec6cb081bfee7a6e1d6ac211b33a820ec2f71 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Fri, 14 Apr 2023 06:21:43 +0200 Subject: [PATCH 029/149] add NYX_REUSE_SNAPSHOT env-var option --- src/afl-forkserver.c | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 33c46b8c..7eb2155e 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -509,6 +509,22 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, } } + if (getenv("NYX_REUSE_SNAPSHOT") != NULL){ + + if (access(getenv("NYX_REUSE_SNAPSHOT"), F_OK) == -1) { + FATAL("NYX_REUSE_SNAPSHOT path does not exist"); + } + + /* stupid sanity check to avoid passing an empty or invalid snapshot directory */ + char* snapshot_file_path = alloc_printf("%s/global.state", getenv("NYX_REUSE_SNAPSHOT")); + if (access(snapshot_file_path, R_OK) == -1) { + FATAL("NYX_REUSE_SNAPSHOT path does not contain a valid Nyx snapshot"); + } + free(snapshot_file_path); + + fsrv->nyx_handlers->nyx_config_set_reuse_snapshot_path(nyx_config, getenv("NYX_REUSE_SNAPSHOT")); + } + fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_bind_cpu_id); ck_free(x); From c34c3e2f5f70d9a445bdbbb2e2f0937e98358607 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Fri, 14 Apr 2023 06:24:46 +0200 Subject: [PATCH 030/149] add some sanity checks and remove duplicate nyx_shutdown calls --- src/afl-forkserver.c | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 7eb2155e..ae2adc3d 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -555,14 +555,12 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, switch (fsrv->nyx_handlers->nyx_exec(fsrv->nyx_runner)) { case Abort: - fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); FATAL("Error: Nyx abort occured..."); break; case IoError: FATAL("Error: QEMU-Nyx has died..."); break; case Error: - fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); FATAL("Error: Nyx runtime error has occured..."); break; default: @@ -1293,8 +1291,14 @@ void afl_fsrv_kill(afl_forkserver_t *fsrv) { #ifdef __linux__ if (fsrv->nyx_mode) { - free(fsrv->nyx_aux_string); - fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); + if (fsrv->nyx_aux_string){ + free(fsrv->nyx_aux_string); + } + + /* check if we actually got a valid nyx runner */ + if (fsrv->nyx_runner) { + fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); + } } @@ -1474,7 +1478,6 @@ afl_fsrv_run_target(afl_forkserver_t *fsrv, u32 timeout, FATAL("FixMe: Nyx InvalidWriteToPayload handler is missing"); break; case Abort: - fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); FATAL("Error: Nyx abort occured..."); case IoError: if (*stop_soon_p) { From b5f7f42cd0a1bce83c8c6d4af9846e7c5da3cdd1 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 14 Apr 2023 10:22:00 +0200 Subject: [PATCH 031/149] update qemuafl, info in afl-plot --- afl-plot | 4 ++-- docs/Changelog.md | 14 +++++++++----- qemu_mode/QEMUAFL_VERSION | 2 +- qemu_mode/qemuafl | 2 +- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/afl-plot b/afl-plot index 90a46d24..230d3bfe 100755 --- a/afl-plot +++ b/afl-plot @@ -287,9 +287,9 @@ $PLOT_EG _EOF_ -) | gnuplot +) | gnuplot || echo "Note: if you see errors concerning 'unknown or ambiguous terminal type' then you need to use a gnuplot that has png support compiled in." -echo "[?] You can also use -g flag to view the plots in an GUI window, and interact with the plots (if you have built afl-plot-ui). Run \"afl-plot-h\" to know more." +echo "[?] You can also use -g flag to view the plots in an GUI window, and interact with the plots (if you have built afl-plot-ui). Run \"afl-plot -h\" to know more." fi diff --git a/docs/Changelog.md b/docs/Changelog.md index 9436fc9f..9f4a8653 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -7,7 +7,6 @@ - afl-fuzz: - ensure temporary file descriptor is closed when not used - added `AFL_NO_WARN_INSTABILITY` - - added `AFL_FRIDA_STATS_INTERVAL` - added time_wo_finds to fuzzer_stats - fixed a crash in pizza (1st april easter egg) mode. Sorry for everyone who was affected! @@ -15,18 +14,23 @@ - option `-p mmopt` now also selects new queue items more often - fix bug in post_process custom mutator implementation - print name of custom mutator in UI + - slight changes that improve fuzzer performance - afl-cc: - add CFI sanitizer variant to gcc targets - llvm 16 + 17 support (thanks to @devnexen!) - support llvm 15 native pcguard changes - support for LLVMFuzzerTestOneInput -1 return + - LTO autoken and llvm_mode: added AFL_LLVM_DICT2FILE_NO_MAIN support - qemu_mode: - fix _RANGES envs to allow hyphens in the filenames - - new custom module: autotoken, grammar free fuzzer for text inputs - - LTO autoken and llvm_mode: added AFL_LLVM_DICT2FILE_NO_MAIN support + - basic riscv support + - frida_mode: + - added `AFL_FRIDA_STATS_INTERVAL` + - fix issue on MacOS + - unicorn_mode: + - updated and minor issues fixed + - new custom module: autotoken, a grammar free fuzzer for text inputs - better sanitizer default options support for all tools - - unicorn_mode: updated and minor issues fixed - - frida_mode: fix issue on MacOS - more minor fixes and cross-platform support ### Version ++4.05c (release) diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index 43dc832b..fa44d173 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -249bf0c872 +0569eff8a1 diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index 249bf0c8..0569eff8 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit 249bf0c8723671a1eebe400a9631d9e69306ff4c +Subproject commit 0569eff8a12dec73642b96757f6b5b51a618a03a From 0c4f0dd4c483ec3cf481958c6b22c2b43b365cef Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 14 Apr 2023 16:09:55 +0200 Subject: [PATCH 032/149] oss-fuzz fix for LLVMFuzzerTestOneInput driver --- utils/aflpp_driver/aflpp_driver.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 19f0d9da..00a94b9c 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -74,7 +74,10 @@ extern unsigned char *__afl_area_ptr; extern unsigned int __afl_map_size; // libFuzzer interface is thin, so we don't include any libFuzzer headers. -__attribute__((weak)) int LLVMFuzzerTestOneInput(const uint8_t *Data, +/* Using the weak attributed on LLVMFuzzerTestOneInput() breaks oss-fuzz but + on the other hand this is what Google needs to make LLVMFuzzerRunDriver() + work. Choose your poison Google! */ +/*__attribute__((weak))*/ int LLVMFuzzerTestOneInput(const uint8_t *Data, size_t Size); __attribute__((weak)) int LLVMFuzzerInitialize(int *argc, char ***argv); __attribute__((weak)) int LLVMFuzzerRunDriver( From 97644836935020b9f42688bb6530f08f536644a9 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 14 Apr 2023 16:27:32 +0200 Subject: [PATCH 033/149] remove warning --- utils/aflpp_driver/aflpp_driver.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 00a94b9c..34294809 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -268,12 +268,15 @@ static int ExecuteFilesOnyByOne(int argc, char **argv, __attribute__((weak)) int main(int argc, char **argv) { +// Enable if LLVMFuzzerTestOneInput() has the weak attribute +/* if (!LLVMFuzzerTestOneInput) { fprintf(stderr, "Error: function LLVMFuzzerTestOneInput() not found!\n"); abort(); } +*/ if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) printf( From 50678ed36962eb9f4d26f50f860b07116290fe42 Mon Sep 17 00:00:00 2001 From: Yaakov Saxon Date: Fri, 14 Apr 2023 11:24:10 -0400 Subject: [PATCH 034/149] Fixing typo: & (background) to && (and) --- test/test-llvm.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-llvm.sh b/test/test-llvm.sh index 68649c23..0e66cc97 100755 --- a/test/test-llvm.sh +++ b/test/test-llvm.sh @@ -265,7 +265,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { echo 00000000000000000000000000000000 > in/in AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V15 -i in -o out -c./test-cmplog -- ./test-c >>errors 2>&1 } >>errors 2>&1 - test -n "$( ls out/default/crashes/id:000000* out/default/hangs/id:000000* 2>/dev/null )" & { + test -n "$( ls out/default/crashes/id:000000* out/default/hangs/id:000000* 2>/dev/null )" && { $ECHO "$GREEN[+] afl-fuzz is working correctly with llvm_mode cmplog" } || { echo CUT------------------------------------------------------------------CUT From 8f6d9d66ef96286661dab876d9b57a36982b2a4d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 15 Apr 2023 08:33:50 +0200 Subject: [PATCH 035/149] fix post_process --- src/afl-fuzz-run.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 26e8549d..4d56f3a7 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -135,8 +135,11 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { if (new_mem != *mem && new_mem != NULL && new_size > 0) { - *mem = afl_realloc((void **)mem, new_size); - memmove(*mem, new_mem, new_size); + u8 *new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), new_size); + if (unlikely(!new_buf)) { PFATAL("alloc"); } + *mem = new_buf; + memcpy(*mem, new_mem, new_size); + afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); } From e12acaa20367f335549c2db97b88ac5c8ffbeab7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 15 Apr 2023 10:12:20 +0200 Subject: [PATCH 036/149] fix custom mutator C examples --- .../examples/custom_mutator_helpers.h | 342 ------------------ custom_mutators/examples/custom_send.c | 8 +- custom_mutators/examples/example.c | 116 +++--- .../examples/post_library_gif.so.c | 53 +-- .../examples/post_library_png.so.c | 24 +- custom_mutators/examples/simple_example.c | 28 +- docs/Changelog.md | 1 + 7 files changed, 73 insertions(+), 499 deletions(-) delete mode 100644 custom_mutators/examples/custom_mutator_helpers.h diff --git a/custom_mutators/examples/custom_mutator_helpers.h b/custom_mutators/examples/custom_mutator_helpers.h deleted file mode 100644 index 62e6efba..00000000 --- a/custom_mutators/examples/custom_mutator_helpers.h +++ /dev/null @@ -1,342 +0,0 @@ -#ifndef CUSTOM_MUTATOR_HELPERS -#define CUSTOM_MUTATOR_HELPERS - -#include "config.h" -#include "types.h" -#include - -#define INITIAL_GROWTH_SIZE (64) - -#define RAND_BELOW(limit) (rand() % (limit)) - -/* Use in a struct: creates a name_buf and a name_size variable. */ -#define BUF_VAR(type, name) \ - type * name##_buf; \ - size_t name##_size; -/* this fills in `&structptr->something_buf, &structptr->something_size`. */ -#define BUF_PARAMS(struct, name) \ - (void **)&struct->name##_buf, &struct->name##_size - -typedef struct { - -} afl_t; - -static void surgical_havoc_mutate(u8 *out_buf, s32 begin, s32 end) { - - static s8 interesting_8[] = {INTERESTING_8}; - static s16 interesting_16[] = {INTERESTING_8, INTERESTING_16}; - static s32 interesting_32[] = {INTERESTING_8, INTERESTING_16, INTERESTING_32}; - - switch (RAND_BELOW(12)) { - - case 0: { - - /* Flip a single bit somewhere. Spooky! */ - - s32 bit_idx = ((RAND_BELOW(end - begin) + begin) << 3) + RAND_BELOW(8); - - out_buf[bit_idx >> 3] ^= 128 >> (bit_idx & 7); - - break; - - } - - case 1: { - - /* Set byte to interesting value. */ - - u8 val = interesting_8[RAND_BELOW(sizeof(interesting_8))]; - out_buf[(RAND_BELOW(end - begin) + begin)] = val; - - break; - - } - - case 2: { - - /* Set word to interesting value, randomly choosing endian. */ - - if (end - begin < 2) break; - - s32 byte_idx = (RAND_BELOW(end - begin) + begin); - - if (byte_idx >= end - 1) break; - - switch (RAND_BELOW(2)) { - - case 0: - *(u16 *)(out_buf + byte_idx) = - interesting_16[RAND_BELOW(sizeof(interesting_16) >> 1)]; - break; - case 1: - *(u16 *)(out_buf + byte_idx) = - SWAP16(interesting_16[RAND_BELOW(sizeof(interesting_16) >> 1)]); - break; - - } - - break; - - } - - case 3: { - - /* Set dword to interesting value, randomly choosing endian. */ - - if (end - begin < 4) break; - - s32 byte_idx = (RAND_BELOW(end - begin) + begin); - - if (byte_idx >= end - 3) break; - - switch (RAND_BELOW(2)) { - - case 0: - *(u32 *)(out_buf + byte_idx) = - interesting_32[RAND_BELOW(sizeof(interesting_32) >> 2)]; - break; - case 1: - *(u32 *)(out_buf + byte_idx) = - SWAP32(interesting_32[RAND_BELOW(sizeof(interesting_32) >> 2)]); - break; - - } - - break; - - } - - case 4: { - - /* Set qword to interesting value, randomly choosing endian. */ - - if (end - begin < 8) break; - - s32 byte_idx = (RAND_BELOW(end - begin) + begin); - - if (byte_idx >= end - 7) break; - - switch (RAND_BELOW(2)) { - - case 0: - *(u64 *)(out_buf + byte_idx) = - (s64)interesting_32[RAND_BELOW(sizeof(interesting_32) >> 2)]; - break; - case 1: - *(u64 *)(out_buf + byte_idx) = SWAP64( - (s64)interesting_32[RAND_BELOW(sizeof(interesting_32) >> 2)]); - break; - - } - - break; - - } - - case 5: { - - /* Randomly subtract from byte. */ - - out_buf[(RAND_BELOW(end - begin) + begin)] -= 1 + RAND_BELOW(ARITH_MAX); - - break; - - } - - case 6: { - - /* Randomly add to byte. */ - - out_buf[(RAND_BELOW(end - begin) + begin)] += 1 + RAND_BELOW(ARITH_MAX); - - break; - - } - - case 7: { - - /* Randomly subtract from word, random endian. */ - - if (end - begin < 2) break; - - s32 byte_idx = (RAND_BELOW(end - begin) + begin); - - if (byte_idx >= end - 1) break; - - if (RAND_BELOW(2)) { - - *(u16 *)(out_buf + byte_idx) -= 1 + RAND_BELOW(ARITH_MAX); - - } else { - - u16 num = 1 + RAND_BELOW(ARITH_MAX); - - *(u16 *)(out_buf + byte_idx) = - SWAP16(SWAP16(*(u16 *)(out_buf + byte_idx)) - num); - - } - - break; - - } - - case 8: { - - /* Randomly add to word, random endian. */ - - if (end - begin < 2) break; - - s32 byte_idx = (RAND_BELOW(end - begin) + begin); - - if (byte_idx >= end - 1) break; - - if (RAND_BELOW(2)) { - - *(u16 *)(out_buf + byte_idx) += 1 + RAND_BELOW(ARITH_MAX); - - } else { - - u16 num = 1 + RAND_BELOW(ARITH_MAX); - - *(u16 *)(out_buf + byte_idx) = - SWAP16(SWAP16(*(u16 *)(out_buf + byte_idx)) + num); - - } - - break; - - } - - case 9: { - - /* Randomly subtract from dword, random endian. */ - - if (end - begin < 4) break; - - s32 byte_idx = (RAND_BELOW(end - begin) + begin); - - if (byte_idx >= end - 3) break; - - if (RAND_BELOW(2)) { - - *(u32 *)(out_buf + byte_idx) -= 1 + RAND_BELOW(ARITH_MAX); - - } else { - - u32 num = 1 + RAND_BELOW(ARITH_MAX); - - *(u32 *)(out_buf + byte_idx) = - SWAP32(SWAP32(*(u32 *)(out_buf + byte_idx)) - num); - - } - - break; - - } - - case 10: { - - /* Randomly add to dword, random endian. */ - - if (end - begin < 4) break; - - s32 byte_idx = (RAND_BELOW(end - begin) + begin); - - if (byte_idx >= end - 3) break; - - if (RAND_BELOW(2)) { - - *(u32 *)(out_buf + byte_idx) += 1 + RAND_BELOW(ARITH_MAX); - - } else { - - u32 num = 1 + RAND_BELOW(ARITH_MAX); - - *(u32 *)(out_buf + byte_idx) = - SWAP32(SWAP32(*(u32 *)(out_buf + byte_idx)) + num); - - } - - break; - - } - - case 11: { - - /* Just set a random byte to a random value. Because, - why not. We use XOR with 1-255 to eliminate the - possibility of a no-op. */ - - out_buf[(RAND_BELOW(end - begin) + begin)] ^= 1 + RAND_BELOW(255); - - break; - - } - - } - -} - -/* This function calculates the next power of 2 greater or equal its argument. - @return The rounded up power of 2 (if no overflow) or 0 on overflow. -*/ -static inline size_t next_pow2(size_t in) { - - if (in == 0 || in > (size_t)-1) - return 0; /* avoid undefined behaviour under-/overflow */ - size_t out = in - 1; - out |= out >> 1; - out |= out >> 2; - out |= out >> 4; - out |= out >> 8; - out |= out >> 16; - return out + 1; - -} - -/* This function makes sure *size is > size_needed after call. - It will realloc *buf otherwise. - *size will grow exponentially as per: - https://blog.mozilla.org/nnethercote/2014/11/04/please-grow-your-buffers-exponentially/ - Will return NULL and free *buf if size_needed is <1 or realloc failed. - @return For convenience, this function returns *buf. - */ -static inline void *maybe_grow(void **buf, size_t *size, size_t size_needed) { - - /* No need to realloc */ - if (likely(size_needed && *size >= size_needed)) return *buf; - - /* No initial size was set */ - if (size_needed < INITIAL_GROWTH_SIZE) size_needed = INITIAL_GROWTH_SIZE; - - /* grow exponentially */ - size_t next_size = next_pow2(size_needed); - - /* handle overflow */ - if (!next_size) { next_size = size_needed; } - - /* alloc */ - *buf = realloc(*buf, next_size); - *size = *buf ? next_size : 0; - - return *buf; - -} - -/* Swaps buf1 ptr and buf2 ptr, as well as their sizes */ -static inline void afl_swap_bufs(void **buf1, size_t *size1, void **buf2, - size_t *size2) { - - void * scratch_buf = *buf1; - size_t scratch_size = *size1; - *buf1 = *buf2; - *size1 = *size2; - *buf2 = scratch_buf; - *size2 = scratch_size; - -} - -#undef INITIAL_GROWTH_SIZE - -#endif - diff --git a/custom_mutators/examples/custom_send.c b/custom_mutators/examples/custom_send.c index 7de72819..9cc4b160 100644 --- a/custom_mutators/examples/custom_send.c +++ b/custom_mutators/examples/custom_send.c @@ -10,21 +10,21 @@ // afl-fuzz -i in -o out -- ./test-instr -f /tmp/foo // -#include "custom_mutator_helpers.h" - #include #include #include #include #include +#include "afl-fuzz.h" + typedef struct my_mutator { - afl_t *afl; + afl_state_t *afl; } my_mutator_t; -my_mutator_t *afl_custom_init(afl_t *afl, unsigned int seed) { +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); if (!data) { diff --git a/custom_mutators/examples/example.c b/custom_mutators/examples/example.c index e680ec8e..42c7469c 100644 --- a/custom_mutators/examples/example.c +++ b/custom_mutators/examples/example.c @@ -7,7 +7,7 @@ */ // You need to use -I/path/to/AFLplusplus/include -I. -#include "custom_mutator_helpers.h" +#include "afl-fuzz.h" #include #include @@ -26,19 +26,14 @@ static const char *commands[] = { typedef struct my_mutator { - afl_t *afl; + afl_state_t *afl; // any additional data here! size_t trim_size_current; int trimmming_steps; int cur_step; - // Reused buffers: - BUF_VAR(u8, fuzz); - BUF_VAR(u8, data); - BUF_VAR(u8, havoc); - BUF_VAR(u8, trim); - BUF_VAR(u8, post_process); + u8 *mutated_out, *post_process_buf, *trim_buf; } my_mutator_t; @@ -53,7 +48,7 @@ typedef struct my_mutator { * There may be multiple instances of this mutator in one afl-fuzz run! * Return NULL on error. */ -my_mutator_t *afl_custom_init(afl_t *afl, unsigned int seed) { +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { srand(seed); // needed also by surgical_havoc_mutate() @@ -65,6 +60,27 @@ my_mutator_t *afl_custom_init(afl_t *afl, unsigned int seed) { } + if ((data->mutated_out = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + if ((data->post_process_buf = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + + if ((data->trim_buf = (u8 *)malloc(MAX_FILE)) == NULL) { + + perror("afl_custom_init malloc"); + return NULL; + + } + data->afl = afl; return data; @@ -96,31 +112,14 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, // the fuzzer size_t mutated_size = DATA_SIZE <= max_size ? DATA_SIZE : max_size; - // maybe_grow is optimized to be quick for reused buffers. - u8 *mutated_out = maybe_grow(BUF_PARAMS(data, fuzz), mutated_size); - if (!mutated_out) { - - *out_buf = NULL; - perror("custom mutator allocation (maybe_grow)"); - return 0; /* afl-fuzz will very likely error out after this. */ - - } + memcpy(data->mutated_out, buf, buf_size); // Randomly select a command string to add as a header to the packet - memcpy(mutated_out, commands[rand() % 3], 3); + memcpy(data->mutated_out, commands[rand() % 3], 3); - // Mutate the payload of the packet - int i; - for (i = 0; i < 8; ++i) { + if (mutated_size > max_size) { mutated_size = max_size; } - // Randomly perform one of the (no len modification) havoc mutations - surgical_havoc_mutate(mutated_out, 3, mutated_size); - - } - - if (max_size > mutated_size) { mutated_size = max_size; } - - *out_buf = mutated_out; + *out_buf = data->mutated_out; return mutated_size; } @@ -144,24 +143,16 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, size_t afl_custom_post_process(my_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf) { - uint8_t *post_process_buf = - maybe_grow(BUF_PARAMS(data, post_process), buf_size + 5); - if (!post_process_buf) { + if (buf_size + 5 > MAX_FILE) { buf_size = MAX_FILE - 5; } - perror("custom mutator realloc failed."); - *out_buf = NULL; - return 0; + memcpy(data->post_process_buf + 5, buf, buf_size); + data->post_process_buf[0] = 'A'; + data->post_process_buf[1] = 'F'; + data->post_process_buf[2] = 'L'; + data->post_process_buf[3] = '+'; + data->post_process_buf[4] = '+'; - } - - memcpy(post_process_buf + 5, buf, buf_size); - post_process_buf[0] = 'A'; - post_process_buf[1] = 'F'; - post_process_buf[2] = 'L'; - post_process_buf[3] = '+'; - post_process_buf[4] = '+'; - - *out_buf = post_process_buf; + *out_buf = data->post_process_buf; return buf_size + 5; @@ -197,13 +188,6 @@ int32_t afl_custom_init_trim(my_mutator_t *data, uint8_t *buf, data->cur_step = 0; - if (!maybe_grow(BUF_PARAMS(data, trim), buf_size)) { - - perror("init_trim grow"); - return -1; - - } - memcpy(data->trim_buf, buf, buf_size); data->trim_size_current = buf_size; @@ -284,27 +268,11 @@ int32_t afl_custom_post_trim(my_mutator_t *data, int success) { size_t afl_custom_havoc_mutation(my_mutator_t *data, u8 *buf, size_t buf_size, u8 **out_buf, size_t max_size) { - if (buf_size == 0) { + *out_buf = buf; // in-place mutation - *out_buf = maybe_grow(BUF_PARAMS(data, havoc), 1); - if (!*out_buf) { + if (buf_size <= sizeof(size_t)) { return buf_size; } - perror("custom havoc: maybe_grow"); - return 0; - - } - - **out_buf = rand() % 256; - buf_size = 1; - - } else { - - // We reuse buf here. It's legal and faster. - *out_buf = buf; - - } - - size_t victim = rand() % buf_size; + size_t victim = rand() % (buf_size - sizeof(size_t)); (*out_buf)[victim] += rand() % 10; return buf_size; @@ -371,9 +339,7 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t *data, void afl_custom_deinit(my_mutator_t *data) { free(data->post_process_buf); - free(data->havoc_buf); - free(data->data_buf); - free(data->fuzz_buf); + free(data->mutated_out); free(data->trim_buf); free(data); diff --git a/custom_mutators/examples/post_library_gif.so.c b/custom_mutators/examples/post_library_gif.so.c index 3cb018a6..6737c627 100644 --- a/custom_mutators/examples/post_library_gif.so.c +++ b/custom_mutators/examples/post_library_gif.so.c @@ -45,9 +45,8 @@ 1) If you don't want to modify the test case, simply set `*out_buf = in_buf` and return the original `len`. - NOTE: the following is currently NOT true, we abort in this case! 2) If you want to skip this test case altogether and have AFL generate a - new one, return 0 or set `*out_buf = NULL`. + new one, return 0. Use this sparingly - it's faster than running the target program with patently useless inputs, but still wastes CPU time. @@ -59,8 +58,6 @@ Note that the buffer will *not* be freed for you. To avoid memory leaks, you need to free it or reuse it on subsequent calls (as shown below). - *** Feel free to reuse the original 'in_buf' BUFFER and return it. *** - Alright. The example below shows a simple postprocessor that tries to make sure that all input files start with "GIF89a". @@ -72,7 +69,7 @@ #include #include #include -#include "alloc-inl.h" +#include "afl-fuzz.h" /* Header that must be present at the beginning of every test case: */ @@ -80,8 +77,7 @@ typedef struct post_state { - unsigned char *buf; - size_t size; + size_t size; } post_state_t; @@ -95,15 +91,6 @@ void *afl_custom_init(void *afl) { } - state->buf = calloc(sizeof(unsigned char), 4096); - if (!state->buf) { - - free(state); - perror("calloc"); - return NULL; - - } - return state; } @@ -113,6 +100,10 @@ void *afl_custom_init(void *afl) { size_t afl_custom_post_process(post_state_t *data, unsigned char *in_buf, unsigned int len, unsigned char **out_buf) { + /* we do in-place modification as we do not increase the size */ + + *out_buf = in_buf; + /* Skip execution altogether for buffers shorter than 6 bytes (just to show how it's done). We can trust len to be sane. */ @@ -120,34 +111,7 @@ size_t afl_custom_post_process(post_state_t *data, unsigned char *in_buf, /* Do nothing for buffers that already start with the expected header. */ - if (!memcmp(in_buf, HEADER, strlen(HEADER))) { - - *out_buf = in_buf; - return len; - - } - - /* Allocate memory for new buffer, reusing previous allocation if - possible. Note we have to use afl-fuzz's own realloc! - We use afl_realloc because it is effective. - You can also work within in_buf, and assign it to *out_buf. */ - - *out_buf = afl_realloc(out_buf, len); - - /* If we're out of memory, the most graceful thing to do is to return the - original buffer and give up on modifying it. Let AFL handle OOM on its - own later on. */ - - if (!*out_buf) { - - *out_buf = in_buf; - return len; - - } - - if (len > strlen(HEADER)) - memcpy(*out_buf + strlen(HEADER), in_buf + strlen(HEADER), - len - strlen(HEADER)); + if (!memcmp(in_buf, HEADER, strlen(HEADER))) { return len; } /* Insert the new header. */ @@ -162,7 +126,6 @@ size_t afl_custom_post_process(post_state_t *data, unsigned char *in_buf, /* Gets called afterwards */ void afl_custom_deinit(post_state_t *data) { - free(data->buf); free(data); } diff --git a/custom_mutators/examples/post_library_png.so.c b/custom_mutators/examples/post_library_png.so.c index cd65b1bc..652da497 100644 --- a/custom_mutators/examples/post_library_png.so.c +++ b/custom_mutators/examples/post_library_png.so.c @@ -30,7 +30,7 @@ #include #include #include -#include "alloc-inl.h" +#include "afl-fuzz.h" /* A macro to round an integer up to 4 kB. */ @@ -53,7 +53,7 @@ void *afl_custom_init(void *afl) { } - state->buf = calloc(sizeof(unsigned char), 4096); + state->buf = calloc(sizeof(unsigned char), MAX_FILE); if (!state->buf) { free(state); @@ -80,21 +80,7 @@ size_t afl_custom_post_process(post_state_t *data, const unsigned char *in_buf, } - /* This is not a good way to do it, if you do not need to grow the buffer - then just work with in_buf instead for speed reasons. - But we want to show how to grow a buffer, so this is how it's done: */ - - unsigned int pos = 8; - unsigned char *new_buf = afl_realloc(out_buf, UP4K(len)); - - if (!new_buf) { - - *out_buf = in_buf; - return len; - - } - - memcpy(new_buf, in_buf, len); + unsigned int pos = 8; /* Minimum size of a zero-length PNG chunk is 12 bytes; if we don't have that, we can bail out. */ @@ -124,7 +110,7 @@ size_t afl_custom_post_process(post_state_t *data, const unsigned char *in_buf, if (real_cksum != file_cksum) { - *(uint32_t *)(new_buf + pos + 8 + chunk_len) = real_cksum; + *(uint32_t *)(data->buf + pos + 8 + chunk_len) = real_cksum; } @@ -134,7 +120,7 @@ size_t afl_custom_post_process(post_state_t *data, const unsigned char *in_buf, } - *out_buf = new_buf; + *out_buf = data->buf; return len; } diff --git a/custom_mutators/examples/simple_example.c b/custom_mutators/examples/simple_example.c index d888ec1f..2c0abe29 100644 --- a/custom_mutators/examples/simple_example.c +++ b/custom_mutators/examples/simple_example.c @@ -1,6 +1,6 @@ // This simple example just creates random buffer <= 100 filled with 'A' // needs -I /path/to/AFLplusplus/include -#include "custom_mutator_helpers.h" +#include "afl-fuzz.h" #include #include @@ -13,14 +13,14 @@ typedef struct my_mutator { - afl_t *afl; + afl_state_t *afl; // Reused buffers: - BUF_VAR(u8, fuzz); + u8 *fuzz_buf; } my_mutator_t; -my_mutator_t *afl_custom_init(afl_t *afl, unsigned int seed) { +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { srand(seed); my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); @@ -31,6 +31,14 @@ my_mutator_t *afl_custom_init(afl_t *afl, unsigned int seed) { } + data->fuzz_buf = (u8 *)malloc(MAX_FILE); + if (!data->fuzz_buf) { + + perror("afl_custom_init malloc"); + return NULL; + + } + data->afl = afl; return data; @@ -44,18 +52,10 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, int size = (rand() % 100) + 1; if (size > max_size) size = max_size; - u8 *mutated_out = maybe_grow(BUF_PARAMS(data, fuzz), size); - if (!mutated_out) { - *out_buf = NULL; - perror("custom mutator allocation (maybe_grow)"); - return 0; /* afl-fuzz will very likely error out after this. */ + memset(data->fuzz_buf, _FIXED_CHAR, size); - } - - memset(mutated_out, _FIXED_CHAR, size); - - *out_buf = mutated_out; + *out_buf = data->fuzz_buf; return size; } diff --git a/docs/Changelog.md b/docs/Changelog.md index 9f4a8653..3337feb9 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -30,6 +30,7 @@ - unicorn_mode: - updated and minor issues fixed - new custom module: autotoken, a grammar free fuzzer for text inputs + - fixed custom mutator C examples - better sanitizer default options support for all tools - more minor fixes and cross-platform support From 120d009e7d7c425019e984c29e027af710a0a7df Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 15 Apr 2023 11:36:58 +0200 Subject: [PATCH 037/149] wasm support --- GNUmakefile | 1 + GNUmakefile.llvm | 5 +++++ instrumentation/afl-compiler-rt.o.c | 2 +- instrumentation/afl-llvm-pass.so.cc | 10 +++++----- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 85f164f5..5bc3f9d5 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -388,6 +388,7 @@ help: @echo NO_NYX - disable building nyx mode dependencies @echo "NO_CORESIGHT - disable building coresight (arm64 only)" @echo NO_UNICORN_ARM64 - disable building unicorn on arm64 + @echo "WAFL_MODE - enable for WASM fuzzing with https://github.com/fgsect/WAFL" @echo AFL_NO_X86 - if compiling on non-intel/amd platforms @echo "LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian)" @echo "==========================================" diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index 09e9b5be..a053403b 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -294,6 +294,11 @@ endif CLANG_CPPFL = `$(LLVM_CONFIG) --cxxflags` -fno-rtti -fPIC $(CXXFLAGS) -Wno-deprecated-declarations CLANG_LFL = `$(LLVM_CONFIG) --ldflags` $(LDFLAGS) +# wasm fuzzing: disable thread-local storage and unset LLVM debug flag +ifdef WAFL_MODE + $(info Compiling libraries for use with WAVM) + CLANG_CPPFL += -DNDEBUG -DNO_TLS +endif # User teor2345 reports that this is required to make things work on MacOS X. ifeq "$(SYS)" "Darwin" diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 26fc7256..e0e40983 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -113,7 +113,7 @@ int __afl_selective_coverage __attribute__((weak)); int __afl_selective_coverage_start_off __attribute__((weak)); static int __afl_selective_coverage_temp = 1; -#if defined(__ANDROID__) || defined(__HAIKU__) +#if defined(__ANDROID__) || defined(__HAIKU__) || defined(NO_TLS) PREV_LOC_T __afl_prev_loc[NGRAM_SIZE_MAX]; PREV_LOC_T __afl_prev_caller[CTX_MAX_K]; u32 __afl_prev_ctx; diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index e8d0b1e5..c59324fd 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -413,7 +413,7 @@ bool AFLCoverage::runOnModule(Module &M) { GlobalVariable *AFLContext = NULL; if (ctx_str || caller_str) -#if defined(__ANDROID__) || defined(__HAIKU__) +#if defined(__ANDROID__) || defined(__HAIKU__) || defined(NO_TLS) AFLContext = new GlobalVariable( M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_ctx"); #else @@ -424,7 +424,7 @@ bool AFLCoverage::runOnModule(Module &M) { #ifdef AFL_HAVE_VECTOR_INTRINSICS if (ngram_size) - #if defined(__ANDROID__) || defined(__HAIKU__) + #if defined(__ANDROID__) || defined(__HAIKU__) || defined(NO_TLS) AFLPrevLoc = new GlobalVariable( M, PrevLocTy, /* isConstant */ false, GlobalValue::ExternalLinkage, /* Initializer */ nullptr, "__afl_prev_loc"); @@ -437,7 +437,7 @@ bool AFLCoverage::runOnModule(Module &M) { #endif else #endif -#if defined(__ANDROID__) || defined(__HAIKU__) +#if defined(__ANDROID__) || defined(__HAIKU__) || defined(NO_TLS) AFLPrevLoc = new GlobalVariable( M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_loc"); #else @@ -448,7 +448,7 @@ bool AFLCoverage::runOnModule(Module &M) { #ifdef AFL_HAVE_VECTOR_INTRINSICS if (ctx_k) - #if defined(__ANDROID__) || defined(__HAIKU__) + #if defined(__ANDROID__) || defined(__HAIKU__) || defined(NO_TLS) AFLPrevCaller = new GlobalVariable( M, PrevCallerTy, /* isConstant */ false, GlobalValue::ExternalLinkage, /* Initializer */ nullptr, "__afl_prev_caller"); @@ -461,7 +461,7 @@ bool AFLCoverage::runOnModule(Module &M) { #endif else #endif -#if defined(__ANDROID__) || defined(__HAIKU__) +#if defined(__ANDROID__) || defined(__HAIKU__) || defined(NO_TLS) AFLPrevCaller = new GlobalVariable(M, Int32Ty, false, GlobalValue::ExternalLinkage, 0, "__afl_prev_caller"); From 2f6242d3f8b09bfbf9e2b172acd0c67015e6ef2b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 15 Apr 2023 12:28:39 +0200 Subject: [PATCH 038/149] update docs --- docs/third_party_tools.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/third_party_tools.md b/docs/third_party_tools.md index 97f2c362..02a40ce5 100644 --- a/docs/third_party_tools.md +++ b/docs/third_party_tools.md @@ -1,5 +1,10 @@ # Tools that help fuzzing with AFL++ +## AFL++ and other development languages + +* [afl-rs](https://github.com/rust-fuzz/afl.rs) - AFL++ for RUST +* [WASM](https://github.com/fgsect/WAFL) - AFL++ for WASM + ## Speeding up fuzzing * [libfiowrapper](https://github.com/marekzmyslowski/libfiowrapper) - if the From 47833bcf9e4b642e090f7cc0da25d1ed99688e5e Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Sun, 16 Apr 2023 04:28:19 +0200 Subject: [PATCH 039/149] fix remove_nyx_tmp_workdir function --- include/common.h | 2 +- src/afl-common.c | 24 +++++++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/include/common.h b/include/common.h index 279a5f47..e03566de 100644 --- a/include/common.h +++ b/include/common.h @@ -150,7 +150,7 @@ void *afl_memmem(const void *haystack, size_t haystacklen, const void *needle, #ifdef __linux__ /* Nyx helper functions to create and remove tmp workdirs */ char* create_nyx_tmp_workdir(void); -void remove_nyx_tmp_workdir(char* nyx_out_dir_path); +void remove_nyx_tmp_workdir(afl_forkserver_t *fsrv, char* nyx_out_dir_path); #endif #endif diff --git a/src/afl-common.c b/src/afl-common.c index 7dbf7129..fe0db94d 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -1381,12 +1381,22 @@ char* create_nyx_tmp_workdir(void) { } /* Vice versa, we remove the tmp workdir for nyx with this helper function. */ -void remove_nyx_tmp_workdir(char* nyx_out_dir_path) { - /* Fix me: This is not recursive, so it will always fail. Use a libnyx helper function instead - * to remove the workdir safely (and not risking to wipe the whole filesystem accidentally). */ - //if (rmdir(nyx_out_dir_path)) { - // PFATAL("Unable to remove nyx workdir"); - //} - free(nyx_out_dir_path); +void remove_nyx_tmp_workdir(afl_forkserver_t *fsrv, char* nyx_out_dir_path) { + char* workdir_path = alloc_printf("%s/workdir", nyx_out_dir_path); + + if (access(workdir_path, R_OK) == 0) { + if(fsrv->nyx_handlers->nyx_remove_work_dir(workdir_path) != true) { + WARNF("Unable to remove nyx workdir (%s)", workdir_path); + } + } + + if (access(nyx_out_dir_path, R_OK) == 0) { + if (rmdir(nyx_out_dir_path)) { + WARNF("Unable to remove nyx workdir (%s)", nyx_out_dir_path); + } + } + + ck_free(workdir_path); + ck_free(nyx_out_dir_path); } #endif From 059d470e8dd0a5339daefe0842f4ad5014717838 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Sun, 16 Apr 2023 04:42:09 +0200 Subject: [PATCH 040/149] improved Nyx tmp dir handling (additional sanity checks) --- include/forkserver.h | 2 + src/afl-analyze.c | 7 +--- src/afl-forkserver.c | 97 ++++++++++++++++++++++++++++++-------------- src/afl-fuzz.c | 8 ---- src/afl-showmap.c | 13 +----- src/afl-tmin.c | 7 +--- 6 files changed, 71 insertions(+), 63 deletions(-) diff --git a/include/forkserver.h b/include/forkserver.h index 7cbad8c8..ba280d38 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -190,6 +190,8 @@ typedef struct afl_forkserver { u32 nyx_id; /* nyx runner id (0 -> master) */ u32 nyx_bind_cpu_id; /* nyx runner cpu id */ char *nyx_aux_string; + bool nyx_use_tmp_workdir; + char *nyx_tmp_workdir_path; #endif } afl_forkserver_t; diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 0bdadfdc..0a4e7fb5 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -1058,7 +1058,7 @@ int main(int argc, char **argv_orig, char **envp) { FATAL("failed to initialize libnyx.so..."); } - fsrv.out_dir_path = create_nyx_tmp_workdir(); + fsrv.nyx_use_tmp_workdir = true; fsrv.nyx_bind_cpu_id = 0; use_argv = argv + optind; @@ -1119,11 +1119,6 @@ int main(int argc, char **argv_orig, char **envp) { OKF("We're done here. Have a nice day!\n"); -#ifdef __linux__ - if (fsrv.nyx_mode) { - remove_nyx_tmp_workdir(fsrv.out_dir_path); - } -#endif afl_shm_deinit(&shm); afl_fsrv_deinit(&fsrv); diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index ae2adc3d..0e705c63 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -126,11 +126,39 @@ nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { fail: FATAL("failed to load libnyx: %s\n", dlerror()); - free(plugin); + ck_free(plugin); return NULL; } +void afl_nyx_runner_kill(afl_forkserver_t *fsrv){ + if (fsrv->nyx_mode) { + + if (fsrv->nyx_aux_string){ + ck_free(fsrv->nyx_aux_string); + } + + /* check if we actually got a valid nyx runner */ + if (fsrv->nyx_runner) { + fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); + } + + /* if we have use a tmp work dir we need to remove it */ + if (fsrv->nyx_use_tmp_workdir && fsrv->nyx_tmp_workdir_path) { + remove_nyx_tmp_workdir(fsrv, fsrv->nyx_tmp_workdir_path); + } + } +} + +/* Wrapper for FATAL() that kills the nyx runner (and removes all created tmp + * files) before exiting. Used before "afl_fsrv_killall()" is registered as + * an atexit() handler. */ +#define NYX_PRE_FATAL(fsrv, x...) \ + do { \ + afl_nyx_runner_kill(fsrv); \ + FATAL(x); \ + } while (0) + #endif /** @@ -168,6 +196,8 @@ void afl_fsrv_init(afl_forkserver_t *fsrv) { fsrv->nyx_runner = NULL; fsrv->nyx_id = 0xFFFFFFFF; fsrv->nyx_bind_cpu_id = 0xFFFFFFFF; + fsrv->nyx_use_tmp_workdir = false; + fsrv->nyx_tmp_workdir_path = NULL; #endif // this structure needs default so we initialize it if this was not done @@ -481,21 +511,24 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (!be_quiet) { ACTF("Spinning up the NYX backend..."); } - if (fsrv->out_dir_path == NULL) { FATAL("Nyx workdir path not found..."); } + if (fsrv->nyx_use_tmp_workdir){ + fsrv->nyx_tmp_workdir_path = create_nyx_tmp_workdir(); + fsrv->out_dir_path = fsrv->nyx_tmp_workdir_path; + } else { + if (fsrv->out_dir_path == NULL) { NYX_PRE_FATAL(fsrv, "Nyx workdir path not found..."); } + } - char *x = alloc_printf("%s/workdir", fsrv->out_dir_path); + char *workdir_path = alloc_printf("%s/workdir", fsrv->out_dir_path); - if (fsrv->nyx_id == 0xFFFFFFFF) { FATAL("Nyx ID is not set..."); } + if (fsrv->nyx_id == 0xFFFFFFFF) {NYX_PRE_FATAL(fsrv, "Nyx ID is not set..."); } if (fsrv->nyx_bind_cpu_id == 0xFFFFFFFF) { - - FATAL("Nyx CPU ID is not set..."); - + NYX_PRE_FATAL(fsrv, "Nyx CPU ID is not set..."); } void* nyx_config = fsrv->nyx_handlers->nyx_config_load(fsrv->target_path); - fsrv->nyx_handlers->nyx_config_set_workdir_path(nyx_config, x); + fsrv->nyx_handlers->nyx_config_set_workdir_path(nyx_config, workdir_path); fsrv->nyx_handlers->nyx_config_set_input_buffer_size(nyx_config, MAX_FILE); fsrv->nyx_handlers->nyx_config_set_input_buffer_write_protection(nyx_config, true); @@ -512,22 +545,36 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (getenv("NYX_REUSE_SNAPSHOT") != NULL){ if (access(getenv("NYX_REUSE_SNAPSHOT"), F_OK) == -1) { - FATAL("NYX_REUSE_SNAPSHOT path does not exist"); + NYX_PRE_FATAL(fsrv, "NYX_REUSE_SNAPSHOT path does not exist"); } /* stupid sanity check to avoid passing an empty or invalid snapshot directory */ char* snapshot_file_path = alloc_printf("%s/global.state", getenv("NYX_REUSE_SNAPSHOT")); if (access(snapshot_file_path, R_OK) == -1) { - FATAL("NYX_REUSE_SNAPSHOT path does not contain a valid Nyx snapshot"); + NYX_PRE_FATAL(fsrv, "NYX_REUSE_SNAPSHOT path does not contain a valid Nyx snapshot"); } - free(snapshot_file_path); + ck_free(snapshot_file_path); + + /* another sanity check to avoid passing a snapshot directory that is + * located in the current workdir (the workdir will be wiped by libnyx on startup) */ + char* outdir_path_real = realpath(fsrv->out_dir_path, NULL); + char* workdir_snapshot_path = alloc_printf("%s/workdir/snapshot", outdir_path_real); + char* reuse_snapshot_path_real = realpath(getenv("NYX_REUSE_SNAPSHOT"), NULL); + + if (strcmp(workdir_snapshot_path, reuse_snapshot_path_real) == 0){ + NYX_PRE_FATAL(fsrv, "NYX_REUSE_SNAPSHOT path is located in current workdir (use another output directory)"); + } + + ck_free(reuse_snapshot_path_real); + ck_free(workdir_snapshot_path); + ck_free(outdir_path_real); fsrv->nyx_handlers->nyx_config_set_reuse_snapshot_path(nyx_config, getenv("NYX_REUSE_SNAPSHOT")); } fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_bind_cpu_id); - ck_free(x); + ck_free(workdir_path); if (fsrv->nyx_runner == NULL) { FATAL("Something went wrong ..."); } @@ -555,13 +602,13 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, switch (fsrv->nyx_handlers->nyx_exec(fsrv->nyx_runner)) { case Abort: - FATAL("Error: Nyx abort occured..."); + NYX_PRE_FATAL(fsrv, "Error: Nyx abort occured..."); break; case IoError: - FATAL("Error: QEMU-Nyx has died..."); + NYX_PRE_FATAL(fsrv, "Error: QEMU-Nyx has died..."); break; case Error: - FATAL("Error: Nyx runtime error has occured..."); + NYX_PRE_FATAL(fsrv, "Error: Nyx runtime error has occured..."); break; default: break; @@ -571,7 +618,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, /* autodict in Nyx mode */ if (!ignore_autodict) { - x = alloc_printf("%s/workdir/dump/afl_autodict.txt", fsrv->out_dir_path); + char* x = alloc_printf("%s/workdir/dump/afl_autodict.txt", fsrv->out_dir_path); int nyx_autodict_fd = open(x, O_RDONLY); ck_free(x); @@ -584,7 +631,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, u8 *dict = ck_alloc(f_len); if (dict == NULL) { - FATAL("Could not allocate %u bytes of autodictionary memory", + NYX_PRE_FATAL(fsrv, "Could not allocate %u bytes of autodictionary memory", f_len); } @@ -602,7 +649,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, } else { - FATAL( + NYX_PRE_FATAL(fsrv, "Reading autodictionary fail at position %u with %u bytes " "left.", offset, len); @@ -1289,19 +1336,7 @@ void afl_fsrv_kill(afl_forkserver_t *fsrv) { fsrv->child_pid = -1; #ifdef __linux__ - if (fsrv->nyx_mode) { - - if (fsrv->nyx_aux_string){ - free(fsrv->nyx_aux_string); - } - - /* check if we actually got a valid nyx runner */ - if (fsrv->nyx_runner) { - fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); - } - - } - + afl_nyx_runner_kill(fsrv); #endif } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 8b4fe1e5..0e380f73 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2191,14 +2191,6 @@ int main(int argc, char **argv_orig, char **envp) { if (!afl->pending_not_fuzzed || !valid_seeds) { - #ifdef __linux__ - if (afl->fsrv.nyx_mode) { - - afl->fsrv.nyx_handlers->nyx_shutdown(afl->fsrv.nyx_runner); - - } - - #endif FATAL("We need at least one valid input seed that does not crash!"); } diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 3ddebaad..832730fd 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1247,7 +1247,7 @@ int main(int argc, char **argv_orig, char **envp) { } - fsrv->out_dir_path = create_nyx_tmp_workdir(); + fsrv->nyx_use_tmp_workdir = true; fsrv->nyx_bind_cpu_id = 0; #endif @@ -1443,12 +1443,6 @@ int main(int argc, char **argv_orig, char **envp) { if (execute_testcases(in_dir) == 0) { -#ifdef __linux__ - if (fsrv->nyx_mode) { - remove_nyx_tmp_workdir(fsrv->out_dir_path); - fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); - } -#endif FATAL("could not read input testcases from %s", in_dir); } @@ -1528,11 +1522,6 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->target_path) { ck_free(fsrv->target_path); } -#ifdef __linux__ - if (fsrv->nyx_mode) { - remove_nyx_tmp_workdir(fsrv->out_dir_path); - } -#endif afl_fsrv_deinit(fsrv); diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 942525d4..98403049 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -1130,7 +1130,7 @@ int main(int argc, char **argv_orig, char **envp) { FATAL("failed to initialize libnyx.so..."); } - fsrv->out_dir_path = create_nyx_tmp_workdir(); + fsrv->nyx_use_tmp_workdir = true; fsrv->nyx_bind_cpu_id = 0; use_argv = argv + optind; @@ -1316,11 +1316,6 @@ int main(int argc, char **argv_orig, char **envp) { OKF("We're done here. Have a nice day!\n"); -#ifdef __linux__ - if (fsrv->nyx_mode) { - remove_nyx_tmp_workdir(fsrv->out_dir_path); - } -#endif remove_shm = 0; afl_shm_deinit(&shm); From 6d4234b3056bec79376c45b8ab40e4d6fb64df04 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Sun, 16 Apr 2023 05:14:32 +0200 Subject: [PATCH 041/149] bump libnyx version --- nyx_mode/LIBNYX_VERSION | 2 +- nyx_mode/libnyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nyx_mode/LIBNYX_VERSION b/nyx_mode/LIBNYX_VERSION index 461499ec..86b32eec 100644 --- a/nyx_mode/LIBNYX_VERSION +++ b/nyx_mode/LIBNYX_VERSION @@ -1 +1 @@ -2822aa1 +2da7f08 diff --git a/nyx_mode/libnyx b/nyx_mode/libnyx index 2822aa1b..2da7f08b 160000 --- a/nyx_mode/libnyx +++ b/nyx_mode/libnyx @@ -1 +1 @@ -Subproject commit 2822aa1b14c5e7e43343abf4c988c4b50f90faf9 +Subproject commit 2da7f08b6e0267ccfe64e1320b24cdb29223459c From d213071e13b1720d06e1a960015db198f363aab5 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Sun, 16 Apr 2023 05:16:01 +0200 Subject: [PATCH 042/149] bump packer version --- nyx_mode/PACKER_VERSION | 2 +- nyx_mode/packer | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nyx_mode/PACKER_VERSION b/nyx_mode/PACKER_VERSION index a8ebe13a..7db88233 100644 --- a/nyx_mode/PACKER_VERSION +++ b/nyx_mode/PACKER_VERSION @@ -1 +1 @@ -86b159b +202bace diff --git a/nyx_mode/packer b/nyx_mode/packer index 86b159ba..202bace8 160000 --- a/nyx_mode/packer +++ b/nyx_mode/packer @@ -1 +1 @@ -Subproject commit 86b159bafc0b2ba8feeaa8761a45b6201d34084f +Subproject commit 202bace888d237e4e8f4507d0eba6791a811554d From 61aeb4486310b4aab66558bd21ead8b6e35501e2 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Sun, 16 Apr 2023 05:19:09 +0200 Subject: [PATCH 043/149] remove redundant access() call --- src/afl-common.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/afl-common.c b/src/afl-common.c index fe0db94d..5692e277 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -1390,10 +1390,8 @@ void remove_nyx_tmp_workdir(afl_forkserver_t *fsrv, char* nyx_out_dir_path) { } } - if (access(nyx_out_dir_path, R_OK) == 0) { - if (rmdir(nyx_out_dir_path)) { - WARNF("Unable to remove nyx workdir (%s)", nyx_out_dir_path); - } + if (rmdir(nyx_out_dir_path)) { + WARNF("Unable to remove nyx workdir (%s)", nyx_out_dir_path); } ck_free(workdir_path); From 0a699d885b513dab06e5be1b655ed03f6a8d592f Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Sun, 16 Apr 2023 06:19:39 +0200 Subject: [PATCH 044/149] add some documentation --- nyx_mode/README.md | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/nyx_mode/README.md b/nyx_mode/README.md index 11698df9..878b2acf 100644 --- a/nyx_mode/README.md +++ b/nyx_mode/README.md @@ -116,11 +116,35 @@ afl-fuzz -i in -o out -Y -S 2 -- ./PACKAGE-DIRECTORY ## AFL++ companion tools (afl-showmap etc.) -Please note that AFL++ companion tools like afl-cmin, afl-showmap, etc. are -not supported with Nyx mode, only afl-fuzz. +AFL++ companion tools support Nyx mode and can be used to analyze or minimize one specific input or an entire output corpus. These tools work similarly to `afl-fuzz`. -For source based instrumentation just use these tools normally, for -binary-only targets use with -Q for qemu_mode. +To run a target with one of these tools, add the `-X` parameter to the command line to enable Nyx mode, and pass the path to a Nyx package directory: + +```shell +afl-tmin -i in_file -o out_file -X -- ./PACKAGE-DIRECTORY +``` + +```shell +afl-analyze -i in_file -X -- ./PACKAGE-DIRECTORY +``` + +```shell +afl-showmap -i in_dir -o out_file -X -- ./PACKAGE-DIRECTORY +``` + +```shell +afl-cmin -i in_dir -o out_dir -X -- ./PACKAGE-DIRECTORY +``` + +On each program startup of one the AFL++ tools in Nyx mode, a Nyx VM is spawned, and a bootstrapping procedure is performed inside the VM to prepare the target environment. As a consequence, due to the bootstrapping procedure, the launch performance is much slower compared to other modes. However, this can be optimized by reusing an existing fuzzing snapshot to avoid the slow re-execution of the bootstrap procedure. + +A fuzzing snapshot is automatically created and stored in the output directory at `out_dir/workdir/snapshot/` by the first parent process of `afl-fuzz` if parallel mode is used. To enable this feature, set the path to an existing snapshot directory in the `NYX_REUSE_SNAPSHOT` environment variable and use the tools as usual: + +```shell +afl-fuzz -i ./in_dir -o ./out_dir -Y -M 0 ./PACKAGE-DIRECTORY + +NYX_REUSE_SNAPSHOT=./out_dir/workdir/snapshot/ afl-analyze -i in_file -X -- ./PACKAGE-DIRECTORY +``` ## Real-world examples From d0b86bf05563dea686d27f14972f448b6f33023b Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Sun, 16 Apr 2023 06:23:38 +0200 Subject: [PATCH 045/149] pass absolute paths to libnyx --- src/afl-forkserver.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 0e705c63..fd4e213d 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -518,7 +518,11 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (fsrv->out_dir_path == NULL) { NYX_PRE_FATAL(fsrv, "Nyx workdir path not found..."); } } - char *workdir_path = alloc_printf("%s/workdir", fsrv->out_dir_path); + /* libnyx expects an absolute path */ + char* outdir_path_absolute = realpath(fsrv->out_dir_path, NULL); + if (outdir_path_absolute == NULL) { NYX_PRE_FATAL(fsrv, "Nyx workdir path cannot be resolved ..."); } + + char *workdir_path = alloc_printf("%s/workdir", outdir_path_absolute); if (fsrv->nyx_id == 0xFFFFFFFF) {NYX_PRE_FATAL(fsrv, "Nyx ID is not set..."); } @@ -557,8 +561,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, /* another sanity check to avoid passing a snapshot directory that is * located in the current workdir (the workdir will be wiped by libnyx on startup) */ - char* outdir_path_real = realpath(fsrv->out_dir_path, NULL); - char* workdir_snapshot_path = alloc_printf("%s/workdir/snapshot", outdir_path_real); + char* workdir_snapshot_path = alloc_printf("%s/workdir/snapshot", outdir_path_absolute); char* reuse_snapshot_path_real = realpath(getenv("NYX_REUSE_SNAPSHOT"), NULL); if (strcmp(workdir_snapshot_path, reuse_snapshot_path_real) == 0){ @@ -567,7 +570,6 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, ck_free(reuse_snapshot_path_real); ck_free(workdir_snapshot_path); - ck_free(outdir_path_real); fsrv->nyx_handlers->nyx_config_set_reuse_snapshot_path(nyx_config, getenv("NYX_REUSE_SNAPSHOT")); } @@ -575,6 +577,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_bind_cpu_id); ck_free(workdir_path); + ck_free(outdir_path_absolute); if (fsrv->nyx_runner == NULL) { FATAL("Something went wrong ..."); } From 56f7e3aa088e715b054f10c01b6b5a7e5acf8931 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 16 Apr 2023 12:42:32 +0200 Subject: [PATCH 046/149] hidden -Y option for nyx, code format --- afl-cmin | 10 +- afl-cmin.bash | 30 ++++-- include/common.h | 4 +- include/forkserver.h | 13 ++- src/afl-analyze.c | 29 +++--- src/afl-common.c | 26 +++-- src/afl-forkserver.c | 155 +++++++++++++++++++++--------- src/afl-showmap.c | 54 +++++++---- src/afl-tmin.c | 27 +++--- utils/aflpp_driver/aflpp_driver.c | 23 ++--- 10 files changed, 245 insertions(+), 126 deletions(-) diff --git a/afl-cmin b/afl-cmin index 12791584..c5e64410 100755 --- a/afl-cmin +++ b/afl-cmin @@ -124,9 +124,9 @@ function usage() { "AFL_FORKSRV_INIT_TMOUT: time the fuzzer waits for the forkserver to come up\n" \ "AFL_KEEP_TRACES: leave the temporary /.traces directory\n" \ "AFL_KILL_SIGNAL: Signal delivered to child processes on timeout (default: SIGKILL)\n" \ -"AFL_FORK_SERVER_KILL_SIGNAL: Signal delivered to fork server processes on termination\n" \ -" (default: SIGTERM). If this is not set and AFL_KILL_SIGNAL is set,\n" \ -" this will be set to the same value as AFL_KILL_SIGNAL.\n" \ +"AFL_FORK_SERVER_KILL_SIGNAL: Signal delivered to fork server processes on\n" \ +" termination (default: SIGTERM). If this is not set and AFL_KILL_SIGNAL is\n" \ +" set, this will be set to the same value as AFL_KILL_SIGNAL.\n" \ "AFL_NO_FORKSRV: run target via execve instead of using the forkserver\n" \ "AFL_CMIN_ALLOW_ANY: write tuples for crashing inputs also\n" \ "AFL_PATH: path for the afl-showmap binary if not found anywhere in PATH\n" \ @@ -157,7 +157,7 @@ BEGIN { # process options Opterr = 1 # default is to diagnose Optind = 1 # skip ARGV[0] - while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUX?")) != -1) { + while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUXY?")) != -1) { if (_go_c == "i") { if (!Optarg) usage() if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} @@ -219,7 +219,7 @@ BEGIN { unicorn_mode = 1 continue } else - if (_go_c == "X") { + if (_go_c == "X" || _go_c == "Y") { if (nyx_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} extra_par = extra_par " -X" nyx_mode = 1 diff --git a/afl-cmin.bash b/afl-cmin.bash index 10c9477a..bcf62eba 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -53,7 +53,7 @@ unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN \ export AFL_QUIET=1 -while getopts "+i:o:f:m:t:eOQUACh" opt; do +while getopts "+i:o:f:m:t:eOQUAChXY" opt; do case "$opt" in @@ -94,6 +94,14 @@ while getopts "+i:o:f:m:t:eOQUACh" opt; do EXTRA_PAR="$EXTRA_PAR -Q" QEMU_MODE=1 ;; + "Y") + EXTRA_PAR="$EXTRA_PAR -X" + NYX_MODE=1 + ;; + "X") + EXTRA_PAR="$EXTRA_PAR -X" + NYX_MODE=1 + ;; "U") EXTRA_PAR="$EXTRA_PAR -U" UNICORN_MODE=1 @@ -128,6 +136,7 @@ Execution control settings: -O - use binary-only instrumentation (FRIDA mode) -Q - use binary-only instrumentation (QEMU mode) -U - use unicorn-based instrumentation (Unicorn mode) + -X - use Nyx mode Minimization settings: @@ -206,17 +215,20 @@ if [ ! "$TIMEOUT" = "none" ]; then fi -if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then +if [ "$NYX_MODE" = "" ]; then + if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then - TNEW="`which "$TARGET_BIN" 2>/dev/null`" + TNEW="`which "$TARGET_BIN" 2>/dev/null`" + + if [ ! -f "$TNEW" -o ! -x "$TNEW" ]; then + echo "[-] Error: binary '$TARGET_BIN' not found or not executable." 1>&2 + exit 1 + fi + + TARGET_BIN="$TNEW" - if [ ! -f "$TNEW" -o ! -x "$TNEW" ]; then - echo "[-] Error: binary '$TARGET_BIN' not found or not executable." 1>&2 - exit 1 fi - TARGET_BIN="$TNEW" - fi grep -aq AFL_DUMP_MAP_SIZE "./$TARGET_BIN" && { @@ -228,7 +240,7 @@ grep -aq AFL_DUMP_MAP_SIZE "./$TARGET_BIN" && { } } -if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$FRIDA_MODE" = "" -a "$UNICORN_MODE" = "" ]; then +if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$FRIDA_MODE" = "" -a "$UNICORN_MODE" = "" -a "$NYX_MODE" = "" ]; then if ! grep -qF "__AFL_SHM_ID" "$TARGET_BIN"; then echo "[-] Error: binary '$TARGET_BIN' doesn't appear to be instrumented." 1>&2 diff --git a/include/common.h b/include/common.h index e03566de..8d85d201 100644 --- a/include/common.h +++ b/include/common.h @@ -149,8 +149,8 @@ void *afl_memmem(const void *haystack, size_t haystacklen, const void *needle, #ifdef __linux__ /* Nyx helper functions to create and remove tmp workdirs */ -char* create_nyx_tmp_workdir(void); -void remove_nyx_tmp_workdir(afl_forkserver_t *fsrv, char* nyx_out_dir_path); +char *create_nyx_tmp_workdir(void); +void remove_nyx_tmp_workdir(afl_forkserver_t *fsrv, char *nyx_out_dir_path); #endif #endif diff --git a/include/forkserver.h b/include/forkserver.h index ba280d38..f5069ce2 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -52,20 +52,25 @@ typedef enum NyxReturnValue { } NyxReturnValue; typedef enum NyxProcessRole { + StandAlone, Parent, Child, + } NyxProcessRole; typedef struct { void *(*nyx_config_load)(const char *sharedir); void (*nyx_config_set_workdir_path)(void *config, const char *workdir); - void (*nyx_config_set_input_buffer_size)(void *config, uint32_t input_buffer_size); - void (*nyx_config_set_input_buffer_write_protection)(void *config, bool input_buffer_write_protection); + void (*nyx_config_set_input_buffer_size)(void *config, + uint32_t input_buffer_size); + void (*nyx_config_set_input_buffer_write_protection)( + void *config, bool input_buffer_write_protection); void (*nyx_config_set_hprintf_fd)(void *config, int32_t hprintf_fd); void (*nyx_config_set_process_role)(void *config, enum NyxProcessRole role); - void (*nyx_config_set_reuse_snapshot_path)(void *config, const char *reuse_snapshot_path); + void (*nyx_config_set_reuse_snapshot_path)(void *config, + const char *reuse_snapshot_path); void *(*nyx_new)(void *config, uint32_t worker_id); void (*nyx_shutdown)(void *qemu_process); @@ -191,7 +196,7 @@ typedef struct afl_forkserver { u32 nyx_bind_cpu_id; /* nyx runner cpu id */ char *nyx_aux_string; bool nyx_use_tmp_workdir; - char *nyx_tmp_workdir_path; + char *nyx_tmp_workdir_path; #endif } afl_forkserver_t; diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 0a4e7fb5..5b122741 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -815,7 +815,7 @@ int main(int argc, char **argv_orig, char **envp) { afl_fsrv_init(&fsrv); - while ((opt = getopt(argc, argv, "+i:f:m:t:eAOQUWXh")) > 0) { + while ((opt = getopt(argc, argv, "+i:f:m:t:eAOQUWXYh")) > 0) { switch (opt) { @@ -966,8 +966,9 @@ int main(int argc, char **argv_orig, char **envp) { fsrv.mem_limit = mem_limit; break; - - #ifdef __linux__ + + case 'Y': // fallthough +#ifdef __linux__ case 'X': /* NYX mode */ if (fsrv.nyx_mode) { FATAL("Multiple -X options not supported"); } @@ -977,11 +978,11 @@ int main(int argc, char **argv_orig, char **envp) { fsrv.nyx_standalone = true; break; - #else +#else case 'X': FATAL("Nyx mode is only availabe on linux..."); break; - #endif +#endif case 'h': usage(argv[0]); @@ -1015,12 +1016,16 @@ int main(int argc, char **argv_orig, char **envp) { set_up_environment(argv); #ifdef __linux__ - if(!fsrv.nyx_mode){ + if (!fsrv.nyx_mode) { + fsrv.target_path = find_binary(argv[optind]); - } - else{ + + } else { + fsrv.target_path = ck_strdup(argv[optind]); + } + #else fsrv.target_path = find_binary(argv[optind]); #endif @@ -1048,6 +1053,7 @@ int main(int argc, char **argv_orig, char **envp) { use_argv = get_cs_argv(argv[0], &target_path, argc - optind, argv + optind); #ifdef __linux__ + } else if (fsrv.nyx_mode) { fsrv.nyx_id = 0; @@ -1055,7 +1061,9 @@ int main(int argc, char **argv_orig, char **envp) { u8 *libnyx_binary = find_afl_binary(argv[0], "libnyx.so"); fsrv.nyx_handlers = afl_load_libnyx_plugin(libnyx_binary); if (fsrv.nyx_handlers == NULL) { + FATAL("failed to initialize libnyx.so..."); + } fsrv.nyx_use_tmp_workdir = true; @@ -1090,9 +1098,7 @@ int main(int argc, char **argv_orig, char **envp) { read_initial_file(); #ifdef __linux__ - if(!fsrv.nyx_mode){ - (void)check_binary_signatures(fsrv.target_path); - } + if (!fsrv.nyx_mode) { (void)check_binary_signatures(fsrv.target_path); } #else (void)check_binary_signatures(fsrv.target_path); #endif @@ -1119,7 +1125,6 @@ int main(int argc, char **argv_orig, char **envp) { OKF("We're done here. Have a nice day!\n"); - afl_shm_deinit(&shm); afl_fsrv_deinit(&fsrv); if (fsrv.target_path) { ck_free(fsrv.target_path); } diff --git a/src/afl-common.c b/src/afl-common.c index 5692e277..a5c48e80 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -1365,36 +1365,46 @@ s32 create_file(u8 *fn) { * etc.). This helper function basically creates both a path to a tmp workdir * and the workdir itself. If the environment variable TMPDIR is set, we use * that as the base directory, otherwise we use /tmp. */ -char* create_nyx_tmp_workdir(void) { +char *create_nyx_tmp_workdir(void) { char *tmpdir = getenv("TMPDIR"); if (!tmpdir) { tmpdir = "/tmp"; } - char* nyx_out_dir_path = alloc_printf("%s/.nyx_tmp_%d/", tmpdir, (u32)getpid()); + char *nyx_out_dir_path = + alloc_printf("%s/.nyx_tmp_%d/", tmpdir, (u32)getpid()); - if (mkdir(nyx_out_dir_path, 0700)) { - PFATAL("Unable to create nyx workdir"); - } + if (mkdir(nyx_out_dir_path, 0700)) { PFATAL("Unable to create nyx workdir"); } return nyx_out_dir_path; + } /* Vice versa, we remove the tmp workdir for nyx with this helper function. */ -void remove_nyx_tmp_workdir(afl_forkserver_t *fsrv, char* nyx_out_dir_path) { - char* workdir_path = alloc_printf("%s/workdir", nyx_out_dir_path); +void remove_nyx_tmp_workdir(afl_forkserver_t *fsrv, char *nyx_out_dir_path) { + + char *workdir_path = alloc_printf("%s/workdir", nyx_out_dir_path); if (access(workdir_path, R_OK) == 0) { - if(fsrv->nyx_handlers->nyx_remove_work_dir(workdir_path) != true) { + + if (fsrv->nyx_handlers->nyx_remove_work_dir(workdir_path) != true) { + WARNF("Unable to remove nyx workdir (%s)", workdir_path); + } + } if (rmdir(nyx_out_dir_path)) { + WARNF("Unable to remove nyx workdir (%s)", nyx_out_dir_path); + } ck_free(workdir_path); ck_free(nyx_out_dir_path); + } + #endif + diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index fd4e213d..aa8c8622 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -50,7 +50,7 @@ #include #ifdef __linux__ -#include + #include /* function to load nyx_helper function from libnyx.so */ @@ -66,22 +66,32 @@ nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { plugin->nyx_config_load = dlsym(handle, "nyx_config_load"); if (plugin->nyx_config_load == NULL) { goto fail; } - plugin->nyx_config_set_workdir_path = dlsym(handle, "nyx_config_set_workdir_path"); + plugin->nyx_config_set_workdir_path = + dlsym(handle, "nyx_config_set_workdir_path"); if (plugin->nyx_config_set_workdir_path == NULL) { goto fail; } - plugin->nyx_config_set_input_buffer_size = dlsym(handle, "nyx_config_set_input_buffer_size"); + plugin->nyx_config_set_input_buffer_size = + dlsym(handle, "nyx_config_set_input_buffer_size"); if (plugin->nyx_config_set_input_buffer_size == NULL) { goto fail; } - plugin->nyx_config_set_input_buffer_write_protection = dlsym(handle, "nyx_config_set_input_buffer_write_protection"); - if (plugin->nyx_config_set_input_buffer_write_protection == NULL) { goto fail; } + plugin->nyx_config_set_input_buffer_write_protection = + dlsym(handle, "nyx_config_set_input_buffer_write_protection"); + if (plugin->nyx_config_set_input_buffer_write_protection == NULL) { - plugin->nyx_config_set_hprintf_fd = dlsym(handle, "nyx_config_set_hprintf_fd"); + goto fail; + + } + + plugin->nyx_config_set_hprintf_fd = + dlsym(handle, "nyx_config_set_hprintf_fd"); if (plugin->nyx_config_set_hprintf_fd == NULL) { goto fail; } - plugin->nyx_config_set_process_role = dlsym(handle, "nyx_config_set_process_role"); + plugin->nyx_config_set_process_role = + dlsym(handle, "nyx_config_set_process_role"); if (plugin->nyx_config_set_process_role == NULL) { goto fail; } - plugin->nyx_config_set_reuse_snapshot_path = dlsym(handle, "nyx_config_set_reuse_snapshot_path"); + plugin->nyx_config_set_reuse_snapshot_path = + dlsym(handle, "nyx_config_set_reuse_snapshot_path"); if (plugin->nyx_config_set_reuse_snapshot_path == NULL) { goto fail; } plugin->nyx_new = dlsym(handle, "nyx_new"); @@ -119,7 +129,6 @@ nyx_plugin_handler_t *afl_load_libnyx_plugin(u8 *libnyx_binary) { plugin->nyx_remove_work_dir = dlsym(handle, "nyx_remove_work_dir"); if (plugin->nyx_remove_work_dir == NULL) { goto fail; } - OKF("libnyx plugin is ready!"); return plugin; @@ -131,33 +140,40 @@ fail: } -void afl_nyx_runner_kill(afl_forkserver_t *fsrv){ +void afl_nyx_runner_kill(afl_forkserver_t *fsrv) { + if (fsrv->nyx_mode) { - if (fsrv->nyx_aux_string){ - ck_free(fsrv->nyx_aux_string); - } + if (fsrv->nyx_aux_string) { ck_free(fsrv->nyx_aux_string); } /* check if we actually got a valid nyx runner */ if (fsrv->nyx_runner) { + fsrv->nyx_handlers->nyx_shutdown(fsrv->nyx_runner); + } /* if we have use a tmp work dir we need to remove it */ if (fsrv->nyx_use_tmp_workdir && fsrv->nyx_tmp_workdir_path) { + remove_nyx_tmp_workdir(fsrv, fsrv->nyx_tmp_workdir_path); + } + } + } -/* Wrapper for FATAL() that kills the nyx runner (and removes all created tmp - * files) before exiting. Used before "afl_fsrv_killall()" is registered as - * an atexit() handler. */ -#define NYX_PRE_FATAL(fsrv, x...) \ - do { \ - afl_nyx_runner_kill(fsrv); \ - FATAL(x); \ - } while (0) + /* Wrapper for FATAL() that kills the nyx runner (and removes all created tmp + * files) before exiting. Used before "afl_fsrv_killall()" is registered as + * an atexit() handler. */ + #define NYX_PRE_FATAL(fsrv, x...) \ + do { \ + \ + afl_nyx_runner_kill(fsrv); \ + FATAL(x); \ + \ + } while (0) #endif @@ -511,70 +527,116 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if (!be_quiet) { ACTF("Spinning up the NYX backend..."); } - if (fsrv->nyx_use_tmp_workdir){ + if (fsrv->nyx_use_tmp_workdir) { + fsrv->nyx_tmp_workdir_path = create_nyx_tmp_workdir(); fsrv->out_dir_path = fsrv->nyx_tmp_workdir_path; + } else { - if (fsrv->out_dir_path == NULL) { NYX_PRE_FATAL(fsrv, "Nyx workdir path not found..."); } + + if (fsrv->out_dir_path == NULL) { + + NYX_PRE_FATAL(fsrv, "Nyx workdir path not found..."); + + } + } /* libnyx expects an absolute path */ - char* outdir_path_absolute = realpath(fsrv->out_dir_path, NULL); - if (outdir_path_absolute == NULL) { NYX_PRE_FATAL(fsrv, "Nyx workdir path cannot be resolved ..."); } + char *outdir_path_absolute = realpath(fsrv->out_dir_path, NULL); + if (outdir_path_absolute == NULL) { + + NYX_PRE_FATAL(fsrv, "Nyx workdir path cannot be resolved ..."); + + } char *workdir_path = alloc_printf("%s/workdir", outdir_path_absolute); - if (fsrv->nyx_id == 0xFFFFFFFF) {NYX_PRE_FATAL(fsrv, "Nyx ID is not set..."); } + if (fsrv->nyx_id == 0xFFFFFFFF) { + + NYX_PRE_FATAL(fsrv, "Nyx ID is not set..."); - if (fsrv->nyx_bind_cpu_id == 0xFFFFFFFF) { - NYX_PRE_FATAL(fsrv, "Nyx CPU ID is not set..."); } - void* nyx_config = fsrv->nyx_handlers->nyx_config_load(fsrv->target_path); + if (fsrv->nyx_bind_cpu_id == 0xFFFFFFFF) { + + NYX_PRE_FATAL(fsrv, "Nyx CPU ID is not set..."); + + } + + void *nyx_config = fsrv->nyx_handlers->nyx_config_load(fsrv->target_path); fsrv->nyx_handlers->nyx_config_set_workdir_path(nyx_config, workdir_path); fsrv->nyx_handlers->nyx_config_set_input_buffer_size(nyx_config, MAX_FILE); - fsrv->nyx_handlers->nyx_config_set_input_buffer_write_protection(nyx_config, true); + fsrv->nyx_handlers->nyx_config_set_input_buffer_write_protection(nyx_config, + true); if (fsrv->nyx_standalone) { + fsrv->nyx_handlers->nyx_config_set_process_role(nyx_config, StandAlone); + } else { + if (fsrv->nyx_parent) { + fsrv->nyx_handlers->nyx_config_set_process_role(nyx_config, Parent); + } else { + fsrv->nyx_handlers->nyx_config_set_process_role(nyx_config, Child); + } + } - if (getenv("NYX_REUSE_SNAPSHOT") != NULL){ + if (getenv("NYX_REUSE_SNAPSHOT") != NULL) { if (access(getenv("NYX_REUSE_SNAPSHOT"), F_OK) == -1) { + NYX_PRE_FATAL(fsrv, "NYX_REUSE_SNAPSHOT path does not exist"); + } - /* stupid sanity check to avoid passing an empty or invalid snapshot directory */ - char* snapshot_file_path = alloc_printf("%s/global.state", getenv("NYX_REUSE_SNAPSHOT")); + /* stupid sanity check to avoid passing an empty or invalid snapshot + * directory */ + char *snapshot_file_path = + alloc_printf("%s/global.state", getenv("NYX_REUSE_SNAPSHOT")); if (access(snapshot_file_path, R_OK) == -1) { - NYX_PRE_FATAL(fsrv, "NYX_REUSE_SNAPSHOT path does not contain a valid Nyx snapshot"); + + NYX_PRE_FATAL( + fsrv, + "NYX_REUSE_SNAPSHOT path does not contain a valid Nyx snapshot"); + } + ck_free(snapshot_file_path); /* another sanity check to avoid passing a snapshot directory that is - * located in the current workdir (the workdir will be wiped by libnyx on startup) */ - char* workdir_snapshot_path = alloc_printf("%s/workdir/snapshot", outdir_path_absolute); - char* reuse_snapshot_path_real = realpath(getenv("NYX_REUSE_SNAPSHOT"), NULL); + * located in the current workdir (the workdir will be wiped by libnyx on + * startup) */ + char *workdir_snapshot_path = + alloc_printf("%s/workdir/snapshot", outdir_path_absolute); + char *reuse_snapshot_path_real = + realpath(getenv("NYX_REUSE_SNAPSHOT"), NULL); + + if (strcmp(workdir_snapshot_path, reuse_snapshot_path_real) == 0) { + + NYX_PRE_FATAL(fsrv, + "NYX_REUSE_SNAPSHOT path is located in current workdir " + "(use another output directory)"); - if (strcmp(workdir_snapshot_path, reuse_snapshot_path_real) == 0){ - NYX_PRE_FATAL(fsrv, "NYX_REUSE_SNAPSHOT path is located in current workdir (use another output directory)"); } ck_free(reuse_snapshot_path_real); ck_free(workdir_snapshot_path); - fsrv->nyx_handlers->nyx_config_set_reuse_snapshot_path(nyx_config, getenv("NYX_REUSE_SNAPSHOT")); + fsrv->nyx_handlers->nyx_config_set_reuse_snapshot_path( + nyx_config, getenv("NYX_REUSE_SNAPSHOT")); + } - fsrv->nyx_runner = fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_bind_cpu_id); + fsrv->nyx_runner = + fsrv->nyx_handlers->nyx_new(nyx_config, fsrv->nyx_bind_cpu_id); ck_free(workdir_path); ck_free(outdir_path_absolute); @@ -621,7 +683,8 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, /* autodict in Nyx mode */ if (!ignore_autodict) { - char* x = alloc_printf("%s/workdir/dump/afl_autodict.txt", fsrv->out_dir_path); + char *x = + alloc_printf("%s/workdir/dump/afl_autodict.txt", fsrv->out_dir_path); int nyx_autodict_fd = open(x, O_RDONLY); ck_free(x); @@ -634,8 +697,9 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, u8 *dict = ck_alloc(f_len); if (dict == NULL) { - NYX_PRE_FATAL(fsrv, "Could not allocate %u bytes of autodictionary memory", - f_len); + NYX_PRE_FATAL( + fsrv, "Could not allocate %u bytes of autodictionary memory", + f_len); } @@ -652,7 +716,8 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, } else { - NYX_PRE_FATAL(fsrv, + NYX_PRE_FATAL( + fsrv, "Reading autodictionary fail at position %u with %u bytes " "left.", offset, len); diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 832730fd..df030672 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -444,8 +444,11 @@ static void showmap_run_target_nyx_mode(afl_forkserver_t *fsrv) { FSRV_RUN_ERROR) { FATAL("Error running target in Nyx mode"); + } + } + #endif /* Execute target application. */ @@ -890,7 +893,7 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_QUIET") != NULL) { be_quiet = true; } - while ((opt = getopt(argc, argv, "+i:o:f:m:t:AeqCZOH:QUWbcrshX")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:AeqCZOH:QUWbcrshXY")) > 0) { switch (opt) { @@ -1078,7 +1081,8 @@ int main(int argc, char **argv_orig, char **envp) { break; - #ifdef __linux__ + case 'Y': // fallthough +#ifdef __linux__ case 'X': /* NYX mode */ if (fsrv->nyx_mode) { FATAL("Multiple -X options not supported"); } @@ -1088,11 +1092,11 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->nyx_standalone = true; break; - #else +#else case 'X': FATAL("Nyx mode is only availabe on linux..."); break; - #endif +#endif case 'b': @@ -1166,12 +1170,16 @@ int main(int argc, char **argv_orig, char **envp) { set_up_environment(fsrv, argv); #ifdef __linux__ - if(!fsrv->nyx_mode){ + if (!fsrv->nyx_mode) { + fsrv->target_path = find_binary(argv[optind]); - } - else{ + + } else { + fsrv->target_path = ck_strdup(argv[optind]); + } + #else fsrv->target_path = find_binary(argv[optind]); #endif @@ -1232,11 +1240,12 @@ int main(int argc, char **argv_orig, char **envp) { get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); #ifdef __linux__ + } else if (fsrv->nyx_mode) { use_argv = ck_alloc(sizeof(char *) * (1)); use_argv[0] = argv[0]; - + fsrv->nyx_id = 0; u8 *libnyx_binary = find_afl_binary(use_argv[0], "libnyx.so"); @@ -1288,9 +1297,12 @@ int main(int argc, char **argv_orig, char **envp) { } #ifdef __linux__ - if(!fsrv->nyx_mode && in_dir){ + if (!fsrv->nyx_mode && in_dir) { + (void)check_binary_signatures(fsrv->target_path); + } + #else if (in_dir) { (void)check_binary_signatures(fsrv->target_path); } #endif @@ -1313,14 +1325,14 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->shmem_fuzz_len = (u32 *)map; fsrv->shmem_fuzz = map + sizeof(u32); - configure_afl_kill_signals( - fsrv, NULL, NULL, (fsrv->qemu_mode || unicorn_mode - #ifdef __linux__ - || fsrv->nyx_mode - #endif - ) - ? SIGKILL - : SIGTERM); + configure_afl_kill_signals(fsrv, NULL, NULL, + (fsrv->qemu_mode || unicorn_mode +#ifdef __linux__ + || fsrv->nyx_mode +#endif + ) + ? SIGKILL + : SIGTERM); if (!fsrv->cs_mode && !fsrv->qemu_mode && !unicorn_mode) { @@ -1464,13 +1476,18 @@ int main(int argc, char **argv_orig, char **envp) { shm_fuzz = deinit_shmem(fsrv, shm_fuzz); #ifdef __linux__ - if(!fsrv->nyx_mode){ + if (!fsrv->nyx_mode) { + #endif showmap_run_target(fsrv, use_argv); #ifdef __linux__ + } else { + showmap_run_target_nyx_mode(fsrv); + } + #endif tcnt = write_results_to_file(fsrv, out_file); if (!quiet_mode) { @@ -1522,7 +1539,6 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->target_path) { ck_free(fsrv->target_path); } - afl_fsrv_deinit(fsrv); if (stdin_file) { ck_free(stdin_file); } diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 98403049..e7442d1d 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -846,7 +846,7 @@ int main(int argc, char **argv_orig, char **envp) { SAYF(cCYA "afl-tmin" VERSION cRST " by Michal Zalewski\n"); - while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeAOQUWXHh")) > 0) { + while ((opt = getopt(argc, argv, "+i:o:f:m:t:B:xeAOQUWXYHh")) > 0) { switch (opt) { @@ -1004,7 +1004,8 @@ int main(int argc, char **argv_orig, char **envp) { break; - #ifdef __linux__ + case 'Y': // fallthough +#ifdef __linux__ case 'X': /* NYX mode */ if (fsrv->nyx_mode) { FATAL("Multiple -X options not supported"); } @@ -1014,11 +1015,11 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->nyx_standalone = true; break; - #else +#else case 'X': FATAL("Nyx mode is only availabe on linux..."); break; - #endif +#endif case 'H': /* Hang Mode */ @@ -1086,12 +1087,16 @@ int main(int argc, char **argv_orig, char **envp) { set_up_environment(fsrv, argv); #ifdef __linux__ - if(!fsrv->nyx_mode){ + if (!fsrv->nyx_mode) { + fsrv->target_path = find_binary(argv[optind]); - } - else{ + + } else { + fsrv->target_path = ck_strdup(argv[optind]); + } + #else fsrv->target_path = find_binary(argv[optind]); #endif @@ -1120,6 +1125,7 @@ int main(int argc, char **argv_orig, char **envp) { get_cs_argv(argv[0], &fsrv->target_path, argc - optind, argv + optind); #ifdef __linux__ + } else if (fsrv->nyx_mode) { fsrv->nyx_id = 0; @@ -1127,7 +1133,9 @@ int main(int argc, char **argv_orig, char **envp) { u8 *libnyx_binary = find_afl_binary(argv[0], "libnyx.so"); fsrv->nyx_handlers = afl_load_libnyx_plugin(libnyx_binary); if (fsrv->nyx_handlers == NULL) { + FATAL("failed to initialize libnyx.so..."); + } fsrv->nyx_use_tmp_workdir = true; @@ -1207,9 +1215,7 @@ int main(int argc, char **argv_orig, char **envp) { read_initial_file(); #ifdef __linux__ - if(!fsrv->nyx_mode){ - (void)check_binary_signatures(fsrv->target_path); - } + if (!fsrv->nyx_mode) { (void)check_binary_signatures(fsrv->target_path); } #else (void)check_binary_signatures(fsrv->target_path); #endif @@ -1316,7 +1322,6 @@ int main(int argc, char **argv_orig, char **envp) { OKF("We're done here. Have a nice day!\n"); - remove_shm = 0; afl_shm_deinit(&shm); if (fsrv->use_shmem_fuzz) shm_fuzz = deinit_shmem(fsrv, shm_fuzz); diff --git a/utils/aflpp_driver/aflpp_driver.c b/utils/aflpp_driver/aflpp_driver.c index 34294809..4e8f466d 100644 --- a/utils/aflpp_driver/aflpp_driver.c +++ b/utils/aflpp_driver/aflpp_driver.c @@ -78,10 +78,10 @@ extern unsigned int __afl_map_size; on the other hand this is what Google needs to make LLVMFuzzerRunDriver() work. Choose your poison Google! */ /*__attribute__((weak))*/ int LLVMFuzzerTestOneInput(const uint8_t *Data, - size_t Size); -__attribute__((weak)) int LLVMFuzzerInitialize(int *argc, char ***argv); -__attribute__((weak)) int LLVMFuzzerRunDriver( - int *argc, char ***argv, int (*callback)(const uint8_t *data, size_t size)); + size_t Size); +__attribute__((weak)) int LLVMFuzzerInitialize(int *argc, char ***argv); +__attribute__((weak)) int LLVMFuzzerRunDriver( + int *argc, char ***argv, int (*callback)(const uint8_t *data, size_t size)); // Default nop ASan hooks for manual poisoning when not linking the ASan // runtime @@ -268,15 +268,16 @@ static int ExecuteFilesOnyByOne(int argc, char **argv, __attribute__((weak)) int main(int argc, char **argv) { -// Enable if LLVMFuzzerTestOneInput() has the weak attribute -/* - if (!LLVMFuzzerTestOneInput) { + // Enable if LLVMFuzzerTestOneInput() has the weak attribute + /* + if (!LLVMFuzzerTestOneInput) { - fprintf(stderr, "Error: function LLVMFuzzerTestOneInput() not found!\n"); - abort(); + fprintf(stderr, "Error: function LLVMFuzzerTestOneInput() not found!\n"); + abort(); - } -*/ + } + + */ if (argc < 2 || strncmp(argv[1], "-h", 2) == 0) printf( From 4d29e484b7b2fcc4ff25774888c8fcbd1a1618c1 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 16 Apr 2023 18:47:50 +0200 Subject: [PATCH 047/149] update nyx readme --- nyx_mode/README.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nyx_mode/README.md b/nyx_mode/README.md index 878b2acf..eee7d363 100644 --- a/nyx_mode/README.md +++ b/nyx_mode/README.md @@ -15,6 +15,7 @@ Underneath it is built upon KVM and QEMU and requires a modern Linux kernel requires an Intel processor (6th generation onwards) and a special 5.10 kernel (see [KVM-Nyx](https://github.com/nyx-fuzz/KVM-Nyx)). + ## Building Nyx mode 1. Install all the packages from [docs/INSTALL.md](../docs/INSTALL.md). @@ -41,6 +42,7 @@ requires an Intel processor (6th generation onwards) and a special 5.10 kernel 5. Optionally, for binary-only fuzzing: set up the required 5.10 kernel, see [KVM-Nyx](https://github.com/nyx-fuzz/KVM-Nyx). + ## Preparing to fuzz a target with Nyx mode For source instrumented fuzzing you can use any afl-cc mode, with LTO even @@ -68,12 +70,21 @@ This will create a directory with all necessary files and the Nyx configuration. The name of the directory will be whatever you choose for `PACKAGE-DIRECTORY` above. -In the final step for the packaging we generate the Nyx configuration: +Note that if the target reads from a file then use the `-file /path/to/file` +parameter to the above command. + +Note that Nyx does **not** support the afl `@@` argument. Instead pass +something like `-file /foo.file -args "--file /foo.file --other-args"` to +the above command. + + +Then the final step: we generate the Nyx package configuration: ```shell python3 nyx_mode/packer/packer/nyx_config_gen.py PACKAGE-DIRECTORY Kernel ``` + ## Fuzzing with Nyx mode All the hard parts are done, fuzzing with Nyx mode is easy - just supply the @@ -114,6 +125,7 @@ afl-fuzz -i in -o out -Y -S 1 -- ./PACKAGE-DIRECTORY afl-fuzz -i in -o out -Y -S 2 -- ./PACKAGE-DIRECTORY ``` + ## AFL++ companion tools (afl-showmap etc.) AFL++ companion tools support Nyx mode and can be used to analyze or minimize one specific input or an entire output corpus. These tools work similarly to `afl-fuzz`. @@ -146,6 +158,7 @@ afl-fuzz -i ./in_dir -o ./out_dir -Y -M 0 ./PACKAGE-DIRECTORY NYX_REUSE_SNAPSHOT=./out_dir/workdir/snapshot/ afl-analyze -i in_file -X -- ./PACKAGE-DIRECTORY ``` + ## Real-world examples ### Fuzzing libxml2 with AFL++ in Nyx-mode From 7f734c96d187312868178e8ead95dc103c557c1f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 17 Apr 2023 10:25:10 +0200 Subject: [PATCH 048/149] v4.06c release --- README.md | 4 ++-- docs/Changelog.md | 5 +++-- include/config.h | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 821b8cb7..c012c400 100644 --- a/README.md +++ b/README.md @@ -2,9 +2,9 @@ AFL++ logo -Release version: [4.05c](https://github.com/AFLplusplus/AFLplusplus/releases) +Release version: [4.06c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.06a +GitHub version: 4.07a Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) diff --git a/docs/Changelog.md b/docs/Changelog.md index 3337feb9..587fb64c 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -3,7 +3,7 @@ This is the list of all noteworthy changes made in every public release of the tool. See README.md for the general instruction manual. -### Version ++4.06a (dev) +### Version ++4.06c (release) - afl-fuzz: - ensure temporary file descriptor is closed when not used - added `AFL_NO_WARN_INSTABILITY` @@ -29,9 +29,10 @@ - fix issue on MacOS - unicorn_mode: - updated and minor issues fixed + - nyx_mode support for all tools + - better sanitizer default options support for all tools - new custom module: autotoken, a grammar free fuzzer for text inputs - fixed custom mutator C examples - - better sanitizer default options support for all tools - more minor fixes and cross-platform support ### Version ++4.05c (release) diff --git a/include/config.h b/include/config.h index e46f515a..b6249a0f 100644 --- a/include/config.h +++ b/include/config.h @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.06a" +#define VERSION "++4.06c" /****************************************************** * * From 02b9e583f2a5dd7d83bd4c02af8d2081532689ed Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 17 Apr 2023 14:41:05 +0200 Subject: [PATCH 049/149] v4.07a init --- docs/Changelog.md | 4 ++++ include/config.h | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 587fb64c..30e76f2c 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -3,6 +3,10 @@ This is the list of all noteworthy changes made in every public release of the tool. See README.md for the general instruction manual. +### Version ++4.07a (dev) + - soon :) + + ### Version ++4.06c (release) - afl-fuzz: - ensure temporary file descriptor is closed when not used diff --git a/include/config.h b/include/config.h index b6249a0f..764c29dc 100644 --- a/include/config.h +++ b/include/config.h @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.06c" +#define VERSION "++4.07a" /****************************************************** * * From 70e30958649f44e104330431cde31e80fbd8557f Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:09:48 +0200 Subject: [PATCH 050/149] added first dummy atnwalk.c file --- custom_mutators/atnwalk/atnwalk.c | 173 ++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 custom_mutators/atnwalk/atnwalk.c diff --git a/custom_mutators/atnwalk/atnwalk.c b/custom_mutators/atnwalk/atnwalk.c new file mode 100644 index 00000000..483ae542 --- /dev/null +++ b/custom_mutators/atnwalk/atnwalk.c @@ -0,0 +1,173 @@ +#include "../../include/afl-fuzz.h" + +#include +#include +#include +#include +#include +#include + + +#define INIT_BUF_SIZE 4096 +#define SOCKET_NAME "/tmp/atnwalk.socket" + +// handshake constants +const uint8_t SERVER_ARE_YOU_ALIVE = 42; +const uint8_t SERVER_YES_I_AM_ALIVE = 213; + +// control bits +const uint8_t SERVER_DECODE_BIT = 0b00000001; +const uint8_t SERVER_ENCODE_BIT = 0b00000010; +const uint8_t SERVER_MUTATE_BIT = 0b00000100; +const uint8_t SERVER_CROSSOVER_BIT = 0b00001000; + + +typedef struct atnwalk_mutator { + uint8_t *decoded_buf; + size_t decoded_size; +} atnwalk_mutator_t; + + +int read_all(int fd, uint8_t *buf, size_t buf_size) { + int n; + size_t offset = 0; + while (offset < buf_size) { + n = read(fd, buf + offset, buf_size - offset); + if (n == -1) { + return 0; + } + offset += n; + } + return 1; +} + + +int write_all(int fd, uint8_t *buf, size_t buf_size) { + int n; + size_t offset = 0; + while (offset < buf_size) { + n = write(fd, buf + offset, buf_size - offset); + if (n == -1) { + return 0; + } + offset += n; + } + return 1; +} + + +/** + * Initialize this custom mutator + * + * @param[in] afl a pointer to the internal state object. Can be ignored for + * now. + * @param[in] seed A seed for this mutator - the same seed should always mutate + * in the same way. + * @return Pointer to the data object this custom mutator instance should use. + * There may be multiple instances of this mutator in one afl-fuzz run! + * Return NULL on error. + */ +atnwalk_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + srand(seed); + atnwalk_mutator_t *data = (atnwalk_mutator_t *) malloc(sizeof(atnwalk_mutator_t)); + if (!data) { + perror("afl_custom_init alloc"); + return NULL; + } + data->decoded_buf = (uint8_t *) malloc(INIT_BUF_SIZE); + data->decoded_size = INIT_BUF_SIZE; + return data; +} + +// TODO: implement +/** + * Perform custom mutations on a given input + * + * (Optional for now. Required in the future) + * + * @param[in] data pointer returned in afl_custom_init for this fuzz case + * @param[in] buf Pointer to input data to be mutated + * @param[in] buf_size Size of input data + * @param[out] out_buf the buffer we will work on. we can reuse *buf. NULL on + * error. + * @param[in] add_buf Buffer containing the additional test case + * @param[in] add_buf_size Size of the additional test case + * @param[in] max_size Maximum size of the mutated output. The mutation must not + * produce data larger than max_size. + * @return Size of the mutated output. + */ +size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf, + uint8_t *add_buf, size_t add_buf_size, size_t max_size) { + struct sockaddr_un addr; + int fd_socket; + ssize_t n; + uint8_t buffer[5]; + + // initialize the socket + fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd_socket == -1) { + perror("socket"); + *out_buf = NULL; + return 0; + } + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); + if (connect(fd_socket, (const struct sockaddr *) &addr, sizeof(addr)) == -1) { + perror("atnwalk server is down"); + *out_buf = NULL; + return 0; + } + + if (!write_all(fd_socket, buffer, 5)) { + perror("write to atnwalk server failed"); + *out_buf = NULL; + return 0; + } + + if (read_all(fd_socket, buffer, 5)) { + perror("read to atnwalk server failed"); + exit(EXIT_FAILURE); + } + + close(fd_socket); +} + +// TODO: implement +/** + * A post-processing function to use right before AFL writes the test case to + * disk in order to execute the target. + * + * (Optional) If this functionality is not needed, simply don't define this + * function. + * + * @param[in] data pointer returned in afl_custom_init for this fuzz case + * @param[in] buf Buffer containing the test case to be executed + * @param[in] buf_size Size of the test case + * @param[out] out_buf Pointer to the buffer containing the test case after + * processing. External library should allocate memory for out_buf. + * The buf pointer may be reused (up to the given buf_size); + * @return Size of the output buffer after processing or the needed amount. + * A return of 0 indicates an error. + */ +size_t afl_custom_post_process(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf) { + data->decoded_buf[0] = 'p'; + data->decoded_buf[1] = 'u'; + data->decoded_buf[2] = 't'; + data->decoded_buf[3] = 's'; + data->decoded_buf[4] = ' '; + data->decoded_buf[5] = ';'; + data->decoded_buf[6] = '\n'; + return 7; +} + +// TODO: implement +/** + * Deinitialize everything + * + * @param data The data ptr from afl_custom_init + */ +void afl_custom_deinit(atnwalk_mutator_t *data) { + free(data->decoded_buf); + free(data); +} From a3bc8d3440acefb7168c849337e4a4841bf11c3d Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:09:48 +0200 Subject: [PATCH 051/149] fixed wrong implementation of control bits in atnwalk.c --- custom_mutators/atnwalk/atnwalk.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/custom_mutators/atnwalk/atnwalk.c b/custom_mutators/atnwalk/atnwalk.c index 483ae542..cc9f9618 100644 --- a/custom_mutators/atnwalk/atnwalk.c +++ b/custom_mutators/atnwalk/atnwalk.c @@ -16,10 +16,10 @@ const uint8_t SERVER_ARE_YOU_ALIVE = 42; const uint8_t SERVER_YES_I_AM_ALIVE = 213; // control bits -const uint8_t SERVER_DECODE_BIT = 0b00000001; -const uint8_t SERVER_ENCODE_BIT = 0b00000010; -const uint8_t SERVER_MUTATE_BIT = 0b00000100; -const uint8_t SERVER_CROSSOVER_BIT = 0b00001000; +const uint8_t SERVER_CROSSOVER_BIT = 0b00000001; +const uint8_t SERVER_MUTATE_BIT = 0b00000010; +const uint8_t SERVER_DECODE_BIT = 0b00000100; +const uint8_t SERVER_ENCODE_BIT = 0b00001000; typedef struct atnwalk_mutator { From 450dbae8cd2ae278e0b381d172cb7694f40278cb Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:09:48 +0200 Subject: [PATCH 052/149] first version with unix domain sockets is ready for testing --- custom_mutators/atnwalk/atnwalk.c | 254 ++++++++++++++++++++++++++---- src/afl-fuzz-run.c | 11 +- 2 files changed, 232 insertions(+), 33 deletions(-) diff --git a/custom_mutators/atnwalk/atnwalk.c b/custom_mutators/atnwalk/atnwalk.c index cc9f9618..584add61 100644 --- a/custom_mutators/atnwalk/atnwalk.c +++ b/custom_mutators/atnwalk/atnwalk.c @@ -1,5 +1,4 @@ -#include "../../include/afl-fuzz.h" - +#include #include #include #include @@ -7,10 +6,10 @@ #include #include - #define INIT_BUF_SIZE 4096 #define SOCKET_NAME "/tmp/atnwalk.socket" + // handshake constants const uint8_t SERVER_ARE_YOU_ALIVE = 42; const uint8_t SERVER_YES_I_AM_ALIVE = 213; @@ -23,8 +22,10 @@ const uint8_t SERVER_ENCODE_BIT = 0b00001000; typedef struct atnwalk_mutator { - uint8_t *decoded_buf; - size_t decoded_size; + uint8_t *fuzz_buf; + size_t fuzz_size; + uint8_t *post_process_buf; + size_t post_process_size; } atnwalk_mutator_t; @@ -55,6 +56,32 @@ int write_all(int fd, uint8_t *buf, size_t buf_size) { return 1; } +void put_uint32(uint8_t *buf, uint32_t val) { + buf[0] = (uint8_t) (val >> 24); + buf[1] = (uint8_t) ((val & 0x00ff0000) >> 16); + buf[2] = (uint8_t) ((val & 0x0000ff00) >> 8); + buf[3] = (uint8_t) (val & 0x000000ff); +} + +uint32_t to_uint32(uint8_t *buf) { + uint32_t val = 0; + val |= (((uint32_t) buf[0]) << 24); + val |= (((uint32_t) buf[1]) << 16); + val |= (((uint32_t) buf[2]) << 8); + val |= ((uint32_t) buf[3]); + return val; +} + +void put_uint64(uint8_t *buf, uint64_t val) { + buf[0] = (uint8_t) (val >> 56); + buf[1] = (uint8_t) ((val & 0x00ff000000000000) >> 48); + buf[2] = (uint8_t) ((val & 0x0000ff0000000000) >> 40); + buf[3] = (uint8_t) ((val & 0x000000ff00000000) >> 32); + buf[4] = (uint8_t) ((val & 0x00000000ff000000) >> 24); + buf[5] = (uint8_t) ((val & 0x0000000000ff0000) >> 16); + buf[6] = (uint8_t) ((val & 0x000000000000ff00) >> 8); + buf[7] = (uint8_t) (val & 0x00000000000000ff); +} /** * Initialize this custom mutator @@ -67,19 +94,21 @@ int write_all(int fd, uint8_t *buf, size_t buf_size) { * There may be multiple instances of this mutator in one afl-fuzz run! * Return NULL on error. */ -atnwalk_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { +atnwalk_mutator_t *afl_custom_init(void *afl, unsigned int seed) { srand(seed); atnwalk_mutator_t *data = (atnwalk_mutator_t *) malloc(sizeof(atnwalk_mutator_t)); if (!data) { perror("afl_custom_init alloc"); return NULL; } - data->decoded_buf = (uint8_t *) malloc(INIT_BUF_SIZE); - data->decoded_size = INIT_BUF_SIZE; + data->fuzz_buf = (uint8_t *) malloc(INIT_BUF_SIZE); + data->fuzz_size = INIT_BUF_SIZE; + data->post_process_buf = (uint8_t *) malloc(INIT_BUF_SIZE); + data->post_process_size = INIT_BUF_SIZE; return data; } -// TODO: implement + /** * Perform custom mutations on a given input * @@ -100,13 +129,12 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u uint8_t *add_buf, size_t add_buf_size, size_t max_size) { struct sockaddr_un addr; int fd_socket; - ssize_t n; - uint8_t buffer[5]; + uint8_t ctrl_buf[8]; + uint8_t wanted; // initialize the socket fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); if (fd_socket == -1) { - perror("socket"); *out_buf = NULL; return 0; } @@ -114,26 +142,127 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u addr.sun_family = AF_UNIX; strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); if (connect(fd_socket, (const struct sockaddr *) &addr, sizeof(addr)) == -1) { - perror("atnwalk server is down"); + close(fd_socket); *out_buf = NULL; return 0; } - if (!write_all(fd_socket, buffer, 5)) { - perror("write to atnwalk server failed"); + // TODO: how to set connection deadline? maybe not required if server already closes the connection? + + // TODO: there should be some kind of loop retrying with different seeds and ultimately giving up on that input? + // maybe this is not necessary, because we may also just return a single byte in case of failure? + + // ask whether the server is alive + ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; + if (!write_all(fd_socket, ctrl_buf, 1)) { + close(fd_socket); *out_buf = NULL; return 0; } - if (read_all(fd_socket, buffer, 5)) { - perror("read to atnwalk server failed"); - exit(EXIT_FAILURE); + // see whether the server replies as expected + if (!read_all(fd_socket, ctrl_buf, 1) || ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { + close(fd_socket); + *out_buf = NULL; + return 0; + } + + // tell the server what we want to do + wanted = SERVER_MUTATE_BIT | SERVER_ENCODE_BIT; + + // 50% chance to perform a crossover if there is an additional buffer available + if ((add_buf_size > 0) && (rand() % 2)) { + wanted |= SERVER_CROSSOVER_BIT; + } + + // tell the server what we want and how much data will be sent + ctrl_buf[0] = wanted; + put_uint32(ctrl_buf + 1, (uint32_t) buf_size); + if (!write_all(fd_socket, ctrl_buf, 5)) { + close(fd_socket); + *out_buf = NULL; + return 0; + } + + // send the data to mutate and encode + if (!write_all(fd_socket, buf, buf_size)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + + if (wanted & SERVER_CROSSOVER_BIT) { + // since we requested crossover, we will first tell how much additional data is to be expected + put_uint32(ctrl_buf, (uint32_t) add_buf_size); + if (!write_all(fd_socket, ctrl_buf, 4)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + + // send the additional data for crossover + if (!write_all(fd_socket, add_buf, add_buf_size)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + + // lastly, a seed is required for crossover so send one + put_uint64(ctrl_buf, (uint64_t) rand()); + if (!write_all(fd_socket, ctrl_buf, 8)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + } + + // since we requested mutation, we need to provide a seed for that + put_uint64(ctrl_buf, (uint64_t) rand()); + if (!write_all(fd_socket, ctrl_buf, 8)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + + // obtain the required buffer size for the data that will be returned + if (!read_all(fd_socket, ctrl_buf, 4)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + size_t new_size = (size_t) to_uint32(ctrl_buf); + + // if the data is too large then we ignore this round + if (new_size > max_size) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + + if (new_size > buf_size) { + // buf is too small, need to use data->fuzz_buf, let's see whether we need to reallocate + if (new_size > data->fuzz_size) { + data->fuzz_size = new_size << 1; + data->fuzz_buf = (uint8_t *) realloc(data->fuzz_buf, data->fuzz_size); + } + *out_buf = data->fuzz_buf; + } else { + // new_size fits into buf, so re-use it + *out_buf = buf; + } + + // obtain the encoded data + if (!read_all(fd_socket, *out_buf, new_size)) { + close(fd_socket); + *out_buf = buf; + return buf_size; } close(fd_socket); + return new_size; } -// TODO: implement + /** * A post-processing function to use right before AFL writes the test case to * disk in order to execute the target. @@ -151,23 +280,90 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u * A return of 0 indicates an error. */ size_t afl_custom_post_process(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf) { - data->decoded_buf[0] = 'p'; - data->decoded_buf[1] = 'u'; - data->decoded_buf[2] = 't'; - data->decoded_buf[3] = 's'; - data->decoded_buf[4] = ' '; - data->decoded_buf[5] = ';'; - data->decoded_buf[6] = '\n'; - return 7; + struct sockaddr_un addr; + int fd_socket; + uint8_t ctrl_buf[8]; + + // initialize the socket + fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd_socket == -1) { + *out_buf = NULL; + return 0; + } + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); + if (connect(fd_socket, (const struct sockaddr *) &addr, sizeof(addr)) == -1) { + close(fd_socket); + *out_buf = NULL; + return 0; + } + + // ask whether the server is alive + ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; + if (!write_all(fd_socket, ctrl_buf, 1)) { + close(fd_socket); + *out_buf = NULL; + return 0; + } + + // see whether the server replies as expected + if (!read_all(fd_socket, ctrl_buf, 1) || ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { + close(fd_socket); + *out_buf = NULL; + return 0; + } + + // tell the server what we want and how much data will be sent + ctrl_buf[0] = SERVER_DECODE_BIT; + put_uint32(ctrl_buf + 1, (uint32_t) buf_size); + if (!write_all(fd_socket, ctrl_buf, 5)) { + close(fd_socket); + *out_buf = NULL; + return 0; + } + + // send the data to decode + if (!write_all(fd_socket, buf, buf_size)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + + // obtain the required buffer size for the data that will be returned + if (!read_all(fd_socket, ctrl_buf, 4)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + size_t new_size = (size_t) to_uint32(ctrl_buf); + + // need to use data->post_process_buf, let's see whether we need to reallocate + if (new_size > data->post_process_size) { + data->post_process_size = new_size << 1; + data->post_process_buf = (uint8_t *) realloc(data->post_process_buf, data->post_process_size); + } + *out_buf = data->post_process_buf; + + // obtain the decoded data + if (!read_all(fd_socket, *out_buf, new_size)) { + close(fd_socket); + *out_buf = buf; + return buf_size; + } + + close(fd_socket); + return new_size; } -// TODO: implement + /** * Deinitialize everything * * @param data The data ptr from afl_custom_init */ void afl_custom_deinit(atnwalk_mutator_t *data) { - free(data->decoded_buf); + free(data->fuzz_buf); + free(data->post_process_buf); free(data); } diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index ee4a3298..26a1ea36 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -130,12 +130,15 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { new_size = afl->max_length; } - - if (new_mem != *mem) { *mem = new_mem; } + // TODO: think about how to enable the change without breaking other implementations + // if (new_mem != *mem) { *mem = new_mem; } /* everything as planned. use the potentially new data. */ - afl_fsrv_write_to_testcase(&afl->fsrv, *mem, new_size); - len = new_size; + // TODO: think about how to enable the change without breaking other implementations + afl_fsrv_write_to_testcase(&afl->fsrv, new_mem, new_size); + + // TODO: think about how to enable the change without breaking other implementations + // len = new_size; } else { From e55b5c54080698ee2efe317321c1f387c225115b Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:09:48 +0200 Subject: [PATCH 053/149] fixed the server handshake commands, works now --- custom_mutators/atnwalk/atnwalk.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/custom_mutators/atnwalk/atnwalk.c b/custom_mutators/atnwalk/atnwalk.c index 584add61..387d8b5d 100644 --- a/custom_mutators/atnwalk/atnwalk.c +++ b/custom_mutators/atnwalk/atnwalk.c @@ -11,8 +11,8 @@ // handshake constants -const uint8_t SERVER_ARE_YOU_ALIVE = 42; -const uint8_t SERVER_YES_I_AM_ALIVE = 213; +const uint8_t SERVER_ARE_YOU_ALIVE = 213; +const uint8_t SERVER_YES_I_AM_ALIVE = 42; // control bits const uint8_t SERVER_CROSSOVER_BIT = 0b00000001; From 529a51c16053125ff0ddce7c6bd149f4ebe65461 Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:09:48 +0200 Subject: [PATCH 054/149] implemented status screen and 50% havoc and 50% splice schedule with limited rounds per queue entry --- custom_mutators/atnwalk/atnwalk.c | 137 +++++++++++++++++++++++++++--- 1 file changed, 124 insertions(+), 13 deletions(-) diff --git a/custom_mutators/atnwalk/atnwalk.c b/custom_mutators/atnwalk/atnwalk.c index 387d8b5d..0194ff18 100644 --- a/custom_mutators/atnwalk/atnwalk.c +++ b/custom_mutators/atnwalk/atnwalk.c @@ -1,3 +1,5 @@ +#include "../../include/afl-fuzz.h" + #include #include #include @@ -6,9 +8,14 @@ #include #include -#define INIT_BUF_SIZE 4096 +#define BUF_SIZE_INIT 4096 #define SOCKET_NAME "/tmp/atnwalk.socket" +// how many errors (e.g. timeouts) to tolerate until moving on to the next queue entry +#define ATNWALK_ERRORS_MAX 1 + +// how many execution timeouts to tolerate until moving on to the next queue entry +#define EXEC_TIMEOUT_MAX 2 // handshake constants const uint8_t SERVER_ARE_YOU_ALIVE = 213; @@ -22,6 +29,14 @@ const uint8_t SERVER_ENCODE_BIT = 0b00001000; typedef struct atnwalk_mutator { + afl_state_t *afl; + uint8_t atnwalk_error_count; + uint64_t prev_timeouts; + uint32_t prev_hits; + uint32_t stage_havoc_cur; + uint32_t stage_havoc_max; + uint32_t stage_splice_cur; + uint32_t stage_splice_max; uint8_t *fuzz_buf; size_t fuzz_size; uint8_t *post_process_buf; @@ -56,6 +71,7 @@ int write_all(int fd, uint8_t *buf, size_t buf_size) { return 1; } + void put_uint32(uint8_t *buf, uint32_t val) { buf[0] = (uint8_t) (val >> 24); buf[1] = (uint8_t) ((val & 0x00ff0000) >> 16); @@ -63,6 +79,7 @@ void put_uint32(uint8_t *buf, uint32_t val) { buf[3] = (uint8_t) (val & 0x000000ff); } + uint32_t to_uint32(uint8_t *buf) { uint32_t val = 0; val |= (((uint32_t) buf[0]) << 24); @@ -72,6 +89,7 @@ uint32_t to_uint32(uint8_t *buf) { return val; } + void put_uint64(uint8_t *buf, uint64_t val) { buf[0] = (uint8_t) (val >> 56); buf[1] = (uint8_t) ((val & 0x00ff000000000000) >> 48); @@ -83,6 +101,7 @@ void put_uint64(uint8_t *buf, uint64_t val) { buf[7] = (uint8_t) (val & 0x00000000000000ff); } + /** * Initialize this custom mutator * @@ -94,21 +113,48 @@ void put_uint64(uint8_t *buf, uint64_t val) { * There may be multiple instances of this mutator in one afl-fuzz run! * Return NULL on error. */ -atnwalk_mutator_t *afl_custom_init(void *afl, unsigned int seed) { +atnwalk_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { srand(seed); atnwalk_mutator_t *data = (atnwalk_mutator_t *) malloc(sizeof(atnwalk_mutator_t)); if (!data) { perror("afl_custom_init alloc"); return NULL; } - data->fuzz_buf = (uint8_t *) malloc(INIT_BUF_SIZE); - data->fuzz_size = INIT_BUF_SIZE; - data->post_process_buf = (uint8_t *) malloc(INIT_BUF_SIZE); - data->post_process_size = INIT_BUF_SIZE; + data->afl = afl; + data->prev_hits = 0; + data->fuzz_buf = (uint8_t *) malloc(BUF_SIZE_INIT); + data->fuzz_size = BUF_SIZE_INIT; + data->post_process_buf = (uint8_t *) malloc(BUF_SIZE_INIT); + data->post_process_size = BUF_SIZE_INIT; return data; } +unsigned int afl_custom_fuzz_count(atnwalk_mutator_t *data, const unsigned char *buf, size_t buf_size) { + // afl_custom_fuzz_count is called exactly once before entering the 'stage-loop' for the current queue entry + // thus, we use it to reset the error count and to initialize stage variables (somewhat not intended by the API, + // but still better than rewriting the whole thing to have a custom mutator stage) + data->atnwalk_error_count = 0; + data->prev_timeouts = data->afl->total_tmouts; + + // it might happen that on the last execution of the splice stage a new path is found + // we need to fix that here and count it + if (data->prev_hits) { + data->afl->stage_finds[STAGE_SPLICE] += data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; + } + data->prev_hits = data->afl->queued_items + data->afl->saved_crashes; + data->stage_havoc_cur = 0; + data->stage_splice_cur = 0; + + // 50% havoc, 50% splice + data->stage_havoc_max = data->afl->stage_max >> 1; + if (data->stage_havoc_max < HAVOC_MIN) { + data->stage_havoc_max = HAVOC_MIN; + } + data->stage_splice_max = data->stage_havoc_max; + return data->stage_havoc_max + data->stage_splice_max; +} + /** * Perform custom mutations on a given input * @@ -132,6 +178,48 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u uint8_t ctrl_buf[8]; uint8_t wanted; + // let's display what's going on in a nice way + if (data->stage_havoc_cur == 0) { + data->afl->stage_name = (uint8_t *) "atnwalk - havoc"; + } + if (data->stage_havoc_cur == data->stage_havoc_max) { + data->afl->stage_name = (uint8_t *) "atnwalk - splice"; + } + + // increase the respective havoc or splice counters + if (data->stage_havoc_cur < data->stage_havoc_max) { + data->stage_havoc_cur++; + data->afl->stage_cycles[STAGE_HAVOC]++; + } else { + // if there is nothing to splice, continue with havoc and skip splicing this time + if (data->afl->ready_for_splicing_count < 1) { + data->stage_havoc_max = data->afl->stage_max; + data->stage_havoc_cur++; + data->afl->stage_cycles[STAGE_HAVOC]++; + } else { + data->stage_splice_cur++; + data->afl->stage_cycles[STAGE_SPLICE]++; + } + } + + // keep track of found new corpus seeds per stage and run the stage twice as long as initially planned + if (data->afl->queued_items + data->afl->saved_crashes > data->prev_hits) { + if (data->stage_splice_cur <= 1) { + data->afl->stage_finds[STAGE_HAVOC] += data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; + } else { + data->afl->stage_finds[STAGE_SPLICE] += + data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; + } + } + data->prev_hits = data->afl->queued_items + data->afl->saved_crashes; + + // check whether this input produces a lot of timeouts, if it does then abandon this queue entry + if (data->afl->total_tmouts - data->prev_timeouts >= EXEC_TIMEOUT_MAX) { + data->afl->stage_max = data->afl->stage_cur; + *out_buf = buf; + return buf_size; + } + // initialize the socket fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); if (fd_socket == -1) { @@ -147,11 +235,6 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u return 0; } - // TODO: how to set connection deadline? maybe not required if server already closes the connection? - - // TODO: there should be some kind of loop retrying with different seeds and ultimately giving up on that input? - // maybe this is not necessary, because we may also just return a single byte in case of failure? - // ask whether the server is alive ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; if (!write_all(fd_socket, ctrl_buf, 1)) { @@ -170,8 +253,8 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u // tell the server what we want to do wanted = SERVER_MUTATE_BIT | SERVER_ENCODE_BIT; - // 50% chance to perform a crossover if there is an additional buffer available - if ((add_buf_size > 0) && (rand() % 2)) { + // perform a crossover if we are splicing + if (data->stage_splice_cur > 0) { wanted |= SERVER_CROSSOVER_BIT; } @@ -196,6 +279,10 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u put_uint32(ctrl_buf, (uint32_t) add_buf_size); if (!write_all(fd_socket, ctrl_buf, 4)) { close(fd_socket); + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } *out_buf = buf; return buf_size; } @@ -203,6 +290,10 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u // send the additional data for crossover if (!write_all(fd_socket, add_buf, add_buf_size)) { close(fd_socket); + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } *out_buf = buf; return buf_size; } @@ -211,6 +302,10 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u put_uint64(ctrl_buf, (uint64_t) rand()); if (!write_all(fd_socket, ctrl_buf, 8)) { close(fd_socket); + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } *out_buf = buf; return buf_size; } @@ -220,6 +315,10 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u put_uint64(ctrl_buf, (uint64_t) rand()); if (!write_all(fd_socket, ctrl_buf, 8)) { close(fd_socket); + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } *out_buf = buf; return buf_size; } @@ -227,6 +326,10 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u // obtain the required buffer size for the data that will be returned if (!read_all(fd_socket, ctrl_buf, 4)) { close(fd_socket); + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } *out_buf = buf; return buf_size; } @@ -235,6 +338,10 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u // if the data is too large then we ignore this round if (new_size > max_size) { close(fd_socket); + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } *out_buf = buf; return buf_size; } @@ -254,6 +361,10 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u // obtain the encoded data if (!read_all(fd_socket, *out_buf, new_size)) { close(fd_socket); + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } *out_buf = buf; return buf_size; } From 9ab902402cd33156257fc0355c0105e7e03f5ba3 Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Mon, 17 Apr 2023 17:09:48 +0200 Subject: [PATCH 055/149] fixed code clones in atnwalk.c, introduced new environment variable AFL_POST_PROCESS_KEEP_ORIGINAL in AFL++ to integrate atnwalk without re-compiling afl-fuzz --- custom_mutators/atnwalk/atnwalk.c | 144 ++++++++++-------------------- include/afl-fuzz.h | 4 +- include/envs.h | 1 + src/afl-fuzz-run.c | 11 +-- src/afl-fuzz-state.c | 7 ++ src/afl-fuzz.c | 2 + 6 files changed, 66 insertions(+), 103 deletions(-) diff --git a/custom_mutators/atnwalk/atnwalk.c b/custom_mutators/atnwalk/atnwalk.c index 0194ff18..901b8a9e 100644 --- a/custom_mutators/atnwalk/atnwalk.c +++ b/custom_mutators/atnwalk/atnwalk.c @@ -1,4 +1,4 @@ -#include "../../include/afl-fuzz.h" +#include "afl-fuzz.h" #include #include @@ -9,7 +9,7 @@ #include #define BUF_SIZE_INIT 4096 -#define SOCKET_NAME "/tmp/atnwalk.socket" +#define SOCKET_NAME "./atnwalk.socket" // how many errors (e.g. timeouts) to tolerate until moving on to the next queue entry #define ATNWALK_ERRORS_MAX 1 @@ -155,6 +155,29 @@ unsigned int afl_custom_fuzz_count(atnwalk_mutator_t *data, const unsigned char return data->stage_havoc_max + data->stage_splice_max; } + +size_t fail_fatal(int fd_socket, uint8_t **out_buf) { + if (fd_socket != -1) { + close(fd_socket); + } + *out_buf = NULL; + return 0; +} + + +size_t fail_gracefully(int fd_socket, atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf) { + if (fd_socket != -1) { + close(fd_socket); + } + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + data->afl->stage_max = data->afl->stage_cur; + } + *out_buf = buf; + return buf_size; +} + + /** * Perform custom mutations on a given input * @@ -202,7 +225,7 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u } } - // keep track of found new corpus seeds per stage and run the stage twice as long as initially planned + // keep track of found new corpus seeds per stage if (data->afl->queued_items + data->afl->saved_crashes > data->prev_hits) { if (data->stage_splice_cur <= 1) { data->afl->stage_finds[STAGE_HAVOC] += data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; @@ -216,38 +239,28 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u // check whether this input produces a lot of timeouts, if it does then abandon this queue entry if (data->afl->total_tmouts - data->prev_timeouts >= EXEC_TIMEOUT_MAX) { data->afl->stage_max = data->afl->stage_cur; - *out_buf = buf; - return buf_size; + return fail_gracefully(-1, data, buf, buf_size, out_buf); } // initialize the socket fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd_socket == -1) { - *out_buf = NULL; - return 0; - } + if (fd_socket == -1) { return fail_fatal(fd_socket, out_buf); } memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); if (connect(fd_socket, (const struct sockaddr *) &addr, sizeof(addr)) == -1) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } // ask whether the server is alive ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; if (!write_all(fd_socket, ctrl_buf, 1)) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } // see whether the server replies as expected if (!read_all(fd_socket, ctrl_buf, 1) || ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } // tell the server what we want to do @@ -262,88 +275,48 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u ctrl_buf[0] = wanted; put_uint32(ctrl_buf + 1, (uint32_t) buf_size); if (!write_all(fd_socket, ctrl_buf, 5)) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } // send the data to mutate and encode if (!write_all(fd_socket, buf, buf_size)) { - close(fd_socket); - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } if (wanted & SERVER_CROSSOVER_BIT) { // since we requested crossover, we will first tell how much additional data is to be expected put_uint32(ctrl_buf, (uint32_t) add_buf_size); if (!write_all(fd_socket, ctrl_buf, 4)) { - close(fd_socket); - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } // send the additional data for crossover if (!write_all(fd_socket, add_buf, add_buf_size)) { - close(fd_socket); - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } // lastly, a seed is required for crossover so send one put_uint64(ctrl_buf, (uint64_t) rand()); if (!write_all(fd_socket, ctrl_buf, 8)) { - close(fd_socket); - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } } // since we requested mutation, we need to provide a seed for that put_uint64(ctrl_buf, (uint64_t) rand()); if (!write_all(fd_socket, ctrl_buf, 8)) { - close(fd_socket); - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } // obtain the required buffer size for the data that will be returned if (!read_all(fd_socket, ctrl_buf, 4)) { - close(fd_socket); - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } size_t new_size = (size_t) to_uint32(ctrl_buf); // if the data is too large then we ignore this round if (new_size > max_size) { - close(fd_socket); - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } if (new_size > buf_size) { @@ -360,13 +333,7 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u // obtain the encoded data if (!read_all(fd_socket, *out_buf, new_size)) { - close(fd_socket); - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } close(fd_socket); @@ -398,54 +365,41 @@ size_t afl_custom_post_process(atnwalk_mutator_t *data, uint8_t *buf, size_t buf // initialize the socket fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); if (fd_socket == -1) { - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } memset(&addr, 0, sizeof(addr)); addr.sun_family = AF_UNIX; strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); if (connect(fd_socket, (const struct sockaddr *) &addr, sizeof(addr)) == -1) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } // ask whether the server is alive ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; if (!write_all(fd_socket, ctrl_buf, 1)) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } // see whether the server replies as expected if (!read_all(fd_socket, ctrl_buf, 1) || ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_fatal(fd_socket, out_buf); } // tell the server what we want and how much data will be sent ctrl_buf[0] = SERVER_DECODE_BIT; put_uint32(ctrl_buf + 1, (uint32_t) buf_size); if (!write_all(fd_socket, ctrl_buf, 5)) { - close(fd_socket); - *out_buf = NULL; - return 0; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } // send the data to decode if (!write_all(fd_socket, buf, buf_size)) { - close(fd_socket); - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } // obtain the required buffer size for the data that will be returned if (!read_all(fd_socket, ctrl_buf, 4)) { - close(fd_socket); - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } size_t new_size = (size_t) to_uint32(ctrl_buf); @@ -458,9 +412,7 @@ size_t afl_custom_post_process(atnwalk_mutator_t *data, uint8_t *buf, size_t buf // obtain the decoded data if (!read_all(fd_socket, *out_buf, new_size)) { - close(fd_socket); - *out_buf = buf; - return buf_size; + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } close(fd_socket); diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 23c20cc4..5d9b17e7 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -386,8 +386,8 @@ typedef struct afl_env_vars { afl_bench_until_crash, afl_debug_child, afl_autoresume, afl_cal_fast, afl_cycle_schedules, afl_expand_havoc, afl_statsd, afl_cmplog_only_new, afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, - afl_keep_timeouts, afl_pizza_mode, afl_no_crash_readme, - afl_no_startup_calibration; + afl_keep_timeouts, afl_pizza_mode, afl_post_process_keep_original, + afl_no_crash_readme, afl_no_startup_calibration; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, diff --git a/include/envs.h b/include/envs.h index 2204a100..1527dfcb 100644 --- a/include/envs.h +++ b/include/envs.h @@ -177,6 +177,7 @@ static char *afl_environment_variables[] = { "AFL_PATH", "AFL_PERFORMANCE_FILE", "AFL_PERSISTENT_RECORD", + "AFL_POST_PROCESS_KEEP_ORIGINAL", "AFL_PRELOAD", "AFL_TARGET_ENV", "AFL_PYTHON_MODULE", diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 26a1ea36..3c3d4817 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -130,15 +130,16 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { new_size = afl->max_length; } - // TODO: think about how to enable the change without breaking other implementations - // if (new_mem != *mem) { *mem = new_mem; } /* everything as planned. use the potentially new data. */ - // TODO: think about how to enable the change without breaking other implementations afl_fsrv_write_to_testcase(&afl->fsrv, new_mem, new_size); - // TODO: think about how to enable the change without breaking other implementations - // len = new_size; + if (likely(!afl->afl_env.afl_post_process_keep_original)) { + + if (new_mem != *mem) { *mem = new_mem; } + len = new_size; + + } } else { diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 0576f84f..10bc2768 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -377,6 +377,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_statsd = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_POST_PROCESS_KEEP_ORIGINAL", + + afl_environment_variable_len)) { + + afl->afl_env.afl_post_process_keep_original = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_TMPDIR", afl_environment_variable_len)) { diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 1f0fcab1..b83af257 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -284,6 +284,8 @@ static void usage(u8 *argv0, int more_help) { PERSISTENT_MSG + "AFL_POST_PROCESS_KEEP_ORIGINAL: save the file as it was prior post-processing to the queue,\n" + " but execute the post-processed one\n" "AFL_PRELOAD: LD_PRELOAD / DYLD_INSERT_LIBRARIES settings for target\n" "AFL_TARGET_ENV: pass extra environment variables to target\n" "AFL_SHUFFLE_QUEUE: reorder the input queue randomly on startup\n" From 4e5f42cab6b8c501eeaf76ec7ca920089f6e0f3a Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 20 Apr 2023 10:39:23 +0200 Subject: [PATCH 056/149] afl-showmap custom mutator support --- GNUmakefile | 2 +- TODO.md | 4 +- afl-cmin | 2 + afl-cmin.bash | 2 + docs/Changelog.md | 3 +- include/afl-fuzz.h | 8 +++- src/afl-showmap.c | 93 +++++++++++++++++++++++++++++++++++++++++++++- 7 files changed, 107 insertions(+), 7 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 5bc3f9d5..0f890308 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -453,7 +453,7 @@ afl-fuzz: $(COMM_HDR) include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/ $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) -lm afl-showmap: src/afl-showmap.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-fuzz-mutators.c src/afl-fuzz-python.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(PYFLAGS) $(LDFLAGS) afl-tmin: src/afl-tmin.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $(COMPILE_STATIC) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o src/afl-performance.o -o $@ $(LDFLAGS) diff --git a/TODO.md b/TODO.md index e7789cf6..dba75070 100644 --- a/TODO.md +++ b/TODO.md @@ -3,14 +3,14 @@ ## Should - splicing selection weighted? - - support afl_custom_{send,post_process}, persistent and deferred fork - server in afl-showmap + - support persistent and deferred fork server in afl-showmap? - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt - afl-plot to support multiple plot_data - parallel builds for source-only targets - get rid of check_binary, replace with more forkserver communication - first fuzzer should be a main automatically? not sure. + - reload fuzz binary on signal ## Maybe diff --git a/afl-cmin b/afl-cmin index c5e64410..e2c26d91 100755 --- a/afl-cmin +++ b/afl-cmin @@ -133,6 +133,8 @@ function usage() { "AFL_PRINT_FILENAMES: If set, the filename currently processed will be " \ "printed to stdout\n" \ "AFL_SKIP_BIN_CHECK: skip afl instrumentation checks for target binary\n" +"AFL_CUSTOM_MUTATOR_LIBRARY: custom mutator library (post_process and send)\n" +"AFL_PYTHON_MODULE: custom mutator library (post_process and send)\n" exit 1 } diff --git a/afl-cmin.bash b/afl-cmin.bash index bcf62eba..5258758e 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -151,6 +151,8 @@ AFL_KEEP_TRACES: leave the temporary \.traces directory AFL_NO_FORKSRV: run target via execve instead of using the forkserver AFL_PATH: last resort location to find the afl-showmap binary AFL_SKIP_BIN_CHECK: skip check for target binary +AFL_CUSTOM_MUTATOR_LIBRARY: custom mutator library (post_process and send) +AFL_PYTHON_MODULE: custom mutator library (post_process and send) _EOF_ exit 1 fi diff --git a/docs/Changelog.md b/docs/Changelog.md index 30e76f2c..5ed5ef2b 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -4,7 +4,8 @@ release of the tool. See README.md for the general instruction manual. ### Version ++4.07a (dev) - - soon :) + - afl-showmap: + - added custom mutator post_process and send support ### Version ++4.06c (release) diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 8b6502b4..ec69ba17 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1103,7 +1103,6 @@ u32 count_bits(afl_state_t *, u8 *); u32 count_bytes(afl_state_t *, u8 *); u32 count_non_255_bytes(afl_state_t *, u8 *); void simplify_trace(afl_state_t *, u8 *); -void classify_counts(afl_forkserver_t *); #ifdef WORD_SIZE_64 void discover_word(u8 *ret, u64 *current, u64 *virgin); #else @@ -1117,6 +1116,9 @@ u8 *describe_op(afl_state_t *, u8, size_t); u8 save_if_interesting(afl_state_t *, void *, u32, u8); u8 has_new_bits(afl_state_t *, u8 *); u8 has_new_bits_unclassified(afl_state_t *, u8 *); +#ifndef AFL_SHOWMAP +void classify_counts(afl_forkserver_t *); +#endif /* Extras */ @@ -1192,11 +1194,13 @@ void fix_up_sync(afl_state_t *); void check_asan_opts(afl_state_t *); void check_binary(afl_state_t *, u8 *); void check_if_tty(afl_state_t *); -void setup_signal_handlers(void); void save_cmdline(afl_state_t *, u32, char **); void read_foreign_testcases(afl_state_t *, int); void write_crash_readme(afl_state_t *afl); u8 check_if_text_buf(u8 *buf, u32 len); +#ifndef AFL_SHOWMAP +void setup_signal_handlers(void); +#endif /* CmpLog */ diff --git a/src/afl-showmap.c b/src/afl-showmap.c index df030672..b5a61de5 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -30,8 +30,10 @@ */ #define AFL_MAIN +#define AFL_SHOWMAP #include "config.h" +#include "afl-fuzz.h" #include "types.h" #include "debug.h" #include "alloc-inl.h" @@ -62,6 +64,8 @@ #include #include +static afl_state_t *afl; + static char *stdin_file; /* stdin file */ static u8 *in_dir = NULL, /* input folder */ @@ -308,12 +312,73 @@ static u32 write_results_to_file(afl_forkserver_t *fsrv, u8 *outfile) { } +void pre_afl_fsrv_write_to_testcase(afl_forkserver_t *fsrv, u8 *mem, u32 len) { + + static u8 buf[MAX_FILE]; + u32 sent = 0; + + if (unlikely(afl->custom_mutators_count)) { + + ssize_t new_size = len; + u8 *new_mem = mem; + u8 *new_buf = NULL; + + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_post_process) { + + new_size = + el->afl_custom_post_process(el->data, new_mem, new_size, &new_buf); + + if (unlikely(!new_buf || new_size <= 0)) { + + return; + + } else { + + new_mem = new_buf; + len = new_size; + + } + + } + + }); + + if (new_mem != mem && new_mem != NULL) { + + mem = buf; + memcpy(mem, new_mem, new_size); + + } + + if (unlikely(afl->custom_mutators_count)) { + + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_fuzz_send) { + + el->afl_custom_fuzz_send(el->data, mem, len); + sent = 1; + + } + + }); + + } + + } + + if (likely(!sent)) { afl_fsrv_write_to_testcase(fsrv, mem, len); } + +} + /* Execute target application. */ static void showmap_run_target_forkserver(afl_forkserver_t *fsrv, u8 *mem, u32 len) { - afl_fsrv_write_to_testcase(fsrv, mem, len); + pre_afl_fsrv_write_to_testcase(fsrv, mem, len); if (!quiet_mode) { SAYF("-- Program output begins --\n" cRST); } @@ -835,6 +900,10 @@ static void usage(u8 *argv0) { "This tool displays raw tuple data captured by AFL instrumentation.\n" "For additional help, consult %s/README.md.\n\n" + "If you use -i mode, then custom mutator post_process send send " + "functionality\n" + "is supported.\n\n" + "Environment variables used:\n" "LD_BIND_LAZY: do not set LD_BIND_NOW env var for target\n" "AFL_CMIN_CRASHES_ONLY: (cmin_mode) only write tuples for crashing " @@ -1266,6 +1335,8 @@ int main(int argc, char **argv_orig, char **envp) { } + afl = calloc(1, sizeof(afl_state_t)); + if (getenv("AFL_FORKSRV_INIT_TMOUT")) { s32 forksrv_init_tmout = atoi(getenv("AFL_FORKSRV_INIT_TMOUT")); @@ -1380,6 +1451,26 @@ int main(int argc, char **argv_orig, char **envp) { } + if (in_dir) { + + afl->fsrv.dev_urandom_fd = open("/dev/urandom", O_RDONLY); + afl->afl_env.afl_custom_mutator_library = + getenv("AFL_CUSTOM_MUTATOR_LIBRARY"); + afl->afl_env.afl_python_module = getenv("AFL_PYTHON_MODULE"); + setup_custom_mutators(afl); + + } else { + + if (getenv("AFL_CUSTOM_MUTATOR_LIBRARY") || getenv("AFL_PYTHON_MODULE")) { + + WARNF( + "Custom mutator environment detected, this is only supported in -i " + "mode!\n"); + + } + + } + if (in_dir) { DIR *dir_in, *dir_out = NULL; From 30495e6bfe4119c9be6597ad0def01e7e0cb8a67 Mon Sep 17 00:00:00 2001 From: eleguevel Date: Fri, 21 Apr 2023 12:00:56 +0200 Subject: [PATCH 057/149] frida mode: add dynamic loaded code exclusion Add the AFL_FRIDA_INST_NO_DYNAMIC_LOAD environment variable and its associated JS function setInstrumentNoDynamicLoad to prevent the instrumentation of late dynamic loaded code. Resolve #1708 --- docs/env_variables.md | 2 ++ frida_mode/README.md | 4 +++- frida_mode/Scripting.md | 6 ++++++ frida_mode/frida.map | 1 + frida_mode/include/ranges.h | 1 + frida_mode/src/js/api.js | 7 +++++++ frida_mode/src/js/js_api.c | 7 +++++++ frida_mode/src/ranges.c | 39 ++++++++++++++++++++++++++++++++++--- frida_mode/ts/lib/afl.ts | 12 ++++++++++++ include/envs.h | 1 + 10 files changed, 76 insertions(+), 4 deletions(-) diff --git a/docs/env_variables.md b/docs/env_variables.md index a6a0ae44..c5995d13 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -677,6 +677,8 @@ support. * `AFL_FRIDA_INST_JIT` - Enable the instrumentation of Just-In-Time compiled code. Code is considered to be JIT if the executable segment is not backed by a file. +* `AFL_FRIDA_INST_NO_DYNAMIC_LOAD` - Don't instrument the code loaded late at + runtime. Strictly limits instrumentation to what has been included. * `AFL_FRIDA_INST_NO_OPTIMIZE` - Don't use optimized inline assembly coverage instrumentation (the default where available). Required to use `AFL_FRIDA_INST_TRACE`. diff --git a/frida_mode/README.md b/frida_mode/README.md index aac13153..49a1fe38 100644 --- a/frida_mode/README.md +++ b/frida_mode/README.md @@ -178,11 +178,13 @@ Default is 256Mb. * `AFL_FRIDA_INST_JIT` - Enable the instrumentation of Just-In-Time compiled code. Code is considered to be JIT if the executable segment is not backed by a file. +* `AFL_FRIDA_INST_NO_DYNAMIC_LOAD` - Don't instrument the code loaded late at + runtime. Strictly limits instrumentation to what has been included. * `AFL_FRIDA_INST_NO_OPTIMIZE` - Don't use optimized inline assembly coverage instrumentation (the default where available). Required to use + `AFL_FRIDA_INST_TRACE`. * `AFL_FRIDA_INST_REGS_FILE` - File to write raw register contents at the start of each block. - `AFL_FRIDA_INST_TRACE`. * `AFL_FRIDA_INST_NO_CACHE` - Don't use a look-up table to cache real to instrumented address block translations. * `AFL_FRIDA_INST_NO_PREFETCH` - Disable prefetching. By default, the child will diff --git a/frida_mode/Scripting.md b/frida_mode/Scripting.md index 023e4a19..dfd09e7b 100644 --- a/frida_mode/Scripting.md +++ b/frida_mode/Scripting.md @@ -844,6 +844,12 @@ class Afl { static setInstrumentLibraries() { Afl.jsApiSetInstrumentLibraries(); } + /** + * See `AFL_FRIDA_INST_NO_DYNAMIC_LOAD` + */ + static setInstrumentNoDynamicLoad() { + Afl.jsApiSetInstrumentNoDynamicLoad(); + } /** * See `AFL_FRIDA_INST_NO_OPTIMIZE` */ diff --git a/frida_mode/frida.map b/frida_mode/frida.map index baf067ab..a98c2096 100644 --- a/frida_mode/frida.map +++ b/frida_mode/frida.map @@ -19,6 +19,7 @@ js_api_set_instrument_jit; js_api_set_instrument_libraries; js_api_set_instrument_instructions; + js_api_set_instrument_no_dynamic_load; js_api_set_instrument_no_optimize; js_api_set_instrument_regs_file; js_api_set_instrument_seed; diff --git a/frida_mode/include/ranges.h b/frida_mode/include/ranges.h index 3bd9eaa6..ca28acd9 100644 --- a/frida_mode/include/ranges.h +++ b/frida_mode/include/ranges.h @@ -6,6 +6,7 @@ extern gboolean ranges_debug_maps; extern gboolean ranges_inst_libs; extern gboolean ranges_inst_jit; +extern gboolean ranges_inst_dynamic_load; void ranges_config(void); void ranges_init(void); diff --git a/frida_mode/src/js/api.js b/frida_mode/src/js/api.js index f9ea1ffb..a65d32df 100644 --- a/frida_mode/src/js/api.js +++ b/frida_mode/src/js/api.js @@ -150,6 +150,12 @@ class Afl { static setInstrumentLibraries() { Afl.jsApiSetInstrumentLibraries(); } + /** + * See `AFL_FRIDA_INST_NO_DYNAMIC_LOAD` + */ + static setInstrumentNoDynamicLoad() { + Afl.jsApiSetInstrumentNoDynamicLoad(); + } /** * See `AFL_FRIDA_INST_NO_OPTIMIZE` */ @@ -342,6 +348,7 @@ Afl.jsApiSetInstrumentDebugFile = Afl.jsApiGetFunction("js_api_set_instrument_de Afl.jsApiSetInstrumentInstructions = Afl.jsApiGetFunction("js_api_set_instrument_instructions", "void", []); Afl.jsApiSetInstrumentJit = Afl.jsApiGetFunction("js_api_set_instrument_jit", "void", []); Afl.jsApiSetInstrumentLibraries = Afl.jsApiGetFunction("js_api_set_instrument_libraries", "void", []); +Afl.jsApiSetInstrumentNoDynamicLoad = Afl.jsApiGetFunction("js_api_set_instrument_no_dynamic_load", "void", []); Afl.jsApiSetInstrumentNoOptimize = Afl.jsApiGetFunction("js_api_set_instrument_no_optimize", "void", []); Afl.jsApiSetInstrumentRegsFile = Afl.jsApiGetFunction("js_api_set_instrument_regs_file", "void", ["pointer"]); Afl.jsApiSetInstrumentSeed = Afl.jsApiGetFunction("js_api_set_instrument_seed", "void", ["uint64"]); diff --git a/frida_mode/src/js/js_api.c b/frida_mode/src/js/js_api.c index 2e996c1c..00278082 100644 --- a/frida_mode/src/js/js_api.c +++ b/frida_mode/src/js/js_api.c @@ -156,6 +156,13 @@ __attribute__((visibility("default"))) void js_api_set_instrument_instructions( } +__attribute__((visibility("default"))) void js_api_set_instrument_no_dynamic_load( + void) { + + ranges_inst_dynamic_load = FALSE; + +} + __attribute__((visibility("default"))) void js_api_set_instrument_no_optimize( void) { diff --git a/frida_mode/src/ranges.c b/frida_mode/src/ranges.c index 72cb9730..e9fc3b4e 100644 --- a/frida_mode/src/ranges.c +++ b/frida_mode/src/ranges.c @@ -18,6 +18,7 @@ typedef struct { gboolean ranges_debug_maps = FALSE; gboolean ranges_inst_libs = FALSE; gboolean ranges_inst_jit = FALSE; +gboolean ranges_inst_dynamic_load = TRUE; static GArray *module_ranges = NULL; static GArray *libs_ranges = NULL; @@ -25,6 +26,7 @@ static GArray *jit_ranges = NULL; static GArray *include_ranges = NULL; static GArray *exclude_ranges = NULL; static GArray *ranges = NULL; +static GArray *whole_memory_ranges = NULL; static void convert_address_token(gchar *token, GumMemoryRange *range) { @@ -387,6 +389,21 @@ static GArray *collect_jit_ranges(void) { } +static GArray *collect_whole_mem_ranges(void) { + + GArray *result; + GumMemoryRange range; + result = g_array_new(false, false, sizeof(GumMemoryRange)); + + range.base_address = 0; + range.size = G_MAXULONG; + + g_array_append_val(result, range); + + return result; + +} + static gboolean intersect_range(GumMemoryRange *rr, GumMemoryRange *ra, GumMemoryRange *rb) { @@ -574,11 +591,17 @@ void ranges_config(void) { if (getenv("AFL_FRIDA_DEBUG_MAPS") != NULL) { ranges_debug_maps = TRUE; } if (getenv("AFL_INST_LIBS") != NULL) { ranges_inst_libs = TRUE; } if (getenv("AFL_FRIDA_INST_JIT") != NULL) { ranges_inst_jit = TRUE; } + if (getenv("AFL_FRIDA_INST_NO_DYNAMIC_LOAD") != NULL) { + + ranges_inst_dynamic_load = FALSE; + + } if (ranges_debug_maps) { ranges_print_debug_maps(); } include_ranges = collect_ranges("AFL_FRIDA_INST_RANGES"); exclude_ranges = collect_ranges("AFL_FRIDA_EXCLUDE_RANGES"); + whole_memory_ranges = collect_whole_mem_ranges(); } @@ -628,10 +651,20 @@ void ranges_init(void) { print_ranges("step4", step4); /* - * After step4, we have the total ranges to be instrumented, we now subtract - * that from the original ranges of the modules to configure stalker. + * After step 4 we have the total ranges to be instrumented, we now subtract + * that either from the original ranges of the modules or from the whole + * memory if AFL_INST_NO_DYNAMIC_LOAD to configure the stalker. */ - step5 = subtract_ranges(module_ranges, step4); + if (ranges_inst_dynamic_load) { + + step5 = subtract_ranges(module_ranges, step4); + + } else { + + step5 = subtract_ranges(whole_memory_ranges, step4); + + } + print_ranges("step5", step5); ranges = merge_ranges(step5); diff --git a/frida_mode/ts/lib/afl.ts b/frida_mode/ts/lib/afl.ts index 6a2350e7..7d1fac6b 100644 --- a/frida_mode/ts/lib/afl.ts +++ b/frida_mode/ts/lib/afl.ts @@ -178,6 +178,13 @@ class Afl { Afl.jsApiSetInstrumentLibraries(); } + /** + * See `AFL_FRIDA_INST_NO_DYNAMIC_LOAD` + */ + public static setInstrumentNoDynamicLoad(): void { + Afl.jsApiSetInstrumentNoDynamicLoad(); + } + /** * See `AFL_FRIDA_INST_NO_OPTIMIZE` */ @@ -443,6 +450,11 @@ class Afl { "void", []); + private static readonly jsApiSetInstrumentNoDynamicLoad = Afl.jsApiGetFunction( + "js_api_set_instrument_no_dynamic_load", + "void", + []); + private static readonly jsApiSetInstrumentNoOptimize = Afl.jsApiGetFunction( "js_api_set_instrument_no_optimize", "void", diff --git a/include/envs.h b/include/envs.h index 066921b9..41eabf60 100644 --- a/include/envs.h +++ b/include/envs.h @@ -65,6 +65,7 @@ static char *afl_environment_variables[] = { "AFL_FRIDA_INST_INSN", "AFL_FRIDA_INST_JIT", "AFL_FRIDA_INST_NO_CACHE", + "AFL_FRIDA_INST_NO_DYNAMIC_LOAD", "AFL_FRIDA_INST_NO_OPTIMIZE", "AFL_FRIDA_INST_NO_PREFETCH", "AFL_FRIDA_INST_NO_PREFETCH_BACKPATCH", From 0a297ed9ef48c1eaf69fdc13bd1016f8f29124be Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 21 Apr 2023 15:09:35 +0200 Subject: [PATCH 058/149] dummy function for afl-showmap --- src/afl-showmap.c | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/afl-showmap.c b/src/afl-showmap.c index b5a61de5..affad7d6 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -140,6 +140,15 @@ static void kill_child() { } +/* dummy function */ +u32 write_to_testcase(afl_state_t *afl, void **mem, u32 a, u32 b) { + + (void)afl; + (void)mem; + return a + b; + +} + static void classify_counts(afl_forkserver_t *fsrv) { u8 *mem = fsrv->trace_bits; From 369ec31f0e1a40f5e617265fc199676175d36814 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 21 Apr 2023 15:15:34 +0200 Subject: [PATCH 059/149] debug output --- GNUmakefile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index 0f890308..23cae65d 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -286,6 +286,8 @@ ifeq "$(shell command -v svn >/dev/null && svn proplist . 2>/dev/null && echo 1 IN_REPO=1 endif +CCVER=$(shell cc -v 2>&1|tail -n 1) + ifeq "$(shell echo 'int main() { return 0;}' | $(CC) $(CFLAGS) -fsanitize=address -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1" ASAN_CFLAGS=-fsanitize=address -fstack-protector-all -fno-omit-frame-pointer -DASAN_BUILD ASAN_LDFLAGS=-fsanitize=address -fstack-protector-all -fno-omit-frame-pointer @@ -431,7 +433,7 @@ endif .PHONY: ready ready: - @echo "[+] Everything seems to be working, ready to compile." + @echo "[+] Everything seems to be working, ready to compile. ($(CCVER))" afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) src/$@.c -o $@ $(LDFLAGS) From 7a8d0a10ce25d20bdd2021920b0b5ebbc0d3f3a6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 21 Apr 2023 15:21:11 +0200 Subject: [PATCH 060/149] add dummy functions to afl-showmap for old gcc compilers --- src/afl-showmap.c | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/src/afl-showmap.c b/src/afl-showmap.c index affad7d6..0b9fc211 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -140,7 +140,7 @@ static void kill_child() { } -/* dummy function */ +/* dummy functions */ u32 write_to_testcase(afl_state_t *afl, void **mem, u32 a, u32 b) { (void)afl; @@ -149,7 +149,30 @@ u32 write_to_testcase(afl_state_t *afl, void **mem, u32 a, u32 b) { } -static void classify_counts(afl_forkserver_t *fsrv) { +void show_stats(afl_state_t *afl) { + + (void)afl; + +} + +void update_bitmap_score(afl_state_t *afl, struct queue_entry *q) { + + (void)afl; + (void)q; + +} + +fsrv_run_result_t fuzz_run_target(afl_state_t *afl, afl_forkserver_t *fsrv, + u32 i) { + + (void)afl; + (void)fsrv; + (void)i; + return 0; + +} + +void classify_counts(afl_forkserver_t *fsrv) { u8 *mem = fsrv->trace_bits; const u8 *map = binary_mode ? count_class_binary : count_class_human; From c49d346e3762fa06bf410d493f70411069d32590 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 21 Apr 2023 15:22:48 +0200 Subject: [PATCH 061/149] remove ubuntu 18.04 from ci, no resources for this on github --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ed1f3228..03e4151d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -14,7 +14,7 @@ jobs: runs-on: "${{ matrix.os }}" strategy: matrix: - os: [ubuntu-22.04, ubuntu-20.04, ubuntu-18.04] + os: [ubuntu-22.04, ubuntu-20.04] env: AFL_SKIP_CPUFREQ: 1 AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 From 779a72ef8c2457430b824f7830eba731745fb6ee Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Fri, 21 Apr 2023 16:46:15 +0200 Subject: [PATCH 062/149] fixed AFL_POST_PROCESS_KEEP_ORIGINAL for version 4.07a --- src/afl-fuzz-run.c | 30 ++++++++++++++++++++---------- 1 file changed, 20 insertions(+), 10 deletions(-) diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index 2d53de93..ac4fb4a9 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -133,14 +133,22 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { } - if (new_mem != *mem && new_mem != NULL && new_size > 0 - && !afl->afl_env.afl_post_process_keep_original) { + if (new_mem != *mem && new_mem != NULL && new_size > 0) { - u8 *new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), new_size); - if (unlikely(!new_buf)) { PFATAL("alloc"); } - *mem = new_buf; - memcpy(*mem, new_mem, new_size); - afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); + new_buf = afl_realloc(AFL_BUF_PARAM(out_scratch), new_size); + if (unlikely(!new_buf)) { PFATAL("alloc"); } + memcpy(new_buf, new_mem, new_size); + + /* if AFL_POST_PROCESS_KEEP_ORIGINAL is set then save the original memory + prior post-processing in new_mem to restore it later */ + if (unlikely(afl->afl_env.afl_post_process_keep_original)) { + + new_mem = *mem; + + } + + *mem = new_buf; + afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); } @@ -162,17 +170,19 @@ write_to_testcase(afl_state_t *afl, void **mem, u32 len, u32 fix) { if (likely(!sent)) { /* everything as planned. use the potentially new data. */ + afl_fsrv_write_to_testcase(&afl->fsrv, *mem, new_size); if (likely(!afl->afl_env.afl_post_process_keep_original)) { - afl_fsrv_write_to_testcase(&afl->fsrv, *mem, new_size); + len = new_size; } else { - afl_fsrv_write_to_testcase(&afl->fsrv, new_mem, new_size); + /* restore the original memory which was saved in new_mem */ + *mem = new_mem; + afl_swap_bufs(AFL_BUF_PARAM(out), AFL_BUF_PARAM(out_scratch)); } - len = new_size; } From d7e6f8cb386a24b5ec090caa6befe27461cd2190 Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Fri, 21 Apr 2023 17:10:19 +0200 Subject: [PATCH 063/149] Create README.md --- custom_mutators/atnwalk/README.md | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 custom_mutators/atnwalk/README.md diff --git a/custom_mutators/atnwalk/README.md b/custom_mutators/atnwalk/README.md new file mode 100644 index 00000000..badb856f --- /dev/null +++ b/custom_mutators/atnwalk/README.md @@ -0,0 +1,45 @@ +# ATNwalk: Grammar-Based Fuzzing using Only Bit-Mutations + +This is a custom mutator integration of ATNwalk that works by communicating via UNIX domain sockets. + +Refer to [https://github.com/atnwalk/testbed](https://github.com/atnwalk/testbed) for detailed instructions on how to get ATNwalk running. + +## Build + +```bash +gcc -I ../../include/ -shared -fPIC -Wall -O3 atnwalk.c -o atnwalk.so +``` + +## Run + +**NOTE:** The commands below just demonstrate an example how running ATNwalk looks like and require a working [testbed](https://github.com/atnwalk/testbed) + +```bash +# create the required a random seed first +mkdir -p ~/campaign/example/seeds +cd ~/campaign/example/seeds +head -c1 /dev/urandom | ~/atnwalk/build/javascript/bin/decode -wb > seed.decoded 2> seed.encoded + +# create the required atnwalk directory and copy the seed +cd ../ +mkdir -p atnwalk/in +cp ./seeds/seed.encoded atnwalk/in/seed +cd atnwalk + +# assign to a single core when benchmarking it, change the CPU number as required +CPU_ID=0 + +# start the ATNwalk server +nohup taskset -c ${CPU_ID} ${HOME}/atnwalk/build/javascript/bin/server 100 > server.log 2>&1 & + +# start AFL++ with ATNwalk +AFL_SKIP_CPUFREQ=1 \ + AFL_DISABLE_TRIM=1 \ + AFL_CUSTOM_MUTATOR_ONLY=1 \ + AFL_CUSTOM_MUTATOR_LIBRARY=${HOME}/AFLplusplus/custom_mutators/atnwalk/atnwalk.so \ + AFL_POST_PROCESS_KEEP_ORIGINAL=1 \ + ~/AFLplusplus/afl-fuzz -t 100 -i in/ -o out -b ${CPU_ID} -- ~/jerryscript/build/bin/jerry + +# make sure to kill the ATNwalk server process after you're done +kill "$(cat atnwalk.pid)" +``` From 228e9527cb0c00644e9601afc0449c586b468576 Mon Sep 17 00:00:00 2001 From: Maik Betka <9078425+voidptr127@users.noreply.github.com> Date: Fri, 21 Apr 2023 17:19:37 +0200 Subject: [PATCH 064/149] fixed formatting with make code-format --- src/afl-fuzz-state.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index cccebeb9..5e736029 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -396,10 +396,10 @@ void read_afl_environment(afl_state_t *afl, char **envp) { } else if (!strncmp(env, "AFL_POST_PROCESS_KEEP_ORIGINAL", - afl_environment_variable_len)) { + afl_environment_variable_len)) { - afl->afl_env.afl_post_process_keep_original = - get_afl_env(afl_environment_variables[i]) ? 1 : 0; + afl->afl_env.afl_post_process_keep_original = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; } else if (!strncmp(env, "AFL_TMPDIR", From 599b4631a3a40930e54e103f8ad1a69499fd1c8b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 22 Apr 2023 11:31:21 +0200 Subject: [PATCH 065/149] typo --- src/afl-fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index f6628851..394f33f9 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -326,7 +326,7 @@ static void usage(u8 *argv0, int more_help) { } #ifdef USE_PYTHON - SAYF("Compiled with %s module support, see docs/custom_mutator.md\n", + SAYF("Compiled with %s module support, see docs/custom_mutators.md\n", (char *)PYTHON_VERSION); #else SAYF("Compiled without Python module support.\n"); From 6bd48a48cbed1f923ff0999ea24af1f548c2e2bc Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 22 Apr 2023 11:39:44 +0200 Subject: [PATCH 066/149] code format --- custom_mutators/README.md | 59 ++- custom_mutators/atnwalk/atnwalk.c | 687 +++++++++++++++++------------- docs/Changelog.md | 4 + frida_mode/src/js/js_api.c | 4 +- 4 files changed, 439 insertions(+), 315 deletions(-) diff --git a/custom_mutators/README.md b/custom_mutators/README.md index 8d01856f..a5a572c0 100644 --- a/custom_mutators/README.md +++ b/custom_mutators/README.md @@ -11,29 +11,6 @@ The `./examples` folder contains examples for custom mutators in python and C. In `./rust`, you will find rust bindings, including a simple example in `./rust/example` and an example for structured fuzzing, based on lain, in`./rust/example_lain`. -## The AFL++ grammar agnostic grammar mutator - -In `./autotokens` you find a token-level fuzzer that does not need to know -anything about the grammar of an input as long as it is in ascii and allows -whitespace. -It is very fast and effective. - -If you are looking for an example of how to effectively create a custom -mutator take a look at this one. - -## The AFL++ Grammar Mutator - -If you use git to clone AFL++, then the following will incorporate our -excellent grammar custom mutator: - -```sh -git submodule update --init -``` - -Read the README in the [Grammar-Mutator] repository on how to use it. - -[Grammar-Mutator]: https://github.com/AFLplusplus/Grammar-Mutator - ## Production-Ready Custom Mutators This directory holds ready to use custom mutators. @@ -47,6 +24,42 @@ and add `AFL_CUSTOM_MUTATOR_ONLY=1` if you only want to use the custom mutator. Multiple custom mutators can be used by separating their paths with `:` in the environment variable. +### The AFL++ grammar agnostic grammar mutator + +In `./autotokens` you find a token-level fuzzer that does not need to know +anything about the grammar of an input as long as it is in ascii and allows +whitespace. +It is very fast and effective. + +If you are looking for an example of how to effectively create a custom +mutator take a look at this one. + +### The AFL++ Grammar Mutator + +If you use git to clone AFL++, then the following will incorporate our +excellent grammar custom mutator: + +```sh +git submodule update --init +``` + +Read the README in the [Grammar-Mutator] repository on how to use it. + +[Grammar-Mutator]: https://github.com/AFLplusplus/Grammar-Mutator + +Note that this custom mutator is not very good though! + +### Other Mutators + +atnwalk and gramatron are grammar custom mutators. Example grammars are +provided. + +honggfuzz, libfuzzer and libafl are partial implementations based on the +mutator implementations of the respective fuzzers. +More for playing than serious usage. + +radamsa is slow and not very good. + ## 3rd Party Custom Mutators ### Superion Mutators diff --git a/custom_mutators/atnwalk/atnwalk.c b/custom_mutators/atnwalk/atnwalk.c index 901b8a9e..c3a2cd95 100644 --- a/custom_mutators/atnwalk/atnwalk.c +++ b/custom_mutators/atnwalk/atnwalk.c @@ -11,10 +11,12 @@ #define BUF_SIZE_INIT 4096 #define SOCKET_NAME "./atnwalk.socket" -// how many errors (e.g. timeouts) to tolerate until moving on to the next queue entry +// how many errors (e.g. timeouts) to tolerate until moving on to the next queue +// entry #define ATNWALK_ERRORS_MAX 1 -// how many execution timeouts to tolerate until moving on to the next queue entry +// how many execution timeouts to tolerate until moving on to the next queue +// entry #define EXEC_TIMEOUT_MAX 2 // handshake constants @@ -27,80 +29,87 @@ const uint8_t SERVER_MUTATE_BIT = 0b00000010; const uint8_t SERVER_DECODE_BIT = 0b00000100; const uint8_t SERVER_ENCODE_BIT = 0b00001000; - typedef struct atnwalk_mutator { - afl_state_t *afl; - uint8_t atnwalk_error_count; - uint64_t prev_timeouts; - uint32_t prev_hits; - uint32_t stage_havoc_cur; - uint32_t stage_havoc_max; - uint32_t stage_splice_cur; - uint32_t stage_splice_max; - uint8_t *fuzz_buf; - size_t fuzz_size; - uint8_t *post_process_buf; - size_t post_process_size; + + afl_state_t *afl; + uint8_t atnwalk_error_count; + uint64_t prev_timeouts; + uint32_t prev_hits; + uint32_t stage_havoc_cur; + uint32_t stage_havoc_max; + uint32_t stage_splice_cur; + uint32_t stage_splice_max; + uint8_t *fuzz_buf; + size_t fuzz_size; + uint8_t *post_process_buf; + size_t post_process_size; + } atnwalk_mutator_t; - int read_all(int fd, uint8_t *buf, size_t buf_size) { - int n; - size_t offset = 0; - while (offset < buf_size) { - n = read(fd, buf + offset, buf_size - offset); - if (n == -1) { - return 0; - } - offset += n; - } - return 1; -} + int n; + size_t offset = 0; + while (offset < buf_size) { + + n = read(fd, buf + offset, buf_size - offset); + if (n == -1) { return 0; } + offset += n; + + } + + return 1; + +} int write_all(int fd, uint8_t *buf, size_t buf_size) { - int n; - size_t offset = 0; - while (offset < buf_size) { - n = write(fd, buf + offset, buf_size - offset); - if (n == -1) { - return 0; - } - offset += n; - } - return 1; -} + int n; + size_t offset = 0; + while (offset < buf_size) { + + n = write(fd, buf + offset, buf_size - offset); + if (n == -1) { return 0; } + offset += n; + + } + + return 1; + +} void put_uint32(uint8_t *buf, uint32_t val) { - buf[0] = (uint8_t) (val >> 24); - buf[1] = (uint8_t) ((val & 0x00ff0000) >> 16); - buf[2] = (uint8_t) ((val & 0x0000ff00) >> 8); - buf[3] = (uint8_t) (val & 0x000000ff); -} + buf[0] = (uint8_t)(val >> 24); + buf[1] = (uint8_t)((val & 0x00ff0000) >> 16); + buf[2] = (uint8_t)((val & 0x0000ff00) >> 8); + buf[3] = (uint8_t)(val & 0x000000ff); + +} uint32_t to_uint32(uint8_t *buf) { - uint32_t val = 0; - val |= (((uint32_t) buf[0]) << 24); - val |= (((uint32_t) buf[1]) << 16); - val |= (((uint32_t) buf[2]) << 8); - val |= ((uint32_t) buf[3]); - return val; -} + uint32_t val = 0; + val |= (((uint32_t)buf[0]) << 24); + val |= (((uint32_t)buf[1]) << 16); + val |= (((uint32_t)buf[2]) << 8); + val |= ((uint32_t)buf[3]); + return val; + +} void put_uint64(uint8_t *buf, uint64_t val) { - buf[0] = (uint8_t) (val >> 56); - buf[1] = (uint8_t) ((val & 0x00ff000000000000) >> 48); - buf[2] = (uint8_t) ((val & 0x0000ff0000000000) >> 40); - buf[3] = (uint8_t) ((val & 0x000000ff00000000) >> 32); - buf[4] = (uint8_t) ((val & 0x00000000ff000000) >> 24); - buf[5] = (uint8_t) ((val & 0x0000000000ff0000) >> 16); - buf[6] = (uint8_t) ((val & 0x000000000000ff00) >> 8); - buf[7] = (uint8_t) (val & 0x00000000000000ff); -} + buf[0] = (uint8_t)(val >> 56); + buf[1] = (uint8_t)((val & 0x00ff000000000000) >> 48); + buf[2] = (uint8_t)((val & 0x0000ff0000000000) >> 40); + buf[3] = (uint8_t)((val & 0x000000ff00000000) >> 32); + buf[4] = (uint8_t)((val & 0x00000000ff000000) >> 24); + buf[5] = (uint8_t)((val & 0x0000000000ff0000) >> 16); + buf[6] = (uint8_t)((val & 0x000000000000ff00) >> 8); + buf[7] = (uint8_t)(val & 0x00000000000000ff); + +} /** * Initialize this custom mutator @@ -114,70 +123,83 @@ void put_uint64(uint8_t *buf, uint64_t val) { * Return NULL on error. */ atnwalk_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { - srand(seed); - atnwalk_mutator_t *data = (atnwalk_mutator_t *) malloc(sizeof(atnwalk_mutator_t)); - if (!data) { - perror("afl_custom_init alloc"); - return NULL; - } - data->afl = afl; - data->prev_hits = 0; - data->fuzz_buf = (uint8_t *) malloc(BUF_SIZE_INIT); - data->fuzz_size = BUF_SIZE_INIT; - data->post_process_buf = (uint8_t *) malloc(BUF_SIZE_INIT); - data->post_process_size = BUF_SIZE_INIT; - return data; + + srand(seed); + atnwalk_mutator_t *data = + (atnwalk_mutator_t *)malloc(sizeof(atnwalk_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + data->afl = afl; + data->prev_hits = 0; + data->fuzz_buf = (uint8_t *)malloc(BUF_SIZE_INIT); + data->fuzz_size = BUF_SIZE_INIT; + data->post_process_buf = (uint8_t *)malloc(BUF_SIZE_INIT); + data->post_process_size = BUF_SIZE_INIT; + return data; + } +unsigned int afl_custom_fuzz_count(atnwalk_mutator_t *data, + const unsigned char *buf, size_t buf_size) { -unsigned int afl_custom_fuzz_count(atnwalk_mutator_t *data, const unsigned char *buf, size_t buf_size) { - // afl_custom_fuzz_count is called exactly once before entering the 'stage-loop' for the current queue entry - // thus, we use it to reset the error count and to initialize stage variables (somewhat not intended by the API, - // but still better than rewriting the whole thing to have a custom mutator stage) - data->atnwalk_error_count = 0; - data->prev_timeouts = data->afl->total_tmouts; + // afl_custom_fuzz_count is called exactly once before entering the + // 'stage-loop' for the current queue entry thus, we use it to reset the error + // count and to initialize stage variables (somewhat not intended by the API, + // but still better than rewriting the whole thing to have a custom mutator + // stage) + data->atnwalk_error_count = 0; + data->prev_timeouts = data->afl->total_tmouts; - // it might happen that on the last execution of the splice stage a new path is found - // we need to fix that here and count it - if (data->prev_hits) { - data->afl->stage_finds[STAGE_SPLICE] += data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; - } - data->prev_hits = data->afl->queued_items + data->afl->saved_crashes; - data->stage_havoc_cur = 0; - data->stage_splice_cur = 0; + // it might happen that on the last execution of the splice stage a new path + // is found we need to fix that here and count it + if (data->prev_hits) { + + data->afl->stage_finds[STAGE_SPLICE] += + data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; + + } + + data->prev_hits = data->afl->queued_items + data->afl->saved_crashes; + data->stage_havoc_cur = 0; + data->stage_splice_cur = 0; + + // 50% havoc, 50% splice + data->stage_havoc_max = data->afl->stage_max >> 1; + if (data->stage_havoc_max < HAVOC_MIN) { data->stage_havoc_max = HAVOC_MIN; } + data->stage_splice_max = data->stage_havoc_max; + return data->stage_havoc_max + data->stage_splice_max; - // 50% havoc, 50% splice - data->stage_havoc_max = data->afl->stage_max >> 1; - if (data->stage_havoc_max < HAVOC_MIN) { - data->stage_havoc_max = HAVOC_MIN; - } - data->stage_splice_max = data->stage_havoc_max; - return data->stage_havoc_max + data->stage_splice_max; } - size_t fail_fatal(int fd_socket, uint8_t **out_buf) { - if (fd_socket != -1) { - close(fd_socket); - } - *out_buf = NULL; - return 0; + + if (fd_socket != -1) { close(fd_socket); } + *out_buf = NULL; + return 0; + } +size_t fail_gracefully(int fd_socket, atnwalk_mutator_t *data, uint8_t *buf, + size_t buf_size, uint8_t **out_buf) { + + if (fd_socket != -1) { close(fd_socket); } + data->atnwalk_error_count++; + if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { + + data->afl->stage_max = data->afl->stage_cur; + + } + + *out_buf = buf; + return buf_size; -size_t fail_gracefully(int fd_socket, atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf) { - if (fd_socket != -1) { - close(fd_socket); - } - data->atnwalk_error_count++; - if (data->atnwalk_error_count > ATNWALK_ERRORS_MAX) { - data->afl->stage_max = data->afl->stage_cur; - } - *out_buf = buf; - return buf_size; } - /** * Perform custom mutations on a given input * @@ -194,153 +216,215 @@ size_t fail_gracefully(int fd_socket, atnwalk_mutator_t *data, uint8_t *buf, siz * produce data larger than max_size. * @return Size of the mutated output. */ -size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf, - uint8_t *add_buf, size_t add_buf_size, size_t max_size) { - struct sockaddr_un addr; - int fd_socket; - uint8_t ctrl_buf[8]; - uint8_t wanted; +size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, + uint8_t **out_buf, uint8_t *add_buf, size_t add_buf_size, + size_t max_size) { - // let's display what's going on in a nice way - if (data->stage_havoc_cur == 0) { - data->afl->stage_name = (uint8_t *) "atnwalk - havoc"; - } - if (data->stage_havoc_cur == data->stage_havoc_max) { - data->afl->stage_name = (uint8_t *) "atnwalk - splice"; - } + struct sockaddr_un addr; + int fd_socket; + uint8_t ctrl_buf[8]; + uint8_t wanted; + + // let's display what's going on in a nice way + if (data->stage_havoc_cur == 0) { + + data->afl->stage_name = (uint8_t *)"atnwalk - havoc"; + + } + + if (data->stage_havoc_cur == data->stage_havoc_max) { + + data->afl->stage_name = (uint8_t *)"atnwalk - splice"; + + } + + // increase the respective havoc or splice counters + if (data->stage_havoc_cur < data->stage_havoc_max) { + + data->stage_havoc_cur++; + data->afl->stage_cycles[STAGE_HAVOC]++; + + } else { + + // if there is nothing to splice, continue with havoc and skip splicing this + // time + if (data->afl->ready_for_splicing_count < 1) { + + data->stage_havoc_max = data->afl->stage_max; + data->stage_havoc_cur++; + data->afl->stage_cycles[STAGE_HAVOC]++; - // increase the respective havoc or splice counters - if (data->stage_havoc_cur < data->stage_havoc_max) { - data->stage_havoc_cur++; - data->afl->stage_cycles[STAGE_HAVOC]++; } else { - // if there is nothing to splice, continue with havoc and skip splicing this time - if (data->afl->ready_for_splicing_count < 1) { - data->stage_havoc_max = data->afl->stage_max; - data->stage_havoc_cur++; - data->afl->stage_cycles[STAGE_HAVOC]++; - } else { - data->stage_splice_cur++; - data->afl->stage_cycles[STAGE_SPLICE]++; - } + + data->stage_splice_cur++; + data->afl->stage_cycles[STAGE_SPLICE]++; + } - // keep track of found new corpus seeds per stage - if (data->afl->queued_items + data->afl->saved_crashes > data->prev_hits) { - if (data->stage_splice_cur <= 1) { - data->afl->stage_finds[STAGE_HAVOC] += data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; - } else { - data->afl->stage_finds[STAGE_SPLICE] += - data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; - } - } - data->prev_hits = data->afl->queued_items + data->afl->saved_crashes; + } + + // keep track of found new corpus seeds per stage + if (data->afl->queued_items + data->afl->saved_crashes > data->prev_hits) { + + if (data->stage_splice_cur <= 1) { + + data->afl->stage_finds[STAGE_HAVOC] += + data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; + + } else { + + data->afl->stage_finds[STAGE_SPLICE] += + data->afl->queued_items + data->afl->saved_crashes - data->prev_hits; - // check whether this input produces a lot of timeouts, if it does then abandon this queue entry - if (data->afl->total_tmouts - data->prev_timeouts >= EXEC_TIMEOUT_MAX) { - data->afl->stage_max = data->afl->stage_cur; - return fail_gracefully(-1, data, buf, buf_size, out_buf); } - // initialize the socket - fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd_socket == -1) { return fail_fatal(fd_socket, out_buf); } - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); - if (connect(fd_socket, (const struct sockaddr *) &addr, sizeof(addr)) == -1) { - return fail_fatal(fd_socket, out_buf); + } + + data->prev_hits = data->afl->queued_items + data->afl->saved_crashes; + + // check whether this input produces a lot of timeouts, if it does then + // abandon this queue entry + if (data->afl->total_tmouts - data->prev_timeouts >= EXEC_TIMEOUT_MAX) { + + data->afl->stage_max = data->afl->stage_cur; + return fail_gracefully(-1, data, buf, buf_size, out_buf); + + } + + // initialize the socket + fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd_socket == -1) { return fail_fatal(fd_socket, out_buf); } + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); + if (connect(fd_socket, (const struct sockaddr *)&addr, sizeof(addr)) == -1) { + + return fail_fatal(fd_socket, out_buf); + + } + + // ask whether the server is alive + ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; + if (!write_all(fd_socket, ctrl_buf, 1)) { + + return fail_fatal(fd_socket, out_buf); + + } + + // see whether the server replies as expected + if (!read_all(fd_socket, ctrl_buf, 1) || + ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { + + return fail_fatal(fd_socket, out_buf); + + } + + // tell the server what we want to do + wanted = SERVER_MUTATE_BIT | SERVER_ENCODE_BIT; + + // perform a crossover if we are splicing + if (data->stage_splice_cur > 0) { wanted |= SERVER_CROSSOVER_BIT; } + + // tell the server what we want and how much data will be sent + ctrl_buf[0] = wanted; + put_uint32(ctrl_buf + 1, (uint32_t)buf_size); + if (!write_all(fd_socket, ctrl_buf, 5)) { + + return fail_fatal(fd_socket, out_buf); + + } + + // send the data to mutate and encode + if (!write_all(fd_socket, buf, buf_size)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + if (wanted & SERVER_CROSSOVER_BIT) { + + // since we requested crossover, we will first tell how much additional data + // is to be expected + put_uint32(ctrl_buf, (uint32_t)add_buf_size); + if (!write_all(fd_socket, ctrl_buf, 4)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + } - // ask whether the server is alive - ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; - if (!write_all(fd_socket, ctrl_buf, 1)) { - return fail_fatal(fd_socket, out_buf); + // send the additional data for crossover + if (!write_all(fd_socket, add_buf, add_buf_size)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + } - // see whether the server replies as expected - if (!read_all(fd_socket, ctrl_buf, 1) || ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { - return fail_fatal(fd_socket, out_buf); - } - - // tell the server what we want to do - wanted = SERVER_MUTATE_BIT | SERVER_ENCODE_BIT; - - // perform a crossover if we are splicing - if (data->stage_splice_cur > 0) { - wanted |= SERVER_CROSSOVER_BIT; - } - - // tell the server what we want and how much data will be sent - ctrl_buf[0] = wanted; - put_uint32(ctrl_buf + 1, (uint32_t) buf_size); - if (!write_all(fd_socket, ctrl_buf, 5)) { - return fail_fatal(fd_socket, out_buf); - } - - // send the data to mutate and encode - if (!write_all(fd_socket, buf, buf_size)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } - - if (wanted & SERVER_CROSSOVER_BIT) { - // since we requested crossover, we will first tell how much additional data is to be expected - put_uint32(ctrl_buf, (uint32_t) add_buf_size); - if (!write_all(fd_socket, ctrl_buf, 4)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } - - // send the additional data for crossover - if (!write_all(fd_socket, add_buf, add_buf_size)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } - - // lastly, a seed is required for crossover so send one - put_uint64(ctrl_buf, (uint64_t) rand()); - if (!write_all(fd_socket, ctrl_buf, 8)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } - } - - // since we requested mutation, we need to provide a seed for that - put_uint64(ctrl_buf, (uint64_t) rand()); + // lastly, a seed is required for crossover so send one + put_uint64(ctrl_buf, (uint64_t)rand()); if (!write_all(fd_socket, ctrl_buf, 8)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + } - // obtain the required buffer size for the data that will be returned - if (!read_all(fd_socket, ctrl_buf, 4)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } - size_t new_size = (size_t) to_uint32(ctrl_buf); + } + + // since we requested mutation, we need to provide a seed for that + put_uint64(ctrl_buf, (uint64_t)rand()); + if (!write_all(fd_socket, ctrl_buf, 8)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + // obtain the required buffer size for the data that will be returned + if (!read_all(fd_socket, ctrl_buf, 4)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + size_t new_size = (size_t)to_uint32(ctrl_buf); + + // if the data is too large then we ignore this round + if (new_size > max_size) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + if (new_size > buf_size) { + + // buf is too small, need to use data->fuzz_buf, let's see whether we need + // to reallocate + if (new_size > data->fuzz_size) { + + data->fuzz_size = new_size << 1; + data->fuzz_buf = (uint8_t *)realloc(data->fuzz_buf, data->fuzz_size); - // if the data is too large then we ignore this round - if (new_size > max_size) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); } - if (new_size > buf_size) { - // buf is too small, need to use data->fuzz_buf, let's see whether we need to reallocate - if (new_size > data->fuzz_size) { - data->fuzz_size = new_size << 1; - data->fuzz_buf = (uint8_t *) realloc(data->fuzz_buf, data->fuzz_size); - } - *out_buf = data->fuzz_buf; - } else { - // new_size fits into buf, so re-use it - *out_buf = buf; - } + *out_buf = data->fuzz_buf; - // obtain the encoded data - if (!read_all(fd_socket, *out_buf, new_size)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } + } else { + + // new_size fits into buf, so re-use it + *out_buf = buf; + + } + + // obtain the encoded data + if (!read_all(fd_socket, *out_buf, new_size)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + close(fd_socket); + return new_size; - close(fd_socket); - return new_size; } - /** * A post-processing function to use right before AFL writes the test case to * disk in order to execute the target. @@ -357,76 +441,99 @@ size_t afl_custom_fuzz(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, u * @return Size of the output buffer after processing or the needed amount. * A return of 0 indicates an error. */ -size_t afl_custom_post_process(atnwalk_mutator_t *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf) { - struct sockaddr_un addr; - int fd_socket; - uint8_t ctrl_buf[8]; +size_t afl_custom_post_process(atnwalk_mutator_t *data, uint8_t *buf, + size_t buf_size, uint8_t **out_buf) { - // initialize the socket - fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); - if (fd_socket == -1) { - return fail_fatal(fd_socket, out_buf); - } - memset(&addr, 0, sizeof(addr)); - addr.sun_family = AF_UNIX; - strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); - if (connect(fd_socket, (const struct sockaddr *) &addr, sizeof(addr)) == -1) { - return fail_fatal(fd_socket, out_buf); - } + struct sockaddr_un addr; + int fd_socket; + uint8_t ctrl_buf[8]; - // ask whether the server is alive - ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; - if (!write_all(fd_socket, ctrl_buf, 1)) { - return fail_fatal(fd_socket, out_buf); - } + // initialize the socket + fd_socket = socket(AF_UNIX, SOCK_STREAM, 0); + if (fd_socket == -1) { return fail_fatal(fd_socket, out_buf); } + memset(&addr, 0, sizeof(addr)); + addr.sun_family = AF_UNIX; + strncpy(addr.sun_path, SOCKET_NAME, sizeof(addr.sun_path) - 1); + if (connect(fd_socket, (const struct sockaddr *)&addr, sizeof(addr)) == -1) { - // see whether the server replies as expected - if (!read_all(fd_socket, ctrl_buf, 1) || ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { - return fail_fatal(fd_socket, out_buf); - } + return fail_fatal(fd_socket, out_buf); - // tell the server what we want and how much data will be sent - ctrl_buf[0] = SERVER_DECODE_BIT; - put_uint32(ctrl_buf + 1, (uint32_t) buf_size); - if (!write_all(fd_socket, ctrl_buf, 5)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } + } - // send the data to decode - if (!write_all(fd_socket, buf, buf_size)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } + // ask whether the server is alive + ctrl_buf[0] = SERVER_ARE_YOU_ALIVE; + if (!write_all(fd_socket, ctrl_buf, 1)) { - // obtain the required buffer size for the data that will be returned - if (!read_all(fd_socket, ctrl_buf, 4)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } - size_t new_size = (size_t) to_uint32(ctrl_buf); + return fail_fatal(fd_socket, out_buf); - // need to use data->post_process_buf, let's see whether we need to reallocate - if (new_size > data->post_process_size) { - data->post_process_size = new_size << 1; - data->post_process_buf = (uint8_t *) realloc(data->post_process_buf, data->post_process_size); - } - *out_buf = data->post_process_buf; + } - // obtain the decoded data - if (!read_all(fd_socket, *out_buf, new_size)) { - return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); - } + // see whether the server replies as expected + if (!read_all(fd_socket, ctrl_buf, 1) || + ctrl_buf[0] != SERVER_YES_I_AM_ALIVE) { + + return fail_fatal(fd_socket, out_buf); + + } + + // tell the server what we want and how much data will be sent + ctrl_buf[0] = SERVER_DECODE_BIT; + put_uint32(ctrl_buf + 1, (uint32_t)buf_size); + if (!write_all(fd_socket, ctrl_buf, 5)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + // send the data to decode + if (!write_all(fd_socket, buf, buf_size)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + // obtain the required buffer size for the data that will be returned + if (!read_all(fd_socket, ctrl_buf, 4)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + size_t new_size = (size_t)to_uint32(ctrl_buf); + + // need to use data->post_process_buf, let's see whether we need to reallocate + if (new_size > data->post_process_size) { + + data->post_process_size = new_size << 1; + data->post_process_buf = + (uint8_t *)realloc(data->post_process_buf, data->post_process_size); + + } + + *out_buf = data->post_process_buf; + + // obtain the decoded data + if (!read_all(fd_socket, *out_buf, new_size)) { + + return fail_gracefully(fd_socket, data, buf, buf_size, out_buf); + + } + + close(fd_socket); + return new_size; - close(fd_socket); - return new_size; } - /** * Deinitialize everything * * @param data The data ptr from afl_custom_init */ void afl_custom_deinit(atnwalk_mutator_t *data) { - free(data->fuzz_buf); - free(data->post_process_buf); - free(data); + + free(data->fuzz_buf); + free(data->post_process_buf); + free(data); + } + diff --git a/docs/Changelog.md b/docs/Changelog.md index 5ed5ef2b..f33acff9 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -4,8 +4,12 @@ release of the tool. See README.md for the general instruction manual. ### Version ++4.07a (dev) + - afl-fuzz: + - new env `AFL_POST_PROCESS_KEEP_ORIGINAL` to keep the orignal + data before post process on finds - afl-showmap: - added custom mutator post_process and send support + - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! ### Version ++4.06c (release) diff --git a/frida_mode/src/js/js_api.c b/frida_mode/src/js/js_api.c index 00278082..288aec95 100644 --- a/frida_mode/src/js/js_api.c +++ b/frida_mode/src/js/js_api.c @@ -156,8 +156,8 @@ __attribute__((visibility("default"))) void js_api_set_instrument_instructions( } -__attribute__((visibility("default"))) void js_api_set_instrument_no_dynamic_load( - void) { +__attribute__((visibility("default"))) void +js_api_set_instrument_no_dynamic_load(void) { ranges_inst_dynamic_load = FALSE; From 531380d6ab9d2467d174ae45dcfa64bbf12230c0 Mon Sep 17 00:00:00 2001 From: Keno Hassler <40292329+kenohassler@users.noreply.github.com> Date: Mon, 24 Apr 2023 17:46:56 +0200 Subject: [PATCH 067/149] llvm-lto: allow skipping initialization --- instrumentation/SanitizerCoverageLTO.so.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index e41f19b6..42583f9e 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1005,7 +1005,8 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( // afl++ START if (dFile.is_open()) dFile.close(); - if (!getenv("AFL_LLVM_LTO_DONTWRITEID") || dictionary.size() || map_addr) { + if (!getenv("AFL_LLVM_LTO_SKIPINIT") && + (!getenv("AFL_LLVM_LTO_DONTWRITEID") || dictionary.size() || map_addr)) { // yes we could create our own function, insert it into ctors ... // but this would be a pain in the butt ... so we use afl-llvm-rt-lto.o From 8c228b0d23e303499dccf3df77c5d0b3a8b59b7b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 24 Apr 2023 18:08:27 +0200 Subject: [PATCH 068/149] afl-showmap -I option --- afl-cmin | 2 +- docs/Changelog.md | 1 + src/afl-showmap.c | 175 ++++++++++++++++++++++++++++++++++++++++------ 3 files changed, 156 insertions(+), 22 deletions(-) diff --git a/afl-cmin b/afl-cmin index e2c26d91..6b36c261 100755 --- a/afl-cmin +++ b/afl-cmin @@ -234,7 +234,7 @@ BEGIN { } # while options if (!mem_limit) mem_limit = "none" - if (!timeout) timeout = "none" + if (!timeout) timeout = "5000" # get program args i = 0 diff --git a/docs/Changelog.md b/docs/Changelog.md index f33acff9..d4e68036 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,6 +9,7 @@ data before post process on finds - afl-showmap: - added custom mutator post_process and send support + - add `-I filelist` option, an alternative to `-i in_dir` - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 0b9fc211..09a1d2dc 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -69,7 +69,9 @@ static afl_state_t *afl; static char *stdin_file; /* stdin file */ static u8 *in_dir = NULL, /* input folder */ - *out_file = NULL, *at_file = NULL; /* Substitution string for @@ */ + *out_file = NULL, /* output file or directory */ + *at_file = NULL, /* Substitution string for @@ */ + *in_filelist = NULL; /* input file list */ static u8 outfile[PATH_MAX]; @@ -878,6 +880,104 @@ u32 execute_testcases(u8 *dir) { } +u32 execute_testcases_filelist(u8 *fn) { + + u32 done = 0; + u8 buf[4096]; + u8 val_buf[2][STRINGIFY_VAL_SIZE_MAX]; + FILE *f; + + if (!be_quiet) { ACTF("Reading from '%s'...", fn); } + + if ((f = fopen(fn, "r")) == NULL) { FATAL("could not open '%s'", fn); } + + while (fgets(buf, sizeof(buf), f) != NULL) { + + struct stat st; + + u8 *fn2 = buf, *fn3; + ; + + while (*fn2 == ' ') { + + ++fn2; + + } + + if (*fn2) { + + while (fn2[strlen(fn2) - 1] == '\r' || fn2[strlen(fn2) - 1] == '\n' || + fn2[strlen(fn2) - 1] == ' ') { + + fn2[strlen(fn2) - 1] = 0; + + } + + } + + if (debug) { printf("Getting coverage for '%s'\n", fn2); } + + if (!*fn2) { continue; } + + if (lstat(fn2, &st) || access(fn2, R_OK)) { + + WARNF("Unable to access '%s'", fn2); + continue; + + } + + if (!S_ISREG(st.st_mode) || !st.st_size) { continue; } + + if ((fn3 = strrchr(fn2, '/'))) { + + ++fn3; + + } else { + + fn3 = fn2; + + } + + if (st.st_size > MAX_FILE && !be_quiet && !quiet_mode) { + + WARNF("Test case '%s' is too big (%s, limit is %s), partial reading", fn2, + stringify_mem_size(val_buf[0], sizeof(val_buf[0]), st.st_size), + stringify_mem_size(val_buf[1], sizeof(val_buf[1]), MAX_FILE)); + + } + + if (!collect_coverage) + snprintf(outfile, sizeof(outfile), "%s/%s", out_file, fn3); + + if (read_file(fn2)) { + + if (wait_for_gdb) { + + fprintf(stderr, "exec: gdb -p %d\n", fsrv->child_pid); + fprintf(stderr, "exec: kill -CONT %d\n", getpid()); + kill(0, SIGSTOP); + + } + + showmap_run_target_forkserver(fsrv, in_data, in_len); + ck_free(in_data); + ++done; + + if (child_crashed && debug) { WARNF("crashed: %s", fn2); } + + if (collect_coverage) + analyze_results(fsrv); + else + tcnt = write_results_to_file(fsrv, outfile); + + } + + } + + return done; + +} + /* Show banner. */ static void show_banner(void) { @@ -920,6 +1020,7 @@ static void usage(u8 *argv0) { " With -C, -o is a file, without -C it must be a " "directory\n" " and each bitmap will be written there individually.\n" + " -I filelist - alternatively to -i, -I is a list of files\n" " -C - collect coverage, writes all edges to -o and gives a " "summary\n" " Must be combined with -i.\n" @@ -932,7 +1033,7 @@ static void usage(u8 *argv0) { "This tool displays raw tuple data captured by AFL instrumentation.\n" "For additional help, consult %s/README.md.\n\n" - "If you use -i mode, then custom mutator post_process send send " + "If you use -i/-I mode, then custom mutator post_process send send " "functionality\n" "is supported.\n\n" @@ -994,7 +1095,7 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_QUIET") != NULL) { be_quiet = true; } - while ((opt = getopt(argc, argv, "+i:o:f:m:t:AeqCZOH:QUWbcrshXY")) > 0) { + while ((opt = getopt(argc, argv, "+i:I:o:f:m:t:AeqCZOH:QUWbcrshXY")) > 0) { switch (opt) { @@ -1012,6 +1113,11 @@ int main(int argc, char **argv_orig, char **envp) { in_dir = optarg; break; + case 'I': + if (in_filelist) { FATAL("Multiple -I options not supported"); } + in_filelist = optarg; + break; + case 'o': if (out_file) { FATAL("Multiple -o options not supported"); } @@ -1234,10 +1340,12 @@ int main(int argc, char **argv_orig, char **envp) { if (optind == argc || !out_file) { usage(argv[0]); } - if (in_dir) { + if (in_dir && in_filelist) { FATAL("you can only specify either -i or -I"); } + + if (in_dir || in_filelist) { if (!out_file && !collect_coverage) - FATAL("for -i you need to specify either -C and/or -o"); + FATAL("for -i/-I you need to specify either -C and/or -o"); } @@ -1294,7 +1402,7 @@ int main(int argc, char **argv_orig, char **envp) { } - if (in_dir) { + if (in_dir || in_filelist) { /* If we don't have a file name chosen yet, use a safe default. */ u8 *use_dir = "."; @@ -1400,7 +1508,7 @@ int main(int argc, char **argv_orig, char **envp) { } #ifdef __linux__ - if (!fsrv->nyx_mode && in_dir) { + if (!fsrv->nyx_mode && (in_dir || in_filelist)) { (void)check_binary_signatures(fsrv->target_path); @@ -1483,7 +1591,7 @@ int main(int argc, char **argv_orig, char **envp) { } - if (in_dir) { + if (in_dir || in_filelist) { afl->fsrv.dev_urandom_fd = open("/dev/urandom", O_RDONLY); afl->afl_env.afl_custom_mutator_library = @@ -1496,33 +1604,46 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_CUSTOM_MUTATOR_LIBRARY") || getenv("AFL_PYTHON_MODULE")) { WARNF( - "Custom mutator environment detected, this is only supported in -i " - "mode!\n"); + "Custom mutator environment detected, this is only supported in " + "-i/-I mode!\n"); } } - if (in_dir) { + if (in_dir || in_filelist) { DIR *dir_in, *dir_out = NULL; + u8 *dn = NULL; if (getenv("AFL_DEBUG_GDB")) wait_for_gdb = true; fsrv->dev_null_fd = open("/dev/null", O_RDWR); if (fsrv->dev_null_fd < 0) { PFATAL("Unable to open /dev/null"); } - // if a queue subdirectory exists switch to that - u8 *dn = alloc_printf("%s/queue", in_dir); - if ((dir_in = opendir(dn)) != NULL) { + if (in_filelist) { - closedir(dir_in); - in_dir = dn; + if (!be_quiet) ACTF("Reading from file list '%s'...", in_filelist); - } else + } else { - ck_free(dn); - if (!be_quiet) ACTF("Reading from directory '%s'...", in_dir); + // if a queue subdirectory exists switch to that + dn = alloc_printf("%s/queue", in_dir); + + if ((dir_in = opendir(dn)) != NULL) { + + closedir(dir_in); + in_dir = dn; + + } else { + + ck_free(dn); + + } + + if (!be_quiet) ACTF("Reading from directory '%s'...", in_dir); + + } if (!collect_coverage) { @@ -1576,9 +1697,21 @@ int main(int argc, char **argv_orig, char **envp) { if (fsrv->support_shmem_fuzz && !fsrv->use_shmem_fuzz) shm_fuzz = deinit_shmem(fsrv, shm_fuzz); - if (execute_testcases(in_dir) == 0) { + if (in_dir) { - FATAL("could not read input testcases from %s", in_dir); + if (execute_testcases(in_dir) == 0) { + + FATAL("could not read input testcases from %s", in_dir); + + } + + } else { + + if (execute_testcases_filelist(in_filelist) == 0) { + + FATAL("could not read input testcases from %s", in_filelist); + + } } From 46237c33322bbe92d35436718b37062f5c1a8cb6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 24 Apr 2023 19:20:52 +0200 Subject: [PATCH 069/149] makefile for atwalk --- custom_mutators/atnwalk/Makefile | 7 +++++++ custom_mutators/atnwalk/README.md | 4 +--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 custom_mutators/atnwalk/Makefile diff --git a/custom_mutators/atnwalk/Makefile b/custom_mutators/atnwalk/Makefile new file mode 100644 index 00000000..bf83cae5 --- /dev/null +++ b/custom_mutators/atnwalk/Makefile @@ -0,0 +1,7 @@ +all: atnwalk.so + +atnwalk.so: atnwalk.c + $(CC) -I ../../include/ -shared -fPIC -O3 -o atnwalk.so atnwalk.c + +clean: + rm -f *.so *.o *~ core diff --git a/custom_mutators/atnwalk/README.md b/custom_mutators/atnwalk/README.md index badb856f..730349a3 100644 --- a/custom_mutators/atnwalk/README.md +++ b/custom_mutators/atnwalk/README.md @@ -6,9 +6,7 @@ Refer to [https://github.com/atnwalk/testbed](https://github.com/atnwalk/testbed ## Build -```bash -gcc -I ../../include/ -shared -fPIC -Wall -O3 atnwalk.c -o atnwalk.so -``` +Just type `make` to build `atnwalk.so`. ## Run From 7c3c0b26d1ae477fbae6944c0de18256621e1993 Mon Sep 17 00:00:00 2001 From: Keno Hassler <40292329+kenohassler@users.noreply.github.com> Date: Mon, 24 Apr 2023 20:21:54 +0200 Subject: [PATCH 070/149] document new env var --- docs/env_variables.md | 6 ++++-- src/afl-cc.c | 2 ++ 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/docs/env_variables.md b/docs/env_variables.md index c5995d13..087ccdb7 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -156,7 +156,7 @@ Available options: - LTO - LTO instrumentation - NATIVE - clang's original pcguard based instrumentation - NGRAM-x - deeper previous location coverage (from NGRAM-2 up to NGRAM-16) - - PCGUARD - our own pcgard based instrumentation (default) + - PCGUARD - our own pcguard based instrumentation (default) #### CMPLOG @@ -240,7 +240,9 @@ combined. the default `0x10000`. A value of 0 or empty sets the map address to be dynamic (the original AFL way, which is slower). - `AFL_LLVM_MAP_DYNAMIC` sets the shared memory address to be dynamic. - + - `AFL_LLVM_LTO_SKIPINIT` skips adding initialization code. Some global vars + (e.g. the highest location ID) are not injected. Needed to instrument with + [WAFL](https://github.com/fgsect/WAFL.git). For more information, see [instrumentation/README.lto.md](../instrumentation/README.lto.md). diff --git a/src/afl-cc.c b/src/afl-cc.c index 7f15ad76..d1001187 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -2099,6 +2099,8 @@ int main(int argc, char **argv, char **envp) { "bb\n" " AFL_REAL_LD: use this lld linker instead of the compiled in " "path\n" + " AFL_LLVM_LTO_SKIPINIT: don't inject initialization code " + "(used in WAFL mode)\n" "If anything fails - be sure to read README.lto.md!\n"); #endif From 7b33148b7553f11dac7a382495a6829fa14f23fe Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 08:23:27 +0200 Subject: [PATCH 071/149] add AFL_LLVM_LTO_SKIPINIT to envs.h --- include/envs.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/envs.h b/include/envs.h index 23599b26..5e68c80b 100644 --- a/include/envs.h +++ b/include/envs.h @@ -160,8 +160,9 @@ static char *afl_environment_variables[] = { "AFL_LLVM_SKIP_NEVERZERO", "AFL_NO_AFFINITY", "AFL_TRY_AFFINITY", - "AFL_LLVM_LTO_STARTID", "AFL_LLVM_LTO_DONTWRITEID", + "AFL_LLVM_LTO_SKIPINIT" + "AFL_LLVM_LTO_STARTID", "AFL_NO_ARITH", "AFL_NO_AUTODICT", "AFL_NO_BUILTIN", From c0ecf7cf61fdca901b041d57e7e2bb78bc8fcf80 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 08:33:51 +0200 Subject: [PATCH 072/149] only reverse reading the queue on restart --- docs/Changelog.md | 4 ++++ src/afl-fuzz-init.c | 30 ++++++++++++++++++++---------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index d4e68036..14323ae0 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -7,6 +7,10 @@ - afl-fuzz: - new env `AFL_POST_PROCESS_KEEP_ORIGINAL` to keep the orignal data before post process on finds + - reverse reading the seeds only on restarts + - afl-cc: + - new env `AFL_LLVM_LTO_SKIPINIT` to support the AFL++ based WASM + (https://github.com/fgsect/WAFL) project - afl-showmap: - added custom mutator post_process and send support - add `-I filelist` option, an alternative to `-i in_dir` diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 01d1e82e..002a26f8 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -718,10 +718,21 @@ void read_testcases(afl_state_t *afl, u8 *directory) { if (nl_cnt) { - i = nl_cnt; + u32 done = 0; + + if (unlikely(afl->in_place_resume)) { + + i = nl_cnt; + + } else { + + i = 0; + + } + do { - --i; + if (unlikely(afl->in_place_resume)) { --i; } struct stat st; u8 dfn[PATH_MAX]; @@ -801,18 +812,17 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } - /* - if (unlikely(afl->schedule >= FAST && afl->schedule <= RARE)) { + if (unlikely(afl->in_place_resume)) { - u64 cksum = hash64(afl->fsrv.trace_bits, afl->fsrv.map_size, - HASH_CONST); afl->queue_top->n_fuzz_entry = cksum % N_FUZZ_SIZE; - afl->n_fuzz[afl->queue_top->n_fuzz_entry] = 1; + if (unlikely(i == 0)) { done = 1; } - } + } else { - */ + if (unlikely(++i == (u32)nl_cnt)) { done = 1; } - } while (i > 0); + } + + } while (!done); } From 7b877e2c1d96efa7486ef4ba7860bec58dd1cd5b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 09:30:25 +0200 Subject: [PATCH 073/149] afl-cmin.bash -T support --- afl-cmin.bash | 107 ++++++++++++++++++++++++++++++++++++++++------ docs/Changelog.md | 2 + 2 files changed, 96 insertions(+), 13 deletions(-) diff --git a/afl-cmin.bash b/afl-cmin.bash index 5258758e..ba7083fa 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -7,6 +7,8 @@ # # Copyright 2014, 2015 Google Inc. All rights reserved. # +# Copyright 2019-2023 AFLplusplus +# # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at: @@ -36,7 +38,7 @@ # array sizes. # -echo "corpus minimization tool for afl-fuzz by Michal Zalewski" +echo "corpus minimization tool for afl-fuzz" echo ######### @@ -46,14 +48,14 @@ echo # Process command-line options... MEM_LIMIT=none -TIMEOUT=none +TIMEOUT=5000 -unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN \ - AFL_CMIN_CRASHES_ONLY AFL_CMIN_ALLOW_ANY QEMU_MODE UNICORN_MODE +unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN F_ARG \ + AFL_CMIN_CRASHES_ONLY AFL_CMIN_ALLOW_ANY QEMU_MODE UNICORN_MODE T_ARG export AFL_QUIET=1 -while getopts "+i:o:f:m:t:eOQUAChXY" opt; do +while getopts "+i:o:f:m:t:T:eOQUAChXY" opt; do case "$opt" in @@ -69,6 +71,7 @@ while getopts "+i:o:f:m:t:eOQUAChXY" opt; do ;; "f") STDIN_FILE="$OPTARG" + F_ARG=1 ;; "m") MEM_LIMIT="$OPTARG" @@ -106,6 +109,9 @@ while getopts "+i:o:f:m:t:eOQUAChXY" opt; do EXTRA_PAR="$EXTRA_PAR -U" UNICORN_MODE=1 ;; + "T") + T_ARG="$OPTARG" + ;; "?") exit 1 ;; @@ -130,9 +136,10 @@ Required parameters: Execution control settings: - -f file - location read by the fuzzed program (stdin) - -m megs - memory limit for child process ($MEM_LIMIT MB) - -t msec - run time limit for child process (none) + -T tasks - how many parallel processes to create (default=1, "all"=nproc) + -f file - location read by the fuzzed program (default: stdin) + -m megs - memory limit for child process (default=$MEM_LIMIT MB) + -t msec - run time limit for child process (default: 5000ms) -O - use binary-only instrumentation (FRIDA mode) -Q - use binary-only instrumentation (QEMU mode) -U - use unicorn-based instrumentation (Unicorn mode) @@ -199,6 +206,11 @@ fi # Check for obvious errors. +if [ ! "$T_ARG" = "" -a ! "$F_ARG" = "" ]; then + echo "[-] Error: -T and -f can not be used together." 1>&2 + exit 1 +fi + if [ ! "$MEM_LIMIT" = "none" ]; then if [ "$MEM_LIMIT" -lt "5" ]; then @@ -233,7 +245,7 @@ if [ "$NYX_MODE" = "" ]; then fi -grep -aq AFL_DUMP_MAP_SIZE "./$TARGET_BIN" && { +grep -aq AFL_DUMP_MAP_SIZE "$TARGET_BIN" && { echo "[!] Trying to obtain the map size of the target ..." MAPSIZE=`AFL_DUMP_MAP_SIZE=1 "./$TARGET_BIN" 2>/dev/null` test -n "$MAPSIZE" && { @@ -299,14 +311,29 @@ if [ ! -x "$SHOWMAP" ]; then exit 1 fi +THREADS= +if [ ! "$T_ARG" = "" ]; then + if [ "$T_ARG" = "all" ]; then + THREADS=$(nproc) + else + if [ "$T_ARG" -gt 0 -a "$T_ARG" -le "$(nproc)" ]; then + THREADS=$T_ARG + else + echo "[-] Error: -T parameter must between 1 and $(nproc) or \"all\"." 1>&2 + fi + fi +fi + IN_COUNT=$((`ls -- "$IN_DIR" 2>/dev/null | wc -l`)) if [ "$IN_COUNT" = "0" ]; then - echo "[+] Hmm, no inputs in the target directory. Nothing to be done." + echo "[-] Hmm, no inputs in the target directory. Nothing to be done." rm -rf "$TRACE_DIR" exit 1 fi +echo "[+] Found $IN_COUNT files for minimizing." + FIRST_FILE=`ls "$IN_DIR" | head -1` # Make sure that we're not dealing with a directory. @@ -355,6 +382,18 @@ else fi +TMPFILE=$OUT_DIR/.list.$$ +if [ ! "$THREADS" = "" ]; then + ls -- "$IN_DIR" > $TMPFILE 2>/dev/null + IN_COUNT=$(cat $TMPFILE | wc -l) + SPLIT=$(($IN_COUNT / $THREADS)) + if [ "$(($IN_COUNT % $THREADS))" -gt 0 ]; then + SPLIT=$(($SPLIT + 1)) + fi + echo "[+] Splitting workload into $THREADS tasks with $SPLIT items on average each." + split -l $SPLIT $TMPFILE $TMPFILE. +fi + # Let's roll! ############################# @@ -363,6 +402,7 @@ fi echo "[*] Obtaining traces for input files in '$IN_DIR'..." +if [ "$THREADS" = "" ]; then ( CUR=0 @@ -386,17 +426,58 @@ echo "[*] Obtaining traces for input files in '$IN_DIR'..." printf "\\r Processing file $CUR/$IN_COUNT... " cp "$IN_DIR/$fn" "$STDIN_FILE" - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" Date: Tue, 25 Apr 2023 11:56:50 +0200 Subject: [PATCH 074/149] fixes --- afl-cmin.bash | 9 +++++++-- src/afl-fuzz-init.c | 7 ++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/afl-cmin.bash b/afl-cmin.bash index ba7083fa..0e2d973d 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -316,12 +316,16 @@ if [ ! "$T_ARG" = "" ]; then if [ "$T_ARG" = "all" ]; then THREADS=$(nproc) else - if [ "$T_ARG" -gt 0 -a "$T_ARG" -le "$(nproc)" ]; then + if [ "$T_ARG" -gt 1 -a "$T_ARG" -le "$(nproc)" ]; then THREADS=$T_ARG else - echo "[-] Error: -T parameter must between 1 and $(nproc) or \"all\"." 1>&2 + echo "[-] Error: -T parameter must between 2 and $(nproc) or \"all\"." 1>&2 fi fi +else + if [ "$F_ARG" = ""]; then + echo "[*] Are you aware of the '-T all' parallelize option that massively improves the speed?" + fi fi IN_COUNT=$((`ls -- "$IN_DIR" 2>/dev/null | wc -l`)) @@ -332,6 +336,7 @@ if [ "$IN_COUNT" = "0" ]; then exit 1 fi +echo "[*] Are you aware that afl-cmin is faster than this afl-cmin.bash script?" echo "[+] Found $IN_COUNT files for minimizing." FIRST_FILE=`ls "$IN_DIR" | head -1` diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 002a26f8..bd591c8f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -756,7 +756,7 @@ void read_testcases(afl_state_t *afl, u8 *directory) { free(nl[i]); /* not tracked */ read_testcases(afl, fn2); ck_free(fn2); - continue; + goto next_entry; } @@ -765,7 +765,7 @@ void read_testcases(afl_state_t *afl, u8 *directory) { if (!S_ISREG(st.st_mode) || !st.st_size || strstr(fn2, "/README.txt")) { ck_free(fn2); - continue; + goto next_entry; } @@ -812,13 +812,14 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } + next_entry: if (unlikely(afl->in_place_resume)) { if (unlikely(i == 0)) { done = 1; } } else { - if (unlikely(++i == (u32)nl_cnt)) { done = 1; } + if (unlikely(++i >= (u32)nl_cnt)) { done = 1; } } From d822181467ec41f1ee2d840c3c5b1918c72ffc86 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 13:13:43 +0200 Subject: [PATCH 075/149] afl-cmin -T support --- afl-cmin | 145 ++++++++++++++++++++++++++++++++++++---------- docs/Changelog.md | 2 +- src/afl-showmap.c | 23 ++++---- 3 files changed, 125 insertions(+), 45 deletions(-) diff --git a/afl-cmin b/afl-cmin index 6b36c261..c8bbd8d7 100755 --- a/afl-cmin +++ b/afl-cmin @@ -103,9 +103,10 @@ function usage() { " -o dir - output directory for minimized files\n" \ "\n" \ "Execution control settings:\n" \ +" -T tasks - how many parallel tasks to run (default: 1, all=nproc)\n" \ " -f file - location read by the fuzzed program (stdin)\n" \ " -m megs - memory limit for child process ("mem_limit" MB)\n" \ -" -t msec - run time limit for child process (default: none)\n" \ +" -t msec - run time limit for child process (default: 5000)\n" \ " -O - use binary-only instrumentation (FRIDA mode)\n" \ " -Q - use binary-only instrumentation (QEMU mode)\n" \ " -U - use unicorn-based instrumentation (unicorn mode)\n" \ @@ -119,7 +120,6 @@ function usage() { "For additional tips, please consult README.md\n" \ "\n" \ "Environment variables used:\n" \ -"AFL_ALLOW_TMP: allow unsafe use of input/output directories under {/var}/tmp\n" \ "AFL_CRASH_EXITCODE: optional child exit code to be interpreted as crash\n" \ "AFL_FORKSRV_INIT_TMOUT: time the fuzzer waits for the forkserver to come up\n" \ "AFL_KEEP_TRACES: leave the temporary /.traces directory\n" \ @@ -159,13 +159,19 @@ BEGIN { # process options Opterr = 1 # default is to diagnose Optind = 1 # skip ARGV[0] - while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUXY?")) != -1) { + while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUXYT:?")) != -1) { if (_go_c == "i") { if (!Optarg) usage() if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} in_dir = Optarg continue } else + if (_go_c == "T") { + if (!Optarg) usage() + if (threads) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + threads = Optarg + continue + } else if (_go_c == "o") { if (!Optarg) usage() if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} @@ -253,21 +259,30 @@ BEGIN { # Do a sanity check to discourage the use of /tmp, since we can't really # handle this safely from an awk script. - if (!ENVIRON["AFL_ALLOW_TMP"]) { - dirlist[0] = in_dir - dirlist[1] = target_bin - dirlist[2] = out_dir - dirlist[3] = stdin_file - "pwd" | getline dirlist[4] # current directory - for (dirind in dirlist) { - dir = dirlist[dirind] + #if (!ENVIRON["AFL_ALLOW_TMP"]) { + # dirlist[0] = in_dir + # dirlist[1] = target_bin + # dirlist[2] = out_dir + # dirlist[3] = stdin_file + # "pwd" | getline dirlist[4] # current directory + # for (dirind in dirlist) { + # dir = dirlist[dirind] + # + # if (dir ~ /^(\/var)?\/tmp/) { + # print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" + # exit 1 + # } + # } + # delete dirlist + #} - if (dir ~ /^(\/var)?\/tmp/) { - print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" - exit 1 - } - } - delete dirlist + if (threads && stdin_file) { + print "[-] Error: -T and -f cannot be used together." > "/dev/stderr" + exit 1 + } + + if (!threads && !stdin_file) { + print "[*] Are you aware of the '-T all' parallelize option that massively improves the speed for large corpuses?" } # If @@ is specified, but there's no -f, let's come up with a temporary input @@ -350,6 +365,18 @@ BEGIN { exit 1 } + if (threads) { + "nproc" | getline nproc + if (threads == "all") { + threads = nproc + } else { + if (!(threads > 1 && threads <= nproc)) { + print "[-] Error: -T option must be between 1 and "nproc" or \"all\"." > "/dev/stderr" + exit 1 + } + } + } + # Check for the more efficient way to copy files... if (0 != system("mkdir -p -m 0700 "trace_dir)) { print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" @@ -459,27 +486,81 @@ BEGIN { # STEP 1: Collecting traces # ############################# + if (threads) { + + inputsperfile = in_count / threads + if (in_count % threads) { + inputsperfile++; + } + + cnt = 0; + tmpfile=out_dir "/.filelist" + for (instance = 1; instance < threads; instance++) { + for (i = 0; i < inputsperfile; i++) { + print in_dir"/"infilesSmallToBigFull[cnt] >> tmpfile"."instance + cnt++ + } + } + for (; cnt < in_count; cnt++) { + print in_dir"/"infilesSmallToBigFull[cnt] >> tmpfile"."threads + } + + print "ls -l "tmpfile"*" + + } + print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." cur = 0; - if (!stdin_file) { - print " Processing "in_count" files (forkserver mode)..." -# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string - retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string) - } else { - print " Processing "in_count" files (forkserver mode)..." -# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 1) { - if (!ENVIRON["AFL_KEEP_TRACES"]) { - system("rm -rf "trace_dir" 2>/dev/null") - system("rmdir "out_dir) + print "[*] Creating " threads " parallel tasks with about " inputsperfile " each." + for (i = 1; i <= threads; i++) { + + if (!stdin_file) { +# print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &" + retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &") + } else { + stdin_file=tmpfile"."i".stdin" +# print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" "tmpfile"."i".done ; } &" + retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" "tmpfile"."i".done ; } &") + } } - exit retval + print "[*] Waiting for parallel tasks to complete ..." + # wait for all processes to finish + ok=0 + while (ok < threads) { + ok=0 + for (i = 1; i <= threads; i++) { + if (system("test -f "tmpfile"."i".done") == 0) { + ok++ + } + } + } + print "[*] Done!" + system("rm -f "tmpfile"*") + } else { + if (!stdin_file) { + print " Processing "in_count" files (forkserver mode)..." +# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string + retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string) + } else { + print " Processing "in_count" files (forkserver mode)..." +# print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" /dev/null") + system("rmdir "out_dir) + } + exit retval + } + } ####################################################### diff --git a/docs/Changelog.md b/docs/Changelog.md index 816a864d..667fd634 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -14,7 +14,7 @@ - afl-showmap: - added custom mutator post_process and send support - add `-I filelist` option, an alternative to `-i in_dir` - - afl-cmin.bash: + - afl-cmin + afl-cmin.bash: - `-T threads` parallel task support, huge speedup! - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 09a1d2dc..d0e01cb1 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -894,9 +894,7 @@ u32 execute_testcases_filelist(u8 *fn) { while (fgets(buf, sizeof(buf), f) != NULL) { struct stat st; - - u8 *fn2 = buf, *fn3; - ; + u8 *fn2 = buf, *fn3; while (*fn2 == ' ') { @@ -904,14 +902,11 @@ u32 execute_testcases_filelist(u8 *fn) { } - if (*fn2) { + while (*fn2 && + (fn2[strlen(fn2) - 1] == '\r' || fn2[strlen(fn2) - 1] == '\n' || + fn2[strlen(fn2) - 1] == ' ')) { - while (fn2[strlen(fn2) - 1] == '\r' || fn2[strlen(fn2) - 1] == '\n' || - fn2[strlen(fn2) - 1] == ' ') { - - fn2[strlen(fn2) - 1] = 0; - - } + fn2[strlen(fn2) - 1] = 0; } @@ -926,6 +921,8 @@ u32 execute_testcases_filelist(u8 *fn) { } + ++done; + if (!S_ISREG(st.st_mode) || !st.st_size) { continue; } if ((fn3 = strrchr(fn2, '/'))) { @@ -946,9 +943,12 @@ u32 execute_testcases_filelist(u8 *fn) { } - if (!collect_coverage) + if (!collect_coverage) { + snprintf(outfile, sizeof(outfile), "%s/%s", out_file, fn3); + } + if (read_file(fn2)) { if (wait_for_gdb) { @@ -961,7 +961,6 @@ u32 execute_testcases_filelist(u8 *fn) { showmap_run_target_forkserver(fsrv, in_data, in_len); ck_free(in_data); - ++done; if (child_crashed && debug) { WARNF("crashed: %s", fn2); } From f3dc56f59a25eca60666393b43e0eef4811ac825 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 14:54:38 +0200 Subject: [PATCH 076/149] update custom mutators --- custom_mutators/honggfuzz/honggfuzz.c | 18 +- custom_mutators/honggfuzz/mangle.c | 1269 ++++++++++++--------- custom_mutators/radamsa/radamsa-mutator.c | 11 +- 3 files changed, 722 insertions(+), 576 deletions(-) diff --git a/custom_mutators/honggfuzz/honggfuzz.c b/custom_mutators/honggfuzz/honggfuzz.c index d7b3c9c5..0dd59aee 100644 --- a/custom_mutators/honggfuzz/honggfuzz.c +++ b/custom_mutators/honggfuzz/honggfuzz.c @@ -3,14 +3,14 @@ #include #include -#include "custom_mutator_helpers.h" +#include "afl-fuzz.h" #include "mangle.h" #define NUMBER_OF_MUTATIONS 5 -uint8_t * queue_input; +uint8_t *queue_input; size_t queue_input_size; -afl_state_t * afl_struct; +afl_state_t *afl_struct; run_t run; honggfuzz_t global; struct _dynfile_t dynfile; @@ -18,8 +18,8 @@ struct _dynfile_t dynfile; typedef struct my_mutator { afl_state_t *afl; - run_t * run; - u8 * mutator_buf; + run_t *run; + u8 *mutator_buf; unsigned int seed; unsigned int extras_cnt, a_extras_cnt; @@ -65,9 +65,9 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { /* When a new queue entry is added we check if there are new dictionary entries to add to honggfuzz structure */ -uint8_t afl_custom_queue_new_entry(my_mutator_t * data, - const uint8_t *filename_new_queue, - const uint8_t *filename_orig_queue) { +void afl_custom_queue_new_entry(my_mutator_t *data, + const uint8_t *filename_new_queue, + const uint8_t *filename_orig_queue) { if (run.global->mutate.dictionaryCnt >= 1024) return; @@ -97,7 +97,7 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t * data, } - return 0; + return; } diff --git a/custom_mutators/honggfuzz/mangle.c b/custom_mutators/honggfuzz/mangle.c index 637d428d..7b7ccb91 100644 --- a/custom_mutators/honggfuzz/mangle.c +++ b/custom_mutators/honggfuzz/mangle.c @@ -39,11 +39,16 @@ #include "libhfcommon/log.h" #include "libhfcommon/util.h" -static inline size_t mangle_LenLeft(run_t* run, size_t off) { - if (off >= run->dynfile->size) { - LOG_F("Offset is too large: off:%zu >= len:%zu", off, run->dynfile->size); - } - return (run->dynfile->size - off - 1); +static inline size_t mangle_LenLeft(run_t *run, size_t off) { + + if (off >= run->dynfile->size) { + + LOG_F("Offset is too large: off:%zu >= len:%zu", off, run->dynfile->size); + + } + + return (run->dynfile->size - off - 1); + } /* @@ -51,196 +56,233 @@ static inline size_t mangle_LenLeft(run_t* run, size_t off) { * Based on an idea by https://twitter.com/gamozolabs */ static inline size_t mangle_getLen(size_t max) { - if (max > _HF_INPUT_MAX_SIZE) { - LOG_F("max (%zu) > _HF_INPUT_MAX_SIZE (%zu)", max, (size_t)_HF_INPUT_MAX_SIZE); - } - if (max == 0) { - LOG_F("max == 0"); - } - if (max == 1) { - return 1; - } - /* Give 50% chance the the uniform distribution */ - if (util_rnd64() & 1) { - return (size_t)util_rndGet(1, max); - } + if (max > _HF_INPUT_MAX_SIZE) { + + LOG_F("max (%zu) > _HF_INPUT_MAX_SIZE (%zu)", max, + (size_t)_HF_INPUT_MAX_SIZE); + + } + + if (max == 0) { LOG_F("max == 0"); } + if (max == 1) { return 1; } + + /* Give 50% chance the the uniform distribution */ + if (util_rnd64() & 1) { return (size_t)util_rndGet(1, max); } + + /* effectively exprand() */ + return (size_t)util_rndGet(1, util_rndGet(1, max)); - /* effectively exprand() */ - return (size_t)util_rndGet(1, util_rndGet(1, max)); } /* Prefer smaller values here, so use mangle_getLen() */ -static inline size_t mangle_getOffSet(run_t* run) { - return mangle_getLen(run->dynfile->size) - 1; +static inline size_t mangle_getOffSet(run_t *run) { + + return mangle_getLen(run->dynfile->size) - 1; + } /* Offset which can be equal to the file size */ -static inline size_t mangle_getOffSetPlus1(run_t* run) { - size_t reqlen = HF_MIN(run->dynfile->size + 1, _HF_INPUT_MAX_SIZE); - return mangle_getLen(reqlen) - 1; +static inline size_t mangle_getOffSetPlus1(run_t *run) { + + size_t reqlen = HF_MIN(run->dynfile->size + 1, _HF_INPUT_MAX_SIZE); + return mangle_getLen(reqlen) - 1; + } -static inline void mangle_Move(run_t* run, size_t off_from, size_t off_to, size_t len) { - if (off_from >= run->dynfile->size) { - return; - } - if (off_to >= run->dynfile->size) { - return; - } - if (off_from == off_to) { - return; - } +static inline void mangle_Move(run_t *run, size_t off_from, size_t off_to, + size_t len) { - size_t len_from = run->dynfile->size - off_from; - len = HF_MIN(len, len_from); + if (off_from >= run->dynfile->size) { return; } + if (off_to >= run->dynfile->size) { return; } + if (off_from == off_to) { return; } - size_t len_to = run->dynfile->size - off_to; - len = HF_MIN(len, len_to); + size_t len_from = run->dynfile->size - off_from; + len = HF_MIN(len, len_from); + + size_t len_to = run->dynfile->size - off_to; + len = HF_MIN(len, len_to); + + memmove(&run->dynfile->data[off_to], &run->dynfile->data[off_from], len); - memmove(&run->dynfile->data[off_to], &run->dynfile->data[off_from], len); } -static inline void mangle_Overwrite( - run_t* run, size_t off, const uint8_t* src, size_t len, bool printable) { - if (len == 0) { - return; - } - size_t maxToCopy = run->dynfile->size - off; - if (len > maxToCopy) { - len = maxToCopy; - } +static inline void mangle_Overwrite(run_t *run, size_t off, const uint8_t *src, + size_t len, bool printable) { + + if (len == 0) { return; } + size_t maxToCopy = run->dynfile->size - off; + if (len > maxToCopy) { len = maxToCopy; } + + memmove(&run->dynfile->data[off], src, len); + if (printable) { util_turnToPrintable(&run->dynfile->data[off], len); } - memmove(&run->dynfile->data[off], src, len); - if (printable) { - util_turnToPrintable(&run->dynfile->data[off], len); - } } -static inline size_t mangle_Inflate(run_t* run, size_t off, size_t len, bool printable) { - if (run->dynfile->size >= run->global->mutate.maxInputSz) { - return 0; - } - if (len > (run->global->mutate.maxInputSz - run->dynfile->size)) { - len = run->global->mutate.maxInputSz - run->dynfile->size; - } +static inline size_t mangle_Inflate(run_t *run, size_t off, size_t len, + bool printable) { - input_setSize(run, run->dynfile->size + len); - mangle_Move(run, off, off + len, run->dynfile->size); - if (printable) { - memset(&run->dynfile->data[off], ' ', len); - } + if (run->dynfile->size >= run->global->mutate.maxInputSz) { return 0; } + if (len > (run->global->mutate.maxInputSz - run->dynfile->size)) { + + len = run->global->mutate.maxInputSz - run->dynfile->size; + + } + + input_setSize(run, run->dynfile->size + len); + mangle_Move(run, off, off + len, run->dynfile->size); + if (printable) { memset(&run->dynfile->data[off], ' ', len); } + + return len; - return len; } -static inline void mangle_Insert( - run_t* run, size_t off, const uint8_t* val, size_t len, bool printable) { - len = mangle_Inflate(run, off, len, printable); +static inline void mangle_Insert(run_t *run, size_t off, const uint8_t *val, + size_t len, bool printable) { + + len = mangle_Inflate(run, off, len, printable); + mangle_Overwrite(run, off, val, len, printable); + +} + +static inline void mangle_UseValue(run_t *run, const uint8_t *val, size_t len, + bool printable) { + + if (util_rnd64() & 1) { + + mangle_Overwrite(run, mangle_getOffSet(run), val, len, printable); + + } else { + + mangle_Insert(run, mangle_getOffSetPlus1(run), val, len, printable); + + } + +} + +static inline void mangle_UseValueAt(run_t *run, size_t off, const uint8_t *val, + size_t len, bool printable) { + + if (util_rnd64() & 1) { + mangle_Overwrite(run, off, val, len, printable); + + } else { + + mangle_Insert(run, off, val, len, printable); + + } + } -static inline void mangle_UseValue(run_t* run, const uint8_t* val, size_t len, bool printable) { - if (util_rnd64() & 1) { - mangle_Overwrite(run, mangle_getOffSet(run), val, len, printable); - } else { - mangle_Insert(run, mangle_getOffSetPlus1(run), val, len, printable); - } +static void mangle_MemSwap(run_t *run, bool printable HF_ATTR_UNUSED) { + + /* No big deal if those two are overlapping */ + size_t off1 = mangle_getOffSet(run); + size_t maxlen1 = run->dynfile->size - off1; + size_t off2 = mangle_getOffSet(run); + size_t maxlen2 = run->dynfile->size - off2; + size_t len = mangle_getLen(HF_MIN(maxlen1, maxlen2)); + + if (off1 == off2) { return; } + + for (size_t i = 0; i < (len / 2); i++) { + + /* + * First - from the head, next from the tail. Don't worry about layout of + * the overlapping part - there's no good solution to that, and it can be + * left somewhat scrambled, while still preserving the entropy + */ + const uint8_t tmp1 = run->dynfile->data[off2 + i]; + run->dynfile->data[off2 + i] = run->dynfile->data[off1 + i]; + run->dynfile->data[off1 + i] = tmp1; + const uint8_t tmp2 = run->dynfile->data[off2 + (len - 1) - i]; + run->dynfile->data[off2 + (len - 1) - i] = + run->dynfile->data[off1 + (len - 1) - i]; + run->dynfile->data[off1 + (len - 1) - i] = tmp2; + + } + } -static inline void mangle_UseValueAt( - run_t* run, size_t off, const uint8_t* val, size_t len, bool printable) { - if (util_rnd64() & 1) { - mangle_Overwrite(run, off, val, len, printable); - } else { - mangle_Insert(run, off, val, len, printable); - } +static void mangle_MemCopy(run_t *run, bool printable HF_ATTR_UNUSED) { + + size_t off = mangle_getOffSet(run); + size_t len = mangle_getLen(run->dynfile->size - off); + + /* Use a temp buf, as Insert/Inflate can change source bytes */ + uint8_t *tmpbuf = (uint8_t *)util_Malloc(len); + defer { + + free(tmpbuf); + + }; + + memmove(tmpbuf, &run->dynfile->data[off], len); + + mangle_UseValue(run, tmpbuf, len, printable); + } -static void mangle_MemSwap(run_t* run, bool printable HF_ATTR_UNUSED) { - /* No big deal if those two are overlapping */ - size_t off1 = mangle_getOffSet(run); - size_t maxlen1 = run->dynfile->size - off1; - size_t off2 = mangle_getOffSet(run); - size_t maxlen2 = run->dynfile->size - off2; - size_t len = mangle_getLen(HF_MIN(maxlen1, maxlen2)); +static void mangle_Bytes(run_t *run, bool printable) { - if (off1 == off2) { - return; - } + uint16_t buf; + if (printable) { + + util_rndBufPrintable((uint8_t *)&buf, sizeof(buf)); + + } else { + + buf = util_rnd64(); + + } + + /* Overwrite with random 1-2-byte values */ + size_t toCopy = util_rndGet(1, 2); + mangle_UseValue(run, (const uint8_t *)&buf, toCopy, printable); - for (size_t i = 0; i < (len / 2); i++) { - /* - * First - from the head, next from the tail. Don't worry about layout of the overlapping - * part - there's no good solution to that, and it can be left somewhat scrambled, - * while still preserving the entropy - */ - const uint8_t tmp1 = run->dynfile->data[off2 + i]; - run->dynfile->data[off2 + i] = run->dynfile->data[off1 + i]; - run->dynfile->data[off1 + i] = tmp1; - const uint8_t tmp2 = run->dynfile->data[off2 + (len - 1) - i]; - run->dynfile->data[off2 + (len - 1) - i] = run->dynfile->data[off1 + (len - 1) - i]; - run->dynfile->data[off1 + (len - 1) - i] = tmp2; - } } -static void mangle_MemCopy(run_t* run, bool printable HF_ATTR_UNUSED) { - size_t off = mangle_getOffSet(run); - size_t len = mangle_getLen(run->dynfile->size - off); +static void mangle_ByteRepeat(run_t *run, bool printable) { - /* Use a temp buf, as Insert/Inflate can change source bytes */ - uint8_t* tmpbuf = (uint8_t*)util_Malloc(len); - defer { - free(tmpbuf); - }; - memmove(tmpbuf, &run->dynfile->data[off], len); + size_t off = mangle_getOffSet(run); + size_t destOff = off + 1; + size_t maxSz = run->dynfile->size - destOff; + + /* No space to repeat */ + if (!maxSz) { + + mangle_Bytes(run, printable); + return; + + } + + size_t len = mangle_getLen(maxSz); + if (util_rnd64() & 0x1) { + + len = mangle_Inflate(run, destOff, len, printable); + + } + + memset(&run->dynfile->data[destOff], run->dynfile->data[off], len); - mangle_UseValue(run, tmpbuf, len, printable); } -static void mangle_Bytes(run_t* run, bool printable) { - uint16_t buf; - if (printable) { - util_rndBufPrintable((uint8_t*)&buf, sizeof(buf)); - } else { - buf = util_rnd64(); - } +static void mangle_Bit(run_t *run, bool printable) { - /* Overwrite with random 1-2-byte values */ - size_t toCopy = util_rndGet(1, 2); - mangle_UseValue(run, (const uint8_t*)&buf, toCopy, printable); -} + size_t off = mangle_getOffSet(run); + run->dynfile->data[off] ^= (uint8_t)(1U << util_rndGet(0, 7)); + if (printable) { util_turnToPrintable(&(run->dynfile->data[off]), 1); } -static void mangle_ByteRepeat(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - size_t destOff = off + 1; - size_t maxSz = run->dynfile->size - destOff; - - /* No space to repeat */ - if (!maxSz) { - mangle_Bytes(run, printable); - return; - } - - size_t len = mangle_getLen(maxSz); - if (util_rnd64() & 0x1) { - len = mangle_Inflate(run, destOff, len, printable); - } - memset(&run->dynfile->data[destOff], run->dynfile->data[off], len); -} - -static void mangle_Bit(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - run->dynfile->data[off] ^= (uint8_t)(1U << util_rndGet(0, 7)); - if (printable) { - util_turnToPrintable(&(run->dynfile->data[off]), 1); - } } static const struct { - const uint8_t val[8]; - const size_t size; + + const uint8_t val[8]; + const size_t size; + } mangleMagicVals[] = { + /* 1B - No endianness */ {"\x00\x00\x00\x00\x00\x00\x00\x00", 1}, {"\x01\x00\x00\x00\x00\x00\x00\x00", 1}, @@ -472,436 +514,543 @@ static const struct { {"\x00\x00\x00\x00\x00\x00\x00\x80", 8}, {"\x01\x00\x00\x00\x00\x00\x00\x80", 8}, {"\xFE\xFF\xFF\xFF\xFF\xFF\xFF\xFF", 8}, + }; -static void mangle_Magic(run_t* run, bool printable) { - uint64_t choice = util_rndGet(0, ARRAYSIZE(mangleMagicVals) - 1); - mangle_UseValue(run, mangleMagicVals[choice].val, mangleMagicVals[choice].size, printable); +static void mangle_Magic(run_t *run, bool printable) { + + uint64_t choice = util_rndGet(0, ARRAYSIZE(mangleMagicVals) - 1); + mangle_UseValue(run, mangleMagicVals[choice].val, + mangleMagicVals[choice].size, printable); + } -static void mangle_StaticDict(run_t* run, bool printable) { - if (run->global->mutate.dictionaryCnt == 0) { - mangle_Bytes(run, printable); - return; - } - uint64_t choice = util_rndGet(0, run->global->mutate.dictionaryCnt - 1); - mangle_UseValue(run, run->global->mutate.dictionary[choice].val, - run->global->mutate.dictionary[choice].len, printable); +static void mangle_StaticDict(run_t *run, bool printable) { + + if (run->global->mutate.dictionaryCnt == 0) { + + mangle_Bytes(run, printable); + return; + + } + + uint64_t choice = util_rndGet(0, run->global->mutate.dictionaryCnt - 1); + mangle_UseValue(run, run->global->mutate.dictionary[choice].val, + run->global->mutate.dictionary[choice].len, printable); + } -static inline const uint8_t* mangle_FeedbackDict(run_t* run, size_t* len) { - if (!run->global->feedback.cmpFeedback) { - return NULL; - } - cmpfeedback_t* cmpf = run->global->feedback.cmpFeedbackMap; - uint32_t cnt = ATOMIC_GET(cmpf->cnt); - if (cnt == 0) { - return NULL; - } - if (cnt > ARRAYSIZE(cmpf->valArr)) { - cnt = ARRAYSIZE(cmpf->valArr); - } - uint32_t choice = util_rndGet(0, cnt - 1); - *len = (size_t)ATOMIC_GET(cmpf->valArr[choice].len); - if (*len == 0) { - return NULL; - } - return cmpf->valArr[choice].val; +static inline const uint8_t *mangle_FeedbackDict(run_t *run, size_t *len) { + + if (!run->global->feedback.cmpFeedback) { return NULL; } + cmpfeedback_t *cmpf = run->global->feedback.cmpFeedbackMap; + uint32_t cnt = ATOMIC_GET(cmpf->cnt); + if (cnt == 0) { return NULL; } + if (cnt > ARRAYSIZE(cmpf->valArr)) { cnt = ARRAYSIZE(cmpf->valArr); } + uint32_t choice = util_rndGet(0, cnt - 1); + *len = (size_t)ATOMIC_GET(cmpf->valArr[choice].len); + if (*len == 0) { return NULL; } + return cmpf->valArr[choice].val; + } -static void mangle_ConstFeedbackDict(run_t* run, bool printable) { - size_t len; - const uint8_t* val = mangle_FeedbackDict(run, &len); - if (val == NULL) { - mangle_Bytes(run, printable); - return; - } - mangle_UseValue(run, val, len, printable); +static void mangle_ConstFeedbackDict(run_t *run, bool printable) { + + size_t len; + const uint8_t *val = mangle_FeedbackDict(run, &len); + if (val == NULL) { + + mangle_Bytes(run, printable); + return; + + } + + mangle_UseValue(run, val, len, printable); + } -static void mangle_MemSet(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - size_t len = mangle_getLen(run->dynfile->size - off); - int val = printable ? (int)util_rndPrintable() : (int)util_rndGet(0, UINT8_MAX); +static void mangle_MemSet(run_t *run, bool printable) { - if (util_rnd64() & 1) { - len = mangle_Inflate(run, off, len, printable); - } + size_t off = mangle_getOffSet(run); + size_t len = mangle_getLen(run->dynfile->size - off); + int val = + printable ? (int)util_rndPrintable() : (int)util_rndGet(0, UINT8_MAX); + + if (util_rnd64() & 1) { len = mangle_Inflate(run, off, len, printable); } + + memset(&run->dynfile->data[off], val, len); - memset(&run->dynfile->data[off], val, len); } -static void mangle_MemClr(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - size_t len = mangle_getLen(run->dynfile->size - off); - int val = printable ? ' ' : 0; +static void mangle_MemClr(run_t *run, bool printable) { - if (util_rnd64() & 1) { - len = mangle_Inflate(run, off, len, printable); - } + size_t off = mangle_getOffSet(run); + size_t len = mangle_getLen(run->dynfile->size - off); + int val = printable ? ' ' : 0; + + if (util_rnd64() & 1) { len = mangle_Inflate(run, off, len, printable); } + + memset(&run->dynfile->data[off], val, len); - memset(&run->dynfile->data[off], val, len); } -static void mangle_RandomBuf(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - size_t len = mangle_getLen(run->dynfile->size - off); +static void mangle_RandomBuf(run_t *run, bool printable) { + + size_t off = mangle_getOffSet(run); + size_t len = mangle_getLen(run->dynfile->size - off); + + if (util_rnd64() & 1) { len = mangle_Inflate(run, off, len, printable); } + + if (printable) { + + util_rndBufPrintable(&run->dynfile->data[off], len); + + } else { + + util_rndBuf(&run->dynfile->data[off], len); + + } + +} + +static inline void mangle_AddSubWithRange(run_t *run, size_t off, size_t varLen, + uint64_t range, bool printable) { + + int64_t delta = (int64_t)util_rndGet(0, range * 2) - (int64_t)range; + + switch (varLen) { + + case 1: { + + run->dynfile->data[off] += delta; + break; - if (util_rnd64() & 1) { - len = mangle_Inflate(run, off, len, printable); } - if (printable) { - util_rndBufPrintable(&run->dynfile->data[off], len); + case 2: { + + int16_t val; + memcpy(&val, &run->dynfile->data[off], sizeof(val)); + if (util_rnd64() & 0x1) { + + val += delta; + + } else { + + /* Foreign endianess */ + val = __builtin_bswap16(val); + val += delta; + val = __builtin_bswap16(val); + + } + + mangle_Overwrite(run, off, (uint8_t *)&val, varLen, printable); + break; + + } + + case 4: { + + int32_t val; + memcpy(&val, &run->dynfile->data[off], sizeof(val)); + if (util_rnd64() & 0x1) { + + val += delta; + + } else { + + /* Foreign endianess */ + val = __builtin_bswap32(val); + val += delta; + val = __builtin_bswap32(val); + + } + + mangle_Overwrite(run, off, (uint8_t *)&val, varLen, printable); + break; + + } + + case 8: { + + int64_t val; + memcpy(&val, &run->dynfile->data[off], sizeof(val)); + if (util_rnd64() & 0x1) { + + val += delta; + + } else { + + /* Foreign endianess */ + val = __builtin_bswap64(val); + val += delta; + val = __builtin_bswap64(val); + + } + + mangle_Overwrite(run, off, (uint8_t *)&val, varLen, printable); + break; + + } + + default: { + + LOG_F("Unknown variable length size: %zu", varLen); + + } + + } + +} + +static void mangle_AddSub(run_t *run, bool printable) { + + size_t off = mangle_getOffSet(run); + + /* 1,2,4,8 */ + size_t varLen = 1U << util_rndGet(0, 3); + if ((run->dynfile->size - off) < varLen) { varLen = 1; } + + uint64_t range; + switch (varLen) { + + case 1: + range = 16; + break; + case 2: + range = 4096; + break; + case 4: + range = 1048576; + break; + case 8: + range = 268435456; + break; + default: + LOG_F("Invalid operand size: %zu", varLen); + + } + + mangle_AddSubWithRange(run, off, varLen, range, printable); + +} + +static void mangle_IncByte(run_t *run, bool printable) { + + size_t off = mangle_getOffSet(run); + if (printable) { + + run->dynfile->data[off] = (run->dynfile->data[off] - 32 + 1) % 95 + 32; + + } else { + + run->dynfile->data[off] += (uint8_t)1UL; + + } + +} + +static void mangle_DecByte(run_t *run, bool printable) { + + size_t off = mangle_getOffSet(run); + if (printable) { + + run->dynfile->data[off] = (run->dynfile->data[off] - 32 + 94) % 95 + 32; + + } else { + + run->dynfile->data[off] -= (uint8_t)1UL; + + } + +} + +static void mangle_NegByte(run_t *run, bool printable) { + + size_t off = mangle_getOffSet(run); + if (printable) { + + run->dynfile->data[off] = 94 - (run->dynfile->data[off] - 32) + 32; + + } else { + + run->dynfile->data[off] = ~(run->dynfile->data[off]); + + } + +} + +static void mangle_Expand(run_t *run, bool printable) { + + size_t off = mangle_getOffSet(run); + size_t len; + if (util_rnd64() % 16) { + + len = mangle_getLen(HF_MIN(16, run->global->mutate.maxInputSz - off)); + + } else { + + len = mangle_getLen(run->global->mutate.maxInputSz - off); + + } + + mangle_Inflate(run, off, len, printable); + +} + +static void mangle_Shrink(run_t *run, bool printable HF_ATTR_UNUSED) { + + if (run->dynfile->size <= 2U) { return; } + + size_t off_start = mangle_getOffSet(run); + size_t len = mangle_LenLeft(run, off_start); + if (len == 0) { return; } + if (util_rnd64() % 16) { + + len = mangle_getLen(HF_MIN(16, len)); + + } else { + + len = mangle_getLen(len); + + } + + size_t off_end = off_start + len; + size_t len_to_move = run->dynfile->size - off_end; + + mangle_Move(run, off_end, off_start, len_to_move); + input_setSize(run, run->dynfile->size - len); + +} + +static void mangle_ASCIINum(run_t *run, bool printable) { + + size_t len = util_rndGet(2, 8); + + char buf[20]; + snprintf(buf, sizeof(buf), "%-19" PRId64, (int64_t)util_rnd64()); + + mangle_UseValue(run, (const uint8_t *)buf, len, printable); + +} + +static void mangle_ASCIINumChange(run_t *run, bool printable) { + + size_t off = mangle_getOffSet(run); + + /* Find a digit */ + for (; off < run->dynfile->size; off++) { + + if (isdigit(run->dynfile->data[off])) { break; } + + } + + size_t left = run->dynfile->size - off; + if (left == 0) { return; } + + size_t len = 0; + uint64_t val = 0; + /* 20 is maximum lenght of a string representing a 64-bit unsigned value */ + for (len = 0; (len < 20) && (len < left); len++) { + + char c = run->dynfile->data[off + len]; + if (!isdigit(c)) { break; } + val *= 10; + val += (c - '0'); + + } + + switch (util_rndGet(0, 7)) { + + case 0: + val++; + break; + case 1: + val--; + break; + case 2: + val *= 2; + break; + case 3: + val /= 2; + break; + case 4: + val = util_rnd64(); + break; + case 5: + val += util_rndGet(1, 256); + break; + case 6: + val -= util_rndGet(1, 256); + break; + case 7: + val = ~(val); + break; + default: + LOG_F("Invalid choice"); + + }; + + char buf[20]; + snprintf(buf, sizeof(buf), "%-19" PRIu64, val); + + mangle_UseValueAt(run, off, (const uint8_t *)buf, len, printable); + +} + +static void mangle_Splice(run_t *run, bool printable) { + + if (run->global->feedback.dynFileMethod == _HF_DYNFILE_NONE) { + + mangle_Bytes(run, printable); + return; + + } + + size_t sz = 0; + const uint8_t *buf = input_getRandomInputAsBuf(run, &sz); + if (!buf) { + + LOG_E("input_getRandomInputAsBuf() returned no input"); + mangle_Bytes(run, printable); + return; + + } + + if (!sz) { + + mangle_Bytes(run, printable); + return; + + } + + size_t remoteOff = mangle_getLen(sz) - 1; + size_t len = mangle_getLen(sz - remoteOff); + mangle_UseValue(run, &buf[remoteOff], len, printable); + +} + +static void mangle_Resize(run_t *run, bool printable) { + + ssize_t oldsz = run->dynfile->size; + ssize_t newsz = 0; + + uint64_t choice = util_rndGet(0, 32); + switch (choice) { + + case 0: /* Set new size arbitrarily */ + newsz = (ssize_t)util_rndGet(1, run->global->mutate.maxInputSz); + break; + case 1 ... 4: /* Increase size by a small value */ + newsz = oldsz + (ssize_t)util_rndGet(0, 8); + break; + case 5: /* Increase size by a larger value */ + newsz = oldsz + (ssize_t)util_rndGet(9, 128); + break; + case 6 ... 9: /* Decrease size by a small value */ + newsz = oldsz - (ssize_t)util_rndGet(0, 8); + break; + case 10: /* Decrease size by a larger value */ + newsz = oldsz - (ssize_t)util_rndGet(9, 128); + break; + case 11 ... 32: /* Do nothing */ + newsz = oldsz; + break; + default: + LOG_F("Illegal value from util_rndGet: %" PRIu64, choice); + break; + + } + + if (newsz < 1) { newsz = 1; } + if (newsz > (ssize_t)run->global->mutate.maxInputSz) { + + newsz = run->global->mutate.maxInputSz; + + } + + input_setSize(run, (size_t)newsz); + if (newsz > oldsz) { + + if (printable) { memset(&run->dynfile->data[oldsz], ' ', newsz - oldsz); } + + } + +} + +void mangle_mangleContent(run_t *run, int speed_factor) { + + static void (*const mangleFuncs[])(run_t * run, bool printable) = { + + mangle_Shrink, mangle_Expand, mangle_Bit, + mangle_IncByte, mangle_DecByte, mangle_NegByte, + mangle_AddSub, mangle_MemSet, mangle_MemClr, + mangle_MemSwap, mangle_MemCopy, mangle_Bytes, + mangle_ASCIINum, mangle_ASCIINumChange, mangle_ByteRepeat, + mangle_Magic, mangle_StaticDict, mangle_ConstFeedbackDict, + mangle_RandomBuf, mangle_Splice, + + }; + + if (run->mutationsPerRun == 0U) { return; } + if (run->dynfile->size == 0U) { + + mangle_Resize(run, /* printable= */ run->global->cfg.only_printable); + + } + + uint64_t changesCnt = run->global->mutate.mutationsPerRun; + + if (speed_factor < 5) { + + changesCnt = util_rndGet(1, run->global->mutate.mutationsPerRun); + + } else if (speed_factor < 10) { + + changesCnt = run->global->mutate.mutationsPerRun; + + } else { + + changesCnt = HF_MIN(speed_factor, 10); + changesCnt = HF_MAX(changesCnt, (run->global->mutate.mutationsPerRun * 5)); + + } + + /* If last coverage acquisition was more than 5 secs ago, use splicing more + * frequently */ + if ((time(NULL) - ATOMIC_GET(run->global->timing.lastCovUpdate)) > 5) { + + if (util_rnd64() & 0x1) { + + mangle_Splice(run, run->global->cfg.only_printable); + + } + + } + + for (uint64_t x = 0; x < changesCnt; x++) { + + if (run->global->feedback.cmpFeedback && (util_rnd64() & 0x1)) { + + /* + * mangle_ConstFeedbackDict() is quite powerful if the dynamic feedback + * dictionary exists. If so, give it 50% chance of being used among all + * mangling functions. + */ + mangle_ConstFeedbackDict( + run, /* printable= */ run->global->cfg.only_printable); + } else { - util_rndBuf(&run->dynfile->data[off], len); + + uint64_t choice = util_rndGet(0, ARRAYSIZE(mangleFuncs) - 1); + mangleFuncs[choice](run, + /* printable= */ run->global->cfg.only_printable); + } + + } + + wmb(); + } -static inline void mangle_AddSubWithRange( - run_t* run, size_t off, size_t varLen, uint64_t range, bool printable) { - int64_t delta = (int64_t)util_rndGet(0, range * 2) - (int64_t)range; - - switch (varLen) { - case 1: { - run->dynfile->data[off] += delta; - break; - } - case 2: { - int16_t val; - memcpy(&val, &run->dynfile->data[off], sizeof(val)); - if (util_rnd64() & 0x1) { - val += delta; - } else { - /* Foreign endianess */ - val = __builtin_bswap16(val); - val += delta; - val = __builtin_bswap16(val); - } - mangle_Overwrite(run, off, (uint8_t*)&val, varLen, printable); - break; - } - case 4: { - int32_t val; - memcpy(&val, &run->dynfile->data[off], sizeof(val)); - if (util_rnd64() & 0x1) { - val += delta; - } else { - /* Foreign endianess */ - val = __builtin_bswap32(val); - val += delta; - val = __builtin_bswap32(val); - } - mangle_Overwrite(run, off, (uint8_t*)&val, varLen, printable); - break; - } - case 8: { - int64_t val; - memcpy(&val, &run->dynfile->data[off], sizeof(val)); - if (util_rnd64() & 0x1) { - val += delta; - } else { - /* Foreign endianess */ - val = __builtin_bswap64(val); - val += delta; - val = __builtin_bswap64(val); - } - mangle_Overwrite(run, off, (uint8_t*)&val, varLen, printable); - break; - } - default: { - LOG_F("Unknown variable length size: %zu", varLen); - } - } -} - -static void mangle_AddSub(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - - /* 1,2,4,8 */ - size_t varLen = 1U << util_rndGet(0, 3); - if ((run->dynfile->size - off) < varLen) { - varLen = 1; - } - - uint64_t range; - switch (varLen) { - case 1: - range = 16; - break; - case 2: - range = 4096; - break; - case 4: - range = 1048576; - break; - case 8: - range = 268435456; - break; - default: - LOG_F("Invalid operand size: %zu", varLen); - } - - mangle_AddSubWithRange(run, off, varLen, range, printable); -} - -static void mangle_IncByte(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - if (printable) { - run->dynfile->data[off] = (run->dynfile->data[off] - 32 + 1) % 95 + 32; - } else { - run->dynfile->data[off] += (uint8_t)1UL; - } -} - -static void mangle_DecByte(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - if (printable) { - run->dynfile->data[off] = (run->dynfile->data[off] - 32 + 94) % 95 + 32; - } else { - run->dynfile->data[off] -= (uint8_t)1UL; - } -} - -static void mangle_NegByte(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - if (printable) { - run->dynfile->data[off] = 94 - (run->dynfile->data[off] - 32) + 32; - } else { - run->dynfile->data[off] = ~(run->dynfile->data[off]); - } -} - -static void mangle_Expand(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - size_t len; - if (util_rnd64() % 16) { - len = mangle_getLen(HF_MIN(16, run->global->mutate.maxInputSz - off)); - } else { - len = mangle_getLen(run->global->mutate.maxInputSz - off); - } - - mangle_Inflate(run, off, len, printable); -} - -static void mangle_Shrink(run_t* run, bool printable HF_ATTR_UNUSED) { - if (run->dynfile->size <= 2U) { - return; - } - - size_t off_start = mangle_getOffSet(run); - size_t len = mangle_LenLeft(run, off_start); - if (len == 0) { - return; - } - if (util_rnd64() % 16) { - len = mangle_getLen(HF_MIN(16, len)); - } else { - len = mangle_getLen(len); - } - size_t off_end = off_start + len; - size_t len_to_move = run->dynfile->size - off_end; - - mangle_Move(run, off_end, off_start, len_to_move); - input_setSize(run, run->dynfile->size - len); -} -static void mangle_ASCIINum(run_t* run, bool printable) { - size_t len = util_rndGet(2, 8); - - char buf[20]; - snprintf(buf, sizeof(buf), "%-19" PRId64, (int64_t)util_rnd64()); - - mangle_UseValue(run, (const uint8_t*)buf, len, printable); -} - -static void mangle_ASCIINumChange(run_t* run, bool printable) { - size_t off = mangle_getOffSet(run); - - /* Find a digit */ - for (; off < run->dynfile->size; off++) { - if (isdigit(run->dynfile->data[off])) { - break; - } - } - size_t left = run->dynfile->size - off; - if (left == 0) { - return; - } - - size_t len = 0; - uint64_t val = 0; - /* 20 is maximum lenght of a string representing a 64-bit unsigned value */ - for (len = 0; (len < 20) && (len < left); len++) { - char c = run->dynfile->data[off + len]; - if (!isdigit(c)) { - break; - } - val *= 10; - val += (c - '0'); - } - - switch (util_rndGet(0, 7)) { - case 0: - val++; - break; - case 1: - val--; - break; - case 2: - val *= 2; - break; - case 3: - val /= 2; - break; - case 4: - val = util_rnd64(); - break; - case 5: - val += util_rndGet(1, 256); - break; - case 6: - val -= util_rndGet(1, 256); - break; - case 7: - val = ~(val); - break; - default: - LOG_F("Invalid choice"); - }; - - char buf[20]; - snprintf(buf, sizeof(buf), "%-19" PRIu64, val); - - mangle_UseValueAt(run, off, (const uint8_t*)buf, len, printable); -} - -static void mangle_Splice(run_t* run, bool printable) { - if (run->global->feedback.dynFileMethod == _HF_DYNFILE_NONE) { - mangle_Bytes(run, printable); - return; - } - - size_t sz = 0; - const uint8_t* buf = input_getRandomInputAsBuf(run, &sz); - if (!buf) { - LOG_E("input_getRandomInputAsBuf() returned no input"); - mangle_Bytes(run, printable); - return; - } - if (!sz) { - mangle_Bytes(run, printable); - return; - } - - size_t remoteOff = mangle_getLen(sz) - 1; - size_t len = mangle_getLen(sz - remoteOff); - mangle_UseValue(run, &buf[remoteOff], len, printable); -} - -static void mangle_Resize(run_t* run, bool printable) { - ssize_t oldsz = run->dynfile->size; - ssize_t newsz = 0; - - uint64_t choice = util_rndGet(0, 32); - switch (choice) { - case 0: /* Set new size arbitrarily */ - newsz = (ssize_t)util_rndGet(1, run->global->mutate.maxInputSz); - break; - case 1 ... 4: /* Increase size by a small value */ - newsz = oldsz + (ssize_t)util_rndGet(0, 8); - break; - case 5: /* Increase size by a larger value */ - newsz = oldsz + (ssize_t)util_rndGet(9, 128); - break; - case 6 ... 9: /* Decrease size by a small value */ - newsz = oldsz - (ssize_t)util_rndGet(0, 8); - break; - case 10: /* Decrease size by a larger value */ - newsz = oldsz - (ssize_t)util_rndGet(9, 128); - break; - case 11 ... 32: /* Do nothing */ - newsz = oldsz; - break; - default: - LOG_F("Illegal value from util_rndGet: %" PRIu64, choice); - break; - } - if (newsz < 1) { - newsz = 1; - } - if (newsz > (ssize_t)run->global->mutate.maxInputSz) { - newsz = run->global->mutate.maxInputSz; - } - - input_setSize(run, (size_t)newsz); - if (newsz > oldsz) { - if (printable) { - memset(&run->dynfile->data[oldsz], ' ', newsz - oldsz); - } - } -} - -void mangle_mangleContent(run_t* run, int speed_factor) { - static void (*const mangleFuncs[])(run_t * run, bool printable) = { - mangle_Shrink, - mangle_Expand, - mangle_Bit, - mangle_IncByte, - mangle_DecByte, - mangle_NegByte, - mangle_AddSub, - mangle_MemSet, - mangle_MemClr, - mangle_MemSwap, - mangle_MemCopy, - mangle_Bytes, - mangle_ASCIINum, - mangle_ASCIINumChange, - mangle_ByteRepeat, - mangle_Magic, - mangle_StaticDict, - mangle_ConstFeedbackDict, - mangle_RandomBuf, - mangle_Splice, - }; - - if (run->mutationsPerRun == 0U) { - return; - } - if (run->dynfile->size == 0U) { - mangle_Resize(run, /* printable= */ run->global->cfg.only_printable); - } - - uint64_t changesCnt = run->global->mutate.mutationsPerRun; - - if (speed_factor < 5) { - changesCnt = util_rndGet(1, run->global->mutate.mutationsPerRun); - } else if (speed_factor < 10) { - changesCnt = run->global->mutate.mutationsPerRun; - } else { - changesCnt = HF_MIN(speed_factor, 10); - changesCnt = HF_MAX(changesCnt, (run->global->mutate.mutationsPerRun * 5)); - } - - /* If last coverage acquisition was more than 5 secs ago, use splicing more frequently */ - if ((time(NULL) - ATOMIC_GET(run->global->timing.lastCovUpdate)) > 5) { - if (util_rnd64() & 0x1) { - mangle_Splice(run, run->global->cfg.only_printable); - } - } - - for (uint64_t x = 0; x < changesCnt; x++) { - if (run->global->feedback.cmpFeedback && (util_rnd64() & 0x1)) { - /* - * mangle_ConstFeedbackDict() is quite powerful if the dynamic feedback dictionary - * exists. If so, give it 50% chance of being used among all mangling functions. - */ - mangle_ConstFeedbackDict(run, /* printable= */ run->global->cfg.only_printable); - } else { - uint64_t choice = util_rndGet(0, ARRAYSIZE(mangleFuncs) - 1); - mangleFuncs[choice](run, /* printable= */ run->global->cfg.only_printable); - } - } - - wmb(); -} diff --git a/custom_mutators/radamsa/radamsa-mutator.c b/custom_mutators/radamsa/radamsa-mutator.c index 624ace3d..466bb5c3 100644 --- a/custom_mutators/radamsa/radamsa-mutator.c +++ b/custom_mutators/radamsa/radamsa-mutator.c @@ -1,6 +1,5 @@ // This simple example just creates random buffer <= 100 filled with 'A' // needs -I /path/to/AFLplusplus/include -//#include "custom_mutator_helpers.h" #include #include @@ -8,19 +7,17 @@ #include #include "radamsa.h" -#include "custom_mutator_helpers.h" +#include "afl-fuzz.h" typedef struct my_mutator { - afl_t *afl; - - u8 *mutator_buf; - + afl_state_t *afl; + u8 *mutator_buf; unsigned int seed; } my_mutator_t; -my_mutator_t *afl_custom_init(afl_t *afl, unsigned int seed) { +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { srand(seed); my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); From 2e23418a09e6c610288414d4ab6841b6f39513af Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 14:55:31 +0200 Subject: [PATCH 077/149] remove symlinks --- .../honggfuzz/custom_mutator_helpers.h | 22 ------------------- .../radamsa/custom_mutator_helpers.h | 1 - 2 files changed, 23 deletions(-) delete mode 100644 custom_mutators/honggfuzz/custom_mutator_helpers.h delete mode 120000 custom_mutators/radamsa/custom_mutator_helpers.h diff --git a/custom_mutators/honggfuzz/custom_mutator_helpers.h b/custom_mutators/honggfuzz/custom_mutator_helpers.h deleted file mode 100644 index 57754697..00000000 --- a/custom_mutators/honggfuzz/custom_mutator_helpers.h +++ /dev/null @@ -1,22 +0,0 @@ -#ifndef CUSTOM_MUTATOR_HELPERS -#define CUSTOM_MUTATOR_HELPERS - -#include "config.h" -#include "types.h" -#include "afl-fuzz.h" -#include - -#define INITIAL_GROWTH_SIZE (64) - -/* Use in a struct: creates a name_buf and a name_size variable. */ -#define BUF_VAR(type, name) \ - type * name##_buf; \ - size_t name##_size; -/* this filles in `&structptr->something_buf, &structptr->something_size`. */ -#define BUF_PARAMS(struct, name) \ - (void **)&struct->name##_buf, &struct->name##_size - -#undef INITIAL_GROWTH_SIZE - -#endif - diff --git a/custom_mutators/radamsa/custom_mutator_helpers.h b/custom_mutators/radamsa/custom_mutator_helpers.h deleted file mode 120000 index f7532ef9..00000000 --- a/custom_mutators/radamsa/custom_mutator_helpers.h +++ /dev/null @@ -1 +0,0 @@ -../examples/custom_mutator_helpers.h \ No newline at end of file From f94a7e88902f1589b105e74ec1a36e3e3bf01f9e Mon Sep 17 00:00:00 2001 From: "Christian Holler (:decoder)" Date: Tue, 25 Apr 2023 16:03:21 +0200 Subject: [PATCH 078/149] Add env var to ignore coverage from dynamically loaded code after forkserver. When using TRACEPC instrumentation, loading code dynamically (e.g. through dlopen()) it can be useful to completely ignore the loaded code, esp. when it cannot be preloaded and is not the target to be tested. This patch allows setting AFL_LLVM_IGNORE_PROBLEMS_COVERAGE=1 to do so. --- instrumentation/afl-compiler-rt.o.c | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index e0e40983..74506e4c 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -1563,17 +1563,27 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { "[-] FATAL: forkserver is already up, but an instrumented dlopen() " "library loaded afterwards. You must AFL_PRELOAD such libraries to " "be able to fuzz them or LD_PRELOAD to run outside of afl-fuzz.\n" - "To ignore this set AFL_IGNORE_PROBLEMS=1 but this will be bad for " - "coverage.\n"); + "To ignore this set AFL_IGNORE_PROBLEMS=1 but this will lead to " + "ambiguous coverage data.\n" + "In addition, you can set AFL_LLVM_IGNORE_PROBLEMS_COVERAGE=1 to " + "ignore the additional coverage instead (use with caution!).\n"); abort(); } else { + u8 ignore_dso_after_fs = !!getenv("AFL_LLVM_IGNORE_PROBLEMS_COVERAGE"); + if (__afl_debug && ignore_dso_after_fs) { + + fprintf(stderr, "Ignoring coverage from dynamically loaded code\n"); + + } + static u32 offset = 5; while (start < stop) { - if (likely(inst_ratio == 100) || R(100) < inst_ratio) { + if (!ignore_dso_after_fs && + (likely(inst_ratio == 100) || R(100) < inst_ratio)) { *(start++) = offset; From 21865c622483d2e2285de3dfad4626c28ca27843 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 16:47:37 +0200 Subject: [PATCH 079/149] rename env to AFL_IGNORE_PROBLEMS_COVERAGE --- docs/FAQ.md | 3 ++- docs/env_variables.md | 3 ++- include/envs.h | 1 + instrumentation/afl-compiler-rt.o.c | 4 ++-- src/afl-fuzz.c | 2 ++ 5 files changed, 9 insertions(+), 4 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 4a9080f8..76350c79 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -229,7 +229,8 @@ If you find an interesting or important question missing, submit it via If this is not a viable option, you can set `AFL_IGNORE_PROBLEMS=1` but then the existing map will be used also for the newly loaded libraries, which allows it to work, however, the efficiency of the fuzzing will be partially - degraded. + degraded. Note that there is additionally `AFL_IGNORE_PROBLEMS_COVERAGE` to + additionally tell AFL++ to ignore any coverage from the late loaded libaries.

diff --git a/docs/env_variables.md b/docs/env_variables.md index 087ccdb7..b1f23159 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -406,7 +406,8 @@ checks or alter some of the more exotic semantics of the tool: - If afl-fuzz encounters an incorrect fuzzing setup during a fuzzing session (not at startup), it will terminate. If you do not want this, then you can - set `AFL_IGNORE_PROBLEMS`. + set `AFL_IGNORE_PROBLEMS`. If you additionally want to also ignore coverage + from late loaded libraries, you can set `AFL_IGNORE_PROBLEMS_COVERAGE`. - When running in the `-M` or `-S` mode, setting `AFL_IMPORT_FIRST` causes the fuzzer to import test cases from other instances before doing anything else. diff --git a/include/envs.h b/include/envs.h index 5e68c80b..fe5ee0e3 100644 --- a/include/envs.h +++ b/include/envs.h @@ -106,6 +106,7 @@ static char *afl_environment_variables[] = { "AFL_HARDEN", "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IGNORE_PROBLEMS", + "AFL_IGNORE_PROBLEMS_COVERAGE", "AFL_IGNORE_TIMEOUTS", "AFL_IGNORE_UNKNOWN_ENVS", "AFL_IMPORT_FIRST", diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 74506e4c..0912e52b 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -1565,13 +1565,13 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { "be able to fuzz them or LD_PRELOAD to run outside of afl-fuzz.\n" "To ignore this set AFL_IGNORE_PROBLEMS=1 but this will lead to " "ambiguous coverage data.\n" - "In addition, you can set AFL_LLVM_IGNORE_PROBLEMS_COVERAGE=1 to " + "In addition, you can set AFL_IGNORE_PROBLEMS_COVERAGE=1 to " "ignore the additional coverage instead (use with caution!).\n"); abort(); } else { - u8 ignore_dso_after_fs = !!getenv("AFL_LLVM_IGNORE_PROBLEMS_COVERAGE"); + u8 ignore_dso_after_fs = !!getenv("AFL_IGNORE_PROBLEMS_COVERAGE"); if (__afl_debug && ignore_dso_after_fs) { fprintf(stderr, "Ignoring coverage from dynamically loaded code\n"); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index ebdbb3fa..c44144f5 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -259,6 +259,8 @@ static void usage(u8 *argv0, int more_help) { "AFL_HANG_TMOUT: override timeout value (in milliseconds)\n" "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: don't warn about core dump handlers\n" "AFL_IGNORE_PROBLEMS: do not abort fuzzing if an incorrect setup is detected\n" + "AFL_IGNORE_PROBLEMS_COVERAGE: if set in addition to AFL_IGNORE_PROBLEMS - also\n + " ignore those libs for coverage\n" "AFL_IGNORE_TIMEOUTS: do not process or save any timeouts\n" "AFL_IGNORE_UNKNOWN_ENVS: don't warn on unknown env vars\n" "AFL_IMPORT_FIRST: sync and import test cases from other fuzzer instances first\n" From 96848398d45e08eaa69be245c986375808d2b3a7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 17:56:36 +0200 Subject: [PATCH 080/149] fix --- src/afl-fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c44144f5..71d2afd8 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -259,7 +259,7 @@ static void usage(u8 *argv0, int more_help) { "AFL_HANG_TMOUT: override timeout value (in milliseconds)\n" "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: don't warn about core dump handlers\n" "AFL_IGNORE_PROBLEMS: do not abort fuzzing if an incorrect setup is detected\n" - "AFL_IGNORE_PROBLEMS_COVERAGE: if set in addition to AFL_IGNORE_PROBLEMS - also\n + "AFL_IGNORE_PROBLEMS_COVERAGE: if set in addition to AFL_IGNORE_PROBLEMS - also\n" " ignore those libs for coverage\n" "AFL_IGNORE_TIMEOUTS: do not process or save any timeouts\n" "AFL_IGNORE_UNKNOWN_ENVS: don't warn on unknown env vars\n" From 432671449f98a675eaf37db52c6318e1edd2423f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 25 Apr 2023 18:19:25 +0200 Subject: [PATCH 081/149] nits --- afl-cmin | 6 ++---- afl-cmin.bash | 2 +- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/afl-cmin b/afl-cmin index c8bbd8d7..63cfdd7e 100755 --- a/afl-cmin +++ b/afl-cmin @@ -281,8 +281,8 @@ BEGIN { exit 1 } - if (!threads && !stdin_file) { - print "[*] Are you aware of the '-T all' parallelize option that massively improves the speed for large corpuses?" + if (!threads && !stdin_file && !nyx_mode) { + print "[*] Are you aware of the '-T all' parallelize option that improves the speed for large/slow corpuses?" } # If @@ is specified, but there's no -f, let's come up with a temporary input @@ -505,8 +505,6 @@ BEGIN { print in_dir"/"infilesSmallToBigFull[cnt] >> tmpfile"."threads } - print "ls -l "tmpfile"*" - } print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." diff --git a/afl-cmin.bash b/afl-cmin.bash index 0e2d973d..d390ff65 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -206,7 +206,7 @@ fi # Check for obvious errors. -if [ ! "$T_ARG" = "" -a ! "$F_ARG" = "" ]; then +if [ ! "$T_ARG" = "" -a ! "$F_ARG" = "" -a ! "$NYX_MODE" == 1 ]; then echo "[-] Error: -T and -f can not be used together." 1>&2 exit 1 fi From b18bc7b98fa23ef805ed2ee3eec04dc1929afd49 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 26 Apr 2023 16:25:03 +0200 Subject: [PATCH 082/149] changelog updates --- TODO.md | 1 - docs/Changelog.md | 8 +++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index dba75070..2efcefea 100644 --- a/TODO.md +++ b/TODO.md @@ -10,7 +10,6 @@ - parallel builds for source-only targets - get rid of check_binary, replace with more forkserver communication - first fuzzer should be a main automatically? not sure. - - reload fuzz binary on signal ## Maybe diff --git a/docs/Changelog.md b/docs/Changelog.md index 667fd634..20b915fa 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -5,9 +5,11 @@ ### Version ++4.07a (dev) - afl-fuzz: + - reverse reading the seeds only on restarts (increases performance) - new env `AFL_POST_PROCESS_KEEP_ORIGINAL` to keep the orignal - data before post process on finds - - reverse reading the seeds only on restarts + data before post process on finds (for atnwalk custom mutator) + - new env `AFL_IGNORE_PROBLEMS_COVERAGE` to ignore coverage from + loaded libs after forkserver initialization (required by Mozilla) - afl-cc: - new env `AFL_LLVM_LTO_SKIPINIT` to support the AFL++ based WASM (https://github.com/fgsect/WAFL) project @@ -15,7 +17,7 @@ - added custom mutator post_process and send support - add `-I filelist` option, an alternative to `-i in_dir` - afl-cmin + afl-cmin.bash: - - `-T threads` parallel task support, huge speedup! + - `-T threads` parallel task support, can be a huge speedup! - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! From 3e84d6a2ae7df5f6b9073a91ccc6acef50b45aab Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 27 Apr 2023 11:49:00 +0200 Subject: [PATCH 083/149] afl++ -> AFL++ --- Dockerfile | 4 +-- GNUmakefile | 12 ++++----- GNUmakefile.gcc_plugin | 6 ++--- GNUmakefile.llvm | 6 ++--- afl-cmin | 2 +- docs/Changelog.md | 2 +- docs/INSTALL.md | 2 +- include/alloc-inl.h | 2 +- instrumentation/SanitizerCoverageLTO.so.cc | 30 +++++++++++----------- instrumentation/afl-llvm-common.cc | 2 +- instrumentation/afl-llvm-dict2file.so.cc | 2 +- qemu_mode/build_qemu_support.sh | 4 +-- src/afl-cc.c | 2 +- src/afl-forkserver.c | 6 ++--- src/afl-fuzz.c | 10 ++++---- src/afl-ld-lto.c | 4 +-- test/test-dlopen.c | 2 +- test/test-gcc-plugin.sh | 2 +- test/test-performance.sh | 4 +-- test/test-pre.sh | 2 +- unicorn_mode/build_unicorn_support.sh | 2 +- 21 files changed, 54 insertions(+), 54 deletions(-) diff --git a/Dockerfile b/Dockerfile index 4e53de40..1b5ffd28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ # FROM ubuntu:22.04 AS aflplusplus -LABEL "maintainer"="afl++ team " +LABEL "maintainer"="AFL++ team " LABEL "about"="AFLplusplus container image" ### Comment out to enable these features @@ -94,4 +94,4 @@ RUN sed -i.bak 's/^ -/ /g' GNUmakefile && \ RUN echo "set encoding=utf-8" > /root/.vimrc && \ echo ". /etc/bash_completion" >> ~/.bashrc && \ echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc && \ - echo "export PS1='"'[afl++ \h] \w \$ '"'" >> ~/.bashrc + echo "export PS1='"'[AFL++ \h] \w \$ '"'" >> ~/.bashrc diff --git a/GNUmakefile b/GNUmakefile index 23cae65d..5900ad61 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -39,7 +39,7 @@ ASAN_OPTIONS=detect_leaks=0 SYS = $(shell uname -s) ARCH = $(shell uname -m) -$(info [*] Compiling afl++ for OS $(SYS) on ARCH $(ARCH)) +$(info [*] Compiling AFL++ for OS $(SYS) on ARCH $(ARCH)) ifdef NO_SPLICING override CFLAGS_OPT += -DNO_SPLICING @@ -359,7 +359,7 @@ performance-test: source-only help: @echo "HELP --- the following make targets exist:" @echo "==========================================" - @echo "all: the main afl++ binaries and llvm/gcc instrumentation" + @echo "all: the main AFL++ binaries and llvm/gcc instrumentation" @echo "binary-only: everything for binary-only fuzzing: frida_mode, nyx_mode, qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator, libtokencap" @echo "source-only: everything for source code fuzzing: nyx_mode, libdislocator, libtokencap" @echo "distrib: everything (for both binary-only and source code fuzzing)" @@ -367,7 +367,7 @@ help: @echo "install: installs everything you have compiled with the build option above" @echo "clean: cleans everything compiled (not downloads when on a checkout)" @echo "deepclean: cleans everything including downloads" - @echo "uninstall: uninstall afl++ from the system" + @echo "uninstall: uninstall AFL++ from the system" @echo "code-format: format the code, do this before you commit and send a PR please!" @echo "tests: this runs the test framework. It is more catered for the developers, but if you run into problems this helps pinpointing the problem" @echo "unit: perform unit tests (based on cmocka and GNU linker)" @@ -749,7 +749,7 @@ endif @echo %.8: % - @echo .TH $* 8 $(BUILD_DATE) "afl++" > $@ + @echo .TH $* 8 $(BUILD_DATE) "AFL++" > $@ @echo .SH NAME >> $@ @echo .B $* >> $@ @echo >> $@ @@ -761,8 +761,8 @@ endif @./$* -hh 2>&1 | tail -n +4 >> $@ @echo >> $@ @echo .SH AUTHOR >> $@ - @echo "afl++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Heiko \"hexcoder-\" Eissfeldt , Andrea Fioraldi and Dominik Maier " >> $@ - @echo The homepage of afl++ is: https://github.com/AFLplusplus/AFLplusplus >> $@ + @echo "AFL++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Dominik Maier , Andrea Fioraldi and Heiko \"hexcoder-\" Eissfeldt " >> $@ + @echo The homepage of AFL++ is: https://github.com/AFLplusplus/AFLplusplus >> $@ @echo >> $@ @echo .SH LICENSE >> $@ @echo Apache License Version 2.0, January 2004 >> $@ diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index 4c4e10c4..41face4c 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -175,7 +175,7 @@ all_done: test_build .NOTPARALLEL: clean %.8: % - @echo .TH $* 8 `date "+%Y-%m-%d"` "afl++" > ./$@ + @echo .TH $* 8 `date "+%Y-%m-%d"` "AFL++" > ./$@ @echo .SH NAME >> ./$@ @echo .B $* >> ./$@ @echo >> ./$@ @@ -187,8 +187,8 @@ all_done: test_build @./$* -h 2>&1 | tail -n +4 >> ./$@ @echo >> ./$@ @echo .SH AUTHOR >> ./$@ - @echo "afl++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Heiko \"hexcoder-\" Eissfeldt , Andrea Fioraldi and Dominik Maier " >> ./$@ - @echo The homepage of afl++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ + @echo "AFL++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Dominik Maier , Andrea Fioraldi and Heiko \"hexcoder-\" Eissfeldt " >> ./$@ + @echo The homepage of AFL++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ @echo >> ./$@ @echo .SH LICENSE >> ./$@ @echo Apache License Version 2.0, January 2004 >> ./$@ diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index a053403b..c1b006ba 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -510,7 +510,7 @@ install: all install -m 644 instrumentation/README.*.md $${DESTDIR}$(DOC_PATH)/ %.8: % - @echo .TH $* 8 $(BUILD_DATE) "afl++" > ./$@ + @echo .TH $* 8 $(BUILD_DATE) "AFL++" > ./$@ @echo .SH NAME >> ./$@ @printf "%s" ".B $* \- " >> ./$@ @./$* -h 2>&1 | head -n 1 | sed -e "s/$$(printf '\e')[^m]*m//g" >> ./$@ @@ -524,8 +524,8 @@ install: all @./$* -h 2>&1 | tail -n +4 >> ./$@ @echo >> ./$@ @echo .SH AUTHOR >> ./$@ - @echo "afl++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Heiko \"hexcoder-\" Eissfeldt , Andrea Fioraldi and Dominik Maier " >> ./$@ - @echo The homepage of afl++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ + @echo "AFL++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Dominik Maier , Andrea Fioraldi and Heiko \"hexcoder-\" Eissfeldt " >> ./$@ + @echo The homepage of AFL++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ @echo >> ./$@ @echo .SH LICENSE >> ./$@ @echo Apache License Version 2.0, January 2004 >> ./$@ diff --git a/afl-cmin b/afl-cmin index 63cfdd7e..ae723c1b 100755 --- a/afl-cmin +++ b/afl-cmin @@ -149,7 +149,7 @@ BEGIN { redirected = 0 } - print "corpus minimization tool for afl++ (awk version)\n" + print "corpus minimization tool for AFL++ (awk version)\n" # defaults extra_par = "" diff --git a/docs/Changelog.md b/docs/Changelog.md index 20b915fa..cd5ed9fc 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -229,7 +229,7 @@ afl-showmap and other tools. - afl-cc: - detect overflow reads on initial input buffer for asan - - new cmplog mode (incompatible with older afl++ versions) + - new cmplog mode (incompatible with older AFL++ versions) - support llvm IR select instrumentation for default PCGUARD and LTO - fix for shared linking on MacOS - better selective instrumentation AFL_LLVM_{ALLOW|DENY}LIST diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 591b7ded..c54cb9ad 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -51,7 +51,7 @@ make source-only These build targets exist: -* all: the main afl++ binaries and llvm/gcc instrumentation +* all: the main AFL++ binaries and llvm/gcc instrumentation * binary-only: everything for binary-only fuzzing: frida_mode, nyx_mode, qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator, libtokencap diff --git a/include/alloc-inl.h b/include/alloc-inl.h index ae37028e..bbb42e88 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -42,7 +42,7 @@ // Be careful! _WANT_ORIGINAL_AFL_ALLOC is not compatible with custom mutators #ifndef _WANT_ORIGINAL_AFL_ALLOC - // afl++ stuff without memory corruption checks - for speed + // AFL++ stuff without memory corruption checks - for speed /* User-facing macro to sprintf() to a dynamically allocated buffer. */ diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 42583f9e..6a719737 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1,4 +1,4 @@ -/* SanitizeCoverage.cpp ported to afl++ LTO :-) */ +/* SanitizeCoverage.cpp ported to AFL++ LTO :-) */ #define AFL_LLVM_PASS @@ -234,7 +234,7 @@ class ModuleSanitizerCoverageLTO SanitizerCoverageOptions Options; - // afl++ START + // AFL++ START // const SpecialCaseList * Allowlist; // const SpecialCaseList * Blocklist; uint32_t autodictionary = 1; @@ -260,7 +260,7 @@ class ModuleSanitizerCoverageLTO Value *MapPtrFixed = NULL; std::ofstream dFile; size_t found = 0; - // afl++ END + // AFL++ END }; @@ -404,7 +404,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( Int8Ty = IRB.getInt8Ty(); Int1Ty = IRB.getInt1Ty(); - /* afl++ START */ + /* AFL++ START */ char *ptr; LLVMContext &Ctx = M.getContext(); Ct = &Ctx; @@ -978,7 +978,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( } - // afl++ END + // AFL++ END SanCovTracePCIndir = M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy); @@ -1002,7 +1002,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( for (auto &F : M) instrumentFunction(F, DTCallback, PDTCallback); - // afl++ START + // AFL++ START if (dFile.is_open()) dFile.close(); if (!getenv("AFL_LLVM_LTO_SKIPINIT") && @@ -1156,7 +1156,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( } - // afl++ END + // AFL++ END // We don't reference these arrays directly in any of our runtime functions, // so we need to prevent them from being dead stripped. @@ -1213,10 +1213,10 @@ static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, // (catchswitch blocks). if (BB->getFirstInsertionPt() == BB->end()) return false; - // afl++ START + // AFL++ START if (!Options.NoPrune && &F.getEntryBlock() == BB && F.size() > 1) return false; - // afl++ END + // AFL++ END if (Options.NoPrune || &F.getEntryBlock() == BB) return true; @@ -1258,10 +1258,10 @@ void ModuleSanitizerCoverageLTO::instrumentFunction( // if (Blocklist && Blocklist->inSection("coverage", "fun", F.getName())) // return; - // afl++ START + // AFL++ START if (!F.size()) return; if (!isInInstrumentList(&F, FMNAME)) return; - // afl++ END + // AFL++ END if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) SplitAllCriticalEdges( @@ -1559,7 +1559,7 @@ bool ModuleSanitizerCoverageLTO::InjectCoverage( for (size_t i = 0, N = AllBlocks.size(); i < N; i++) { - // afl++ START + // AFL++ START if (BlockList.size()) { int skip = 0; @@ -1581,7 +1581,7 @@ bool ModuleSanitizerCoverageLTO::InjectCoverage( } - // afl++ END + // AFL++ END InjectCoverageAtBlock(F, *AllBlocks[i], i, IsLeafFunc); @@ -1647,7 +1647,7 @@ void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, if (Options.TracePCGuard) { - // afl++ START + // AFL++ START ++afl_global_id; if (dFile.is_open()) { @@ -1711,7 +1711,7 @@ void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, // done :) inst++; - // afl++ END + // AFL++ END /* XXXXXXXXXXXXXXXXXXX diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index 5d82aa25..7f17b02d 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -584,7 +584,7 @@ bool isInInstrumentList(llvm::Function *F, std::string Filename) { } // Calculate the number of average collisions that would occur if all -// location IDs would be assigned randomly (like normal afl/afl++). +// location IDs would be assigned randomly (like normal afl/AFL++). // This uses the "balls in bins" algorithm. unsigned long long int calculateCollisions(uint32_t edges) { diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index 97f1d47f..cf368e35 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -744,7 +744,7 @@ static void registerAFLdict2filePass(const PassManagerBuilder &, } static RegisterPass X("afl-dict2file", - "afl++ dict2file instrumentation pass", + "AFL++ dict2file instrumentation pass", false, false); static RegisterStandardPasses RegisterAFLdict2filePass( diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index a064fe58..f59cba78 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -356,7 +356,7 @@ fi if ! command -v "$CROSS" > /dev/null ; then if [ "$CPU_TARGET" = "$(uname -m)" ] ; then - echo "[+] Building afl++ qemu support libraries with CC=$CC" + echo "[+] Building AFL++ qemu support libraries with CC=$CC" echo "[+] Building libcompcov ..." make -C libcompcov && echo "[+] libcompcov ready" echo "[+] Building unsigaction ..." @@ -371,7 +371,7 @@ if ! command -v "$CROSS" > /dev/null ; then echo "[!] Cross compiler $CROSS could not be found, cannot compile libcompcov libqasan and unsigaction" fi else - echo "[+] Building afl++ qemu support libraries with CC=\"$CROSS $CROSS_FLAGS\"" + echo "[+] Building AFL++ qemu support libraries with CC=\"$CROSS $CROSS_FLAGS\"" echo "[+] Building libcompcov ..." make -C libcompcov CC="$CROSS $CROSS_FLAGS" && echo "[+] libcompcov ready" echo "[+] Building unsigaction ..." diff --git a/src/afl-cc.c b/src/afl-cc.c index d1001187..99ce39d4 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -642,7 +642,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } //#if LLVM_MAJOR >= 13 - // // Use the old pass manager in LLVM 14 which the afl++ passes still + // // Use the old pass manager in LLVM 14 which the AFL++ passes still // use. cc_params[cc_par_cnt++] = "-flegacy-pass-manager"; //#endif diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index aa8c8622..30c8901c 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -489,7 +489,7 @@ static void report_error_and_exit(int error) { break; case FS_ERROR_OLD_CMPLOG: FATAL( - "the -c cmplog target was instrumented with an too old afl++ " + "the -c cmplog target was instrumented with an too old AFL++ " "version, you need to recompile it."); break; case FS_ERROR_OLD_CMPLOG_QEMU: @@ -987,7 +987,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if ((status & FS_OPT_ENABLED) == FS_OPT_ENABLED) { - // workaround for recent afl++ versions + // workaround for recent AFL++ versions if ((status & FS_OPT_OLD_AFLPP_WORKAROUND) == FS_OPT_OLD_AFLPP_WORKAROUND) status = (status & 0xf0ffffff); @@ -1059,7 +1059,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, FATAL( "Target's coverage map size of %u is larger than the one this " - "afl++ is set with (%u). Either set AFL_MAP_SIZE=%u and restart " + "AFL++ is set with (%u). Either set AFL_MAP_SIZE=%u and restart " " afl-fuzz, or change MAP_SIZE_POW2 in config.h and recompile " "afl-fuzz", tmp_map_size, fsrv->map_size, tmp_map_size); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 71d2afd8..646dc50b 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1280,16 +1280,16 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->fsrv.mem_limit && afl->shm.cmplog_mode) afl->fsrv.mem_limit += 260; - OKF("afl++ is maintained by Marc \"van Hauser\" Heuse, Heiko \"hexcoder\" " - "Eißfeldt, Andrea Fioraldi and Dominik Maier"); - OKF("afl++ is open source, get it at " + OKF("AFL++ is maintained by Marc \"van Hauser\" Heuse, Dominik Maier, Andrea " + "Fioraldi and Heiko \"hexcoder\" Eißfeldt"); + OKF("AFL++ is open source, get it at " "https://github.com/AFLplusplus/AFLplusplus"); - OKF("NOTE: afl++ >= v3 has changed defaults and behaviours - see README.md"); + OKF("NOTE: AFL++ >= v3 has changed defaults and behaviours - see README.md"); #ifdef __linux__ if (afl->fsrv.nyx_mode) { - OKF("afl++ Nyx mode is enabled (developed and mainted by Sergej Schumilo)"); + OKF("AFL++ Nyx mode is enabled (developed and mainted by Sergej Schumilo)"); OKF("Nyx is open source, get it at https://github.com/Nyx-Fuzz"); } diff --git a/src/afl-ld-lto.c b/src/afl-ld-lto.c index 5438bd9f..420dd817 100644 --- a/src/afl-ld-lto.c +++ b/src/afl-ld-lto.c @@ -2,7 +2,7 @@ american fuzzy lop++ - wrapper for llvm 11+ lld ----------------------------------------------- - Written by Marc Heuse for afl++ + Written by Marc Heuse for AFL++ Maintained by Marc Heuse , Heiko Eißfeldt @@ -210,7 +210,7 @@ static void edit_params(int argc, char **argv) { if (strcmp(argv[i], "--afl") == 0) { - if (!be_quiet) OKF("afl++ test command line flag detected, exiting."); + if (!be_quiet) OKF("AFL++ test command line flag detected, exiting."); exit(0); } diff --git a/test/test-dlopen.c b/test/test-dlopen.c index b81bab13..39442f93 100644 --- a/test/test-dlopen.c +++ b/test/test-dlopen.c @@ -28,7 +28,7 @@ int main(int argc, char **argv) { } - // must use deferred forkserver as otherwise afl++ instrumentation aborts + // must use deferred forkserver as otherwise AFL++ instrumentation aborts // because all dlopen() of instrumented libs must be before the forkserver __AFL_INIT(); diff --git a/test/test-gcc-plugin.sh b/test/test-gcc-plugin.sh index 54e6987f..3690a80a 100755 --- a/test/test-gcc-plugin.sh +++ b/test/test-gcc-plugin.sh @@ -23,7 +23,7 @@ test -e ../afl-gcc-fast -a -e ../afl-compiler-rt.o && { $ECHO "$GREEN[+] gcc_plugin run reported $TUPLES instrumented locations which is fine" } || { $ECHO "$RED[!] gcc_plugin instrumentation produces a weird numbers: $TUPLES" - $ECHO "$YELLOW[-] this is a known issue in gcc, not afl++. It is not flagged as an error because travis builds would all fail otherwise :-(" + $ECHO "$YELLOW[-] this is a known issue in gcc, not AFL++. It is not flagged as an error because travis builds would all fail otherwise :-(" #CODE=1 } test "$TUPLES" -lt 2 && SKIP=1 diff --git a/test/test-performance.sh b/test/test-performance.sh index d61e2f2a..50957141 100755 --- a/test/test-performance.sh +++ b/test/test-performance.sh @@ -7,7 +7,7 @@ FILE=$AFL_PERFORMANCE_FILE test -z "$FILE" && FILE=.afl_performance test -e $FILE || { - echo Warning: This script measure the performance of afl++ and saves the result for future comparisons into $FILE + echo Warning: This script measure the performance of AFL++ and saves the result for future comparisons into $FILE echo Press ENTER to continue or CONTROL-C to abort read IN } @@ -74,7 +74,7 @@ afl-system-config > /dev/null 2>&1 echo Performance settings applied. echo -$ECHO "${RESET}${GREY}[*] starting afl++ performance test framework ..." +$ECHO "${RESET}${GREY}[*] starting AFL++ performance test framework ..." $ECHO "$BLUE[*] Testing: ${AFL_GCC}" GCC=x diff --git a/test/test-pre.sh b/test/test-pre.sh index b8b286e5..1ca9dfb5 100755 --- a/test/test-pre.sh +++ b/test/test-pre.sh @@ -133,7 +133,7 @@ MEM_LIMIT=none export PATH="${PATH}:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" -$ECHO "${RESET}${GREY}[*] starting afl++ test framework ..." +$ECHO "${RESET}${GREY}[*] starting AFL++ test framework ..." test -z "$SYS" && $ECHO "$YELLOW[-] uname -m did not succeed" diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index 53ec2481..d3d16ad5 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -182,7 +182,7 @@ git pull sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null git checkout "$UNICORNAFL_VERSION" || exit 1 -echo "[*] making sure afl++ header files match" +echo "[*] making sure AFL++ header files match" cp "../../include/config.h" "./include" || exit 1 echo "[*] Configuring Unicorn build..." From a25439cfa1521065ff9775c2314ed80a31fba6f2 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 27 Apr 2023 11:50:12 +0200 Subject: [PATCH 084/149] update readme --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c012c400..863c2fce 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Repository: AFL++ is maintained by: * Marc "van Hauser" Heuse -* Heiko "hexcoder-" Eißfeldt * Andrea Fioraldi * Dominik Maier +* Heiko "hexcoder-" Eißfeldt * Documentation: Jana Aydinbas Originally developed by Michał "lcamtuf" Zalewski. From e983e2e9cfb9e4c8489dc35f28bca502ec241c27 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 27 Apr 2023 16:24:43 +0200 Subject: [PATCH 085/149] more debug --- src/afl-fuzz-init.c | 32 ++++++++++++++++++++++++++------ 1 file changed, 26 insertions(+), 6 deletions(-) diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index bd591c8f..baf56a5f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -716,6 +716,8 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } + // if (getenv("MYTEST")) afl->in_place_resume = 1; + if (nl_cnt) { u32 done = 0; @@ -827,6 +829,8 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } + // if (getenv("MYTEST")) afl->in_place_resume = 0; + free(nl); /* not tracked */ if (!afl->queued_items && directory == NULL) { @@ -908,8 +912,10 @@ void perform_dry_run(afl_state_t *afl) { if (res == afl->crash_mode || res == FSRV_RUN_NOBITS) { - SAYF(cGRA " len = %u, map size = %u, exec speed = %llu us\n" cRST, - q->len, q->bitmap_size, q->exec_us); + SAYF(cGRA + " len = %u, map size = %u, exec speed = %llu us, hash = " + "%016llx\n" cRST, + q->len, q->bitmap_size, q->exec_us, q->exec_cksum); } @@ -1164,14 +1170,14 @@ void perform_dry_run(afl_state_t *afl) { u32 duplicates = 0, i; - for (idx = 0; idx < afl->queued_items; idx++) { + for (idx = 0; idx < afl->queued_items - 1; idx++) { q = afl->queue_buf[idx]; if (!q || q->disabled || q->cal_failed || !q->exec_cksum) { continue; } - u32 done = 0; + for (i = idx + 1; - i < afl->queued_items && !done && likely(afl->queue_buf[i]); i++) { + likely(i < afl->queued_items && afl->queue_buf[i] && !done); ++i) { struct queue_entry *p = afl->queue_buf[i]; if (p->disabled || p->cal_failed || !p->exec_cksum) { continue; } @@ -1194,6 +1200,13 @@ void perform_dry_run(afl_state_t *afl) { p->disabled = 1; p->perf_score = 0; + if (afl->debug) { + + WARNF("Same coverage - %s is kept active, %s is disabled.", + q->fname, p->fname); + + } + } else { if (!q->was_fuzzed) { @@ -1207,7 +1220,14 @@ void perform_dry_run(afl_state_t *afl) { q->disabled = 1; q->perf_score = 0; - done = 1; + if (afl->debug) { + + WARNF("Same coverage - %s is kept active, %s is disabled.", + p->fname, q->fname); + + } + + done = 1; // end inner loop because outer loop entry is disabled now } From a2daef29f9c323c0a6a7a64013aadb79ffd3e534 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 27 Apr 2023 17:57:22 +0200 Subject: [PATCH 086/149] slightly different weighting algo (#1719) * better seed selection * slightly different weighting calculation * remove unnecessary memset --- include/afl-fuzz.h | 4 +- src/afl-fuzz-queue.c | 92 ++++++++++++++++++++++++++++++-------------- 2 files changed, 65 insertions(+), 31 deletions(-) diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 831a0dbc..8fb7ecb1 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1223,7 +1223,7 @@ double rand_next_percent(afl_state_t *afl); static inline u32 rand_below(afl_state_t *afl, u32 limit) { - if (limit <= 1) return 0; + if (unlikely(limit <= 1)) return 0; /* The boundary not being necessarily a power of 2, we need to ensure the result uniformity. */ @@ -1256,7 +1256,7 @@ static inline u32 rand_below(afl_state_t *afl, u32 limit) { expand havoc mode */ static inline u32 rand_below_datalen(afl_state_t *afl, u32 limit) { - if (limit <= 1) return 0; + if (unlikely(limit <= 1)) return 0; switch (rand_below(afl, 3)) { diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 8ad7cd97..b10bf749 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -49,11 +49,13 @@ inline u32 select_next_queue_entry(afl_state_t *afl) { u32 s = rand_below(afl, afl->queued_items); double p = rand_next_percent(afl); + /* fprintf(stderr, "select: p=%f s=%u ... p < prob[s]=%f ? s=%u : alias[%u]=%u" " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p < afl->alias_probability[s] ? s : afl->alias_table[s]); */ + return (p < afl->alias_probability[s] ? s : afl->alias_table[s]); } @@ -87,25 +89,28 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, void create_alias_table(afl_state_t *afl) { - u32 n = afl->queued_items, i = 0, a, g; + u32 n = afl->queued_items, i = 0, nSmall = 0, nLarge = n - 1; double sum = 0; + double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double)); + u32 *Small = (int *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); + u32 *Large = (int *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); + afl->alias_table = (u32 *)afl_realloc((void **)&afl->alias_table, n * sizeof(u32)); afl->alias_probability = (double *)afl_realloc( (void **)&afl->alias_probability, n * sizeof(double)); - double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double)); - int *S = (int *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); - int *L = (int *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); - if (!P || !S || !L || !afl->alias_table || !afl->alias_probability) { + if (!P || !Small || !Large || !afl->alias_table || !afl->alias_probability) { FATAL("could not acquire memory for alias table"); } - memset((void *)afl->alias_table, 0, n * sizeof(u32)); memset((void *)afl->alias_probability, 0, n * sizeof(double)); + memset((void *)afl->alias_table, 0, n * sizeof(u32)); + memset((void *)Small, 0, n * sizeof(u32)); + memset((void *)Large, 0, n * sizeof(u32)); if (likely(afl->schedule < RARE)) { @@ -166,7 +171,15 @@ void create_alias_table(afl_state_t *afl) { for (i = 0; i < n; i++) { // weight is always 0 for disabled entries - P[i] = (afl->queue_buf[i]->weight * n) / sum; + if (unlikely(afl->queue_buf[i]->disabled)) { + + P[i] = 0; + + } else { + + P[i] = (afl->queue_buf[i]->weight * n) / sum; + + } } @@ -176,60 +189,81 @@ void create_alias_table(afl_state_t *afl) { struct queue_entry *q = afl->queue_buf[i]; - if (likely(!q->disabled)) { q->perf_score = calculate_score(afl, q); } + if (likely(!q->disabled)) { - sum += q->perf_score; + q->perf_score = calculate_score(afl, q); + sum += q->perf_score; + + } } for (i = 0; i < n; i++) { // perf_score is always 0 for disabled entries - P[i] = (afl->queue_buf[i]->perf_score * n) / sum; + if (unlikely(afl->queue_buf[i]->disabled)) { + + P[i] = 0; + + } else { + + P[i] = (afl->queue_buf[i]->perf_score * n) / sum; + + } } } - int nS = 0, nL = 0, s; - for (s = (s32)n - 1; s >= 0; --s) { + // Done collecting weightings in P, now create the arrays. - if (P[s] < 1) { + for (s32 j = (s32)(n - 1); j >= 0; j--) { - S[nS++] = s; + if (P[j] < 1) { + + Small[nSmall++] = (u32)j; } else { - L[nL++] = s; + Large[nLarge--] = (u32)j; } } - while (nS && nL) { + while (nSmall && nLarge != n - 1) { - a = S[--nS]; - g = L[--nL]; - afl->alias_probability[a] = P[a]; - afl->alias_table[a] = g; - P[g] = P[g] + P[a] - 1; - if (P[g] < 1) { + u32 small = Small[--nSmall]; + u32 large = Large[++nLarge]; - S[nS++] = g; + afl->alias_probability[small] = P[small]; + afl->alias_table[small] = large; + + P[large] = P[large] - (1 - P[small]); + + if (P[large] < 1) { + + Small[nSmall++] = large; } else { - L[nL++] = g; + Large[nLarge--] = large; } } - while (nL) - afl->alias_probability[L[--nL]] = 1; + while (nSmall) { - while (nS) - afl->alias_probability[S[--nS]] = 1; + afl->alias_probability[Small[--nSmall]] = 1; + + } + + while (nLarge != n - 1) { + + afl->alias_probability[Large[++nLarge]] = 1; + + } afl->reinit_table = 0; @@ -264,7 +298,7 @@ void create_alias_table(afl_state_t *afl) { */ /* fprintf(stderr, " entry alias probability perf_score weight - filename\n"); for (u32 i = 0; i < n; ++i) fprintf(stderr, " %5u %5u %11u + filename\n"); for (i = 0; i < n; ++i) fprintf(stderr, " %5u %5u %11u %0.9f %0.9f %s\n", i, afl->alias_table[i], afl->alias_probability[i], afl->queue_buf[i]->perf_score, afl->queue_buf[i]->weight, afl->queue_buf[i]->fname); From 6172bc7312f85276101edbf78d2dd702f9ddfb49 Mon Sep 17 00:00:00 2001 From: fxlb Date: Thu, 27 Apr 2023 16:00:26 +0000 Subject: [PATCH 087/149] Add "Hangs saved" to afl-whatsup (#1717) The hangs could show long or infinite loops. This is important. Co-authored-by: van Hauser --- afl-whatsup | 3 +++ 1 file changed, 3 insertions(+) diff --git a/afl-whatsup b/afl-whatsup index cec1ae28..6f29ab24 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -88,6 +88,7 @@ TOTAL_TIME=0 TOTAL_EXECS=0 TOTAL_EPS=0 TOTAL_CRASHES=0 +TOTAL_HANGS=0 TOTAL_PFAV=0 TOTAL_PENDING=0 @@ -190,6 +191,7 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do TOTAL_EPS=$((TOTAL_EPS + EXEC_SEC)) TOTAL_EXECS=$((TOTAL_EXECS + execs_done)) TOTAL_CRASHES=$((TOTAL_CRASHES + saved_crashes)) + TOTAL_HANGS=$((TOTAL_HANGS + saved_hangs)) TOTAL_PENDING=$((TOTAL_PENDING + pending_total)) TOTAL_PFAV=$((TOTAL_PFAV + pending_favs)) @@ -301,6 +303,7 @@ if [ "$ALIVE_CNT" -gt "1" ]; then fi echo " Crashes saved : $TOTAL_CRASHES" +echo " Hangs saved : $TOTAL_HANGS" echo "Cycles without finds : $TOTAL_WCOP" echo " Time without finds : $TOTAL_LAST_FIND" echo From 6cad585bdc5c335cc2894c97e9aaf6d5fff88e1f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 27 Apr 2023 18:57:28 +0200 Subject: [PATCH 088/149] nits --- src/afl-showmap.c | 2 +- test/test-llvm.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/afl-showmap.c b/src/afl-showmap.c index d0e01cb1..f60acb2d 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1287,7 +1287,7 @@ int main(int argc, char **argv_orig, char **envp) { break; - case 'Y': // fallthough + case 'Y': // fallthrough #ifdef __linux__ case 'X': /* NYX mode */ diff --git a/test/test-llvm.sh b/test/test-llvm.sh index 0e66cc97..714bda93 100755 --- a/test/test-llvm.sh +++ b/test/test-llvm.sh @@ -263,7 +263,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { { mkdir -p in echo 00000000000000000000000000000000 > in/in - AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V15 -i in -o out -c./test-cmplog -- ./test-c >>errors 2>&1 + AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V30 -i in -o out -c./test-cmplog -- ./test-c >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/crashes/id:000000* out/default/hangs/id:000000* 2>/dev/null )" && { $ECHO "$GREEN[+] afl-fuzz is working correctly with llvm_mode cmplog" From 41b0fe7280372031753fc5f11b9a03b214189155 Mon Sep 17 00:00:00 2001 From: Nick Potenski Date: Thu, 27 Apr 2023 11:57:55 -0500 Subject: [PATCH 089/149] afl-showmap: Start a only a single fork server (#1718) A forkserver is started by afl_fsrv_get_mapsize() when dynamically finding the map size. When an input directory option is specified a second fork server was also started. This commit re-arranges the inits for several forkserver struct members so that we can re-use the server started by the get_mapsize() call when not in coresight/qemu/unicorn modes and just start the server otherwise. --- src/afl-showmap.c | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/src/afl-showmap.c b/src/afl-showmap.c index f60acb2d..9c029035 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1421,6 +1421,14 @@ int main(int argc, char **argv_orig, char **envp) { // If @@ are in the target args, replace them and also set use_stdin=false. detect_file_args(argv + optind, stdin_file, &fsrv->use_stdin); + fsrv->dev_null_fd = open("/dev/null", O_RDWR); + if (fsrv->dev_null_fd < 0) { PFATAL("Unable to open /dev/null"); } + + fsrv->out_file = stdin_file; + fsrv->out_fd = + open(stdin_file, O_RDWR | O_CREAT | O_EXCL, DEFAULT_PERMISSION); + if (fsrv->out_fd < 0) { PFATAL("Unable to create '%s'", stdin_file); } + } else { // If @@ are in the target args, replace them and also set use_stdin=false. @@ -1588,6 +1596,14 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->map_size = map_size; + } else { + + afl_fsrv_start(fsrv, use_argv, &stop_soon, + (get_afl_env("AFL_DEBUG_CHILD") || + get_afl_env("AFL_DEBUG_CHILD_OUTPUT")) + ? 1 + : 0); + } if (in_dir || in_filelist) { @@ -1617,9 +1633,6 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_DEBUG_GDB")) wait_for_gdb = true; - fsrv->dev_null_fd = open("/dev/null", O_RDWR); - if (fsrv->dev_null_fd < 0) { PFATAL("Unable to open /dev/null"); } - if (in_filelist) { if (!be_quiet) ACTF("Reading from file list '%s'...", in_filelist); @@ -1666,10 +1679,6 @@ int main(int argc, char **argv_orig, char **envp) { } atexit(at_exit_handler); - fsrv->out_file = stdin_file; - fsrv->out_fd = - open(stdin_file, O_RDWR | O_CREAT | O_EXCL, DEFAULT_PERMISSION); - if (fsrv->out_fd < 0) { PFATAL("Unable to create '%s'", out_file); } if (get_afl_env("AFL_DEBUG")) { @@ -1685,12 +1694,6 @@ int main(int argc, char **argv_orig, char **envp) { } - afl_fsrv_start(fsrv, use_argv, &stop_soon, - (get_afl_env("AFL_DEBUG_CHILD") || - get_afl_env("AFL_DEBUG_CHILD_OUTPUT")) - ? 1 - : 0); - map_size = fsrv->map_size; if (fsrv->support_shmem_fuzz && !fsrv->use_shmem_fuzz) From e956f23a77b776a5c11344889503c833adbf1052 Mon Sep 17 00:00:00 2001 From: "Christian Holler (:decoder)" Date: Fri, 28 Apr 2023 11:35:22 +0200 Subject: [PATCH 090/149] Source Code Coverage support for Nyx (Part 1) (#1720) * Additional source code reformatting in afl-compiler-rt * Add source code coverage support to afl-compiler-rt (for use with Nyx) --- GNUmakefile.llvm | 5 + instrumentation/README.llvm.md | 24 +++ instrumentation/afl-compiler-rt.o.c | 261 +++++++++++++++++++++++++++- src/afl-cc.c | 26 ++- 4 files changed, 308 insertions(+), 8 deletions(-) diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index c1b006ba..2bb4e7f8 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -274,6 +274,11 @@ ifndef LLVM_DEBUG CFLAGS_SAFE += -Wno-deprecated endif +ifdef CODE_COVERAGE + override CFLAGS_SAFE += -D__AFL_CODE_COVERAGE=1 + override LDFLAGS += -ldl +endif + override CFLAGS += $(CFLAGS_SAFE) ifdef AFL_TRACE_PC diff --git a/instrumentation/README.llvm.md b/instrumentation/README.llvm.md index c0677474..126cf1a2 100644 --- a/instrumentation/README.llvm.md +++ b/instrumentation/README.llvm.md @@ -280,3 +280,27 @@ Please note that the default counter implementations are not thread safe! Support for thread safe counters in mode LLVM CLASSIC can be activated with setting `AFL_LLVM_THREADSAFE_INST=1`. + +## 8) Source code coverage through instrumentation + +Measuring source code coverage is a common task in fuzzing, but it is very +difficut to do in some situations (e.g. when using snapshot fuzzing). + +When using the `AFL_LLVM_INSTRUMENT=llvm-codecov` option, afl-cc will use +native trace-pc-guard instrumentation but additionally select options that +are required to utilize the instrumentation for source code coverage. + +In particular, it will switch the instrumentation to be per basic block +instead of instrumenting edges, disable all guard pruning and enable the +experimental pc-table support that allows the runtime to gather 100% of +instrumented basic blocks at start, including their locations. + +Note: You must compile AFL with the `CODE_COVERAGE=1` option to enable the +respective parts in the AFL compiler runtime. Support is currently only +implemented for Nyx, but can in theory also work without Nyx. + +Note: You might have to adjust `MAP_SIZE_POW2` in include/config.h to ensure +that your coverage map is large enough to hold all basic blocks of your +target program without any collisions. + +More documentation on how to utilize this with Nyx will follow. diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 0912e52b..3f8b519b 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -14,6 +14,16 @@ */ +#ifdef __AFL_CODE_COVERAGE + #ifndef _GNU_SOURCE + #define _GNU_SOURCE + #endif + #ifndef __USE_GNU + #define __USE_GNU + #endif + #include +#endif + #ifdef __ANDROID__ #include "android-ashmem.h" #endif @@ -105,6 +115,44 @@ u32 __afl_dictionary_len; u64 __afl_map_addr; u32 __afl_first_final_loc; +#ifdef __AFL_CODE_COVERAGE +typedef struct afl_module_info_t afl_module_info_t; + +struct afl_module_info_t { + + // A unique id starting with 0 + u32 id; + + // Name and base address of the module + char *name; + uintptr_t base_address; + + // PC Guard start/stop + u32 start; + u32 stop; + + // PC Table begin/end + const uintptr_t *pcs_beg; + const uintptr_t *pcs_end; + + u8 mapped; + + afl_module_info_t *next; + +}; + +typedef struct { + + uintptr_t PC, PCFlags; + +} PCTableEntry; + +afl_module_info_t *__afl_module_info = NULL; + +u32 __afl_pcmap_size = 0; +uintptr_t *__afl_pcmap_ptr = NULL; +#endif // __AFL_CODE_COVERAGE + /* 1 if we are running in afl, and the forkserver was started, else 0 */ u32 __afl_connected = 0; @@ -496,11 +544,12 @@ static void __afl_map_shm(void) { if (__afl_map_size && __afl_map_size > MAP_SIZE) { - u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); - if (!map_env || atoi((char *)map_env) < MAP_SIZE) { + u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); + if (!map_env || atoi((char *)map_env) < MAP_SIZE) { - send_forkserver_error(FS_ERROR_MAP_SIZE); - _exit(1); + fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); + send_forkserver_error(FS_ERROR_MAP_SIZE); + _exit(1); } @@ -512,13 +561,13 @@ static void __afl_map_shm(void) { if (!__afl_area_ptr || __afl_area_ptr == (void *)-1) { - if (__afl_map_addr) + if (__afl_map_addr) send_forkserver_error(FS_ERROR_MAP_ADDR); else send_forkserver_error(FS_ERROR_SHMAT); perror("shmat for map"); - _exit(1); + _exit(1); } @@ -678,6 +727,27 @@ static void __afl_map_shm(void) { } +#ifdef __AFL_CODE_COVERAGE + char *pcmap_id_str = getenv("__AFL_PCMAP_SHM_ID"); + + if (pcmap_id_str) { + + __afl_pcmap_size = __afl_map_size * sizeof(void *); + u32 shm_id = atoi(pcmap_id_str); + + __afl_pcmap_ptr = (uintptr_t *)shmat(shm_id, NULL, 0); + + if (__afl_debug) { + + fprintf(stderr, "DEBUG: Received %p via shmat for pcmap\n", + __afl_pcmap_ptr); + + } + + } + +#endif // __AFL_CODE_COVERAGE + } /* unmap SHM. */ @@ -686,6 +756,17 @@ static void __afl_unmap_shm(void) { if (!__afl_already_initialized_shm) return; +#ifdef __AFL_CODE_COVERAGE + if (__afl_pcmap_size) { + + shmdt((void *)__afl_pcmap_ptr); + __afl_pcmap_ptr = NULL; + __afl_pcmap_size = 0; + + } + +#endif // __AFL_CODE_COVERAGE + char *id_str = getenv(SHM_ENV_VAR); if (id_str) { @@ -1507,6 +1588,102 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *guard) { } +#ifdef __AFL_CODE_COVERAGE +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + + if (__afl_debug) { + + fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init called\n"); + + } + + // If for whatever reason, we cannot get dlinfo here, then pc_guard_init also + // couldn't get it and we'd end up attributing to the wrong module. + Dl_info dlinfo; + if (!dladdr(__builtin_return_address(0), &dlinfo)) { + + fprintf(stderr, + "WARNING: Ignoring __sanitizer_cov_pcs_init callback due to " + "missing module info\n"); + return; + + } + + afl_module_info_t *last_module_info = __afl_module_info; + while (last_module_info && last_module_info->next) { + + last_module_info = last_module_info->next; + + } + + if (!last_module_info) { + + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init called with no module info?!\n"); + abort(); + + } + + last_module_info->pcs_beg = pcs_beg; + last_module_info->pcs_end = pcs_end; + + // Now update the pcmap. If this is the last module coming in, after all + // pre-loaded code, then this will also map all of our delayed previous + // modules. + + if (!__afl_pcmap_ptr) { return; } + + for (afl_module_info_t *mod_info = __afl_module_info; mod_info; + mod_info = mod_info->next) { + + if (mod_info->mapped) { continue; } + + PCTableEntry *start = (PCTableEntry *)(mod_info->pcs_beg); + PCTableEntry *end = (PCTableEntry *)(mod_info->pcs_end); + + u32 in_module_index = 0; + + while (start < end) { + + if (mod_info->start + in_module_index >= __afl_map_size) { + + fprintf(stderr, "ERROR: __sanitizer_cov_pcs_init out of bounds?!\n"); + abort(); + + } + + uintptr_t PC = start->PC; + + // This is what `GetPreviousInstructionPc` in sanitizer runtime does + // for x86/x86-64. Needs more work for ARM and other archs. + PC = PC - 1; + + // Calculate relative offset in module + PC = PC - mod_info->base_address; + + __afl_pcmap_ptr[mod_info->start + in_module_index] = PC; + + start++; + in_module_index++; + + } + + mod_info->mapped = 1; + + if (__afl_debug) { + + fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init initialized %u PCs\n", + in_module_index); + + } + + } + +} + +#endif // __AFL_CODE_COVERAGE + /* Init callback. Populates instrumentation IDs. Note that we're using ID of 0 as a special value to indicate non-instrumented bits. That may still touch the bitmap, but in a fairly harmless way. */ @@ -1538,6 +1715,62 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { if (start == stop || *start) { return; } +#ifdef __AFL_CODE_COVERAGE + u32 *orig_start = start; + afl_module_info_t *mod_info = NULL; + + Dl_info dlinfo; + if (dladdr(__builtin_return_address(0), &dlinfo)) { + + if (__afl_already_initialized_forkserver) { + + fprintf(stderr, "[pcmap] Error: Module was not preloaded: %s\n", + dlinfo.dli_fname); + + } else { + + afl_module_info_t *last_module_info = __afl_module_info; + while (last_module_info && last_module_info->next) { + + last_module_info = last_module_info->next; + + } + + mod_info = malloc(sizeof(afl_module_info_t)); + + mod_info->id = last_module_info ? last_module_info->id + 1 : 0; + mod_info->name = strdup(dlinfo.dli_fname); + mod_info->base_address = (uintptr_t)dlinfo.dli_fbase; + mod_info->start = 0; + mod_info->stop = 0; + mod_info->pcs_beg = NULL; + mod_info->pcs_end = NULL; + mod_info->mapped = 0; + mod_info->next = NULL; + + if (last_module_info) { + + last_module_info->next = mod_info; + + } else { + + __afl_module_info = mod_info; + + } + + fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", dlinfo.dli_fname, + dlinfo.dli_fbase); + + } + + } else { + + fprintf(stderr, "[pcmap] dladdr call failed\n"); + + } + +#endif // __AFL_CODE_COVERAGE + x = getenv("AFL_INST_RATIO"); if (x) { @@ -1625,6 +1858,22 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } +#ifdef __AFL_CODE_COVERAGE + if (mod_info) { + + mod_info->start = *orig_start; + mod_info->stop = *(stop - 1); + if (__afl_debug) { + + fprintf(stderr, "DEBUG: [pcmap] Start Index: %u Stop Index: %u\n", + mod_info->start, mod_info->stop); + + } + + } + +#endif // __AFL_CODE_COVERAGE + if (__afl_debug) { fprintf(stderr, diff --git a/src/afl-cc.c b/src/afl-cc.c index 99ce39d4..b11a041d 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -76,6 +76,7 @@ enum { INSTRUMENT_OPT_NGRAM = 16, INSTRUMENT_OPT_CALLER = 32, INSTRUMENT_OPT_CTX_K = 64, + INSTRUMENT_OPT_CODECOV = 128, }; @@ -751,7 +752,15 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else if (instrument_mode == INSTRUMENT_LLVMNATIVE) { #if LLVM_MAJOR >= 4 - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; + if (instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { + #if LLVM_MAJOR >= 6 + cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"; + #else + FATAL("pcguard instrumentation with pc-table requires llvm 6.0.1+"); + #endif + } else { + cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; + } #else FATAL("pcguard instrumentation requires llvm 4.0.1+"); #endif @@ -1682,6 +1691,18 @@ int main(int argc, char **argv, char **envp) { } + if (strncasecmp(ptr2, "llvmcodecov", strlen("llvmcodecov")) == 0 || + strncasecmp(ptr2, "llvm-codecov", strlen("llvm-codecov")) == 0) { + + if (!instrument_mode || instrument_mode == INSTRUMENT_LLVMNATIVE) { + instrument_mode = INSTRUMENT_LLVMNATIVE; + instrument_opt_mode |= INSTRUMENT_OPT_CODECOV; + } else + FATAL("main instrumentation mode already set with %s", + instrument_mode_string[instrument_mode]); + + } + if (strncasecmp(ptr2, "cfg", strlen("cfg")) == 0 || strncasecmp(ptr2, "instrim", strlen("instrim")) == 0) { @@ -2241,7 +2262,8 @@ int main(int argc, char **argv, char **envp) { "(requires LLVM 11 or higher)"); #endif - if (instrument_opt_mode && instrument_mode != INSTRUMENT_CLASSIC) + if (instrument_opt_mode && instrument_opt_mode != INSTRUMENT_OPT_CODECOV && + instrument_mode != INSTRUMENT_CLASSIC) FATAL( "CALLER, CTX and NGRAM instrumentation options can only be used with " "the LLVM CLASSIC instrumentation mode."); From 5813a4319c88848b2a1c47c12fe27f5e14dcad44 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 28 Apr 2023 11:42:21 +0200 Subject: [PATCH 091/149] doc, code format --- GNUmakefile | 3 ++- docs/INSTALL.md | 10 ++++------ instrumentation/afl-compiler-rt.o.c | 14 ++++++------- src/afl-cc.c | 31 +++++++++++++++++++++-------- 4 files changed, 36 insertions(+), 22 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 5900ad61..56b8bb42 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -379,6 +379,7 @@ help: @echo Known build environment options: @echo "==========================================" @echo STATIC - compile AFL++ static + @echo CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md) @echo ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes @echo UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for debug purposes @echo DEBUG - no optimization, -ggdb3, all warnings and -Werror @@ -394,7 +395,7 @@ help: @echo AFL_NO_X86 - if compiling on non-intel/amd platforms @echo "LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian)" @echo "==========================================" - @echo e.g.: make ASAN_BUILD=1 + @echo e.g.: make LLVM_CONFIG=llvm-config-16 .PHONY: test_x86 ifndef AFL_NO_X86 diff --git a/docs/INSTALL.md b/docs/INSTALL.md index c54cb9ad..637e8658 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -79,22 +79,20 @@ make STATIC=1 These build options exist: * STATIC - compile AFL++ static +* CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md) * ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes -* UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for - debug purposes +* UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for debug purposes * DEBUG - no optimization, -ggdb3, all warnings and -Werror * LLVM_DEBUG - shows llvm deprecation warnings * PROFILING - compile afl-fuzz with profiling information * INTROSPECTION - compile afl-fuzz with mutation introspection * NO_PYTHON - disable python support -* NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for - normal fuzzing +* NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for normal fuzzing * NO_NYX - disable building nyx mode dependencies * NO_CORESIGHT - disable building coresight (arm64 only) * NO_UNICORN_ARM64 - disable building unicorn on arm64 * AFL_NO_X86 - if compiling on non-intel/amd platforms -* LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config - (e.g., Debian) +* LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian) e.g.: `make LLVM_CONFIG=llvm-config-14` diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 3f8b519b..5372fae0 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -544,12 +544,12 @@ static void __afl_map_shm(void) { if (__afl_map_size && __afl_map_size > MAP_SIZE) { - u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); - if (!map_env || atoi((char *)map_env) < MAP_SIZE) { + u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); + if (!map_env || atoi((char *)map_env) < MAP_SIZE) { - fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); - send_forkserver_error(FS_ERROR_MAP_SIZE); - _exit(1); + fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); + send_forkserver_error(FS_ERROR_MAP_SIZE); + _exit(1); } @@ -561,13 +561,13 @@ static void __afl_map_shm(void) { if (!__afl_area_ptr || __afl_area_ptr == (void *)-1) { - if (__afl_map_addr) + if (__afl_map_addr) send_forkserver_error(FS_ERROR_MAP_ADDR); else send_forkserver_error(FS_ERROR_SHMAT); perror("shmat for map"); - _exit(1); + _exit(1); } diff --git a/src/afl-cc.c b/src/afl-cc.c index b11a041d..19314555 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -752,15 +752,21 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else if (instrument_mode == INSTRUMENT_LLVMNATIVE) { #if LLVM_MAJOR >= 4 - if (instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { + if (instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { + #if LLVM_MAJOR >= 6 - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"; + cc_params[cc_par_cnt++] = + "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"; #else FATAL("pcguard instrumentation with pc-table requires llvm 6.0.1+"); #endif - } else { + + } else { + cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; - } + + } + #else FATAL("pcguard instrumentation requires llvm 4.0.1+"); #endif @@ -1660,13 +1666,17 @@ int main(int argc, char **argv, char **envp) { instrument_mode = INSTRUMENT_CLASSIC; lto_mode = 1; - } else if (!instrument_mode || instrument_mode == INSTRUMENT_AFL) + } else if (!instrument_mode || instrument_mode == INSTRUMENT_AFL) { instrument_mode = INSTRUMENT_AFL; - else + + } else { + FATAL("main instrumentation mode already set with %s", instrument_mode_string[instrument_mode]); + } + } if (strncasecmp(ptr2, "pc-guard", strlen("pc-guard")) == 0 || @@ -1695,12 +1705,17 @@ int main(int argc, char **argv, char **envp) { strncasecmp(ptr2, "llvm-codecov", strlen("llvm-codecov")) == 0) { if (!instrument_mode || instrument_mode == INSTRUMENT_LLVMNATIVE) { + instrument_mode = INSTRUMENT_LLVMNATIVE; - instrument_opt_mode |= INSTRUMENT_OPT_CODECOV; - } else + instrument_opt_mode |= INSTRUMENT_OPT_CODECOV; + + } else { + FATAL("main instrumentation mode already set with %s", instrument_mode_string[instrument_mode]); + } + } if (strncasecmp(ptr2, "cfg", strlen("cfg")) == 0 || From 74be9ab5ce61d5b561faf688c245143da1a0141e Mon Sep 17 00:00:00 2001 From: vH Date: Fri, 28 Apr 2023 14:55:35 +0200 Subject: [PATCH 092/149] llvm 17 changes --- instrumentation/SanitizerCoverageLTO.so.cc | 6 ++++-- instrumentation/SanitizerCoveragePCGUARD.so.cc | 2 +- instrumentation/afl-llvm-common.h | 2 ++ instrumentation/afl-llvm-dict2file.so.cc | 4 +++- instrumentation/cmplog-routines-pass.cc | 4 +++- instrumentation/cmplog-switches-pass.cc | 4 +++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 42583f9e..e779bb79 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -19,8 +19,8 @@ #include "llvm/ADT/SmallVector.h" #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" + #include "llvm/Analysis/EHPersonalities.h" #endif -#include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/BasicBlock.h" @@ -49,7 +49,9 @@ #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 85b1ddd5..8be9e329 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -15,8 +15,8 @@ #include "llvm/ADT/SmallVector.h" #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" + #include "llvm/Analysis/EHPersonalities.h" #endif -#include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" diff --git a/instrumentation/afl-llvm-common.h b/instrumentation/afl-llvm-common.h index 16a13da5..c9324460 100644 --- a/instrumentation/afl-llvm-common.h +++ b/instrumentation/afl-llvm-common.h @@ -22,7 +22,9 @@ typedef long double max_align_t; #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" +#if LLVM_VERSION_MAJOR < 17 #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index 97f1d47f..97155cd6 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -53,7 +53,9 @@ #include "llvm/IR/Verifier.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ValueTracking.h" diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 0498156d..39db5aa4 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -38,7 +38,9 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index cd0ae76d..38de669d 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -39,7 +39,9 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" From 00c86b7cb155a266c84c9a62b33697fa3f367386 Mon Sep 17 00:00:00 2001 From: vH Date: Fri, 28 Apr 2023 14:56:52 +0200 Subject: [PATCH 093/149] llvm 17 changes --- instrumentation/SanitizerCoverageLTO.so.cc | 6 ++++-- instrumentation/SanitizerCoveragePCGUARD.so.cc | 2 +- instrumentation/afl-llvm-common.h | 2 ++ instrumentation/afl-llvm-dict2file.so.cc | 4 +++- instrumentation/cmplog-routines-pass.cc | 4 +++- instrumentation/cmplog-switches-pass.cc | 4 +++- 6 files changed, 16 insertions(+), 6 deletions(-) diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 6a719737..1ee85234 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -19,8 +19,8 @@ #include "llvm/ADT/SmallVector.h" #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" + #include "llvm/Analysis/EHPersonalities.h" #endif -#include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ValueTracking.h" #include "llvm/IR/BasicBlock.h" @@ -49,7 +49,9 @@ #include "llvm/Support/VirtualFileSystem.h" #include "llvm/Support/raw_ostream.h" #include "llvm/Transforms/Instrumentation.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 85b1ddd5..8be9e329 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -15,8 +15,8 @@ #include "llvm/ADT/SmallVector.h" #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" + #include "llvm/Analysis/EHPersonalities.h" #endif -#include "llvm/Analysis/EHPersonalities.h" #include "llvm/Analysis/PostDominators.h" #include "llvm/IR/CFG.h" #include "llvm/IR/Constant.h" diff --git a/instrumentation/afl-llvm-common.h b/instrumentation/afl-llvm-common.h index 16a13da5..c9324460 100644 --- a/instrumentation/afl-llvm-common.h +++ b/instrumentation/afl-llvm-common.h @@ -22,7 +22,9 @@ typedef long double max_align_t; #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" +#if LLVM_VERSION_MAJOR < 17 #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #if LLVM_VERSION_MAJOR > 3 || \ (LLVM_VERSION_MAJOR == 3 && LLVM_VERSION_MINOR > 4) diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index cf368e35..8ee13010 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -53,7 +53,9 @@ #include "llvm/IR/Verifier.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Analysis/LoopInfo.h" #include "llvm/Analysis/ValueTracking.h" diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 0498156d..39db5aa4 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -38,7 +38,9 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" diff --git a/instrumentation/cmplog-switches-pass.cc b/instrumentation/cmplog-switches-pass.cc index cd0ae76d..38de669d 100644 --- a/instrumentation/cmplog-switches-pass.cc +++ b/instrumentation/cmplog-switches-pass.cc @@ -39,7 +39,9 @@ #include "llvm/IR/Module.h" #include "llvm/Support/Debug.h" #include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/IPO/PassManagerBuilder.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/IPO/PassManagerBuilder.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Pass.h" #include "llvm/Analysis/ValueTracking.h" From f567a89dae29afb2e421d649f0e750e77913f08c Mon Sep 17 00:00:00 2001 From: vH Date: Fri, 28 Apr 2023 15:39:01 +0200 Subject: [PATCH 094/149] more llvm 17 --- instrumentation/SanitizerCoverageLTO.so.cc | 2 ++ instrumentation/SanitizerCoveragePCGUARD.so.cc | 2 ++ 2 files changed, 4 insertions(+) diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index 1ee85234..b3b0d2cd 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -20,6 +20,8 @@ #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" #include "llvm/Analysis/EHPersonalities.h" +#else + #include "llvm/IR/EHPersonalities.h" #endif #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ValueTracking.h" diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 8be9e329..41c38283 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -16,6 +16,8 @@ #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" #include "llvm/Analysis/EHPersonalities.h" +#else + #include "llvm/IR/EHPersonalities.h" #endif #include "llvm/Analysis/PostDominators.h" #include "llvm/IR/CFG.h" From ed96f9b209ceed9e0295bd0bce452bd74e797f1f Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 28 Apr 2023 16:02:09 +0200 Subject: [PATCH 095/149] add frida mode tutorial --- docs/tutorials.md | 4 ++++ frida_mode/README.md | 2 ++ 2 files changed, 6 insertions(+) diff --git a/docs/tutorials.md b/docs/tutorials.md index 758fddab..342080fd 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -20,6 +20,10 @@ training, then we can highly recommend the following: * [https://github.com/antonio-morales/Fuzzing101](https://github.com/antonio-morales/Fuzzing101) +Here is good workflow description for frida_mode: + +* [https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html](https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html) + If you are interested in fuzzing structured data (where you define what the structure is), these links have you covered (some are outdated though): diff --git a/frida_mode/README.md b/frida_mode/README.md index 49a1fe38..bfca443c 100644 --- a/frida_mode/README.md +++ b/frida_mode/README.md @@ -7,6 +7,8 @@ variables. In FRIDA mode, binary programs are instrumented, similarly to QEMU mode. +A tutorial can be found at [https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html](https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html) + ## Current progress As FRIDA mode is new, it is missing a lot of features. The design is such that From 9065d4ba86ecdafeade50e5235ee1e99f4179692 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 1 May 2023 08:38:13 +0200 Subject: [PATCH 096/149] fix effector map --- src/afl-fuzz-one.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index ee562f96..442240a9 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -842,6 +842,7 @@ u8 fuzz_one_original(afl_state_t *afl) { eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); if (unlikely(!eff_map)) { PFATAL("alloc"); } + memset(eff_map, 0, sizeof(len)); eff_map[0] = 1; if (EFF_APOS(len - 1) != 0) { @@ -3570,6 +3571,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); if (unlikely(!eff_map)) { PFATAL("alloc"); } + memset(eff_map, 0, sizeof(len)); eff_map[0] = 1; if (EFF_APOS(len - 1) != 0) { From fcab3ec99026e92b688a69de476a0763942a9d67 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 1 May 2023 08:55:37 +0200 Subject: [PATCH 097/149] docs --- docs/FAQ.md | 8 ++++++++ docs/best_practices.md | 5 +++++ 2 files changed, 13 insertions(+) diff --git a/docs/FAQ.md b/docs/FAQ.md index 76350c79..8178db46 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -171,6 +171,14 @@ If you find an interesting or important question missing, submit it via The more "unstable" edges there are, the harder it is for AFL++ to identify valid new paths. + If you fuzz in persistent mode (`AFL_LOOP` or `LLVMFuzzerTestOneInput()` + harnesses, a large number of unstable edges can mean that the target keeps + internal state and therefore it is possible that crashes cannot be replayed. + In such a case do either **not** fuzz in persistent mode (remove `AFL_LOOP()` + from your harness or call `LLVMFuzzerTestOneInput()` harnesses with `@@`), + or set a low `AFL_LOOP` value, e.g. 100, and enable `AFL_PERSISTENT_RECORD` + in `config.h` with the same value. + A value above 90% is usually fine and a value above 80% is also still ok, and even a value above 20% can still result in successful finds of bugs. However, it is recommended that for values below 90% or 80% you should take diff --git a/docs/best_practices.md b/docs/best_practices.md index 133c645e..459fcaf7 100644 --- a/docs/best_practices.md +++ b/docs/best_practices.md @@ -131,6 +131,11 @@ jitter, or is a hash map function etc., then it should not be instrumented. To be able to exclude these functions (based on AFL++'s measured stability), the following process will allow to identify functions with variable edges. +Note that this is only useful for non-persistent targets! +If a persistent target is unstable whereas when run non-persistent is fine, +then this means that the target is keeping internal state, which is bad for +fuzzing. Fuzz such targets **without** persistent mode. + Four steps are required to do this and it also requires quite some knowledge of coding and/or disassembly and is effectively possible only with `afl-clang-fast` `PCGUARD` and `afl-clang-lto` `LTO` instrumentation. From 2cd07abca9c7b843bbd2085e0e4d852d41169092 Mon Sep 17 00:00:00 2001 From: lazymio Date: Mon, 1 May 2023 13:12:05 +0200 Subject: [PATCH 098/149] Should memset EFF_ALEN(len) of eff_map (#1722) --- src/afl-fuzz-one.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 442240a9..a9902087 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -842,7 +842,7 @@ u8 fuzz_one_original(afl_state_t *afl) { eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); if (unlikely(!eff_map)) { PFATAL("alloc"); } - memset(eff_map, 0, sizeof(len)); + memset(eff_map, 0, EFF_ALEN(len)); eff_map[0] = 1; if (EFF_APOS(len - 1) != 0) { @@ -3571,7 +3571,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); if (unlikely(!eff_map)) { PFATAL("alloc"); } - memset(eff_map, 0, sizeof(len)); + memset(eff_map, 0, EFF_ALEN(len)); eff_map[0] = 1; if (EFF_APOS(len - 1) != 0) { From 22db79aefafb48fed48199a86a39babdee795870 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 1 May 2023 15:07:49 +0200 Subject: [PATCH 099/149] fix reallocs --- include/alloc-inl.h | 7 +++---- src/afl-fuzz.c | 39 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 4 deletions(-) diff --git a/include/alloc-inl.h b/include/alloc-inl.h index bbb42e88..1e9a192b 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -704,12 +704,11 @@ static inline void *afl_realloc(void **buf, size_t size_needed) { *buf = NULL; return NULL; - } else { - - new_buf = newer_buf; - } + new_buf = newer_buf; + memset(((u8 *)new_buf) + current_size, 0, next_size - current_size); + new_buf->complete_size = next_size; *buf = (void *)(new_buf->buf); return *buf; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 646dc50b..c02479cf 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1979,6 +1979,7 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->non_instrumented_mode || afl->fsrv.qemu_mode || afl->fsrv.frida_mode || afl->fsrv.cs_mode || afl->unicorn_mode) { + u32 old_map_size = map_size; map_size = afl->fsrv.real_map_size = afl->fsrv.map_size = MAP_SIZE; afl->virgin_bits = ck_realloc(afl->virgin_bits, map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, map_size); @@ -1990,6 +1991,18 @@ int main(int argc, char **argv_orig, char **envp) { afl->first_trace = ck_realloc(afl->first_trace, map_size); afl->map_tmp_buf = ck_realloc(afl->map_tmp_buf, map_size); + if (old_map_size < map_size) { + + memset(afl->var_bytes + old_map_size, 0, map_size - old_map_size); + memset(afl->top_rated + old_map_size, 0, map_size - old_map_size); + memset(afl->clean_trace + old_map_size, 0, map_size - old_map_size); + memset(afl->clean_trace_custom + old_map_size, 0, + map_size - old_map_size); + memset(afl->first_trace + old_map_size, 0, map_size - old_map_size); + memset(afl->map_tmp_buf + old_map_size, 0, map_size - old_map_size); + + } + } afl->argv = use_argv; @@ -2017,6 +2030,7 @@ int main(int argc, char **argv_orig, char **envp) { OKF("Re-initializing maps to %u bytes", new_map_size); + u32 old_map_size = map_size; afl->virgin_bits = ck_realloc(afl->virgin_bits, new_map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, new_map_size); afl->virgin_crash = ck_realloc(afl->virgin_crash, new_map_size); @@ -2029,6 +2043,18 @@ int main(int argc, char **argv_orig, char **envp) { afl->first_trace = ck_realloc(afl->first_trace, new_map_size); afl->map_tmp_buf = ck_realloc(afl->map_tmp_buf, new_map_size); + if (old_map_size < new_map_size) { + + memset(afl->var_bytes + old_map_size, 0, new_map_size - old_map_size); + memset(afl->top_rated + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace_custom + old_map_size, 0, + new_map_size - old_map_size); + memset(afl->first_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->map_tmp_buf + old_map_size, 0, new_map_size - old_map_size); + + } + afl_fsrv_kill(&afl->fsrv); afl_shm_deinit(&afl->shm); afl->fsrv.map_size = new_map_size; @@ -2079,6 +2105,7 @@ int main(int argc, char **argv_orig, char **envp) { OKF("Re-initializing maps to %u bytes due cmplog", new_map_size); + u32 old_map_size = map_size; afl->virgin_bits = ck_realloc(afl->virgin_bits, new_map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, new_map_size); afl->virgin_crash = ck_realloc(afl->virgin_crash, new_map_size); @@ -2091,6 +2118,18 @@ int main(int argc, char **argv_orig, char **envp) { afl->first_trace = ck_realloc(afl->first_trace, new_map_size); afl->map_tmp_buf = ck_realloc(afl->map_tmp_buf, new_map_size); + if (old_map_size < new_map_size) { + + memset(afl->var_bytes + old_map_size, 0, new_map_size - old_map_size); + memset(afl->top_rated + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace_custom + old_map_size, 0, + new_map_size - old_map_size); + memset(afl->first_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->map_tmp_buf + old_map_size, 0, new_map_size - old_map_size); + + } + afl_fsrv_kill(&afl->fsrv); afl_fsrv_kill(&afl->cmplog_fsrv); afl_shm_deinit(&afl->shm); From a7b7f3cde9b3a420ea5ac32f7309e8a856a01e94 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 2 May 2023 18:25:56 +0200 Subject: [PATCH 100/149] fix afl-system-config for macos --- afl-system-config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/afl-system-config b/afl-system-config index bf6397fa..b50bb06e 100755 --- a/afl-system-config +++ b/afl-system-config @@ -110,7 +110,7 @@ if [ "$PLATFORM" = "Darwin" ] ; then sysctl kern.sysv.shmall=131072000 echo Settings applied. echo - if [ $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ] ; then + if $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ; then echo echo Unloading the default crash reporter SL=/System/Library; PL=com.apple.ReportCrash @@ -119,6 +119,7 @@ if [ "$PLATFORM" = "Darwin" ] ; then echo fi echo It is recommended to disable System Integrity Protection for increased performance. + echo See: https://developer.apple.com/documentation/security/disabling_and_enabling_system_integrity_protection echo DONE=1 fi From f516926f006545d45162eaef723d786a427721f8 Mon Sep 17 00:00:00 2001 From: Moshe Kaplan Date: Thu, 4 May 2023 11:23:30 -0400 Subject: [PATCH 101/149] afl-fuzz.c: Document -i - in --help (#1725) afl-fuzz.c: Document `-i -` in `--help`, to write that `-i` can be passed '-' to resume the prior fuzzing job. Also reference AFL_AUTORESUME so users know they can set that parameter to sidestep the issue entirely. --- src/afl-fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c02479cf..c5206282 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -124,7 +124,7 @@ static void usage(u8 *argv0, int more_help) { "\n%s [ options ] -- /path/to/fuzzed_app [ ... ]\n\n" "Required parameters:\n" - " -i dir - input directory with test cases\n" + " -i dir - input directory with test cases (or '-' to resume, also see AFL_AUTORESUME)\n" " -o dir - output directory for fuzzer findings\n\n" "Execution control settings:\n" From 396157dedae2049f830c49eb81ef9617275333ee Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 5 May 2023 13:52:54 +0200 Subject: [PATCH 102/149] tritondse custom mutator attempt --- custom_mutators/aflpp_tritondse/README.md | 17 +++ .../aflpp_tritondse/aflpp_tritondse.py | 106 ++++++++++++++++++ src/afl-fuzz-one.c | 18 +-- 3 files changed, 133 insertions(+), 8 deletions(-) create mode 100644 custom_mutators/aflpp_tritondse/README.md create mode 100644 custom_mutators/aflpp_tritondse/aflpp_tritondse.py diff --git a/custom_mutators/aflpp_tritondse/README.md b/custom_mutators/aflpp_tritondse/README.md new file mode 100644 index 00000000..8a5dd02b --- /dev/null +++ b/custom_mutators/aflpp_tritondse/README.md @@ -0,0 +1,17 @@ +# An AFL++ custom mutator using TritonDSE + +## Installing the requirements + +`pip3 install tritondse` + +## How to run with an example + +``` +../../afl-cc -o ../../test-instr ../../test-instr.c +mkdir -p in +echo aaaa > in/in +TRITON_DSE_TARGET=../../test-instr AFL_CUSTOM_MUTATOR_ONLY=1 AFL_SYNC_TIME=1 AFL_PYTHON_MODULE=aflpp_tritondse PYTHONPATH=. ../../afl-fuzz -i in -o out -- ../../test-instr +``` + +Note that this custom mutator works differently, new finds are synced +after 10-60 seconds to the fuzzing instance. diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py new file mode 100644 index 00000000..33bf8a9f --- /dev/null +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -0,0 +1,106 @@ +import sys +import os +import logging + +from tritondse import Config +from tritondse import CoverageStrategy +from tritondse import ProcessState +from tritondse import Program +from tritondse import Seed +from tritondse import SeedFormat +from tritondse import SymbolicExecutor +from tritondse import SymbolicExplorator + + +#logging.basicConfig(level=logging.INFO) + +is_debug = False +out_path = "out/tritondse/queue" +input_file = None +prog = None +config = None +dse = None +cycle = 0 +count = 0 +hashes = set() + +def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): + #logging.info(f"[PRE-EXEC] Processing seed: {se.seed.hash}, \ + # ({repr(se.seed.content)})") + global count + global hasshes + if se.seed.hash not in hashes: + hashes.add(se.seed.hash) + filename = out_path + "/id:" + f"{count:06}" + "," + se.seed.hash + if not os.path.exists(filename): + with open(filename, 'wb') as file: + file.write(se.seed.content) + count += 1 + if input_file: + with open(input_file, 'wb') as file: + file.write(se.seed.content) + + +def init(seed): + global prog + global config + global dse + global input_file + global is_debug + # Load the program (LIEF-based program loader). + prog = Program(os.environ['TRITON_DSE_TARGET']) + # Set the configuration. + argv = None + try: + foo = os.environ['AFL_DEBUG'] + is_debug = True + except KeyError: + pass + try: + argv_list = os.environ['TRITON_DSE_TARGET_ARGV'] + argv = argv_list.split() + except KeyError: + pass + try: + foo = os.environ['TRITON_DSE_TARGET_INPUT'] + input_file = foo + except KeyError: + pass + config = Config(coverage_strategy = CoverageStrategy.PATH, + debug = is_debug, + pipe_stdout = is_debug, + pipe_stderr = is_debug, + execution_timeout = 1, + program_argv = argv, + smt_timeout= 50, + seed_format = SeedFormat.RAW) + # Create an instance of the Symbolic Explorator + dse = SymbolicExplorator(config, prog) + # Add callbacks. + dse.callback_manager.register_pre_execution_callback(pre_exec_hook) + # Create the output directory + os.makedirs(out_path, exist_ok=True) + + +#def fuzz(buf, add_buf, max_size): +# return b"" + + +def queue_new_entry(filename_new_queue, filename_orig_queue): + global dse + global cycle + # Add seed to the worklist. + with open(filename_new_queue, "rb") as file: + seed = file.read() + seed = Seed(seed) + dse.add_input_seed(seed) + if is_debug: + print("NEW FILE " + filename_new_queue + " count " + str(cycle)) + cycle += 1 + # Start exploration! + #dse.step() + dse.explore() + pass + +def splice_optout(): + pass diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index a9902087..c6e9a295 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -2048,20 +2048,22 @@ custom_mutator_stage: afl->queue_cur->stats_mutated += afl->stage_max; #endif - if (likely(afl->custom_only)) { - - /* Skip other stages */ - ret_val = 0; - goto abandon_entry; - - } - /**************** * RANDOM HAVOC * ****************/ havoc_stage: + if (unlikely(afl->custom_only)) { + + /* Force UI update */ + show_stats(afl); + /* Skip other stages */ + ret_val = 0; + goto abandon_entry; + + } + afl->stage_cur_byte = -1; /* The havoc stage mutation code is also invoked when splicing files; if the From f585f262669c14d8b7037d4a34eaa9eb7aef38c5 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 5 May 2023 14:04:53 +0200 Subject: [PATCH 103/149] tritondse fixes --- .../aflpp_tritondse/aflpp_tritondse.py | 20 +++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index 33bf8a9f..49f67d75 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -1,6 +1,7 @@ import sys import os import logging +import hashlib from tritondse import Config from tritondse import CoverageStrategy @@ -92,14 +93,17 @@ def queue_new_entry(filename_new_queue, filename_orig_queue): # Add seed to the worklist. with open(filename_new_queue, "rb") as file: seed = file.read() - seed = Seed(seed) - dse.add_input_seed(seed) - if is_debug: - print("NEW FILE " + filename_new_queue + " count " + str(cycle)) - cycle += 1 - # Start exploration! - #dse.step() - dse.explore() + hash = hashlib.md5(seed).hexdigest() + if hash not in hashes: + hashes.add(hash) + if is_debug: + print("NEW FILE " + filename_new_queue + " hash " + hash + " count " + str(cycle)) + cycle += 1 + seed = Seed(seed) + dse.add_input_seed(seed) + # Start exploration! + #dse.step() + dse.explore() pass def splice_optout(): From 2c421d48fa668d23a20d06bf974fe05be14e5591 Mon Sep 17 00:00:00 2001 From: Sergej Schumilo Date: Fri, 5 May 2023 14:08:01 +0200 Subject: [PATCH 104/149] update libnyx (#1727) --- nyx_mode/LIBNYX_VERSION | 2 +- nyx_mode/libnyx | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nyx_mode/LIBNYX_VERSION b/nyx_mode/LIBNYX_VERSION index 86b32eec..ed88ec10 100644 --- a/nyx_mode/LIBNYX_VERSION +++ b/nyx_mode/LIBNYX_VERSION @@ -1 +1 @@ -2da7f08 +c8a72dc diff --git a/nyx_mode/libnyx b/nyx_mode/libnyx index 2da7f08b..c8a72dc3 160000 --- a/nyx_mode/libnyx +++ b/nyx_mode/libnyx @@ -1 +1 @@ -Subproject commit 2da7f08b6e0267ccfe64e1320b24cdb29223459c +Subproject commit c8a72dc3b302b4037b03738454e1dc8e2bb18796 From 001d9d3d20890941120cc16e9109b4c561da39d1 Mon Sep 17 00:00:00 2001 From: Moshe Kaplan Date: Fri, 5 May 2023 10:02:00 -0400 Subject: [PATCH 105/149] GNUmakefile: Update LLVM instructions (#1728) Update LLVM instructions, because versions higher than 14 are supported and to be explicit that LLD is also required --- GNUmakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GNUmakefile b/GNUmakefile index 56b8bb42..794ebeab 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -316,7 +316,7 @@ all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu @test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler" @test -e afl-llvm-pass.so && echo "[+] LLVM basic mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" - @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be built, it is optional, if you want it, please install LLVM 11-14. More information at instrumentation/README.lto.md on how to build it" + @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be built, it is optional, if you want it, please install LLVM and LLD 11+. More information at instrumentation/README.lto.md on how to build it" ifneq "$(SYS)" "Darwin" @test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this" endif From c092892488a1ed8e5213b9dcdf3d4da617fe0dd2 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sat, 6 May 2023 09:26:24 +0200 Subject: [PATCH 106/149] disable macos in the ci, works fine for me --- .github/workflows/ci.yml | 40 ++++++++++++++++++++-------------------- nyx_mode/libnyx | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03e4151d..fdf618b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,23 +36,23 @@ jobs: run: make distrib ASAN_BUILD=1 NO_NYX=1 - name: run tests run: sudo -E ./afl-system-config; make tests - macos: - runs-on: macOS-latest - env: - AFL_MAP_SIZE: 65536 - AFL_SKIP_CPUFREQ: 1 - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 - steps: - - uses: actions/checkout@v3 - - name: install - run: brew install make gcc llvm - - name: fix install - run: cd /usr/local/bin; ln -s gcc-11 gcc; ln -s g++-11 g++; which gcc; gcc -v - - name: build - run: export PATH=/usr/local/Cellar/llvm/*/":$PATH"; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; sudo -E ./afl-system-config; gmake ASAN_BUILD=1 - - name: frida - run: export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; cd frida_mode; gmake - - name: run tests - run: sudo -E ./afl-system-config; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export PATH=/usr/local/Cellar/llvm/*/":/usr/local/bin:$PATH"; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; gmake tests - - name: force frida test for MacOS - run: export AFL_PATH=`pwd`; /usr/local/bin/gcc -o test-instr test-instr.c; mkdir in; echo > in/in; AFL_NO_UI=1 ./afl-fuzz -O -i in -o out -V 5 -- ./test-instr + # macos: + # runs-on: macOS-latest + # env: + # AFL_MAP_SIZE: 65536 + # AFL_SKIP_CPUFREQ: 1 + # AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 + # steps: + # - uses: actions/checkout@v3 + # - name: install + # run: brew install make gcc llvm + # - name: fix install + # run: cd /usr/local/bin; ln -s gcc-11 gcc; ln -s g++-11 g++; which gcc; gcc -v + # - name: build + # run: export PATH=/usr/local/Cellar/llvm/*/":$PATH"; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; sudo -E ./afl-system-config; gmake ASAN_BUILD=1 + # - name: frida + # run: export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; cd frida_mode; gmake + # - name: run tests + # run: sudo -E ./afl-system-config; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export PATH=/usr/local/Cellar/llvm/*/":/usr/local/bin:$PATH"; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; gmake tests + # - name: force frida test for MacOS + # run: export AFL_PATH=`pwd`; /usr/local/bin/gcc -o test-instr test-instr.c; mkdir in; echo > in/in; AFL_NO_UI=1 ./afl-fuzz -O -i in -o out -V 5 -- ./test-instr diff --git a/nyx_mode/libnyx b/nyx_mode/libnyx index c8a72dc3..2da7f08b 160000 --- a/nyx_mode/libnyx +++ b/nyx_mode/libnyx @@ -1 +1 @@ -Subproject commit c8a72dc3b302b4037b03738454e1dc8e2bb18796 +Subproject commit 2da7f08b6e0267ccfe64e1320b24cdb29223459c From c97caa6e1095a4bce8f0c32108e6e33f7ac240e4 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 9 May 2023 14:17:09 +0200 Subject: [PATCH 107/149] fix makefile --- GNUmakefile | 2 +- src/afl-fuzz.c | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 794ebeab..31374c10 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -379,7 +379,7 @@ help: @echo Known build environment options: @echo "==========================================" @echo STATIC - compile AFL++ static - @echo CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md) + @echo "CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md)" @echo ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes @echo UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for debug purposes @echo DEBUG - no optimization, -ggdb3, all warnings and -Werror diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c5206282..f982258f 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -124,7 +124,8 @@ static void usage(u8 *argv0, int more_help) { "\n%s [ options ] -- /path/to/fuzzed_app [ ... ]\n\n" "Required parameters:\n" - " -i dir - input directory with test cases (or '-' to resume, also see AFL_AUTORESUME)\n" + " -i dir - input directory with test cases (or '-' to resume, " + "also see AFL_AUTORESUME)\n" " -o dir - output directory for fuzzer findings\n\n" "Execution control settings:\n" From 70da0c2e405102dc044cb4bed0f4f1e847c90d0b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 10 May 2023 16:09:18 +0200 Subject: [PATCH 108/149] better tritondse support --- .../aflpp_tritondse/aflpp_tritondse.py | 54 +++++++++-- docs/custom_mutators.md | 28 ++++++ include/envs.h | 4 + src/afl-fuzz.c | 91 ++++++++++++++----- 4 files changed, 145 insertions(+), 32 deletions(-) diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index 49f67d75..9584b368 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -7,6 +7,7 @@ from tritondse import Config from tritondse import CoverageStrategy from tritondse import ProcessState from tritondse import Program +from tritondse import CleLoader from tritondse import Seed from tritondse import SeedFormat from tritondse import SymbolicExecutor @@ -16,7 +17,7 @@ from tritondse import SymbolicExplorator #logging.basicConfig(level=logging.INFO) is_debug = False -out_path = "out/tritondse/queue" +out_path = "" input_file = None prog = None config = None @@ -29,28 +30,38 @@ def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): #logging.info(f"[PRE-EXEC] Processing seed: {se.seed.hash}, \ # ({repr(se.seed.content)})") global count - global hasshes + global hashes + print('DEBUG - prehook') if se.seed.hash not in hashes: hashes.add(se.seed.hash) filename = out_path + "/id:" + f"{count:06}" + "," + se.seed.hash if not os.path.exists(filename): + if is_debug: + print('Creating queue input ' + filename) with open(filename, 'wb') as file: file.write(se.seed.content) count += 1 + else: + print('has hash: ' + se.seed.hash) if input_file: + if is_debug: + print('Writing to ' + input_file + ' the content: ' + str(se.seed.content)) with open(input_file, 'wb') as file: file.write(se.seed.content) + else: + print('no input!') def init(seed): global prog global config global dse + global out_path global input_file global is_debug # Load the program (LIEF-based program loader). - prog = Program(os.environ['TRITON_DSE_TARGET']) - # Set the configuration. + prog = CleLoader(os.environ['AFL_CUSTOM_INFO_PROGRAM']) + # Process other configuration environment variables. argv = None try: foo = os.environ['AFL_DEBUG'] @@ -58,15 +69,42 @@ def init(seed): except KeyError: pass try: - argv_list = os.environ['TRITON_DSE_TARGET_ARGV'] - argv = argv_list.split() + foo = os.environ['AFL_CUSTOM_INFO_OUT'] + out_path = foo + '/../tritondse/queue' except KeyError: pass try: - foo = os.environ['TRITON_DSE_TARGET_INPUT'] + foo = os.environ['AFL_CUSTOM_INFO_PROGRAM_INPUT'] input_file = foo except KeyError: pass + try: + argv_list = os.environ['AFL_CUSTOM_INFO_PROGRAM_ARGV'] + argv_tmp = [ os.environ['AFL_CUSTOM_INFO_PROGRAM'] ] + argv_tmp += argv_list.split() + argv = [] + # now check for @@ + for item in argv_tmp: + if "@@" in item: + input_file = out_path + '/../.input' + argv.append(input_file) + else: + argv.append(item) + except KeyError: + pass + # Create the output directory + os.makedirs(out_path, exist_ok=True) + # Debug + if is_debug: + print('DEBUG target: ' + os.environ['AFL_CUSTOM_INFO_PROGRAM']) + if argv: + print('DEBUG argv: ') + print(argv) + if input_file: + print('DEBUG input_file: ' + input_file) + print('DEBUG out_path: ' + out_path) + print('') + # Now set up TritonDSE config = Config(coverage_strategy = CoverageStrategy.PATH, debug = is_debug, pipe_stdout = is_debug, @@ -79,8 +117,6 @@ def init(seed): dse = SymbolicExplorator(config, prog) # Add callbacks. dse.callback_manager.register_pre_execution_callback(pre_exec_hook) - # Create the output directory - os.makedirs(out_path, exist_ok=True) #def fuzz(buf, add_buf, max_size): diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index a1de479e..3f7e9e6e 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -304,6 +304,34 @@ Note: for some distributions, you might also need the package `python[3]-apt`. In case your setup is different, set the necessary variables like this: `PYTHON_INCLUDE=/path/to/python/include LDFLAGS=-L/path/to/python/lib make`. +### Helpers + +For C/C++ custom mutators you get a pointer to `afl_state_t *afl` in the +`afl_custom_init()` which contains all information that you need. +Note that if you access it, you need to recompile your custom mutator if +you update AFL++ because the structure might have changed! + +For mutators written in Python, Rust, GO, etc. there are a few environment +variables set to help you to get started: + +`AFL_CUSTOM_INFO_PROGRAM` - the program name of the target that is executed. +If your custom mutator is used with modes like Qemu (`-Q`), this will still +contain the target program, not afl-qemu-trace. + +`AFL_CUSTOM_INFO_PROGRAM_INPUT` - if the `-f` parameter is used with afl-fuzz +then this value is found in this environment variable. + +`AFL_CUSTOM_INFO_PROGRAM_ARGV` - this contains the parameters given to the +target program and still has the `@@` identifier in there. + +Note: If `AFL_CUSTOM_INFO_PROGRAM_INPUT` is empty and `AFL_CUSTOM_INFO_PROGRAM_ARGV` +is either empty or does not contain `@@` then the target gets the input via +`stdin`. + +`AFL_CUSTOM_INFO_OUT` - This is the output directory for this fuzzer instance, +so if `afl-fuzz` was called with `-o out -S foobar`, then this will be set to +`out/foobar`. + ### Custom Mutator Preparation For C/C++ mutators, the source code must be compiled as a shared object: diff --git a/include/envs.h b/include/envs.h index fe5ee0e3..edfd06e4 100644 --- a/include/envs.h +++ b/include/envs.h @@ -37,6 +37,10 @@ static char *afl_environment_variables[] = { "AFL_CRASH_EXITCODE", "AFL_CUSTOM_MUTATOR_LIBRARY", "AFL_CUSTOM_MUTATOR_ONLY", + "AFL_CUSTOM_INFO_PROGRAM", + "AFL_CUSTOM_INFO_PROGRAM_ARGV", + "AFL_CUSTOM_INFO_PROGRAM_INPUT", + "AFL_CUSTOM_INFO_OUT", "AFL_CXX", "AFL_CYCLE_SCHEDULES", "AFL_DEBUG", diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index f982258f..4339ddd2 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1530,29 +1530,6 @@ int main(int argc, char **argv_orig, char **envp) { } - if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { - - if (afl->custom_only) { - - FATAL("Custom mutators are incompatible with MOpt (-L)"); - - } - - u32 custom_fuzz = 0; - LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { - - if (el->afl_custom_fuzz) { custom_fuzz = 1; } - - }); - - if (custom_fuzz) { - - WARNF("afl_custom_fuzz is incompatible with MOpt (-L)"); - - } - - } - if (afl->afl_env.afl_max_det_extras) { s32 max_det_extras = atoi(afl->afl_env.afl_max_det_extras); @@ -1827,8 +1804,76 @@ int main(int argc, char **argv_orig, char **envp) { printf("DEBUG: rand %06d is %u\n", counter, rand_below(afl, 65536)); #endif + if (!getenv("AFL_CUSTOM_INFO_PROGRAM")) { + + setenv("AFL_CUSTOM_INFO_PROGRAM", argv[optind], 1); + + } + + if (!getenv("AFL_CUSTOM_INFO_PROGRAM_INPUT") && afl->fsrv.out_file) { + + setenv("AFL_CUSTOM_INFO_PROGRAM_INPUT", afl->fsrv.out_file, 1); + + } + + { + + u8 envbuf[8096] = "", tmpbuf[8096] = ""; + for (s32 i = optind + 1; i < argc; ++i) { + + strcpy(tmpbuf, envbuf); + if (strchr(argv[i], ' ') && !strchr(argv[i], '"') && + !strchr(argv[i], '\'')) { + + if (!strchr(argv[i], '\'')) { + + snprintf(envbuf, sizeof(tmpbuf), "%s '%s'", tmpbuf, argv[i]); + + } else { + + snprintf(envbuf, sizeof(tmpbuf), "%s \"%s\"", tmpbuf, argv[i]); + + } + + } else { + + snprintf(envbuf, sizeof(tmpbuf), "%s %s", tmpbuf, argv[i]); + + } + + } + + setenv("AFL_CUSTOM_INFO_PROGRAM_ARGV", envbuf + 1, 1); + + } + + setenv("AFL_CUSTOM_INFO_OUT", afl->out_dir, 1); // same as __AFL_OUT_DIR + setup_custom_mutators(afl); + if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { + + if (afl->custom_only) { + + FATAL("Custom mutators are incompatible with MOpt (-L)"); + + } + + u32 custom_fuzz = 0; + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_fuzz) { custom_fuzz = 1; } + + }); + + if (custom_fuzz) { + + WARNF("afl_custom_fuzz is incompatible with MOpt (-L)"); + + } + + } + write_setup_file(afl, argc, argv); setup_cmdline_file(afl, argv + optind); From eaf59d5a194f5e5469a86158aeb0e936111ad790 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 11 May 2023 07:55:17 +0200 Subject: [PATCH 109/149] next steps for tritondse --- .../aflpp_tritondse/aflpp_tritondse.py | 50 ++++++++++--------- 1 file changed, 26 insertions(+), 24 deletions(-) diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index 9584b368..e0219f0b 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -3,19 +3,17 @@ import os import logging import hashlib +from tritondse import CleLoader +from tritondse import CompositeData from tritondse import Config from tritondse import CoverageStrategy from tritondse import ProcessState from tritondse import Program -from tritondse import CleLoader from tritondse import Seed from tritondse import SeedFormat from tritondse import SymbolicExecutor from tritondse import SymbolicExplorator - -#logging.basicConfig(level=logging.INFO) - is_debug = False out_path = "" input_file = None @@ -25,13 +23,11 @@ dse = None cycle = 0 count = 0 hashes = set() +format = SeedFormat.RAW def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): - #logging.info(f"[PRE-EXEC] Processing seed: {se.seed.hash}, \ - # ({repr(se.seed.content)})") global count global hashes - print('DEBUG - prehook') if se.seed.hash not in hashes: hashes.add(se.seed.hash) filename = out_path + "/id:" + f"{count:06}" + "," + se.seed.hash @@ -39,26 +35,26 @@ def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): if is_debug: print('Creating queue input ' + filename) with open(filename, 'wb') as file: - file.write(se.seed.content) + if input_file: + file.write(se.seed.content.files[input_file]) + else: + file.write(se.seed.content) count += 1 - else: - print('has hash: ' + se.seed.hash) - if input_file: - if is_debug: - print('Writing to ' + input_file + ' the content: ' + str(se.seed.content)) - with open(input_file, 'wb') as file: - file.write(se.seed.content) - else: - print('no input!') + #if input_file: + # if is_debug: + # print('Writing to ' + input_file + ' the content: ' + str(se.seed.content)) + # with open(input_file, 'wb') as file: + # file.write(se.seed.content) def init(seed): - global prog global config global dse - global out_path + global format global input_file global is_debug + global out_path + global prog # Load the program (LIEF-based program loader). prog = CleLoader(os.environ['AFL_CUSTOM_INFO_PROGRAM']) # Process other configuration environment variables. @@ -104,6 +100,8 @@ def init(seed): print('DEBUG input_file: ' + input_file) print('DEBUG out_path: ' + out_path) print('') + if input_file: + format = SeedFormat.COMPOSITE # Now set up TritonDSE config = Config(coverage_strategy = CoverageStrategy.PATH, debug = is_debug, @@ -112,7 +110,7 @@ def init(seed): execution_timeout = 1, program_argv = argv, smt_timeout= 50, - seed_format = SeedFormat.RAW) + seed_format = format) # Create an instance of the Symbolic Explorator dse = SymbolicExplorator(config, prog) # Add callbacks. @@ -124,18 +122,22 @@ def init(seed): def queue_new_entry(filename_new_queue, filename_orig_queue): - global dse global cycle + global dse # Add seed to the worklist. with open(filename_new_queue, "rb") as file: - seed = file.read() - hash = hashlib.md5(seed).hexdigest() + data = file.read() + hash = hashlib.md5(data).hexdigest() if hash not in hashes: hashes.add(hash) if is_debug: print("NEW FILE " + filename_new_queue + " hash " + hash + " count " + str(cycle)) cycle += 1 - seed = Seed(seed) + if input_file: + seed = Seed(CompositeData(files={"stdin": b"", # nothing on stdin + input_file: data})) + else: + seed = Seed(data) dse.add_input_seed(seed) # Start exploration! #dse.step() From 3a98d7af18e6ebf12e7cce2eb78bdb9b9927be3e Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Thu, 11 May 2023 21:02:46 +0200 Subject: [PATCH 110/149] qemuafl: Persistent mode for PPC32 targets --- docs/Changelog.md | 2 ++ qemu_mode/QEMUAFL_VERSION | 2 +- qemu_mode/qemuafl | 2 +- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index cd5ed9fc..1fe714e8 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -18,6 +18,8 @@ - add `-I filelist` option, an alternative to `-i in_dir` - afl-cmin + afl-cmin.bash: - `-T threads` parallel task support, can be a huge speedup! + - qemuafl: + - Persistent mode support for ppc32 tragets by @worksbutnottested - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index fa44d173..043a9f82 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -0569eff8a1 +b0abbe2 diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index 0569eff8..b0abbe2e 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit 0569eff8a12dec73642b96757f6b5b51a618a03a +Subproject commit b0abbe2e74ed74ff6ff25b5ea3110d27ba978001 From a752b159212db458d77cd13c46fdfbde01045d91 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 12 May 2023 08:29:31 +0200 Subject: [PATCH 111/149] update qemu_mode --- docs/Changelog.md | 4 ++-- qemu_mode/QEMUAFL_VERSION | 2 +- qemu_mode/qemuafl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 1fe714e8..e85de763 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -18,8 +18,8 @@ - add `-I filelist` option, an alternative to `-i in_dir` - afl-cmin + afl-cmin.bash: - `-T threads` parallel task support, can be a huge speedup! - - qemuafl: - - Persistent mode support for ppc32 tragets by @worksbutnottested + - qemu_mode: + - Persistent mode +QASAN support for ppc32 tragets by @worksbutnottested - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index 043a9f82..44ea5345 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -b0abbe2 +a1321713c7 diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index b0abbe2e..a1321713 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit b0abbe2e74ed74ff6ff25b5ea3110d27ba978001 +Subproject commit a1321713c7502c152dd7527555e0f8a800d55225 From 93c821aaa3df0cf20f892ce72447ff022161c8ab Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 12 May 2023 08:39:11 +0200 Subject: [PATCH 112/149] afl-clang-lto incomptable with -flto=thin --- docs/Changelog.md | 1 + src/afl-cc.c | 9 +++++++++ 2 files changed, 10 insertions(+) diff --git a/docs/Changelog.md b/docs/Changelog.md index e85de763..799c13af 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -13,6 +13,7 @@ - afl-cc: - new env `AFL_LLVM_LTO_SKIPINIT` to support the AFL++ based WASM (https://github.com/fgsect/WAFL) project + - error and print help if afl-clan-lto is used with lto=thin - afl-showmap: - added custom mutator post_process and send support - add `-I filelist` option, an alternative to `-i in_dir` diff --git a/src/afl-cc.c b/src/afl-cc.c index 19314555..13ca751e 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -853,6 +853,15 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (cur[0] != '-') { non_dash = 1; } if (!strncmp(cur, "--afl", 5)) continue; + + if (lto_mode && !strncmp(cur, "-flto=thin", 10)) { + + FATAL( + "afl-clang-lto cannot work with -flto=thin. Switch to -flto=full or " + "use afl-clang-fast!"); + + } + if (lto_mode && !strncmp(cur, "-fuse-ld=", 9)) continue; if (lto_mode && !strncmp(cur, "--ld-path=", 10)) continue; if (!strncmp(cur, "-fno-unroll", 11)) continue; From 7f636dbfc247fbe75910fa8fb681ea55d230ba79 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 12 May 2023 15:58:20 +0200 Subject: [PATCH 113/149] add @responsefile support for afl-cc --- docs/Changelog.md | 1 + src/afl-cc.c | 460 ++++++++++++++++++++++++++++++---------------- 2 files changed, 300 insertions(+), 161 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 799c13af..3602af50 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -11,6 +11,7 @@ - new env `AFL_IGNORE_PROBLEMS_COVERAGE` to ignore coverage from loaded libs after forkserver initialization (required by Mozilla) - afl-cc: + - added @responsefile support - new env `AFL_LLVM_LTO_SKIPINIT` to support the AFL++ based WASM (https://github.com/fgsect/WAFL) project - error and print help if afl-clan-lto is used with lto=thin diff --git a/src/afl-cc.c b/src/afl-cc.c index 13ca751e..972ac8cd 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -31,6 +31,8 @@ #include #include #include +#include +#include #if (LLVM_MAJOR - 0 == 0) #undef LLVM_MAJOR @@ -376,15 +378,304 @@ void parse_fsanitize(char *string) { } +static u8 fortify_set = 0, asan_set = 0, x_set = 0, bit_mode = 0, + shared_linking = 0, preprocessor_only = 0, have_unroll = 0, + have_o = 0, have_pic = 0, have_c = 0, partial_linking = 0, + non_dash = 0; + +static void process_params(u32 argc, char **argv) { + + if (cc_par_cnt + argc >= 1024) { FATAL("Too many command line parameters"); } + + if (lto_mode && argc > 1) { + + u32 idx; + for (idx = 1; idx < argc; idx++) { + + if (!strncasecmp(argv[idx], "-fpic", 5)) have_pic = 1; + + } + + } + + // for (u32 x = 0; x < argc; ++x) fprintf(stderr, "[%u] %s\n", x, argv[x]); + + /* Process the argument list. */ + + u8 skip_next = 0; + while (--argc) { + + u8 *cur = *(++argv); + + if (skip_next) { + + skip_next = 0; + continue; + + } + + if (cur[0] != '-') { non_dash = 1; } + if (!strncmp(cur, "--afl", 5)) continue; + + if (lto_mode && !strncmp(cur, "-flto=thin", 10)) { + + FATAL( + "afl-clang-lto cannot work with -flto=thin. Switch to -flto=full or " + "use afl-clang-fast!"); + + } + + if (lto_mode && !strncmp(cur, "-fuse-ld=", 9)) continue; + if (lto_mode && !strncmp(cur, "--ld-path=", 10)) continue; + if (!strncmp(cur, "-fno-unroll", 11)) continue; + if (strstr(cur, "afl-compiler-rt") || strstr(cur, "afl-llvm-rt")) continue; + if (!strcmp(cur, "-Wl,-z,defs") || !strcmp(cur, "-Wl,--no-undefined") || + !strcmp(cur, "--no-undefined")) { + + continue; + + } + + if (compiler_mode == GCC_PLUGIN && !strcmp(cur, "-pipe")) { continue; } + + if (!strcmp(cur, "-z") || !strcmp(cur, "-Wl,-z")) { + + u8 *param = *(argv + 1); + if (!strcmp(param, "defs") || !strcmp(param, "-Wl,defs")) { + + skip_next = 1; + continue; + + } + + } + + if ((compiler_mode == GCC || compiler_mode == GCC_PLUGIN) && + !strncmp(cur, "-stdlib=", 8)) { + + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } + continue; + + } + + if (!strncmp(cur, "-fsanitize-coverage-", 20) && strstr(cur, "list=")) { + + have_instr_list = 1; + + } + + if (!strncmp(cur, "-fsanitize=", strlen("-fsanitize=")) && + strchr(cur, ',')) { + + parse_fsanitize(cur); + if (!cur || strlen(cur) <= strlen("-fsanitize=")) { continue; } + + } else if ((!strncmp(cur, "-fsanitize=fuzzer-", + + strlen("-fsanitize=fuzzer-")) || + !strncmp(cur, "-fsanitize-coverage", + strlen("-fsanitize-coverage"))) && + (strncmp(cur, "sanitize-coverage-allow", + strlen("sanitize-coverage-allow")) && + strncmp(cur, "sanitize-coverage-deny", + strlen("sanitize-coverage-deny")) && + instrument_mode != INSTRUMENT_LLVMNATIVE)) { + + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } + continue; + + } + + if (need_aflpplib || !strcmp(cur, "-fsanitize=fuzzer")) { + + u8 *afllib = find_object("libAFLDriver.a", argv[0]); + + if (!be_quiet) { + + OKF("Found '-fsanitize=fuzzer', replacing with libAFLDriver.a"); + + } + + if (!afllib) { + + if (!be_quiet) { + + WARNF( + "Cannot find 'libAFLDriver.a' to replace '-fsanitize=fuzzer' in " + "the flags - this will fail!"); + + } + + } else { + + cc_params[cc_par_cnt++] = afllib; + +#ifdef __APPLE__ + cc_params[cc_par_cnt++] = "-undefined"; + cc_params[cc_par_cnt++] = "dynamic_lookup"; +#endif + + } + + if (need_aflpplib) { + + need_aflpplib = 0; + + } else { + + continue; + + } + + } + + if (!strcmp(cur, "-m32")) bit_mode = 32; + if (!strcmp(cur, "armv7a-linux-androideabi")) bit_mode = 32; + if (!strcmp(cur, "-m64")) bit_mode = 64; + + if (!strcmp(cur, "-fsanitize=address") || !strcmp(cur, "-fsanitize=memory")) + asan_set = 1; + + if (strstr(cur, "FORTIFY_SOURCE")) fortify_set = 1; + + if (!strcmp(cur, "-x")) x_set = 1; + if (!strcmp(cur, "-E")) preprocessor_only = 1; + if (!strcmp(cur, "-shared")) shared_linking = 1; + if (!strcmp(cur, "-dynamiclib")) shared_linking = 1; + if (!strcmp(cur, "--target=wasm32-wasi")) passthrough = 1; + if (!strcmp(cur, "-Wl,-r")) partial_linking = 1; + if (!strcmp(cur, "-Wl,-i")) partial_linking = 1; + if (!strcmp(cur, "-Wl,--relocatable")) partial_linking = 1; + if (!strcmp(cur, "-r")) partial_linking = 1; + if (!strcmp(cur, "--relocatable")) partial_linking = 1; + if (!strcmp(cur, "-c")) have_c = 1; + + if (!strncmp(cur, "-O", 2)) have_o = 1; + if (!strncmp(cur, "-funroll-loop", 13)) have_unroll = 1; + + if (*cur == '@') { + + // response file support. + // we have two choices - move everything to the command line or + // rewrite the response files to temporary files and delete them + // afterwards. We choose the first for easiness. + // We do *not* support quotes in the rsp files to cope with spaces in + // filenames etc! If you need that then send a patch! + u8 *filename = cur + 1; + if (debug) { DEBUGF("response file=%s\n", filename); } + FILE *f = fopen(filename, "r"); + struct stat st; + + // Check not found or empty? let the compiler complain if so. + if (!f || fstat(fileno(f), &st) < 0 || st.st_size < 1) { + + cc_params[cc_par_cnt++] = cur; + continue; + + } + + u8 *tmpbuf = malloc(st.st_size + 1), *ptr; + char **args = malloc(sizeof(char *) * (st.st_size >> 1)); + int count = 1, cont = 0, cont_act = 0; + + while (fgets(tmpbuf, st.st_size, f)) { + + ptr = tmpbuf; + // no leading whitespace + while (isspace(*ptr)) { + + ++ptr; + cont_act = 0; + + } + + // no comments, no empty lines + if (*ptr == '#' || *ptr == '\n' || !*ptr) { continue; } + // remove LF + if (ptr[strlen(ptr) - 1] == '\n') { ptr[strlen(ptr) - 1] = 0; } + // remove CR + if (*ptr && ptr[strlen(ptr) - 1] == '\r') { ptr[strlen(ptr) - 1] = 0; } + // handle \ at end of line + if (*ptr && ptr[strlen(ptr) - 1] == '\\') { + + cont = 1; + ptr[strlen(ptr) - 1] = 0; + + } + + // remove whitespace at end + while (*ptr && isspace(ptr[strlen(ptr) - 1])) { + + ptr[strlen(ptr) - 1] = 0; + cont = 0; + + } + + if (*ptr) { + + do { + + u8 *value = ptr; + while (*ptr && !isspace(*ptr)) { + + ++ptr; + + } + + while (*ptr && isspace(*ptr)) { + + *ptr++ = 0; + + } + + if (cont_act) { + + u32 len = strlen(args[count - 1]) + strlen(value) + 1; + u8 *tmp = malloc(len); + snprintf(tmp, len, "%s%s", args[count - 1], value); + free(args[count - 1]); + args[count - 1] = tmp; + cont_act = 0; + + } else { + + args[count++] = strdup(value); + + } + + } while (*ptr); + + } + + if (cont) { + + cont_act = 1; + cont = 0; + + } + + } + + if (count) { process_params(count, args); } + + // we cannot free args[] + free(tmpbuf); + + continue; + + } + + cc_params[cc_par_cnt++] = cur; + + } + +} + /* Copy argv to cc_params, making the necessary edits. */ static void edit_params(u32 argc, char **argv, char **envp) { - u8 fortify_set = 0, asan_set = 0, x_set = 0, bit_mode = 0, shared_linking = 0, - preprocessor_only = 0, have_unroll = 0, have_o = 0, have_pic = 0, - have_c = 0, partial_linking = 0; - - cc_params = ck_alloc((argc + 128) * sizeof(u8 *)); + cc_params = ck_alloc(1024 * sizeof(u8 *)); if (lto_mode) { @@ -831,168 +1122,15 @@ static void edit_params(u32 argc, char **argv, char **envp) { } - if (!have_pic) cc_params[cc_par_cnt++] = "-fPIC"; - } } - /* Detect stray -v calls from ./configure scripts. */ + /* Inspect the command line parameters. */ - u8 skip_next = 0, non_dash = 0; - while (--argc) { + process_params(argc, argv); - u8 *cur = *(++argv); - - if (skip_next) { - - skip_next = 0; - continue; - - } - - if (cur[0] != '-') { non_dash = 1; } - if (!strncmp(cur, "--afl", 5)) continue; - - if (lto_mode && !strncmp(cur, "-flto=thin", 10)) { - - FATAL( - "afl-clang-lto cannot work with -flto=thin. Switch to -flto=full or " - "use afl-clang-fast!"); - - } - - if (lto_mode && !strncmp(cur, "-fuse-ld=", 9)) continue; - if (lto_mode && !strncmp(cur, "--ld-path=", 10)) continue; - if (!strncmp(cur, "-fno-unroll", 11)) continue; - if (strstr(cur, "afl-compiler-rt") || strstr(cur, "afl-llvm-rt")) continue; - if (!strcmp(cur, "-Wl,-z,defs") || !strcmp(cur, "-Wl,--no-undefined") || - !strcmp(cur, "--no-undefined")) { - - continue; - - } - - if (compiler_mode == GCC_PLUGIN && !strcmp(cur, "-pipe")) { continue; } - - if (!strcmp(cur, "-z") || !strcmp(cur, "-Wl,-z")) { - - u8 *param = *(argv + 1); - if (!strcmp(param, "defs") || !strcmp(param, "-Wl,defs")) { - - skip_next = 1; - continue; - - } - - } - - if ((compiler_mode == GCC || compiler_mode == GCC_PLUGIN) && - !strncmp(cur, "-stdlib=", 8)) { - - if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } - continue; - - } - - if (!strncmp(cur, "-fsanitize-coverage-", 20) && strstr(cur, "list=")) { - - have_instr_list = 1; - - } - - if (!strncmp(cur, "-fsanitize=", strlen("-fsanitize=")) && - strchr(cur, ',')) { - - parse_fsanitize(cur); - if (!cur || strlen(cur) <= strlen("-fsanitize=")) { continue; } - - } else if ((!strncmp(cur, "-fsanitize=fuzzer-", - - strlen("-fsanitize=fuzzer-")) || - !strncmp(cur, "-fsanitize-coverage", - strlen("-fsanitize-coverage"))) && - (strncmp(cur, "sanitize-coverage-allow", - strlen("sanitize-coverage-allow")) && - strncmp(cur, "sanitize-coverage-deny", - strlen("sanitize-coverage-deny")) && - instrument_mode != INSTRUMENT_LLVMNATIVE)) { - - if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } - continue; - - } - - if (need_aflpplib || !strcmp(cur, "-fsanitize=fuzzer")) { - - u8 *afllib = find_object("libAFLDriver.a", argv[0]); - - if (!be_quiet) { - - OKF("Found '-fsanitize=fuzzer', replacing with libAFLDriver.a"); - - } - - if (!afllib) { - - if (!be_quiet) { - - WARNF( - "Cannot find 'libAFLDriver.a' to replace '-fsanitize=fuzzer' in " - "the flags - this will fail!"); - - } - - } else { - - cc_params[cc_par_cnt++] = afllib; - -#ifdef __APPLE__ - cc_params[cc_par_cnt++] = "-undefined"; - cc_params[cc_par_cnt++] = "dynamic_lookup"; -#endif - - } - - if (need_aflpplib) { - - need_aflpplib = 0; - - } else { - - continue; - - } - - } - - if (!strcmp(cur, "-m32")) bit_mode = 32; - if (!strcmp(cur, "armv7a-linux-androideabi")) bit_mode = 32; - if (!strcmp(cur, "-m64")) bit_mode = 64; - - if (!strcmp(cur, "-fsanitize=address") || !strcmp(cur, "-fsanitize=memory")) - asan_set = 1; - - if (strstr(cur, "FORTIFY_SOURCE")) fortify_set = 1; - - if (!strcmp(cur, "-x")) x_set = 1; - if (!strcmp(cur, "-E")) preprocessor_only = 1; - if (!strcmp(cur, "-shared")) shared_linking = 1; - if (!strcmp(cur, "-dynamiclib")) shared_linking = 1; - if (!strcmp(cur, "--target=wasm32-wasi")) passthrough = 1; - if (!strcmp(cur, "-Wl,-r")) partial_linking = 1; - if (!strcmp(cur, "-Wl,-i")) partial_linking = 1; - if (!strcmp(cur, "-Wl,--relocatable")) partial_linking = 1; - if (!strcmp(cur, "-r")) partial_linking = 1; - if (!strcmp(cur, "--relocatable")) partial_linking = 1; - if (!strcmp(cur, "-c")) have_c = 1; - - if (!strncmp(cur, "-O", 2)) have_o = 1; - if (!strncmp(cur, "-funroll-loop", 13)) have_unroll = 1; - - cc_params[cc_par_cnt++] = cur; - - } + if (!have_pic) { cc_params[cc_par_cnt++] = "-fPIC"; } // in case LLVM is installed not via a package manager or "make install" // e.g. compiled download or compiled from github then its ./lib directory From c4b1566ba35c697cda7822bd0cf30e2e3eeee0c7 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 15 May 2023 11:51:37 +0300 Subject: [PATCH 114/149] push to stable (#1734) * afl++ -> AFL++ * update readme * more debug * slightly different weighting algo (#1719) * better seed selection * slightly different weighting calculation * remove unnecessary memset * Add "Hangs saved" to afl-whatsup (#1717) The hangs could show long or infinite loops. This is important. Co-authored-by: van Hauser * nits * afl-showmap: Start a only a single fork server (#1718) A forkserver is started by afl_fsrv_get_mapsize() when dynamically finding the map size. When an input directory option is specified a second fork server was also started. This commit re-arranges the inits for several forkserver struct members so that we can re-use the server started by the get_mapsize() call when not in coresight/qemu/unicorn modes and just start the server otherwise. * Source Code Coverage support for Nyx (Part 1) (#1720) * Additional source code reformatting in afl-compiler-rt * Add source code coverage support to afl-compiler-rt (for use with Nyx) * doc, code format * llvm 17 changes * more llvm 17 * add frida mode tutorial * fix effector map * docs * Should memset EFF_ALEN(len) of eff_map (#1722) * fix reallocs * fix afl-system-config for macos * afl-fuzz.c: Document -i - in --help (#1725) afl-fuzz.c: Document `-i -` in `--help`, to write that `-i` can be passed '-' to resume the prior fuzzing job. Also reference AFL_AUTORESUME so users know they can set that parameter to sidestep the issue entirely. * tritondse custom mutator attempt * tritondse fixes * update libnyx (#1727) * GNUmakefile: Update LLVM instructions (#1728) Update LLVM instructions, because versions higher than 14 are supported and to be explicit that LLD is also required * disable macos in the ci, works fine for me * fix makefile * better tritondse support * next steps for tritondse * qemuafl: Persistent mode for PPC32 targets * update qemu_mode * afl-clang-lto incomptable with -flto=thin * add @responsefile support for afl-cc --------- Co-authored-by: fxlb Co-authored-by: Nick Potenski Co-authored-by: Christian Holler (:decoder) Co-authored-by: lazymio Co-authored-by: Moshe Kaplan Co-authored-by: Sergej Schumilo Co-authored-by: Dominik Maier --- .github/workflows/ci.yml | 40 +- Dockerfile | 4 +- GNUmakefile | 17 +- GNUmakefile.gcc_plugin | 6 +- GNUmakefile.llvm | 11 +- README.md | 2 +- afl-cmin | 2 +- afl-system-config | 3 +- afl-whatsup | 3 + custom_mutators/aflpp_tritondse/README.md | 17 + .../aflpp_tritondse/aflpp_tritondse.py | 148 ++++++ docs/Changelog.md | 6 +- docs/FAQ.md | 8 + docs/INSTALL.md | 12 +- docs/best_practices.md | 5 + docs/custom_mutators.md | 28 + docs/tutorials.md | 4 + frida_mode/README.md | 2 + include/afl-fuzz.h | 4 +- include/alloc-inl.h | 9 +- include/envs.h | 4 + instrumentation/README.llvm.md | 24 + instrumentation/SanitizerCoverageLTO.so.cc | 32 +- .../SanitizerCoveragePCGUARD.so.cc | 2 + instrumentation/afl-compiler-rt.o.c | 249 +++++++++ instrumentation/afl-llvm-common.cc | 2 +- instrumentation/afl-llvm-dict2file.so.cc | 2 +- nyx_mode/LIBNYX_VERSION | 2 +- qemu_mode/QEMUAFL_VERSION | 2 +- qemu_mode/build_qemu_support.sh | 4 +- qemu_mode/qemuafl | 2 +- src/afl-cc.c | 498 ++++++++++++------ src/afl-forkserver.c | 6 +- src/afl-fuzz-init.c | 32 +- src/afl-fuzz-one.c | 20 +- src/afl-fuzz-queue.c | 92 +++- src/afl-fuzz.c | 143 ++++- src/afl-ld-lto.c | 4 +- src/afl-showmap.c | 31 +- test/test-dlopen.c | 2 +- test/test-gcc-plugin.sh | 2 +- test/test-llvm.sh | 2 +- test/test-performance.sh | 4 +- test/test-pre.sh | 2 +- unicorn_mode/build_unicorn_support.sh | 2 +- 45 files changed, 1165 insertions(+), 331 deletions(-) create mode 100644 custom_mutators/aflpp_tritondse/README.md create mode 100644 custom_mutators/aflpp_tritondse/aflpp_tritondse.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 03e4151d..fdf618b9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,23 +36,23 @@ jobs: run: make distrib ASAN_BUILD=1 NO_NYX=1 - name: run tests run: sudo -E ./afl-system-config; make tests - macos: - runs-on: macOS-latest - env: - AFL_MAP_SIZE: 65536 - AFL_SKIP_CPUFREQ: 1 - AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 - steps: - - uses: actions/checkout@v3 - - name: install - run: brew install make gcc llvm - - name: fix install - run: cd /usr/local/bin; ln -s gcc-11 gcc; ln -s g++-11 g++; which gcc; gcc -v - - name: build - run: export PATH=/usr/local/Cellar/llvm/*/":$PATH"; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; sudo -E ./afl-system-config; gmake ASAN_BUILD=1 - - name: frida - run: export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; cd frida_mode; gmake - - name: run tests - run: sudo -E ./afl-system-config; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export PATH=/usr/local/Cellar/llvm/*/":/usr/local/bin:$PATH"; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; gmake tests - - name: force frida test for MacOS - run: export AFL_PATH=`pwd`; /usr/local/bin/gcc -o test-instr test-instr.c; mkdir in; echo > in/in; AFL_NO_UI=1 ./afl-fuzz -O -i in -o out -V 5 -- ./test-instr + # macos: + # runs-on: macOS-latest + # env: + # AFL_MAP_SIZE: 65536 + # AFL_SKIP_CPUFREQ: 1 + # AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1 + # steps: + # - uses: actions/checkout@v3 + # - name: install + # run: brew install make gcc llvm + # - name: fix install + # run: cd /usr/local/bin; ln -s gcc-11 gcc; ln -s g++-11 g++; which gcc; gcc -v + # - name: build + # run: export PATH=/usr/local/Cellar/llvm/*/":$PATH"; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; sudo -E ./afl-system-config; gmake ASAN_BUILD=1 + # - name: frida + # run: export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; cd frida_mode; gmake + # - name: run tests + # run: sudo -E ./afl-system-config; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export PATH=/usr/local/Cellar/llvm/*/":/usr/local/bin:$PATH"; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; gmake tests + # - name: force frida test for MacOS + # run: export AFL_PATH=`pwd`; /usr/local/bin/gcc -o test-instr test-instr.c; mkdir in; echo > in/in; AFL_NO_UI=1 ./afl-fuzz -O -i in -o out -V 5 -- ./test-instr diff --git a/Dockerfile b/Dockerfile index 4e53de40..1b5ffd28 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ # FROM ubuntu:22.04 AS aflplusplus -LABEL "maintainer"="afl++ team " +LABEL "maintainer"="AFL++ team " LABEL "about"="AFLplusplus container image" ### Comment out to enable these features @@ -94,4 +94,4 @@ RUN sed -i.bak 's/^ -/ /g' GNUmakefile && \ RUN echo "set encoding=utf-8" > /root/.vimrc && \ echo ". /etc/bash_completion" >> ~/.bashrc && \ echo 'alias joe="joe --wordwrap --joe_state -nobackup"' >> ~/.bashrc && \ - echo "export PS1='"'[afl++ \h] \w \$ '"'" >> ~/.bashrc + echo "export PS1='"'[AFL++ \h] \w \$ '"'" >> ~/.bashrc diff --git a/GNUmakefile b/GNUmakefile index 23cae65d..31374c10 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -39,7 +39,7 @@ ASAN_OPTIONS=detect_leaks=0 SYS = $(shell uname -s) ARCH = $(shell uname -m) -$(info [*] Compiling afl++ for OS $(SYS) on ARCH $(ARCH)) +$(info [*] Compiling AFL++ for OS $(SYS) on ARCH $(ARCH)) ifdef NO_SPLICING override CFLAGS_OPT += -DNO_SPLICING @@ -316,7 +316,7 @@ all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu @test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler" @test -e afl-llvm-pass.so && echo "[+] LLVM basic mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be built, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" - @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be built, it is optional, if you want it, please install LLVM 11-14. More information at instrumentation/README.lto.md on how to build it" + @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be built, it is optional, if you want it, please install LLVM and LLD 11+. More information at instrumentation/README.lto.md on how to build it" ifneq "$(SYS)" "Darwin" @test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this" endif @@ -359,7 +359,7 @@ performance-test: source-only help: @echo "HELP --- the following make targets exist:" @echo "==========================================" - @echo "all: the main afl++ binaries and llvm/gcc instrumentation" + @echo "all: the main AFL++ binaries and llvm/gcc instrumentation" @echo "binary-only: everything for binary-only fuzzing: frida_mode, nyx_mode, qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator, libtokencap" @echo "source-only: everything for source code fuzzing: nyx_mode, libdislocator, libtokencap" @echo "distrib: everything (for both binary-only and source code fuzzing)" @@ -367,7 +367,7 @@ help: @echo "install: installs everything you have compiled with the build option above" @echo "clean: cleans everything compiled (not downloads when on a checkout)" @echo "deepclean: cleans everything including downloads" - @echo "uninstall: uninstall afl++ from the system" + @echo "uninstall: uninstall AFL++ from the system" @echo "code-format: format the code, do this before you commit and send a PR please!" @echo "tests: this runs the test framework. It is more catered for the developers, but if you run into problems this helps pinpointing the problem" @echo "unit: perform unit tests (based on cmocka and GNU linker)" @@ -379,6 +379,7 @@ help: @echo Known build environment options: @echo "==========================================" @echo STATIC - compile AFL++ static + @echo "CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md)" @echo ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes @echo UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for debug purposes @echo DEBUG - no optimization, -ggdb3, all warnings and -Werror @@ -394,7 +395,7 @@ help: @echo AFL_NO_X86 - if compiling on non-intel/amd platforms @echo "LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian)" @echo "==========================================" - @echo e.g.: make ASAN_BUILD=1 + @echo e.g.: make LLVM_CONFIG=llvm-config-16 .PHONY: test_x86 ifndef AFL_NO_X86 @@ -749,7 +750,7 @@ endif @echo %.8: % - @echo .TH $* 8 $(BUILD_DATE) "afl++" > $@ + @echo .TH $* 8 $(BUILD_DATE) "AFL++" > $@ @echo .SH NAME >> $@ @echo .B $* >> $@ @echo >> $@ @@ -761,8 +762,8 @@ endif @./$* -hh 2>&1 | tail -n +4 >> $@ @echo >> $@ @echo .SH AUTHOR >> $@ - @echo "afl++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Heiko \"hexcoder-\" Eissfeldt , Andrea Fioraldi and Dominik Maier " >> $@ - @echo The homepage of afl++ is: https://github.com/AFLplusplus/AFLplusplus >> $@ + @echo "AFL++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Dominik Maier , Andrea Fioraldi and Heiko \"hexcoder-\" Eissfeldt " >> $@ + @echo The homepage of AFL++ is: https://github.com/AFLplusplus/AFLplusplus >> $@ @echo >> $@ @echo .SH LICENSE >> $@ @echo Apache License Version 2.0, January 2004 >> $@ diff --git a/GNUmakefile.gcc_plugin b/GNUmakefile.gcc_plugin index 4c4e10c4..41face4c 100644 --- a/GNUmakefile.gcc_plugin +++ b/GNUmakefile.gcc_plugin @@ -175,7 +175,7 @@ all_done: test_build .NOTPARALLEL: clean %.8: % - @echo .TH $* 8 `date "+%Y-%m-%d"` "afl++" > ./$@ + @echo .TH $* 8 `date "+%Y-%m-%d"` "AFL++" > ./$@ @echo .SH NAME >> ./$@ @echo .B $* >> ./$@ @echo >> ./$@ @@ -187,8 +187,8 @@ all_done: test_build @./$* -h 2>&1 | tail -n +4 >> ./$@ @echo >> ./$@ @echo .SH AUTHOR >> ./$@ - @echo "afl++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Heiko \"hexcoder-\" Eissfeldt , Andrea Fioraldi and Dominik Maier " >> ./$@ - @echo The homepage of afl++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ + @echo "AFL++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Dominik Maier , Andrea Fioraldi and Heiko \"hexcoder-\" Eissfeldt " >> ./$@ + @echo The homepage of AFL++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ @echo >> ./$@ @echo .SH LICENSE >> ./$@ @echo Apache License Version 2.0, January 2004 >> ./$@ diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index a053403b..2bb4e7f8 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -274,6 +274,11 @@ ifndef LLVM_DEBUG CFLAGS_SAFE += -Wno-deprecated endif +ifdef CODE_COVERAGE + override CFLAGS_SAFE += -D__AFL_CODE_COVERAGE=1 + override LDFLAGS += -ldl +endif + override CFLAGS += $(CFLAGS_SAFE) ifdef AFL_TRACE_PC @@ -510,7 +515,7 @@ install: all install -m 644 instrumentation/README.*.md $${DESTDIR}$(DOC_PATH)/ %.8: % - @echo .TH $* 8 $(BUILD_DATE) "afl++" > ./$@ + @echo .TH $* 8 $(BUILD_DATE) "AFL++" > ./$@ @echo .SH NAME >> ./$@ @printf "%s" ".B $* \- " >> ./$@ @./$* -h 2>&1 | head -n 1 | sed -e "s/$$(printf '\e')[^m]*m//g" >> ./$@ @@ -524,8 +529,8 @@ install: all @./$* -h 2>&1 | tail -n +4 >> ./$@ @echo >> ./$@ @echo .SH AUTHOR >> ./$@ - @echo "afl++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Heiko \"hexcoder-\" Eissfeldt , Andrea Fioraldi and Dominik Maier " >> ./$@ - @echo The homepage of afl++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ + @echo "AFL++ was written by Michal \"lcamtuf\" Zalewski and is maintained by Marc \"van Hauser\" Heuse , Dominik Maier , Andrea Fioraldi and Heiko \"hexcoder-\" Eissfeldt " >> ./$@ + @echo The homepage of AFL++ is: https://github.com/AFLplusplus/AFLplusplus >> ./$@ @echo >> ./$@ @echo .SH LICENSE >> ./$@ @echo Apache License Version 2.0, January 2004 >> ./$@ diff --git a/README.md b/README.md index c012c400..863c2fce 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,9 @@ Repository: AFL++ is maintained by: * Marc "van Hauser" Heuse -* Heiko "hexcoder-" Eißfeldt * Andrea Fioraldi * Dominik Maier +* Heiko "hexcoder-" Eißfeldt * Documentation: Jana Aydinbas Originally developed by Michał "lcamtuf" Zalewski. diff --git a/afl-cmin b/afl-cmin index 63cfdd7e..ae723c1b 100755 --- a/afl-cmin +++ b/afl-cmin @@ -149,7 +149,7 @@ BEGIN { redirected = 0 } - print "corpus minimization tool for afl++ (awk version)\n" + print "corpus minimization tool for AFL++ (awk version)\n" # defaults extra_par = "" diff --git a/afl-system-config b/afl-system-config index bf6397fa..b50bb06e 100755 --- a/afl-system-config +++ b/afl-system-config @@ -110,7 +110,7 @@ if [ "$PLATFORM" = "Darwin" ] ; then sysctl kern.sysv.shmall=131072000 echo Settings applied. echo - if [ $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ] ; then + if $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ; then echo echo Unloading the default crash reporter SL=/System/Library; PL=com.apple.ReportCrash @@ -119,6 +119,7 @@ if [ "$PLATFORM" = "Darwin" ] ; then echo fi echo It is recommended to disable System Integrity Protection for increased performance. + echo See: https://developer.apple.com/documentation/security/disabling_and_enabling_system_integrity_protection echo DONE=1 fi diff --git a/afl-whatsup b/afl-whatsup index cec1ae28..6f29ab24 100755 --- a/afl-whatsup +++ b/afl-whatsup @@ -88,6 +88,7 @@ TOTAL_TIME=0 TOTAL_EXECS=0 TOTAL_EPS=0 TOTAL_CRASHES=0 +TOTAL_HANGS=0 TOTAL_PFAV=0 TOTAL_PENDING=0 @@ -190,6 +191,7 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do TOTAL_EPS=$((TOTAL_EPS + EXEC_SEC)) TOTAL_EXECS=$((TOTAL_EXECS + execs_done)) TOTAL_CRASHES=$((TOTAL_CRASHES + saved_crashes)) + TOTAL_HANGS=$((TOTAL_HANGS + saved_hangs)) TOTAL_PENDING=$((TOTAL_PENDING + pending_total)) TOTAL_PFAV=$((TOTAL_PFAV + pending_favs)) @@ -301,6 +303,7 @@ if [ "$ALIVE_CNT" -gt "1" ]; then fi echo " Crashes saved : $TOTAL_CRASHES" +echo " Hangs saved : $TOTAL_HANGS" echo "Cycles without finds : $TOTAL_WCOP" echo " Time without finds : $TOTAL_LAST_FIND" echo diff --git a/custom_mutators/aflpp_tritondse/README.md b/custom_mutators/aflpp_tritondse/README.md new file mode 100644 index 00000000..8a5dd02b --- /dev/null +++ b/custom_mutators/aflpp_tritondse/README.md @@ -0,0 +1,17 @@ +# An AFL++ custom mutator using TritonDSE + +## Installing the requirements + +`pip3 install tritondse` + +## How to run with an example + +``` +../../afl-cc -o ../../test-instr ../../test-instr.c +mkdir -p in +echo aaaa > in/in +TRITON_DSE_TARGET=../../test-instr AFL_CUSTOM_MUTATOR_ONLY=1 AFL_SYNC_TIME=1 AFL_PYTHON_MODULE=aflpp_tritondse PYTHONPATH=. ../../afl-fuzz -i in -o out -- ../../test-instr +``` + +Note that this custom mutator works differently, new finds are synced +after 10-60 seconds to the fuzzing instance. diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py new file mode 100644 index 00000000..e0219f0b --- /dev/null +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -0,0 +1,148 @@ +import sys +import os +import logging +import hashlib + +from tritondse import CleLoader +from tritondse import CompositeData +from tritondse import Config +from tritondse import CoverageStrategy +from tritondse import ProcessState +from tritondse import Program +from tritondse import Seed +from tritondse import SeedFormat +from tritondse import SymbolicExecutor +from tritondse import SymbolicExplorator + +is_debug = False +out_path = "" +input_file = None +prog = None +config = None +dse = None +cycle = 0 +count = 0 +hashes = set() +format = SeedFormat.RAW + +def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): + global count + global hashes + if se.seed.hash not in hashes: + hashes.add(se.seed.hash) + filename = out_path + "/id:" + f"{count:06}" + "," + se.seed.hash + if not os.path.exists(filename): + if is_debug: + print('Creating queue input ' + filename) + with open(filename, 'wb') as file: + if input_file: + file.write(se.seed.content.files[input_file]) + else: + file.write(se.seed.content) + count += 1 + #if input_file: + # if is_debug: + # print('Writing to ' + input_file + ' the content: ' + str(se.seed.content)) + # with open(input_file, 'wb') as file: + # file.write(se.seed.content) + + +def init(seed): + global config + global dse + global format + global input_file + global is_debug + global out_path + global prog + # Load the program (LIEF-based program loader). + prog = CleLoader(os.environ['AFL_CUSTOM_INFO_PROGRAM']) + # Process other configuration environment variables. + argv = None + try: + foo = os.environ['AFL_DEBUG'] + is_debug = True + except KeyError: + pass + try: + foo = os.environ['AFL_CUSTOM_INFO_OUT'] + out_path = foo + '/../tritondse/queue' + except KeyError: + pass + try: + foo = os.environ['AFL_CUSTOM_INFO_PROGRAM_INPUT'] + input_file = foo + except KeyError: + pass + try: + argv_list = os.environ['AFL_CUSTOM_INFO_PROGRAM_ARGV'] + argv_tmp = [ os.environ['AFL_CUSTOM_INFO_PROGRAM'] ] + argv_tmp += argv_list.split() + argv = [] + # now check for @@ + for item in argv_tmp: + if "@@" in item: + input_file = out_path + '/../.input' + argv.append(input_file) + else: + argv.append(item) + except KeyError: + pass + # Create the output directory + os.makedirs(out_path, exist_ok=True) + # Debug + if is_debug: + print('DEBUG target: ' + os.environ['AFL_CUSTOM_INFO_PROGRAM']) + if argv: + print('DEBUG argv: ') + print(argv) + if input_file: + print('DEBUG input_file: ' + input_file) + print('DEBUG out_path: ' + out_path) + print('') + if input_file: + format = SeedFormat.COMPOSITE + # Now set up TritonDSE + config = Config(coverage_strategy = CoverageStrategy.PATH, + debug = is_debug, + pipe_stdout = is_debug, + pipe_stderr = is_debug, + execution_timeout = 1, + program_argv = argv, + smt_timeout= 50, + seed_format = format) + # Create an instance of the Symbolic Explorator + dse = SymbolicExplorator(config, prog) + # Add callbacks. + dse.callback_manager.register_pre_execution_callback(pre_exec_hook) + + +#def fuzz(buf, add_buf, max_size): +# return b"" + + +def queue_new_entry(filename_new_queue, filename_orig_queue): + global cycle + global dse + # Add seed to the worklist. + with open(filename_new_queue, "rb") as file: + data = file.read() + hash = hashlib.md5(data).hexdigest() + if hash not in hashes: + hashes.add(hash) + if is_debug: + print("NEW FILE " + filename_new_queue + " hash " + hash + " count " + str(cycle)) + cycle += 1 + if input_file: + seed = Seed(CompositeData(files={"stdin": b"", # nothing on stdin + input_file: data})) + else: + seed = Seed(data) + dse.add_input_seed(seed) + # Start exploration! + #dse.step() + dse.explore() + pass + +def splice_optout(): + pass diff --git a/docs/Changelog.md b/docs/Changelog.md index 20b915fa..3602af50 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -11,13 +11,17 @@ - new env `AFL_IGNORE_PROBLEMS_COVERAGE` to ignore coverage from loaded libs after forkserver initialization (required by Mozilla) - afl-cc: + - added @responsefile support - new env `AFL_LLVM_LTO_SKIPINIT` to support the AFL++ based WASM (https://github.com/fgsect/WAFL) project + - error and print help if afl-clan-lto is used with lto=thin - afl-showmap: - added custom mutator post_process and send support - add `-I filelist` option, an alternative to `-i in_dir` - afl-cmin + afl-cmin.bash: - `-T threads` parallel task support, can be a huge speedup! + - qemu_mode: + - Persistent mode +QASAN support for ppc32 tragets by @worksbutnottested - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! @@ -229,7 +233,7 @@ afl-showmap and other tools. - afl-cc: - detect overflow reads on initial input buffer for asan - - new cmplog mode (incompatible with older afl++ versions) + - new cmplog mode (incompatible with older AFL++ versions) - support llvm IR select instrumentation for default PCGUARD and LTO - fix for shared linking on MacOS - better selective instrumentation AFL_LLVM_{ALLOW|DENY}LIST diff --git a/docs/FAQ.md b/docs/FAQ.md index 76350c79..8178db46 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -171,6 +171,14 @@ If you find an interesting or important question missing, submit it via The more "unstable" edges there are, the harder it is for AFL++ to identify valid new paths. + If you fuzz in persistent mode (`AFL_LOOP` or `LLVMFuzzerTestOneInput()` + harnesses, a large number of unstable edges can mean that the target keeps + internal state and therefore it is possible that crashes cannot be replayed. + In such a case do either **not** fuzz in persistent mode (remove `AFL_LOOP()` + from your harness or call `LLVMFuzzerTestOneInput()` harnesses with `@@`), + or set a low `AFL_LOOP` value, e.g. 100, and enable `AFL_PERSISTENT_RECORD` + in `config.h` with the same value. + A value above 90% is usually fine and a value above 80% is also still ok, and even a value above 20% can still result in successful finds of bugs. However, it is recommended that for values below 90% or 80% you should take diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 591b7ded..637e8658 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -51,7 +51,7 @@ make source-only These build targets exist: -* all: the main afl++ binaries and llvm/gcc instrumentation +* all: the main AFL++ binaries and llvm/gcc instrumentation * binary-only: everything for binary-only fuzzing: frida_mode, nyx_mode, qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator, libtokencap @@ -79,22 +79,20 @@ make STATIC=1 These build options exist: * STATIC - compile AFL++ static +* CODE_COVERAGE - compile the target for code coverage (see docs/instrumentation/README.llvm.md) * ASAN_BUILD - compiles AFL++ with memory sanitizer for debug purposes -* UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for - debug purposes +* UBSAN_BUILD - compiles AFL++ tools with undefined behaviour sanitizer for debug purposes * DEBUG - no optimization, -ggdb3, all warnings and -Werror * LLVM_DEBUG - shows llvm deprecation warnings * PROFILING - compile afl-fuzz with profiling information * INTROSPECTION - compile afl-fuzz with mutation introspection * NO_PYTHON - disable python support -* NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for - normal fuzzing +* NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for normal fuzzing * NO_NYX - disable building nyx mode dependencies * NO_CORESIGHT - disable building coresight (arm64 only) * NO_UNICORN_ARM64 - disable building unicorn on arm64 * AFL_NO_X86 - if compiling on non-intel/amd platforms -* LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config - (e.g., Debian) +* LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config (e.g., Debian) e.g.: `make LLVM_CONFIG=llvm-config-14` diff --git a/docs/best_practices.md b/docs/best_practices.md index 133c645e..459fcaf7 100644 --- a/docs/best_practices.md +++ b/docs/best_practices.md @@ -131,6 +131,11 @@ jitter, or is a hash map function etc., then it should not be instrumented. To be able to exclude these functions (based on AFL++'s measured stability), the following process will allow to identify functions with variable edges. +Note that this is only useful for non-persistent targets! +If a persistent target is unstable whereas when run non-persistent is fine, +then this means that the target is keeping internal state, which is bad for +fuzzing. Fuzz such targets **without** persistent mode. + Four steps are required to do this and it also requires quite some knowledge of coding and/or disassembly and is effectively possible only with `afl-clang-fast` `PCGUARD` and `afl-clang-lto` `LTO` instrumentation. diff --git a/docs/custom_mutators.md b/docs/custom_mutators.md index a1de479e..3f7e9e6e 100644 --- a/docs/custom_mutators.md +++ b/docs/custom_mutators.md @@ -304,6 +304,34 @@ Note: for some distributions, you might also need the package `python[3]-apt`. In case your setup is different, set the necessary variables like this: `PYTHON_INCLUDE=/path/to/python/include LDFLAGS=-L/path/to/python/lib make`. +### Helpers + +For C/C++ custom mutators you get a pointer to `afl_state_t *afl` in the +`afl_custom_init()` which contains all information that you need. +Note that if you access it, you need to recompile your custom mutator if +you update AFL++ because the structure might have changed! + +For mutators written in Python, Rust, GO, etc. there are a few environment +variables set to help you to get started: + +`AFL_CUSTOM_INFO_PROGRAM` - the program name of the target that is executed. +If your custom mutator is used with modes like Qemu (`-Q`), this will still +contain the target program, not afl-qemu-trace. + +`AFL_CUSTOM_INFO_PROGRAM_INPUT` - if the `-f` parameter is used with afl-fuzz +then this value is found in this environment variable. + +`AFL_CUSTOM_INFO_PROGRAM_ARGV` - this contains the parameters given to the +target program and still has the `@@` identifier in there. + +Note: If `AFL_CUSTOM_INFO_PROGRAM_INPUT` is empty and `AFL_CUSTOM_INFO_PROGRAM_ARGV` +is either empty or does not contain `@@` then the target gets the input via +`stdin`. + +`AFL_CUSTOM_INFO_OUT` - This is the output directory for this fuzzer instance, +so if `afl-fuzz` was called with `-o out -S foobar`, then this will be set to +`out/foobar`. + ### Custom Mutator Preparation For C/C++ mutators, the source code must be compiled as a shared object: diff --git a/docs/tutorials.md b/docs/tutorials.md index 758fddab..342080fd 100644 --- a/docs/tutorials.md +++ b/docs/tutorials.md @@ -20,6 +20,10 @@ training, then we can highly recommend the following: * [https://github.com/antonio-morales/Fuzzing101](https://github.com/antonio-morales/Fuzzing101) +Here is good workflow description for frida_mode: + +* [https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html](https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html) + If you are interested in fuzzing structured data (where you define what the structure is), these links have you covered (some are outdated though): diff --git a/frida_mode/README.md b/frida_mode/README.md index 49a1fe38..bfca443c 100644 --- a/frida_mode/README.md +++ b/frida_mode/README.md @@ -7,6 +7,8 @@ variables. In FRIDA mode, binary programs are instrumented, similarly to QEMU mode. +A tutorial can be found at [https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html](https://blog.quarkslab.com/android-greybox-fuzzing-with-afl-frida-mode.html) + ## Current progress As FRIDA mode is new, it is missing a lot of features. The design is such that diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 831a0dbc..8fb7ecb1 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1223,7 +1223,7 @@ double rand_next_percent(afl_state_t *afl); static inline u32 rand_below(afl_state_t *afl, u32 limit) { - if (limit <= 1) return 0; + if (unlikely(limit <= 1)) return 0; /* The boundary not being necessarily a power of 2, we need to ensure the result uniformity. */ @@ -1256,7 +1256,7 @@ static inline u32 rand_below(afl_state_t *afl, u32 limit) { expand havoc mode */ static inline u32 rand_below_datalen(afl_state_t *afl, u32 limit) { - if (limit <= 1) return 0; + if (unlikely(limit <= 1)) return 0; switch (rand_below(afl, 3)) { diff --git a/include/alloc-inl.h b/include/alloc-inl.h index ae37028e..1e9a192b 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -42,7 +42,7 @@ // Be careful! _WANT_ORIGINAL_AFL_ALLOC is not compatible with custom mutators #ifndef _WANT_ORIGINAL_AFL_ALLOC - // afl++ stuff without memory corruption checks - for speed + // AFL++ stuff without memory corruption checks - for speed /* User-facing macro to sprintf() to a dynamically allocated buffer. */ @@ -704,12 +704,11 @@ static inline void *afl_realloc(void **buf, size_t size_needed) { *buf = NULL; return NULL; - } else { - - new_buf = newer_buf; - } + new_buf = newer_buf; + memset(((u8 *)new_buf) + current_size, 0, next_size - current_size); + new_buf->complete_size = next_size; *buf = (void *)(new_buf->buf); return *buf; diff --git a/include/envs.h b/include/envs.h index fe5ee0e3..edfd06e4 100644 --- a/include/envs.h +++ b/include/envs.h @@ -37,6 +37,10 @@ static char *afl_environment_variables[] = { "AFL_CRASH_EXITCODE", "AFL_CUSTOM_MUTATOR_LIBRARY", "AFL_CUSTOM_MUTATOR_ONLY", + "AFL_CUSTOM_INFO_PROGRAM", + "AFL_CUSTOM_INFO_PROGRAM_ARGV", + "AFL_CUSTOM_INFO_PROGRAM_INPUT", + "AFL_CUSTOM_INFO_OUT", "AFL_CXX", "AFL_CYCLE_SCHEDULES", "AFL_DEBUG", diff --git a/instrumentation/README.llvm.md b/instrumentation/README.llvm.md index c0677474..126cf1a2 100644 --- a/instrumentation/README.llvm.md +++ b/instrumentation/README.llvm.md @@ -280,3 +280,27 @@ Please note that the default counter implementations are not thread safe! Support for thread safe counters in mode LLVM CLASSIC can be activated with setting `AFL_LLVM_THREADSAFE_INST=1`. + +## 8) Source code coverage through instrumentation + +Measuring source code coverage is a common task in fuzzing, but it is very +difficut to do in some situations (e.g. when using snapshot fuzzing). + +When using the `AFL_LLVM_INSTRUMENT=llvm-codecov` option, afl-cc will use +native trace-pc-guard instrumentation but additionally select options that +are required to utilize the instrumentation for source code coverage. + +In particular, it will switch the instrumentation to be per basic block +instead of instrumenting edges, disable all guard pruning and enable the +experimental pc-table support that allows the runtime to gather 100% of +instrumented basic blocks at start, including their locations. + +Note: You must compile AFL with the `CODE_COVERAGE=1` option to enable the +respective parts in the AFL compiler runtime. Support is currently only +implemented for Nyx, but can in theory also work without Nyx. + +Note: You might have to adjust `MAP_SIZE_POW2` in include/config.h to ensure +that your coverage map is large enough to hold all basic blocks of your +target program without any collisions. + +More documentation on how to utilize this with Nyx will follow. diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index e779bb79..b3b0d2cd 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1,4 +1,4 @@ -/* SanitizeCoverage.cpp ported to afl++ LTO :-) */ +/* SanitizeCoverage.cpp ported to AFL++ LTO :-) */ #define AFL_LLVM_PASS @@ -20,6 +20,8 @@ #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" #include "llvm/Analysis/EHPersonalities.h" +#else + #include "llvm/IR/EHPersonalities.h" #endif #include "llvm/Analysis/PostDominators.h" #include "llvm/Analysis/ValueTracking.h" @@ -236,7 +238,7 @@ class ModuleSanitizerCoverageLTO SanitizerCoverageOptions Options; - // afl++ START + // AFL++ START // const SpecialCaseList * Allowlist; // const SpecialCaseList * Blocklist; uint32_t autodictionary = 1; @@ -262,7 +264,7 @@ class ModuleSanitizerCoverageLTO Value *MapPtrFixed = NULL; std::ofstream dFile; size_t found = 0; - // afl++ END + // AFL++ END }; @@ -406,7 +408,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( Int8Ty = IRB.getInt8Ty(); Int1Ty = IRB.getInt1Ty(); - /* afl++ START */ + /* AFL++ START */ char *ptr; LLVMContext &Ctx = M.getContext(); Ct = &Ctx; @@ -980,7 +982,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( } - // afl++ END + // AFL++ END SanCovTracePCIndir = M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy); @@ -1004,7 +1006,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( for (auto &F : M) instrumentFunction(F, DTCallback, PDTCallback); - // afl++ START + // AFL++ START if (dFile.is_open()) dFile.close(); if (!getenv("AFL_LLVM_LTO_SKIPINIT") && @@ -1158,7 +1160,7 @@ bool ModuleSanitizerCoverageLTO::instrumentModule( } - // afl++ END + // AFL++ END // We don't reference these arrays directly in any of our runtime functions, // so we need to prevent them from being dead stripped. @@ -1215,10 +1217,10 @@ static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, // (catchswitch blocks). if (BB->getFirstInsertionPt() == BB->end()) return false; - // afl++ START + // AFL++ START if (!Options.NoPrune && &F.getEntryBlock() == BB && F.size() > 1) return false; - // afl++ END + // AFL++ END if (Options.NoPrune || &F.getEntryBlock() == BB) return true; @@ -1260,10 +1262,10 @@ void ModuleSanitizerCoverageLTO::instrumentFunction( // if (Blocklist && Blocklist->inSection("coverage", "fun", F.getName())) // return; - // afl++ START + // AFL++ START if (!F.size()) return; if (!isInInstrumentList(&F, FMNAME)) return; - // afl++ END + // AFL++ END if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) SplitAllCriticalEdges( @@ -1561,7 +1563,7 @@ bool ModuleSanitizerCoverageLTO::InjectCoverage( for (size_t i = 0, N = AllBlocks.size(); i < N; i++) { - // afl++ START + // AFL++ START if (BlockList.size()) { int skip = 0; @@ -1583,7 +1585,7 @@ bool ModuleSanitizerCoverageLTO::InjectCoverage( } - // afl++ END + // AFL++ END InjectCoverageAtBlock(F, *AllBlocks[i], i, IsLeafFunc); @@ -1649,7 +1651,7 @@ void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, if (Options.TracePCGuard) { - // afl++ START + // AFL++ START ++afl_global_id; if (dFile.is_open()) { @@ -1713,7 +1715,7 @@ void ModuleSanitizerCoverageLTO::InjectCoverageAtBlock(Function &F, // done :) inst++; - // afl++ END + // AFL++ END /* XXXXXXXXXXXXXXXXXXX diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 8be9e329..41c38283 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -16,6 +16,8 @@ #if LLVM_VERSION_MAJOR < 17 #include "llvm/ADT/Triple.h" #include "llvm/Analysis/EHPersonalities.h" +#else + #include "llvm/IR/EHPersonalities.h" #endif #include "llvm/Analysis/PostDominators.h" #include "llvm/IR/CFG.h" diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 0912e52b..5372fae0 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -14,6 +14,16 @@ */ +#ifdef __AFL_CODE_COVERAGE + #ifndef _GNU_SOURCE + #define _GNU_SOURCE + #endif + #ifndef __USE_GNU + #define __USE_GNU + #endif + #include +#endif + #ifdef __ANDROID__ #include "android-ashmem.h" #endif @@ -105,6 +115,44 @@ u32 __afl_dictionary_len; u64 __afl_map_addr; u32 __afl_first_final_loc; +#ifdef __AFL_CODE_COVERAGE +typedef struct afl_module_info_t afl_module_info_t; + +struct afl_module_info_t { + + // A unique id starting with 0 + u32 id; + + // Name and base address of the module + char *name; + uintptr_t base_address; + + // PC Guard start/stop + u32 start; + u32 stop; + + // PC Table begin/end + const uintptr_t *pcs_beg; + const uintptr_t *pcs_end; + + u8 mapped; + + afl_module_info_t *next; + +}; + +typedef struct { + + uintptr_t PC, PCFlags; + +} PCTableEntry; + +afl_module_info_t *__afl_module_info = NULL; + +u32 __afl_pcmap_size = 0; +uintptr_t *__afl_pcmap_ptr = NULL; +#endif // __AFL_CODE_COVERAGE + /* 1 if we are running in afl, and the forkserver was started, else 0 */ u32 __afl_connected = 0; @@ -499,6 +547,7 @@ static void __afl_map_shm(void) { u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); if (!map_env || atoi((char *)map_env) < MAP_SIZE) { + fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); send_forkserver_error(FS_ERROR_MAP_SIZE); _exit(1); @@ -678,6 +727,27 @@ static void __afl_map_shm(void) { } +#ifdef __AFL_CODE_COVERAGE + char *pcmap_id_str = getenv("__AFL_PCMAP_SHM_ID"); + + if (pcmap_id_str) { + + __afl_pcmap_size = __afl_map_size * sizeof(void *); + u32 shm_id = atoi(pcmap_id_str); + + __afl_pcmap_ptr = (uintptr_t *)shmat(shm_id, NULL, 0); + + if (__afl_debug) { + + fprintf(stderr, "DEBUG: Received %p via shmat for pcmap\n", + __afl_pcmap_ptr); + + } + + } + +#endif // __AFL_CODE_COVERAGE + } /* unmap SHM. */ @@ -686,6 +756,17 @@ static void __afl_unmap_shm(void) { if (!__afl_already_initialized_shm) return; +#ifdef __AFL_CODE_COVERAGE + if (__afl_pcmap_size) { + + shmdt((void *)__afl_pcmap_ptr); + __afl_pcmap_ptr = NULL; + __afl_pcmap_size = 0; + + } + +#endif // __AFL_CODE_COVERAGE + char *id_str = getenv(SHM_ENV_VAR); if (id_str) { @@ -1507,6 +1588,102 @@ void __sanitizer_cov_trace_pc_guard(uint32_t *guard) { } +#ifdef __AFL_CODE_COVERAGE +void __sanitizer_cov_pcs_init(const uintptr_t *pcs_beg, + const uintptr_t *pcs_end) { + + if (__afl_debug) { + + fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init called\n"); + + } + + // If for whatever reason, we cannot get dlinfo here, then pc_guard_init also + // couldn't get it and we'd end up attributing to the wrong module. + Dl_info dlinfo; + if (!dladdr(__builtin_return_address(0), &dlinfo)) { + + fprintf(stderr, + "WARNING: Ignoring __sanitizer_cov_pcs_init callback due to " + "missing module info\n"); + return; + + } + + afl_module_info_t *last_module_info = __afl_module_info; + while (last_module_info && last_module_info->next) { + + last_module_info = last_module_info->next; + + } + + if (!last_module_info) { + + fprintf(stderr, + "ERROR: __sanitizer_cov_pcs_init called with no module info?!\n"); + abort(); + + } + + last_module_info->pcs_beg = pcs_beg; + last_module_info->pcs_end = pcs_end; + + // Now update the pcmap. If this is the last module coming in, after all + // pre-loaded code, then this will also map all of our delayed previous + // modules. + + if (!__afl_pcmap_ptr) { return; } + + for (afl_module_info_t *mod_info = __afl_module_info; mod_info; + mod_info = mod_info->next) { + + if (mod_info->mapped) { continue; } + + PCTableEntry *start = (PCTableEntry *)(mod_info->pcs_beg); + PCTableEntry *end = (PCTableEntry *)(mod_info->pcs_end); + + u32 in_module_index = 0; + + while (start < end) { + + if (mod_info->start + in_module_index >= __afl_map_size) { + + fprintf(stderr, "ERROR: __sanitizer_cov_pcs_init out of bounds?!\n"); + abort(); + + } + + uintptr_t PC = start->PC; + + // This is what `GetPreviousInstructionPc` in sanitizer runtime does + // for x86/x86-64. Needs more work for ARM and other archs. + PC = PC - 1; + + // Calculate relative offset in module + PC = PC - mod_info->base_address; + + __afl_pcmap_ptr[mod_info->start + in_module_index] = PC; + + start++; + in_module_index++; + + } + + mod_info->mapped = 1; + + if (__afl_debug) { + + fprintf(stderr, "DEBUG: __sanitizer_cov_pcs_init initialized %u PCs\n", + in_module_index); + + } + + } + +} + +#endif // __AFL_CODE_COVERAGE + /* Init callback. Populates instrumentation IDs. Note that we're using ID of 0 as a special value to indicate non-instrumented bits. That may still touch the bitmap, but in a fairly harmless way. */ @@ -1538,6 +1715,62 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { if (start == stop || *start) { return; } +#ifdef __AFL_CODE_COVERAGE + u32 *orig_start = start; + afl_module_info_t *mod_info = NULL; + + Dl_info dlinfo; + if (dladdr(__builtin_return_address(0), &dlinfo)) { + + if (__afl_already_initialized_forkserver) { + + fprintf(stderr, "[pcmap] Error: Module was not preloaded: %s\n", + dlinfo.dli_fname); + + } else { + + afl_module_info_t *last_module_info = __afl_module_info; + while (last_module_info && last_module_info->next) { + + last_module_info = last_module_info->next; + + } + + mod_info = malloc(sizeof(afl_module_info_t)); + + mod_info->id = last_module_info ? last_module_info->id + 1 : 0; + mod_info->name = strdup(dlinfo.dli_fname); + mod_info->base_address = (uintptr_t)dlinfo.dli_fbase; + mod_info->start = 0; + mod_info->stop = 0; + mod_info->pcs_beg = NULL; + mod_info->pcs_end = NULL; + mod_info->mapped = 0; + mod_info->next = NULL; + + if (last_module_info) { + + last_module_info->next = mod_info; + + } else { + + __afl_module_info = mod_info; + + } + + fprintf(stderr, "[pcmap] Module: %s Base Address: %p\n", dlinfo.dli_fname, + dlinfo.dli_fbase); + + } + + } else { + + fprintf(stderr, "[pcmap] dladdr call failed\n"); + + } + +#endif // __AFL_CODE_COVERAGE + x = getenv("AFL_INST_RATIO"); if (x) { @@ -1625,6 +1858,22 @@ void __sanitizer_cov_trace_pc_guard_init(uint32_t *start, uint32_t *stop) { } +#ifdef __AFL_CODE_COVERAGE + if (mod_info) { + + mod_info->start = *orig_start; + mod_info->stop = *(stop - 1); + if (__afl_debug) { + + fprintf(stderr, "DEBUG: [pcmap] Start Index: %u Stop Index: %u\n", + mod_info->start, mod_info->stop); + + } + + } + +#endif // __AFL_CODE_COVERAGE + if (__afl_debug) { fprintf(stderr, diff --git a/instrumentation/afl-llvm-common.cc b/instrumentation/afl-llvm-common.cc index 5d82aa25..7f17b02d 100644 --- a/instrumentation/afl-llvm-common.cc +++ b/instrumentation/afl-llvm-common.cc @@ -584,7 +584,7 @@ bool isInInstrumentList(llvm::Function *F, std::string Filename) { } // Calculate the number of average collisions that would occur if all -// location IDs would be assigned randomly (like normal afl/afl++). +// location IDs would be assigned randomly (like normal afl/AFL++). // This uses the "balls in bins" algorithm. unsigned long long int calculateCollisions(uint32_t edges) { diff --git a/instrumentation/afl-llvm-dict2file.so.cc b/instrumentation/afl-llvm-dict2file.so.cc index 97155cd6..8ee13010 100644 --- a/instrumentation/afl-llvm-dict2file.so.cc +++ b/instrumentation/afl-llvm-dict2file.so.cc @@ -746,7 +746,7 @@ static void registerAFLdict2filePass(const PassManagerBuilder &, } static RegisterPass X("afl-dict2file", - "afl++ dict2file instrumentation pass", + "AFL++ dict2file instrumentation pass", false, false); static RegisterStandardPasses RegisterAFLdict2filePass( diff --git a/nyx_mode/LIBNYX_VERSION b/nyx_mode/LIBNYX_VERSION index 86b32eec..ed88ec10 100644 --- a/nyx_mode/LIBNYX_VERSION +++ b/nyx_mode/LIBNYX_VERSION @@ -1 +1 @@ -2da7f08 +c8a72dc diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index fa44d173..44ea5345 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -0569eff8a1 +a1321713c7 diff --git a/qemu_mode/build_qemu_support.sh b/qemu_mode/build_qemu_support.sh index a064fe58..f59cba78 100755 --- a/qemu_mode/build_qemu_support.sh +++ b/qemu_mode/build_qemu_support.sh @@ -356,7 +356,7 @@ fi if ! command -v "$CROSS" > /dev/null ; then if [ "$CPU_TARGET" = "$(uname -m)" ] ; then - echo "[+] Building afl++ qemu support libraries with CC=$CC" + echo "[+] Building AFL++ qemu support libraries with CC=$CC" echo "[+] Building libcompcov ..." make -C libcompcov && echo "[+] libcompcov ready" echo "[+] Building unsigaction ..." @@ -371,7 +371,7 @@ if ! command -v "$CROSS" > /dev/null ; then echo "[!] Cross compiler $CROSS could not be found, cannot compile libcompcov libqasan and unsigaction" fi else - echo "[+] Building afl++ qemu support libraries with CC=\"$CROSS $CROSS_FLAGS\"" + echo "[+] Building AFL++ qemu support libraries with CC=\"$CROSS $CROSS_FLAGS\"" echo "[+] Building libcompcov ..." make -C libcompcov CC="$CROSS $CROSS_FLAGS" && echo "[+] libcompcov ready" echo "[+] Building unsigaction ..." diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index 0569eff8..a1321713 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit 0569eff8a12dec73642b96757f6b5b51a618a03a +Subproject commit a1321713c7502c152dd7527555e0f8a800d55225 diff --git a/src/afl-cc.c b/src/afl-cc.c index d1001187..972ac8cd 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -31,6 +31,8 @@ #include #include #include +#include +#include #if (LLVM_MAJOR - 0 == 0) #undef LLVM_MAJOR @@ -76,6 +78,7 @@ enum { INSTRUMENT_OPT_NGRAM = 16, INSTRUMENT_OPT_CALLER = 32, INSTRUMENT_OPT_CTX_K = 64, + INSTRUMENT_OPT_CODECOV = 128, }; @@ -375,15 +378,304 @@ void parse_fsanitize(char *string) { } +static u8 fortify_set = 0, asan_set = 0, x_set = 0, bit_mode = 0, + shared_linking = 0, preprocessor_only = 0, have_unroll = 0, + have_o = 0, have_pic = 0, have_c = 0, partial_linking = 0, + non_dash = 0; + +static void process_params(u32 argc, char **argv) { + + if (cc_par_cnt + argc >= 1024) { FATAL("Too many command line parameters"); } + + if (lto_mode && argc > 1) { + + u32 idx; + for (idx = 1; idx < argc; idx++) { + + if (!strncasecmp(argv[idx], "-fpic", 5)) have_pic = 1; + + } + + } + + // for (u32 x = 0; x < argc; ++x) fprintf(stderr, "[%u] %s\n", x, argv[x]); + + /* Process the argument list. */ + + u8 skip_next = 0; + while (--argc) { + + u8 *cur = *(++argv); + + if (skip_next) { + + skip_next = 0; + continue; + + } + + if (cur[0] != '-') { non_dash = 1; } + if (!strncmp(cur, "--afl", 5)) continue; + + if (lto_mode && !strncmp(cur, "-flto=thin", 10)) { + + FATAL( + "afl-clang-lto cannot work with -flto=thin. Switch to -flto=full or " + "use afl-clang-fast!"); + + } + + if (lto_mode && !strncmp(cur, "-fuse-ld=", 9)) continue; + if (lto_mode && !strncmp(cur, "--ld-path=", 10)) continue; + if (!strncmp(cur, "-fno-unroll", 11)) continue; + if (strstr(cur, "afl-compiler-rt") || strstr(cur, "afl-llvm-rt")) continue; + if (!strcmp(cur, "-Wl,-z,defs") || !strcmp(cur, "-Wl,--no-undefined") || + !strcmp(cur, "--no-undefined")) { + + continue; + + } + + if (compiler_mode == GCC_PLUGIN && !strcmp(cur, "-pipe")) { continue; } + + if (!strcmp(cur, "-z") || !strcmp(cur, "-Wl,-z")) { + + u8 *param = *(argv + 1); + if (!strcmp(param, "defs") || !strcmp(param, "-Wl,defs")) { + + skip_next = 1; + continue; + + } + + } + + if ((compiler_mode == GCC || compiler_mode == GCC_PLUGIN) && + !strncmp(cur, "-stdlib=", 8)) { + + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } + continue; + + } + + if (!strncmp(cur, "-fsanitize-coverage-", 20) && strstr(cur, "list=")) { + + have_instr_list = 1; + + } + + if (!strncmp(cur, "-fsanitize=", strlen("-fsanitize=")) && + strchr(cur, ',')) { + + parse_fsanitize(cur); + if (!cur || strlen(cur) <= strlen("-fsanitize=")) { continue; } + + } else if ((!strncmp(cur, "-fsanitize=fuzzer-", + + strlen("-fsanitize=fuzzer-")) || + !strncmp(cur, "-fsanitize-coverage", + strlen("-fsanitize-coverage"))) && + (strncmp(cur, "sanitize-coverage-allow", + strlen("sanitize-coverage-allow")) && + strncmp(cur, "sanitize-coverage-deny", + strlen("sanitize-coverage-deny")) && + instrument_mode != INSTRUMENT_LLVMNATIVE)) { + + if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } + continue; + + } + + if (need_aflpplib || !strcmp(cur, "-fsanitize=fuzzer")) { + + u8 *afllib = find_object("libAFLDriver.a", argv[0]); + + if (!be_quiet) { + + OKF("Found '-fsanitize=fuzzer', replacing with libAFLDriver.a"); + + } + + if (!afllib) { + + if (!be_quiet) { + + WARNF( + "Cannot find 'libAFLDriver.a' to replace '-fsanitize=fuzzer' in " + "the flags - this will fail!"); + + } + + } else { + + cc_params[cc_par_cnt++] = afllib; + +#ifdef __APPLE__ + cc_params[cc_par_cnt++] = "-undefined"; + cc_params[cc_par_cnt++] = "dynamic_lookup"; +#endif + + } + + if (need_aflpplib) { + + need_aflpplib = 0; + + } else { + + continue; + + } + + } + + if (!strcmp(cur, "-m32")) bit_mode = 32; + if (!strcmp(cur, "armv7a-linux-androideabi")) bit_mode = 32; + if (!strcmp(cur, "-m64")) bit_mode = 64; + + if (!strcmp(cur, "-fsanitize=address") || !strcmp(cur, "-fsanitize=memory")) + asan_set = 1; + + if (strstr(cur, "FORTIFY_SOURCE")) fortify_set = 1; + + if (!strcmp(cur, "-x")) x_set = 1; + if (!strcmp(cur, "-E")) preprocessor_only = 1; + if (!strcmp(cur, "-shared")) shared_linking = 1; + if (!strcmp(cur, "-dynamiclib")) shared_linking = 1; + if (!strcmp(cur, "--target=wasm32-wasi")) passthrough = 1; + if (!strcmp(cur, "-Wl,-r")) partial_linking = 1; + if (!strcmp(cur, "-Wl,-i")) partial_linking = 1; + if (!strcmp(cur, "-Wl,--relocatable")) partial_linking = 1; + if (!strcmp(cur, "-r")) partial_linking = 1; + if (!strcmp(cur, "--relocatable")) partial_linking = 1; + if (!strcmp(cur, "-c")) have_c = 1; + + if (!strncmp(cur, "-O", 2)) have_o = 1; + if (!strncmp(cur, "-funroll-loop", 13)) have_unroll = 1; + + if (*cur == '@') { + + // response file support. + // we have two choices - move everything to the command line or + // rewrite the response files to temporary files and delete them + // afterwards. We choose the first for easiness. + // We do *not* support quotes in the rsp files to cope with spaces in + // filenames etc! If you need that then send a patch! + u8 *filename = cur + 1; + if (debug) { DEBUGF("response file=%s\n", filename); } + FILE *f = fopen(filename, "r"); + struct stat st; + + // Check not found or empty? let the compiler complain if so. + if (!f || fstat(fileno(f), &st) < 0 || st.st_size < 1) { + + cc_params[cc_par_cnt++] = cur; + continue; + + } + + u8 *tmpbuf = malloc(st.st_size + 1), *ptr; + char **args = malloc(sizeof(char *) * (st.st_size >> 1)); + int count = 1, cont = 0, cont_act = 0; + + while (fgets(tmpbuf, st.st_size, f)) { + + ptr = tmpbuf; + // no leading whitespace + while (isspace(*ptr)) { + + ++ptr; + cont_act = 0; + + } + + // no comments, no empty lines + if (*ptr == '#' || *ptr == '\n' || !*ptr) { continue; } + // remove LF + if (ptr[strlen(ptr) - 1] == '\n') { ptr[strlen(ptr) - 1] = 0; } + // remove CR + if (*ptr && ptr[strlen(ptr) - 1] == '\r') { ptr[strlen(ptr) - 1] = 0; } + // handle \ at end of line + if (*ptr && ptr[strlen(ptr) - 1] == '\\') { + + cont = 1; + ptr[strlen(ptr) - 1] = 0; + + } + + // remove whitespace at end + while (*ptr && isspace(ptr[strlen(ptr) - 1])) { + + ptr[strlen(ptr) - 1] = 0; + cont = 0; + + } + + if (*ptr) { + + do { + + u8 *value = ptr; + while (*ptr && !isspace(*ptr)) { + + ++ptr; + + } + + while (*ptr && isspace(*ptr)) { + + *ptr++ = 0; + + } + + if (cont_act) { + + u32 len = strlen(args[count - 1]) + strlen(value) + 1; + u8 *tmp = malloc(len); + snprintf(tmp, len, "%s%s", args[count - 1], value); + free(args[count - 1]); + args[count - 1] = tmp; + cont_act = 0; + + } else { + + args[count++] = strdup(value); + + } + + } while (*ptr); + + } + + if (cont) { + + cont_act = 1; + cont = 0; + + } + + } + + if (count) { process_params(count, args); } + + // we cannot free args[] + free(tmpbuf); + + continue; + + } + + cc_params[cc_par_cnt++] = cur; + + } + +} + /* Copy argv to cc_params, making the necessary edits. */ static void edit_params(u32 argc, char **argv, char **envp) { - u8 fortify_set = 0, asan_set = 0, x_set = 0, bit_mode = 0, shared_linking = 0, - preprocessor_only = 0, have_unroll = 0, have_o = 0, have_pic = 0, - have_c = 0, partial_linking = 0; - - cc_params = ck_alloc((argc + 128) * sizeof(u8 *)); + cc_params = ck_alloc(1024 * sizeof(u8 *)); if (lto_mode) { @@ -642,7 +934,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } //#if LLVM_MAJOR >= 13 - // // Use the old pass manager in LLVM 14 which the afl++ passes still + // // Use the old pass manager in LLVM 14 which the AFL++ passes still // use. cc_params[cc_par_cnt++] = "-flegacy-pass-manager"; //#endif @@ -751,7 +1043,21 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else if (instrument_mode == INSTRUMENT_LLVMNATIVE) { #if LLVM_MAJOR >= 4 - cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; + if (instrument_opt_mode & INSTRUMENT_OPT_CODECOV) { + + #if LLVM_MAJOR >= 6 + cc_params[cc_par_cnt++] = + "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"; + #else + FATAL("pcguard instrumentation with pc-table requires llvm 6.0.1+"); + #endif + + } else { + + cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; + + } + #else FATAL("pcguard instrumentation requires llvm 4.0.1+"); #endif @@ -816,159 +1122,15 @@ static void edit_params(u32 argc, char **argv, char **envp) { } - if (!have_pic) cc_params[cc_par_cnt++] = "-fPIC"; - } } - /* Detect stray -v calls from ./configure scripts. */ + /* Inspect the command line parameters. */ - u8 skip_next = 0, non_dash = 0; - while (--argc) { + process_params(argc, argv); - u8 *cur = *(++argv); - - if (skip_next) { - - skip_next = 0; - continue; - - } - - if (cur[0] != '-') { non_dash = 1; } - if (!strncmp(cur, "--afl", 5)) continue; - if (lto_mode && !strncmp(cur, "-fuse-ld=", 9)) continue; - if (lto_mode && !strncmp(cur, "--ld-path=", 10)) continue; - if (!strncmp(cur, "-fno-unroll", 11)) continue; - if (strstr(cur, "afl-compiler-rt") || strstr(cur, "afl-llvm-rt")) continue; - if (!strcmp(cur, "-Wl,-z,defs") || !strcmp(cur, "-Wl,--no-undefined") || - !strcmp(cur, "--no-undefined")) { - - continue; - - } - - if (compiler_mode == GCC_PLUGIN && !strcmp(cur, "-pipe")) { continue; } - - if (!strcmp(cur, "-z") || !strcmp(cur, "-Wl,-z")) { - - u8 *param = *(argv + 1); - if (!strcmp(param, "defs") || !strcmp(param, "-Wl,defs")) { - - skip_next = 1; - continue; - - } - - } - - if ((compiler_mode == GCC || compiler_mode == GCC_PLUGIN) && - !strncmp(cur, "-stdlib=", 8)) { - - if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } - continue; - - } - - if (!strncmp(cur, "-fsanitize-coverage-", 20) && strstr(cur, "list=")) { - - have_instr_list = 1; - - } - - if (!strncmp(cur, "-fsanitize=", strlen("-fsanitize=")) && - strchr(cur, ',')) { - - parse_fsanitize(cur); - if (!cur || strlen(cur) <= strlen("-fsanitize=")) { continue; } - - } else if ((!strncmp(cur, "-fsanitize=fuzzer-", - - strlen("-fsanitize=fuzzer-")) || - !strncmp(cur, "-fsanitize-coverage", - strlen("-fsanitize-coverage"))) && - (strncmp(cur, "sanitize-coverage-allow", - strlen("sanitize-coverage-allow")) && - strncmp(cur, "sanitize-coverage-deny", - strlen("sanitize-coverage-deny")) && - instrument_mode != INSTRUMENT_LLVMNATIVE)) { - - if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } - continue; - - } - - if (need_aflpplib || !strcmp(cur, "-fsanitize=fuzzer")) { - - u8 *afllib = find_object("libAFLDriver.a", argv[0]); - - if (!be_quiet) { - - OKF("Found '-fsanitize=fuzzer', replacing with libAFLDriver.a"); - - } - - if (!afllib) { - - if (!be_quiet) { - - WARNF( - "Cannot find 'libAFLDriver.a' to replace '-fsanitize=fuzzer' in " - "the flags - this will fail!"); - - } - - } else { - - cc_params[cc_par_cnt++] = afllib; - -#ifdef __APPLE__ - cc_params[cc_par_cnt++] = "-undefined"; - cc_params[cc_par_cnt++] = "dynamic_lookup"; -#endif - - } - - if (need_aflpplib) { - - need_aflpplib = 0; - - } else { - - continue; - - } - - } - - if (!strcmp(cur, "-m32")) bit_mode = 32; - if (!strcmp(cur, "armv7a-linux-androideabi")) bit_mode = 32; - if (!strcmp(cur, "-m64")) bit_mode = 64; - - if (!strcmp(cur, "-fsanitize=address") || !strcmp(cur, "-fsanitize=memory")) - asan_set = 1; - - if (strstr(cur, "FORTIFY_SOURCE")) fortify_set = 1; - - if (!strcmp(cur, "-x")) x_set = 1; - if (!strcmp(cur, "-E")) preprocessor_only = 1; - if (!strcmp(cur, "-shared")) shared_linking = 1; - if (!strcmp(cur, "-dynamiclib")) shared_linking = 1; - if (!strcmp(cur, "--target=wasm32-wasi")) passthrough = 1; - if (!strcmp(cur, "-Wl,-r")) partial_linking = 1; - if (!strcmp(cur, "-Wl,-i")) partial_linking = 1; - if (!strcmp(cur, "-Wl,--relocatable")) partial_linking = 1; - if (!strcmp(cur, "-r")) partial_linking = 1; - if (!strcmp(cur, "--relocatable")) partial_linking = 1; - if (!strcmp(cur, "-c")) have_c = 1; - - if (!strncmp(cur, "-O", 2)) have_o = 1; - if (!strncmp(cur, "-funroll-loop", 13)) have_unroll = 1; - - cc_params[cc_par_cnt++] = cur; - - } + if (!have_pic) { cc_params[cc_par_cnt++] = "-fPIC"; } // in case LLVM is installed not via a package manager or "make install" // e.g. compiled download or compiled from github then its ./lib directory @@ -1651,13 +1813,17 @@ int main(int argc, char **argv, char **envp) { instrument_mode = INSTRUMENT_CLASSIC; lto_mode = 1; - } else if (!instrument_mode || instrument_mode == INSTRUMENT_AFL) + } else if (!instrument_mode || instrument_mode == INSTRUMENT_AFL) { instrument_mode = INSTRUMENT_AFL; - else + + } else { + FATAL("main instrumentation mode already set with %s", instrument_mode_string[instrument_mode]); + } + } if (strncasecmp(ptr2, "pc-guard", strlen("pc-guard")) == 0 || @@ -1682,6 +1848,23 @@ int main(int argc, char **argv, char **envp) { } + if (strncasecmp(ptr2, "llvmcodecov", strlen("llvmcodecov")) == 0 || + strncasecmp(ptr2, "llvm-codecov", strlen("llvm-codecov")) == 0) { + + if (!instrument_mode || instrument_mode == INSTRUMENT_LLVMNATIVE) { + + instrument_mode = INSTRUMENT_LLVMNATIVE; + instrument_opt_mode |= INSTRUMENT_OPT_CODECOV; + + } else { + + FATAL("main instrumentation mode already set with %s", + instrument_mode_string[instrument_mode]); + + } + + } + if (strncasecmp(ptr2, "cfg", strlen("cfg")) == 0 || strncasecmp(ptr2, "instrim", strlen("instrim")) == 0) { @@ -2241,7 +2424,8 @@ int main(int argc, char **argv, char **envp) { "(requires LLVM 11 or higher)"); #endif - if (instrument_opt_mode && instrument_mode != INSTRUMENT_CLASSIC) + if (instrument_opt_mode && instrument_opt_mode != INSTRUMENT_OPT_CODECOV && + instrument_mode != INSTRUMENT_CLASSIC) FATAL( "CALLER, CTX and NGRAM instrumentation options can only be used with " "the LLVM CLASSIC instrumentation mode."); diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index aa8c8622..30c8901c 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -489,7 +489,7 @@ static void report_error_and_exit(int error) { break; case FS_ERROR_OLD_CMPLOG: FATAL( - "the -c cmplog target was instrumented with an too old afl++ " + "the -c cmplog target was instrumented with an too old AFL++ " "version, you need to recompile it."); break; case FS_ERROR_OLD_CMPLOG_QEMU: @@ -987,7 +987,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, if ((status & FS_OPT_ENABLED) == FS_OPT_ENABLED) { - // workaround for recent afl++ versions + // workaround for recent AFL++ versions if ((status & FS_OPT_OLD_AFLPP_WORKAROUND) == FS_OPT_OLD_AFLPP_WORKAROUND) status = (status & 0xf0ffffff); @@ -1059,7 +1059,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, FATAL( "Target's coverage map size of %u is larger than the one this " - "afl++ is set with (%u). Either set AFL_MAP_SIZE=%u and restart " + "AFL++ is set with (%u). Either set AFL_MAP_SIZE=%u and restart " " afl-fuzz, or change MAP_SIZE_POW2 in config.h and recompile " "afl-fuzz", tmp_map_size, fsrv->map_size, tmp_map_size); diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index bd591c8f..baf56a5f 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -716,6 +716,8 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } + // if (getenv("MYTEST")) afl->in_place_resume = 1; + if (nl_cnt) { u32 done = 0; @@ -827,6 +829,8 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } + // if (getenv("MYTEST")) afl->in_place_resume = 0; + free(nl); /* not tracked */ if (!afl->queued_items && directory == NULL) { @@ -908,8 +912,10 @@ void perform_dry_run(afl_state_t *afl) { if (res == afl->crash_mode || res == FSRV_RUN_NOBITS) { - SAYF(cGRA " len = %u, map size = %u, exec speed = %llu us\n" cRST, - q->len, q->bitmap_size, q->exec_us); + SAYF(cGRA + " len = %u, map size = %u, exec speed = %llu us, hash = " + "%016llx\n" cRST, + q->len, q->bitmap_size, q->exec_us, q->exec_cksum); } @@ -1164,14 +1170,14 @@ void perform_dry_run(afl_state_t *afl) { u32 duplicates = 0, i; - for (idx = 0; idx < afl->queued_items; idx++) { + for (idx = 0; idx < afl->queued_items - 1; idx++) { q = afl->queue_buf[idx]; if (!q || q->disabled || q->cal_failed || !q->exec_cksum) { continue; } - u32 done = 0; + for (i = idx + 1; - i < afl->queued_items && !done && likely(afl->queue_buf[i]); i++) { + likely(i < afl->queued_items && afl->queue_buf[i] && !done); ++i) { struct queue_entry *p = afl->queue_buf[i]; if (p->disabled || p->cal_failed || !p->exec_cksum) { continue; } @@ -1194,6 +1200,13 @@ void perform_dry_run(afl_state_t *afl) { p->disabled = 1; p->perf_score = 0; + if (afl->debug) { + + WARNF("Same coverage - %s is kept active, %s is disabled.", + q->fname, p->fname); + + } + } else { if (!q->was_fuzzed) { @@ -1207,7 +1220,14 @@ void perform_dry_run(afl_state_t *afl) { q->disabled = 1; q->perf_score = 0; - done = 1; + if (afl->debug) { + + WARNF("Same coverage - %s is kept active, %s is disabled.", + p->fname, q->fname); + + } + + done = 1; // end inner loop because outer loop entry is disabled now } diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index ee562f96..c6e9a295 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -842,6 +842,7 @@ u8 fuzz_one_original(afl_state_t *afl) { eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); if (unlikely(!eff_map)) { PFATAL("alloc"); } + memset(eff_map, 0, EFF_ALEN(len)); eff_map[0] = 1; if (EFF_APOS(len - 1) != 0) { @@ -2047,20 +2048,22 @@ custom_mutator_stage: afl->queue_cur->stats_mutated += afl->stage_max; #endif - if (likely(afl->custom_only)) { - - /* Skip other stages */ - ret_val = 0; - goto abandon_entry; - - } - /**************** * RANDOM HAVOC * ****************/ havoc_stage: + if (unlikely(afl->custom_only)) { + + /* Force UI update */ + show_stats(afl); + /* Skip other stages */ + ret_val = 0; + goto abandon_entry; + + } + afl->stage_cur_byte = -1; /* The havoc stage mutation code is also invoked when splicing files; if the @@ -3570,6 +3573,7 @@ static u8 mopt_common_fuzzing(afl_state_t *afl, MOpt_globals_t MOpt_globals) { eff_map = afl_realloc(AFL_BUF_PARAM(eff), EFF_ALEN(len)); if (unlikely(!eff_map)) { PFATAL("alloc"); } + memset(eff_map, 0, EFF_ALEN(len)); eff_map[0] = 1; if (EFF_APOS(len - 1) != 0) { diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 8ad7cd97..b10bf749 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -49,11 +49,13 @@ inline u32 select_next_queue_entry(afl_state_t *afl) { u32 s = rand_below(afl, afl->queued_items); double p = rand_next_percent(afl); + /* fprintf(stderr, "select: p=%f s=%u ... p < prob[s]=%f ? s=%u : alias[%u]=%u" " ==> %u\n", p, s, afl->alias_probability[s], s, s, afl->alias_table[s], p < afl->alias_probability[s] ? s : afl->alias_table[s]); */ + return (p < afl->alias_probability[s] ? s : afl->alias_table[s]); } @@ -87,25 +89,28 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, void create_alias_table(afl_state_t *afl) { - u32 n = afl->queued_items, i = 0, a, g; + u32 n = afl->queued_items, i = 0, nSmall = 0, nLarge = n - 1; double sum = 0; + double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double)); + u32 *Small = (int *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); + u32 *Large = (int *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); + afl->alias_table = (u32 *)afl_realloc((void **)&afl->alias_table, n * sizeof(u32)); afl->alias_probability = (double *)afl_realloc( (void **)&afl->alias_probability, n * sizeof(double)); - double *P = (double *)afl_realloc(AFL_BUF_PARAM(out), n * sizeof(double)); - int *S = (int *)afl_realloc(AFL_BUF_PARAM(out_scratch), n * sizeof(u32)); - int *L = (int *)afl_realloc(AFL_BUF_PARAM(in_scratch), n * sizeof(u32)); - if (!P || !S || !L || !afl->alias_table || !afl->alias_probability) { + if (!P || !Small || !Large || !afl->alias_table || !afl->alias_probability) { FATAL("could not acquire memory for alias table"); } - memset((void *)afl->alias_table, 0, n * sizeof(u32)); memset((void *)afl->alias_probability, 0, n * sizeof(double)); + memset((void *)afl->alias_table, 0, n * sizeof(u32)); + memset((void *)Small, 0, n * sizeof(u32)); + memset((void *)Large, 0, n * sizeof(u32)); if (likely(afl->schedule < RARE)) { @@ -166,7 +171,15 @@ void create_alias_table(afl_state_t *afl) { for (i = 0; i < n; i++) { // weight is always 0 for disabled entries - P[i] = (afl->queue_buf[i]->weight * n) / sum; + if (unlikely(afl->queue_buf[i]->disabled)) { + + P[i] = 0; + + } else { + + P[i] = (afl->queue_buf[i]->weight * n) / sum; + + } } @@ -176,60 +189,81 @@ void create_alias_table(afl_state_t *afl) { struct queue_entry *q = afl->queue_buf[i]; - if (likely(!q->disabled)) { q->perf_score = calculate_score(afl, q); } + if (likely(!q->disabled)) { - sum += q->perf_score; + q->perf_score = calculate_score(afl, q); + sum += q->perf_score; + + } } for (i = 0; i < n; i++) { // perf_score is always 0 for disabled entries - P[i] = (afl->queue_buf[i]->perf_score * n) / sum; + if (unlikely(afl->queue_buf[i]->disabled)) { + + P[i] = 0; + + } else { + + P[i] = (afl->queue_buf[i]->perf_score * n) / sum; + + } } } - int nS = 0, nL = 0, s; - for (s = (s32)n - 1; s >= 0; --s) { + // Done collecting weightings in P, now create the arrays. - if (P[s] < 1) { + for (s32 j = (s32)(n - 1); j >= 0; j--) { - S[nS++] = s; + if (P[j] < 1) { + + Small[nSmall++] = (u32)j; } else { - L[nL++] = s; + Large[nLarge--] = (u32)j; } } - while (nS && nL) { + while (nSmall && nLarge != n - 1) { - a = S[--nS]; - g = L[--nL]; - afl->alias_probability[a] = P[a]; - afl->alias_table[a] = g; - P[g] = P[g] + P[a] - 1; - if (P[g] < 1) { + u32 small = Small[--nSmall]; + u32 large = Large[++nLarge]; - S[nS++] = g; + afl->alias_probability[small] = P[small]; + afl->alias_table[small] = large; + + P[large] = P[large] - (1 - P[small]); + + if (P[large] < 1) { + + Small[nSmall++] = large; } else { - L[nL++] = g; + Large[nLarge--] = large; } } - while (nL) - afl->alias_probability[L[--nL]] = 1; + while (nSmall) { - while (nS) - afl->alias_probability[S[--nS]] = 1; + afl->alias_probability[Small[--nSmall]] = 1; + + } + + while (nLarge != n - 1) { + + afl->alias_probability[Large[++nLarge]] = 1; + + } afl->reinit_table = 0; @@ -264,7 +298,7 @@ void create_alias_table(afl_state_t *afl) { */ /* fprintf(stderr, " entry alias probability perf_score weight - filename\n"); for (u32 i = 0; i < n; ++i) fprintf(stderr, " %5u %5u %11u + filename\n"); for (i = 0; i < n; ++i) fprintf(stderr, " %5u %5u %11u %0.9f %0.9f %s\n", i, afl->alias_table[i], afl->alias_probability[i], afl->queue_buf[i]->perf_score, afl->queue_buf[i]->weight, afl->queue_buf[i]->fname); diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 71d2afd8..4339ddd2 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -124,7 +124,8 @@ static void usage(u8 *argv0, int more_help) { "\n%s [ options ] -- /path/to/fuzzed_app [ ... ]\n\n" "Required parameters:\n" - " -i dir - input directory with test cases\n" + " -i dir - input directory with test cases (or '-' to resume, " + "also see AFL_AUTORESUME)\n" " -o dir - output directory for fuzzer findings\n\n" "Execution control settings:\n" @@ -1280,16 +1281,16 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->fsrv.mem_limit && afl->shm.cmplog_mode) afl->fsrv.mem_limit += 260; - OKF("afl++ is maintained by Marc \"van Hauser\" Heuse, Heiko \"hexcoder\" " - "Eißfeldt, Andrea Fioraldi and Dominik Maier"); - OKF("afl++ is open source, get it at " + OKF("AFL++ is maintained by Marc \"van Hauser\" Heuse, Dominik Maier, Andrea " + "Fioraldi and Heiko \"hexcoder\" Eißfeldt"); + OKF("AFL++ is open source, get it at " "https://github.com/AFLplusplus/AFLplusplus"); - OKF("NOTE: afl++ >= v3 has changed defaults and behaviours - see README.md"); + OKF("NOTE: AFL++ >= v3 has changed defaults and behaviours - see README.md"); #ifdef __linux__ if (afl->fsrv.nyx_mode) { - OKF("afl++ Nyx mode is enabled (developed and mainted by Sergej Schumilo)"); + OKF("AFL++ Nyx mode is enabled (developed and mainted by Sergej Schumilo)"); OKF("Nyx is open source, get it at https://github.com/Nyx-Fuzz"); } @@ -1529,29 +1530,6 @@ int main(int argc, char **argv_orig, char **envp) { } - if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { - - if (afl->custom_only) { - - FATAL("Custom mutators are incompatible with MOpt (-L)"); - - } - - u32 custom_fuzz = 0; - LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { - - if (el->afl_custom_fuzz) { custom_fuzz = 1; } - - }); - - if (custom_fuzz) { - - WARNF("afl_custom_fuzz is incompatible with MOpt (-L)"); - - } - - } - if (afl->afl_env.afl_max_det_extras) { s32 max_det_extras = atoi(afl->afl_env.afl_max_det_extras); @@ -1826,8 +1804,76 @@ int main(int argc, char **argv_orig, char **envp) { printf("DEBUG: rand %06d is %u\n", counter, rand_below(afl, 65536)); #endif + if (!getenv("AFL_CUSTOM_INFO_PROGRAM")) { + + setenv("AFL_CUSTOM_INFO_PROGRAM", argv[optind], 1); + + } + + if (!getenv("AFL_CUSTOM_INFO_PROGRAM_INPUT") && afl->fsrv.out_file) { + + setenv("AFL_CUSTOM_INFO_PROGRAM_INPUT", afl->fsrv.out_file, 1); + + } + + { + + u8 envbuf[8096] = "", tmpbuf[8096] = ""; + for (s32 i = optind + 1; i < argc; ++i) { + + strcpy(tmpbuf, envbuf); + if (strchr(argv[i], ' ') && !strchr(argv[i], '"') && + !strchr(argv[i], '\'')) { + + if (!strchr(argv[i], '\'')) { + + snprintf(envbuf, sizeof(tmpbuf), "%s '%s'", tmpbuf, argv[i]); + + } else { + + snprintf(envbuf, sizeof(tmpbuf), "%s \"%s\"", tmpbuf, argv[i]); + + } + + } else { + + snprintf(envbuf, sizeof(tmpbuf), "%s %s", tmpbuf, argv[i]); + + } + + } + + setenv("AFL_CUSTOM_INFO_PROGRAM_ARGV", envbuf + 1, 1); + + } + + setenv("AFL_CUSTOM_INFO_OUT", afl->out_dir, 1); // same as __AFL_OUT_DIR + setup_custom_mutators(afl); + if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { + + if (afl->custom_only) { + + FATAL("Custom mutators are incompatible with MOpt (-L)"); + + } + + u32 custom_fuzz = 0; + LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { + + if (el->afl_custom_fuzz) { custom_fuzz = 1; } + + }); + + if (custom_fuzz) { + + WARNF("afl_custom_fuzz is incompatible with MOpt (-L)"); + + } + + } + write_setup_file(afl, argc, argv); setup_cmdline_file(afl, argv + optind); @@ -1979,6 +2025,7 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->non_instrumented_mode || afl->fsrv.qemu_mode || afl->fsrv.frida_mode || afl->fsrv.cs_mode || afl->unicorn_mode) { + u32 old_map_size = map_size; map_size = afl->fsrv.real_map_size = afl->fsrv.map_size = MAP_SIZE; afl->virgin_bits = ck_realloc(afl->virgin_bits, map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, map_size); @@ -1990,6 +2037,18 @@ int main(int argc, char **argv_orig, char **envp) { afl->first_trace = ck_realloc(afl->first_trace, map_size); afl->map_tmp_buf = ck_realloc(afl->map_tmp_buf, map_size); + if (old_map_size < map_size) { + + memset(afl->var_bytes + old_map_size, 0, map_size - old_map_size); + memset(afl->top_rated + old_map_size, 0, map_size - old_map_size); + memset(afl->clean_trace + old_map_size, 0, map_size - old_map_size); + memset(afl->clean_trace_custom + old_map_size, 0, + map_size - old_map_size); + memset(afl->first_trace + old_map_size, 0, map_size - old_map_size); + memset(afl->map_tmp_buf + old_map_size, 0, map_size - old_map_size); + + } + } afl->argv = use_argv; @@ -2017,6 +2076,7 @@ int main(int argc, char **argv_orig, char **envp) { OKF("Re-initializing maps to %u bytes", new_map_size); + u32 old_map_size = map_size; afl->virgin_bits = ck_realloc(afl->virgin_bits, new_map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, new_map_size); afl->virgin_crash = ck_realloc(afl->virgin_crash, new_map_size); @@ -2029,6 +2089,18 @@ int main(int argc, char **argv_orig, char **envp) { afl->first_trace = ck_realloc(afl->first_trace, new_map_size); afl->map_tmp_buf = ck_realloc(afl->map_tmp_buf, new_map_size); + if (old_map_size < new_map_size) { + + memset(afl->var_bytes + old_map_size, 0, new_map_size - old_map_size); + memset(afl->top_rated + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace_custom + old_map_size, 0, + new_map_size - old_map_size); + memset(afl->first_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->map_tmp_buf + old_map_size, 0, new_map_size - old_map_size); + + } + afl_fsrv_kill(&afl->fsrv); afl_shm_deinit(&afl->shm); afl->fsrv.map_size = new_map_size; @@ -2079,6 +2151,7 @@ int main(int argc, char **argv_orig, char **envp) { OKF("Re-initializing maps to %u bytes due cmplog", new_map_size); + u32 old_map_size = map_size; afl->virgin_bits = ck_realloc(afl->virgin_bits, new_map_size); afl->virgin_tmout = ck_realloc(afl->virgin_tmout, new_map_size); afl->virgin_crash = ck_realloc(afl->virgin_crash, new_map_size); @@ -2091,6 +2164,18 @@ int main(int argc, char **argv_orig, char **envp) { afl->first_trace = ck_realloc(afl->first_trace, new_map_size); afl->map_tmp_buf = ck_realloc(afl->map_tmp_buf, new_map_size); + if (old_map_size < new_map_size) { + + memset(afl->var_bytes + old_map_size, 0, new_map_size - old_map_size); + memset(afl->top_rated + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->clean_trace_custom + old_map_size, 0, + new_map_size - old_map_size); + memset(afl->first_trace + old_map_size, 0, new_map_size - old_map_size); + memset(afl->map_tmp_buf + old_map_size, 0, new_map_size - old_map_size); + + } + afl_fsrv_kill(&afl->fsrv); afl_fsrv_kill(&afl->cmplog_fsrv); afl_shm_deinit(&afl->shm); diff --git a/src/afl-ld-lto.c b/src/afl-ld-lto.c index 5438bd9f..420dd817 100644 --- a/src/afl-ld-lto.c +++ b/src/afl-ld-lto.c @@ -2,7 +2,7 @@ american fuzzy lop++ - wrapper for llvm 11+ lld ----------------------------------------------- - Written by Marc Heuse for afl++ + Written by Marc Heuse for AFL++ Maintained by Marc Heuse , Heiko Eißfeldt @@ -210,7 +210,7 @@ static void edit_params(int argc, char **argv) { if (strcmp(argv[i], "--afl") == 0) { - if (!be_quiet) OKF("afl++ test command line flag detected, exiting."); + if (!be_quiet) OKF("AFL++ test command line flag detected, exiting."); exit(0); } diff --git a/src/afl-showmap.c b/src/afl-showmap.c index d0e01cb1..9c029035 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1287,7 +1287,7 @@ int main(int argc, char **argv_orig, char **envp) { break; - case 'Y': // fallthough + case 'Y': // fallthrough #ifdef __linux__ case 'X': /* NYX mode */ @@ -1421,6 +1421,14 @@ int main(int argc, char **argv_orig, char **envp) { // If @@ are in the target args, replace them and also set use_stdin=false. detect_file_args(argv + optind, stdin_file, &fsrv->use_stdin); + fsrv->dev_null_fd = open("/dev/null", O_RDWR); + if (fsrv->dev_null_fd < 0) { PFATAL("Unable to open /dev/null"); } + + fsrv->out_file = stdin_file; + fsrv->out_fd = + open(stdin_file, O_RDWR | O_CREAT | O_EXCL, DEFAULT_PERMISSION); + if (fsrv->out_fd < 0) { PFATAL("Unable to create '%s'", stdin_file); } + } else { // If @@ are in the target args, replace them and also set use_stdin=false. @@ -1588,6 +1596,14 @@ int main(int argc, char **argv_orig, char **envp) { fsrv->map_size = map_size; + } else { + + afl_fsrv_start(fsrv, use_argv, &stop_soon, + (get_afl_env("AFL_DEBUG_CHILD") || + get_afl_env("AFL_DEBUG_CHILD_OUTPUT")) + ? 1 + : 0); + } if (in_dir || in_filelist) { @@ -1617,9 +1633,6 @@ int main(int argc, char **argv_orig, char **envp) { if (getenv("AFL_DEBUG_GDB")) wait_for_gdb = true; - fsrv->dev_null_fd = open("/dev/null", O_RDWR); - if (fsrv->dev_null_fd < 0) { PFATAL("Unable to open /dev/null"); } - if (in_filelist) { if (!be_quiet) ACTF("Reading from file list '%s'...", in_filelist); @@ -1666,10 +1679,6 @@ int main(int argc, char **argv_orig, char **envp) { } atexit(at_exit_handler); - fsrv->out_file = stdin_file; - fsrv->out_fd = - open(stdin_file, O_RDWR | O_CREAT | O_EXCL, DEFAULT_PERMISSION); - if (fsrv->out_fd < 0) { PFATAL("Unable to create '%s'", out_file); } if (get_afl_env("AFL_DEBUG")) { @@ -1685,12 +1694,6 @@ int main(int argc, char **argv_orig, char **envp) { } - afl_fsrv_start(fsrv, use_argv, &stop_soon, - (get_afl_env("AFL_DEBUG_CHILD") || - get_afl_env("AFL_DEBUG_CHILD_OUTPUT")) - ? 1 - : 0); - map_size = fsrv->map_size; if (fsrv->support_shmem_fuzz && !fsrv->use_shmem_fuzz) diff --git a/test/test-dlopen.c b/test/test-dlopen.c index b81bab13..39442f93 100644 --- a/test/test-dlopen.c +++ b/test/test-dlopen.c @@ -28,7 +28,7 @@ int main(int argc, char **argv) { } - // must use deferred forkserver as otherwise afl++ instrumentation aborts + // must use deferred forkserver as otherwise AFL++ instrumentation aborts // because all dlopen() of instrumented libs must be before the forkserver __AFL_INIT(); diff --git a/test/test-gcc-plugin.sh b/test/test-gcc-plugin.sh index 54e6987f..3690a80a 100755 --- a/test/test-gcc-plugin.sh +++ b/test/test-gcc-plugin.sh @@ -23,7 +23,7 @@ test -e ../afl-gcc-fast -a -e ../afl-compiler-rt.o && { $ECHO "$GREEN[+] gcc_plugin run reported $TUPLES instrumented locations which is fine" } || { $ECHO "$RED[!] gcc_plugin instrumentation produces a weird numbers: $TUPLES" - $ECHO "$YELLOW[-] this is a known issue in gcc, not afl++. It is not flagged as an error because travis builds would all fail otherwise :-(" + $ECHO "$YELLOW[-] this is a known issue in gcc, not AFL++. It is not flagged as an error because travis builds would all fail otherwise :-(" #CODE=1 } test "$TUPLES" -lt 2 && SKIP=1 diff --git a/test/test-llvm.sh b/test/test-llvm.sh index 0e66cc97..714bda93 100755 --- a/test/test-llvm.sh +++ b/test/test-llvm.sh @@ -263,7 +263,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { { mkdir -p in echo 00000000000000000000000000000000 > in/in - AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V15 -i in -o out -c./test-cmplog -- ./test-c >>errors 2>&1 + AFL_BENCH_UNTIL_CRASH=1 ../afl-fuzz -m none -V30 -i in -o out -c./test-cmplog -- ./test-c >>errors 2>&1 } >>errors 2>&1 test -n "$( ls out/default/crashes/id:000000* out/default/hangs/id:000000* 2>/dev/null )" && { $ECHO "$GREEN[+] afl-fuzz is working correctly with llvm_mode cmplog" diff --git a/test/test-performance.sh b/test/test-performance.sh index d61e2f2a..50957141 100755 --- a/test/test-performance.sh +++ b/test/test-performance.sh @@ -7,7 +7,7 @@ FILE=$AFL_PERFORMANCE_FILE test -z "$FILE" && FILE=.afl_performance test -e $FILE || { - echo Warning: This script measure the performance of afl++ and saves the result for future comparisons into $FILE + echo Warning: This script measure the performance of AFL++ and saves the result for future comparisons into $FILE echo Press ENTER to continue or CONTROL-C to abort read IN } @@ -74,7 +74,7 @@ afl-system-config > /dev/null 2>&1 echo Performance settings applied. echo -$ECHO "${RESET}${GREY}[*] starting afl++ performance test framework ..." +$ECHO "${RESET}${GREY}[*] starting AFL++ performance test framework ..." $ECHO "$BLUE[*] Testing: ${AFL_GCC}" GCC=x diff --git a/test/test-pre.sh b/test/test-pre.sh index b8b286e5..1ca9dfb5 100755 --- a/test/test-pre.sh +++ b/test/test-pre.sh @@ -133,7 +133,7 @@ MEM_LIMIT=none export PATH="${PATH}:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" -$ECHO "${RESET}${GREY}[*] starting afl++ test framework ..." +$ECHO "${RESET}${GREY}[*] starting AFL++ test framework ..." test -z "$SYS" && $ECHO "$YELLOW[-] uname -m did not succeed" diff --git a/unicorn_mode/build_unicorn_support.sh b/unicorn_mode/build_unicorn_support.sh index 53ec2481..d3d16ad5 100755 --- a/unicorn_mode/build_unicorn_support.sh +++ b/unicorn_mode/build_unicorn_support.sh @@ -182,7 +182,7 @@ git pull sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null git checkout "$UNICORNAFL_VERSION" || exit 1 -echo "[*] making sure afl++ header files match" +echo "[*] making sure AFL++ header files match" cp "../../include/config.h" "./include" || exit 1 echo "[*] Configuring Unicorn build..." From 6d23df2c7c5246eb2e3da393b99a9c06bac972c1 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 15 May 2023 17:13:20 +0200 Subject: [PATCH 115/149] add target_intelligence --- utils/target_intelligence/README.md | 61 +++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 utils/target_intelligence/README.md diff --git a/utils/target_intelligence/README.md b/utils/target_intelligence/README.md new file mode 100644 index 00000000..086c9e20 --- /dev/null +++ b/utils/target_intelligence/README.md @@ -0,0 +1,61 @@ +# Target Intelligence + +These are some ideas you can do so that your target that you are fuzzing can +give helpful feedback to AFL++. + +## Add to the AFL++ dictionary from your target + +For this you target must be compiled for CMPLOG (`AFL_LLVM_CMPLOG=1`). + +Add in your source code: + +``` +__attribute__((weak)) void __cmplog_rtn_hook_strn(u8 *ptr1, u8 *ptr2, u64 len); +__attribute__((weak)) void __cmplog_ins_hook1(uint8_t arg1, uint8_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook2(uint16_t arg1, uint16_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook4(uint32_t arg1, uint32_t arg2, uint8_t attr); +__attribute__((weak)) void __cmplog_ins_hook8(uint64_t arg1, uint64_t arg2, uint8_t attr); + +int in_your_function(...) { + + // to add two strings to the AFL++ dictionary: + if (__cmplog_rtn_hook_strn) + __cmplog_rtn_hook_strn(string1, length_of_string1, string2, length_of_string2); + + // to add two 32 bit integers to the AFL++ dictionary: + if (__cmplog_ins_hook4) + __cmplog_ins_hook4(first_32_bit_var, second_32_bit_var, 0); + +} +``` + +Note that this only makes sense if these values are in-depth processed in the +target in a way that AFL++ CMPLOG cannot uncover these, e.g. if these values +are transformed by a matrix computation. + +Fixed values are always better to give to afl-fuzz via a `-x dictionary`. + +## Add inputs to AFL++ dictionary from your target + +If for whatever reason you want your target to propose new inputs to AFL++, +then this is actually very easy. +The environment variable `AFL_CUSTOM_INFO_OUT` contains the output directory +of this run - including the fuzzer instance name (e.g. `default`), so if you +run `afl-fuzz -o out -S foobar`, the value would be `out/foobar`). + +To show afl-fuzz an input it should consider just do the following: + +1. create the directory `$AFL_CUSTOM_INFO_OUT/../target/queue` +2. create any new inputs you want afl-fuzz to notice in that directory with the + following naming convention: `id:NUMBER-OF-LENGTH-SIX-WITH-LEADING-ZEROES,whatever` + where that number has to be increasing. + e.g.: +``` + id:000000,first_file + id:000001,second_file + id:000002,third_file + etc. +``` + +Note that this will not work in nyx_mode because afl-fuzz cannot see inside the +virtual machine. From 1ad63a6a32d966f1ac05ff40163ef7f747011307 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 16 May 2023 12:20:58 +0200 Subject: [PATCH 116/149] fix tritondse --- .../aflpp_tritondse/aflpp_tritondse.py | 68 ++++++++++++++++++- 1 file changed, 65 insertions(+), 3 deletions(-) diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index e0219f0b..48367bc7 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -22,14 +22,17 @@ config = None dse = None cycle = 0 count = 0 +finding = 0 hashes = set() format = SeedFormat.RAW def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): global count global hashes + global finding if se.seed.hash not in hashes: hashes.add(se.seed.hash) + finding = 1 filename = out_path + "/id:" + f"{count:06}" + "," + se.seed.hash if not os.path.exists(filename): if is_debug: @@ -47,6 +50,59 @@ def pre_exec_hook(se: SymbolicExecutor, state: ProcessState): # file.write(se.seed.content) +#def rtn_open(se: SymbolicExecutor, pstate: ProcessState, pc): +# """ +# The open behavior. +# """ +# logging.debug('open hooked') +# +# # Get arguments +# arg0 = pstate.get_argument_value(0) # const char *pathname +# flags = pstate.get_argument_value(1) # int flags +# mode = pstate.get_argument_value(2) # int mode +# arg0s = pstate.memory.read_string(arg0) +# +# # Concretize the whole path name +# pstate.concretize_memory_bytes(arg0, len(arg0s)+1) # Concretize the whole string + \0 +# +# # We use flags as concrete value +# pstate.concretize_argument(1) +# +# # Use the flags to open the file in the write mode. +# mode = "" +# if (flags & 0xFF) == 0x00: # O_RDONLY +# mode = "r" +# elif (flags & 0xFF) == 0x01: # O_WRONLY +# mode = "w" +# elif (flags & 0xFF) == 0x02: # O_RDWR +# mode = "r+" +# +# if (flags & 0x0100): # O_CREAT +# mode += "x" +# if (flags & 0x0200): # O_APPEND +# mode = "a" # replace completely value +# +# if se.seed.is_file_defined(arg0s) and "r" in mode: # input file and opened in reading +# logging.info(f"opening an input file: {arg0s}") +# # Program is opening an input +# data = se.seed.get_file_input(arg0s) +# filedesc = pstate.create_file_descriptor(arg0s, io.BytesIO(data)) +# fd = filedesc.id +# else: +# # Try to open it as a regular file +# try: +# fd = open(arg0s, mode) # use the mode here +# filedesc = pstate.create_file_descriptor(arg0s, fd) +# fd = filedesc.id +# except Exception as e: +# logging.debug(f"Failed to open {arg0s} {e}") +# fd = pstate.minus_one +# +# pstate.write_register("rax", fd) # write the return value +# pstate.cpu.program_counter = pstate.pop_stack_value() # pop the return value +# se.skip_instruction() # skip the current instruction so that the engine go straight fetching the next instruction + + def init(seed): global config global dse @@ -115,10 +171,16 @@ def init(seed): dse = SymbolicExplorator(config, prog) # Add callbacks. dse.callback_manager.register_pre_execution_callback(pre_exec_hook) + #dse.callback_manager.register_function_callback("open", rtn_open) -#def fuzz(buf, add_buf, max_size): -# return b"" +def fuzz(buf, add_buf, max_size): + global finding + finding = 1 + while finding == 1: + finding = 0 + dse.step() + return b"" def queue_new_entry(filename_new_queue, filename_orig_queue): @@ -141,7 +203,7 @@ def queue_new_entry(filename_new_queue, filename_orig_queue): dse.add_input_seed(seed) # Start exploration! #dse.step() - dse.explore() + #dse.explore() pass def splice_optout(): From 49997e60cba8dc260d45cc0ce68fa810588e2f23 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 16 May 2023 12:33:58 +0200 Subject: [PATCH 117/149] fix --- custom_mutators/aflpp_tritondse/aflpp_tritondse.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index 48367bc7..cef28f34 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -206,5 +206,11 @@ def queue_new_entry(filename_new_queue, filename_orig_queue): #dse.explore() pass + +# we simulate just doing one single fuzz in the custom mutator +def fuzz_count(buf): + return 1 + + def splice_optout(): pass From dfdc6fd12cdae1fe2dab1183f20d3c312a7f2f6d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 16 May 2023 14:54:02 +0200 Subject: [PATCH 118/149] add missing envs in the docs --- docs/env_variables.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/env_variables.md b/docs/env_variables.md index b1f23159..0f0869d2 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -619,6 +619,14 @@ The QEMU wrapper used to instrument binary-only code supports several settings: - Setting `AFL_INST_LIBS` causes the translator to also instrument the code inside any dynamically linked libraries (notably including glibc). + - You can use `AFL_QEMU_INST_RANGES=0xaaaa-0xbbbb,0xcccc-0xdddd` to just + instrument specific memory locations, e.g. a specific library. + Excluding ranges takes priority over any included ranges or `AFL_INST_LIBS`. + + - You can use `AFL_QEMU_EXCLUDE_RANGES=0xaaaa-0xbbbb,0xcccc-0xdddd` to **NOT** + instrument specific memory locations, e.g. a specific library. + Excluding ranges takes priority over any included ranges or `AFL_INST_LIBS`. + - It is possible to set `AFL_INST_RATIO` to skip the instrumentation on some of the basic blocks, which can be useful when dealing with very complex binaries. From 1d0694df86a3ce70ffac2846f36605eac9300abe Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 17 May 2023 15:25:26 +0200 Subject: [PATCH 119/149] add symqemu custom mutator --- custom_mutators/symcc/README.md | 2 + custom_mutators/symqemu/Makefile | 14 + custom_mutators/symqemu/README.md | 11 + custom_mutators/symqemu/symqemu.c | 446 ++++++++++++++++++++++++++++++ docs/Changelog.md | 3 + instrumentation/afl-llvm-common.h | 2 +- test-instr.c | 9 +- 7 files changed, 483 insertions(+), 4 deletions(-) create mode 100644 custom_mutators/symqemu/Makefile create mode 100644 custom_mutators/symqemu/README.md create mode 100644 custom_mutators/symqemu/symqemu.c diff --git a/custom_mutators/symcc/README.md b/custom_mutators/symcc/README.md index 364a348e..a6839a37 100644 --- a/custom_mutators/symcc/README.md +++ b/custom_mutators/symcc/README.md @@ -5,6 +5,8 @@ This uses the symcc to find new paths into the target. Note that this is a just a proof of concept example! It is better to use the fuzzing helpers of symcc, symqemu, Fuzzolic, etc. rather than this. +Also the symqemu custom mutator is better than this. + To use this custom mutator follow the steps in the symcc repository [https://github.com/eurecom-s3/symcc/](https://github.com/eurecom-s3/symcc/) on how to build symcc and how to instrument a target binary (the same target diff --git a/custom_mutators/symqemu/Makefile b/custom_mutators/symqemu/Makefile new file mode 100644 index 00000000..3361ab0f --- /dev/null +++ b/custom_mutators/symqemu/Makefile @@ -0,0 +1,14 @@ + +ifdef DEBUG + CFLAGS += -DDEBUG +endif + +all: symqemu-mutator.so + +CFLAGS += -O3 -funroll-loops + +symqemu-mutator.so: symqemu.c + $(CC) $(CFLAGS) $(CPPFLAGS) -g -I../../include -shared -fPIC -o symqemu-mutator.so symqemu.c + +clean: + rm -f symqemu-mutator.so *.o *~ core diff --git a/custom_mutators/symqemu/README.md b/custom_mutators/symqemu/README.md new file mode 100644 index 00000000..0d5cd4d7 --- /dev/null +++ b/custom_mutators/symqemu/README.md @@ -0,0 +1,11 @@ +# custum mutator: symqemu + +This uses the symcc to find new paths into the target. + +To use this custom mutator follow the steps in the symqemu repository +[https://github.com/eurecom-s3/symqemu/](https://github.com/eurecom-s3/symqemu/) +on how to build symqemu-x86_x64 and put it in your `PATH`. + +just type `make` to build this custom mutator. + +```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/symqemu/symqemu-mutator.so AFL_SYNC_TIME=1 afl-fuzz ...``` diff --git a/custom_mutators/symqemu/symqemu.c b/custom_mutators/symqemu/symqemu.c new file mode 100644 index 00000000..9030397b --- /dev/null +++ b/custom_mutators/symqemu/symqemu.c @@ -0,0 +1,446 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include "config.h" +#include "debug.h" +#include "afl-fuzz.h" +#include "common.h" + +afl_state_t *afl_struct; +static u32 debug = 0; + +#define DBG(x...) \ + if (debug) { fprintf(stderr, x); } + +typedef struct my_mutator { + + afl_state_t *afl; + u8 *mutator_buf; + u8 *out_dir; + u8 *queue_dir; + u8 *target; + u8 *symqemu; + u8 *input_file; + u32 counter; + u32 seed; + u32 argc; + u8 **argv; + +} my_mutator_t; + +my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { + + if (getenv("AFL_DEBUG")) debug = 1; + + my_mutator_t *data = calloc(1, sizeof(my_mutator_t)); + if (!data) { + + perror("afl_custom_init alloc"); + return NULL; + + } + + char *path = getenv("PATH"); + char *exec_name = "symqemu-x86_64"; + char *token = strtok(path, ":"); + char exec_path[4096]; + + while (token != NULL && data->symqemu == NULL) { + + snprintf(exec_path, sizeof(exec_path), "%s/%s", token, exec_name); + if (access(exec_path, X_OK) == 0) { + + data->symqemu = (u8 *)strdup(exec_path); + break; + + } + + token = strtok(NULL, ":"); + + } + + if (!data->symqemu) FATAL("symqemu binary %s not found", exec_name); + DBG("Found %s\n", data->symqemu); + + if (getenv("AFL_CUSTOM_MUTATOR_ONLY")) + FATAL("the symqemu module cannot be used with AFL_CUSTOM_MUTATOR_ONLY."); + + if ((data->mutator_buf = malloc(MAX_FILE)) == NULL) { + + free(data); + perror("mutator_buf alloc"); + return NULL; + + } + + data->target = getenv("AFL_CUSTOM_INFO_PROGRAM"); + + u8 *path_tmp = getenv("AFL_CUSTOM_INFO_OUT"); + u32 len = strlen(path_tmp) + 32; + u8 *symqemu_path = malloc(len); + data->out_dir = malloc(len); + data->queue_dir = malloc(len); + snprintf(symqemu_path, len, "%s/../symqemu", path_tmp); + snprintf(data->out_dir, len, "%s/../symqemu/out", path_tmp); + snprintf(data->queue_dir, len, "%s/../symqemu/queue", path_tmp); + + mkdir(symqemu_path, 0755); + mkdir(data->out_dir, 0755); + mkdir(data->queue_dir, 0755); + + setenv("SYMCC_OUTPUT_DIR", data->out_dir, 1); + + data->input_file = getenv("AFL_CUSTOM_INFO_PROGRAM_INPUT"); + + u8 *tmp = NULL; + if ((tmp = getenv("AFL_CUSTOM_INFO_PROGRAM_ARGV")) && *tmp) { + + int argc = 0, index = 2; + for (u32 i = 0; i < strlen(tmp); ++i) + if (isspace(tmp[i])) ++argc; + + data->argv = (u8 **)malloc((argc + 4) * sizeof(u8 **)); + u8 *p = strdup(tmp); + + do { + + data->argv[index] = p; + while (*p && !isspace(*p)) + ++p; + if (*p) { + + *p++ = 0; + while (isspace(*p)) + ++p; + + } + + if (strcmp(data->argv[index], "@@") == 0) { + + if (!data->input_file) { + + u32 ilen = strlen(symqemu_path) + 32; + data->input_file = malloc(ilen); + snprintf(data->input_file, ilen, "%s/.input", symqemu_path); + + } + + data->argv[index] = data->input_file; + + } + + DBG("%d: %s\n", index, data->argv[index]); + index++; + + } while (*p); + + data->argv[index] = NULL; + data->argc = index; + + } else { + + data->argv = (u8 **)malloc(8 * sizeof(u8 **)); + data->argc = 2; + data->argv[2] = NULL; + + } + + data->argv[0] = data->symqemu; + data->argv[1] = data->target; + + DBG("out_dir=%s, queue_dir=%s, target=%s, input_file=%s, argc=%u\n", + data->out_dir, data->queue_dir, data->target, + data->input_file ? (char *)data->input_file : (char *)"", + data->argc); + + if (data->input_file) { setenv("SYMCC_INPUT_FILE", data->input_file, 1); } + + data->afl = afl; + data->seed = seed; + afl_struct = afl; + + if (debug) { + + fprintf(stderr, "["); + for (u32 i = 0; i <= data->argc; ++i) + fprintf(stderr, " \"%s\"", + data->argv[i] ? (char *)data->argv[i] : ""); + fprintf(stderr, " ]\n"); + + } + + OKF("Custom mutator symqemu loaded - note that the initial startup of " + "afl-fuzz will be delayed the more starting seeds are present. This is " + "fine, do not worry!"); + + return data; + +} + +/* When a new queue entry is added we run this input with the symqemu + instrumented binary */ +uint8_t afl_custom_queue_new_entry(my_mutator_t *data, + const uint8_t *filename_new_queue, + const uint8_t *filename_orig_queue) { + + int pipefd[2]; + struct stat st; + if (data->afl->afl_env.afl_no_ui) + ACTF("Sending to symqemu: %s", filename_new_queue); + u8 *fn = alloc_printf("%s", filename_new_queue); + if (!(stat(fn, &st) == 0 && S_ISREG(st.st_mode) && st.st_size)) { + + ck_free(fn); + PFATAL("Couldn't find enqueued file: %s", fn); + + } + + if (afl_struct->fsrv.use_stdin) { + + if (pipe(pipefd) == -1) { + + ck_free(fn); + PFATAL( + "Couldn't create a pipe for interacting with symqemu child process"); + + } + + } + + int fd = open(fn, O_RDONLY); + if (fd < 0) return 0; + ssize_t r = read(fd, data->mutator_buf, MAX_FILE); + DBG("fn=%s, fd=%d, size=%ld\n", fn, fd, r); + ck_free(fn); + close(fd); + + if (data->input_file) { + + fd = open(data->input_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); + ssize_t s = write(fd, data->mutator_buf, r); + close(fd); + DBG("wrote %zd/%zd to %s\n", s, r, data->input_file); + + } + + int pid = fork(); + + if (pid == -1) return 0; + + if (pid) { + + if (!data->input_file || afl_struct->fsrv.use_stdin) { + + close(pipefd[0]); + + if (fd >= 0) { + + if (r <= 0) { + + close(pipefd[1]); + return 0; + + } + + if (r > fcntl(pipefd[1], F_GETPIPE_SZ)) + fcntl(pipefd[1], F_SETPIPE_SZ, MAX_FILE); + ck_write(pipefd[1], data->mutator_buf, r, filename_new_queue); + + } else { + + ck_free(fn); + close(pipefd[1]); + PFATAL( + "Something happened to the enqueued file before sending its " + "contents to symqemu binary"); + + } + + close(pipefd[1]); + + } + + pid = waitpid(pid, NULL, 0); + DBG("symqemu finished executing!\n"); + + // At this point we need to transfer files to output dir, since their names + // collide and symqemu will just overwrite them + + struct dirent **nl; + int32_t items = scandir(data->out_dir, &nl, NULL, NULL); + u8 *origin_name = basename(filename_new_queue); + u8 source_name[4096], destination_name[4096]; + int32_t i; + + if (items > 0) { + + for (i = 0; i < (u32)items; ++i) { + + // symqemu output files start with a digit + if (!isdigit(nl[i]->d_name[0])) continue; + + struct stat st; + snprintf(source_name, sizeof(source_name), "%s/%s", data->out_dir, + nl[i]->d_name); + DBG("file=%s\n", source_name); + + if (stat(source_name, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + + snprintf(destination_name, sizeof(destination_name), "%s/id:%06u,%s", + data->queue_dir, data->counter++, nl[i]->d_name); + DBG("src=%s dst=%s\n", source_name, destination_name); + rename(source_name, destination_name); + + } + + free(nl[i]); + + } + + free(nl); + + } + + DBG("Done!\n"); + + } else /* (pid == 0) */ { // child + + if (afl_struct->fsrv.use_stdin) { + + close(pipefd[1]); + dup2(pipefd[0], 0); + + } + + DBG("exec=%s\n", data->target); + if (!debug) { + + close(1); + close(2); + dup2(afl_struct->fsrv.dev_null_fd, 1); + dup2(afl_struct->fsrv.dev_null_fd, 2); + + } + + execvp((char *)data->argv[0], (char **)data->argv); + fprintf(stderr, "Executing: ["); + for (u32 i = 0; i <= data->argc; ++i) + fprintf(stderr, " \"%s\"", + data->argv[i] ? (char *)data->argv[i] : ""); + fprintf(stderr, " ]\n"); + FATAL("Failed to execute %s %s\n", data->argv[0], data->argv[1]); + exit(-1); + + } + + return 0; + +} + +/* +uint32_t afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf, + size_t buf_size) { + + uint32_t count = 0, i; + struct dirent **nl; + int32_t items = scandir(data->out_dir, &nl, NULL, NULL); + + if (items > 0) { + + for (i = 0; i < (u32)items; ++i) { + + struct stat st; + u8 * fn = alloc_printf("%s/%s", data->out_dir, nl[i]->d_name); + DBG("test=%s\n", fn); + if (stat(fn, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + + DBG("found=%s\n", fn); + count++; + + } + + ck_free(fn); + free(nl[i]); + + } + + free(nl); + + } + + DBG("dir=%s, count=%u\n", data->out_dir, count); + return count; + +} + +*/ + +// here we actually just read the files generated from symqemu +/* +size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, + u8 **out_buf, uint8_t *add_buf, size_t add_buf_size, + size_t max_size) { + + struct dirent **nl; + int32_t i, done = 0, items = scandir(data->out_dir, &nl, NULL, NULL); + ssize_t size = 0; + + if (items <= 0) return 0; + + for (i = 0; i < (u32)items; ++i) { + + struct stat st; + u8 * fn = alloc_printf("%s/%s", data->out_dir, nl[i]->d_name); + + if (done == 0) { + + if (stat(fn, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + + int fd = open(fn, O_RDONLY); + + if (fd >= 0) { + + size = read(fd, data->mutator_buf, max_size); + *out_buf = data->mutator_buf; + + close(fd); + done = 1; + + } + + } + + unlink(fn); + + } + + ck_free(fn); + free(nl[i]); + + } + + free(nl); + DBG("FUZZ size=%lu\n", size); + return (uint32_t)size; + +} + +*/ + +/** + * Deinitialize everything + * + * @param data The data ptr from afl_custom_init + */ +void afl_custom_deinit(my_mutator_t *data) { + + free(data->mutator_buf); + free(data); + +} + diff --git a/docs/Changelog.md b/docs/Changelog.md index 3602af50..e99747f6 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -23,6 +23,9 @@ - qemu_mode: - Persistent mode +QASAN support for ppc32 tragets by @worksbutnottested - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! + - two new custom mutators are now available: + - TritonDSE in custom_mutators/aflpp_tritondse + - SymQEMU in custom_mutators/symqemu ### Version ++4.06c (release) diff --git a/instrumentation/afl-llvm-common.h b/instrumentation/afl-llvm-common.h index c9324460..23f67179 100644 --- a/instrumentation/afl-llvm-common.h +++ b/instrumentation/afl-llvm-common.h @@ -23,7 +23,7 @@ typedef long double max_align_t; #include "llvm/Support/Debug.h" #include "llvm/Support/MathExtras.h" #if LLVM_VERSION_MAJOR < 17 -#include "llvm/Transforms/IPO/PassManagerBuilder.h" + #include "llvm/Transforms/IPO/PassManagerBuilder.h" #endif #if LLVM_VERSION_MAJOR > 3 || \ diff --git a/test-instr.c b/test-instr.c index 1d9f2e6e..eda5189c 100644 --- a/test-instr.c +++ b/test-instr.c @@ -24,7 +24,7 @@ int main(int argc, char **argv) { - int fd = 0; + int fd = 0, cnt; char buff[8]; char *buf = buff; @@ -32,7 +32,6 @@ int main(int argc, char **argv) { if (argc == 2) { buf = argv[1]; - printf("Input %s - ", buf); } else { @@ -47,15 +46,19 @@ int main(int argc, char **argv) { } - if (read(fd, buf, sizeof(buf)) < 1) { + if ((cnt = read(fd, buf, sizeof(buf) - 1)) < 1) { printf("Hum?\n"); return 1; } + buf[cnt] = 0; + } + if (getenv("AFL_DEBUG")) fprintf(stderr, "test-instr: %s\n", buf); + // we support three input cases (plus a 4th if stdin is used but there is no // input) switch (buf[0]) { From 3e3adb4d377dcfa1e878ab00f061a894923bd642 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 17 May 2023 18:39:54 +0200 Subject: [PATCH 120/149] enforce python setting detection --- GNUmakefile | 34 +++++++++++++++---------------- custom_mutators/symqemu/README.md | 2 +- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 31374c10..bfc2aa03 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -180,13 +180,13 @@ AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) ifneq "$(shell command -v python3m 2>/dev/null)" "" ifneq "$(shell command -v python3m-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python3m-config --includes) - PYTHON_VERSION ?= $(strip $(shell python3m --version 2>&1)) + PYTHON_INCLUDE := $(shell python3m-config --includes) + PYTHON_VERSION := $(strip $(shell python3m --version 2>&1)) # Starting with python3.8, we need to pass the `embed` flag. Earlier versions didn't know this flag. ifeq "$(shell python3m-config --embed --libs 2>/dev/null | grep -q lpython && echo 1 )" "1" - PYTHON_LIB ?= $(shell python3m-config --libs --embed --ldflags) + PYTHON_LIB := $(shell python3m-config --libs --embed --ldflags) else - PYTHON_LIB ?= $(shell python3m-config --ldflags) + PYTHON_LIB := $(shell python3m-config --ldflags) endif endif endif @@ -194,13 +194,13 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python3 2>/dev/null)" "" ifneq "$(shell command -v python3-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python3-config --includes) - PYTHON_VERSION ?= $(strip $(shell python3 --version 2>&1)) + PYTHON_INCLUDE := $(shell python3-config --includes) + PYTHON_VERSION := $(strip $(shell python3 --version 2>&1)) # Starting with python3.8, we need to pass the `embed` flag. Earlier versions didn't know this flag. ifeq "$(shell python3-config --embed --libs 2>/dev/null | grep -q lpython && echo 1 )" "1" - PYTHON_LIB ?= $(shell python3-config --libs --embed --ldflags) + PYTHON_LIB := $(shell python3-config --libs --embed --ldflags) else - PYTHON_LIB ?= $(shell python3-config --ldflags) + PYTHON_LIB := $(shell python3-config --ldflags) endif endif endif @@ -209,9 +209,9 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python 2>/dev/null)" "" ifneq "$(shell command -v python-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python-config --includes) - PYTHON_LIB ?= $(shell python-config --ldflags) - PYTHON_VERSION ?= $(strip $(shell python --version 2>&1)) + PYTHON_INCLUDE := $(shell python-config --includes) + PYTHON_LIB := $(shell python-config --ldflags) + PYTHON_VERSION := $(strip $(shell python --version 2>&1)) endif endif endif @@ -220,9 +220,9 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python3.7 2>/dev/null)" "" ifneq "$(shell command -v python3.7-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python3.7-config --includes) - PYTHON_LIB ?= $(shell python3.7-config --ldflags) - PYTHON_VERSION ?= $(strip $(shell python3.7 --version 2>&1)) + PYTHON_INCLUDE := $(shell python3.7-config --includes) + PYTHON_LIB := $(shell python3.7-config --ldflags) + PYTHON_VERSION := $(strip $(shell python3.7 --version 2>&1)) endif endif endif @@ -231,9 +231,9 @@ endif ifeq "$(PYTHON_INCLUDE)" "" ifneq "$(shell command -v python2.7 2>/dev/null)" "" ifneq "$(shell command -v python2.7-config 2>/dev/null)" "" - PYTHON_INCLUDE ?= $(shell python2.7-config --includes) - PYTHON_LIB ?= $(shell python2.7-config --ldflags) - PYTHON_VERSION ?= $(strip $(shell python2.7 --version 2>&1)) + PYTHON_INCLUDE := $(shell python2.7-config --includes) + PYTHON_LIB := $(shell python2.7-config --ldflags) + PYTHON_VERSION := $(strip $(shell python2.7 --version 2>&1)) endif endif endif diff --git a/custom_mutators/symqemu/README.md b/custom_mutators/symqemu/README.md index 0d5cd4d7..55ce05c5 100644 --- a/custom_mutators/symqemu/README.md +++ b/custom_mutators/symqemu/README.md @@ -8,4 +8,4 @@ on how to build symqemu-x86_x64 and put it in your `PATH`. just type `make` to build this custom mutator. -```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/symqemu/symqemu-mutator.so AFL_SYNC_TIME=1 afl-fuzz ...``` +```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/symqemu/symqemu-mutator.so AFL_SYNC_TIME=1 AFL_DISABLE_TRIM=1 afl-fuzz ...``` From f664eb58c50ab9b6b130139613a35ff7a7297f1b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 17 May 2023 19:21:41 +0200 Subject: [PATCH 121/149] fix debug build --- GNUmakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index bfc2aa03..4ecdae52 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -139,13 +139,13 @@ endif ifdef DEBUG $(info Compiling DEBUG version of binaries) - override CFLAGS += -ggdb3 -O0 -Wall -Wextra -Werror $(CFLAGS_OPT) + override CFLAGS += -ggdb3 -O0 -Wall -Wextra -Werror -Wno-error=format-truncation= $(CFLAGS_OPT) else CFLAGS ?= -O2 $(CFLAGS_OPT) # -funroll-loops is slower on modern compilers endif override CFLAGS += -g -Wno-pointer-sign -Wno-variadic-macros -Wall -Wextra -Wno-pointer-arith \ - -fPIC -I include/ -DAFL_PATH=\"$(HELPER_PATH)\" \ + -fPIC -I include/ -DAFL_PATH=\"$(HELPER_PATH)\" -Wno-format-truncation \ -DBIN_PATH=\"$(BIN_PATH)\" -DDOC_PATH=\"$(DOC_PATH)\" # -fstack-protector From abd6eace9d767e4db6019e8eb69080d2352015c9 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 18 May 2023 10:32:15 +0200 Subject: [PATCH 122/149] improved symqemu custom mutator --- custom_mutators/symqemu/README.md | 2 +- custom_mutators/symqemu/symqemu.c | 241 ++++++++++++------------------ include/afl-fuzz.h | 1 + src/afl-fuzz-one.c | 1 + 4 files changed, 99 insertions(+), 146 deletions(-) diff --git a/custom_mutators/symqemu/README.md b/custom_mutators/symqemu/README.md index 55ce05c5..b7702c06 100644 --- a/custom_mutators/symqemu/README.md +++ b/custom_mutators/symqemu/README.md @@ -8,4 +8,4 @@ on how to build symqemu-x86_x64 and put it in your `PATH`. just type `make` to build this custom mutator. -```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/symqemu/symqemu-mutator.so AFL_SYNC_TIME=1 AFL_DISABLE_TRIM=1 afl-fuzz ...``` +```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/symqemu/symqemu-mutator.so AFL_DISABLE_TRIM=1 afl-fuzz ...``` diff --git a/custom_mutators/symqemu/symqemu.c b/custom_mutators/symqemu/symqemu.c index 9030397b..163ae240 100644 --- a/custom_mutators/symqemu/symqemu.c +++ b/custom_mutators/symqemu/symqemu.c @@ -13,6 +13,9 @@ afl_state_t *afl_struct; static u32 debug = 0; +static u32 found_items = 0; + +#define SYMQEMU_LOCATION "symqemu" #define DBG(x...) \ if (debug) { fprintf(stderr, x); } @@ -22,7 +25,6 @@ typedef struct my_mutator { afl_state_t *afl; u8 *mutator_buf; u8 *out_dir; - u8 *queue_dir; u8 *target; u8 *symqemu; u8 *input_file; @@ -67,8 +69,13 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { if (!data->symqemu) FATAL("symqemu binary %s not found", exec_name); DBG("Found %s\n", data->symqemu); - if (getenv("AFL_CUSTOM_MUTATOR_ONLY")) - FATAL("the symqemu module cannot be used with AFL_CUSTOM_MUTATOR_ONLY."); + if (getenv("AFL_CUSTOM_MUTATOR_ONLY")) { + + WARNF( + "the symqemu module is not very effective with " + "AFL_CUSTOM_MUTATOR_ONLY."); + + } if ((data->mutator_buf = malloc(MAX_FILE)) == NULL) { @@ -84,14 +91,11 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { u32 len = strlen(path_tmp) + 32; u8 *symqemu_path = malloc(len); data->out_dir = malloc(len); - data->queue_dir = malloc(len); - snprintf(symqemu_path, len, "%s/../symqemu", path_tmp); - snprintf(data->out_dir, len, "%s/../symqemu/out", path_tmp); - snprintf(data->queue_dir, len, "%s/../symqemu/queue", path_tmp); + snprintf(symqemu_path, len, "%s/%s", path_tmp, SYMQEMU_LOCATION); + snprintf(data->out_dir, len, "%s/out", symqemu_path, path_tmp); - mkdir(symqemu_path, 0755); - mkdir(data->out_dir, 0755); - mkdir(data->queue_dir, 0755); + (void)mkdir(symqemu_path, 0755); + (void)mkdir(data->out_dir, 0755); setenv("SYMCC_OUTPUT_DIR", data->out_dir, 1); @@ -153,8 +157,8 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { data->argv[0] = data->symqemu; data->argv[1] = data->target; - DBG("out_dir=%s, queue_dir=%s, target=%s, input_file=%s, argc=%u\n", - data->out_dir, data->queue_dir, data->target, + DBG("out_dir=%s, target=%s, input_file=%s, argc=%u\n", data->out_dir, + data->target, data->input_file ? (char *)data->input_file : (char *)"", data->argc); @@ -174,29 +178,39 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { } - OKF("Custom mutator symqemu loaded - note that the initial startup of " - "afl-fuzz will be delayed the more starting seeds are present. This is " - "fine, do not worry!"); - return data; } -/* When a new queue entry is added we run this input with the symqemu - instrumented binary */ -uint8_t afl_custom_queue_new_entry(my_mutator_t *data, - const uint8_t *filename_new_queue, - const uint8_t *filename_orig_queue) { +/* No need to receive a splicing item */ +void afl_custom_splice_optout(void *data) { + + (void)(data); + +} + +u32 afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf, size_t buf_size) { + + if (likely(!afl_struct->queue_cur->favored || + afl_struct->queue_cur->was_fuzzed)) { + + return 0; + + } int pipefd[2]; struct stat st; - if (data->afl->afl_env.afl_no_ui) - ACTF("Sending to symqemu: %s", filename_new_queue); - u8 *fn = alloc_printf("%s", filename_new_queue); - if (!(stat(fn, &st) == 0 && S_ISREG(st.st_mode) && st.st_size)) { - ck_free(fn); - PFATAL("Couldn't find enqueued file: %s", fn); + if (afl_struct->afl_env.afl_no_ui) { + + ACTF("Sending to symqemu: %s", afl_struct->queue_cur->fname); + + } + + if (!(stat(afl_struct->queue_cur->fname, &st) == 0 && S_ISREG(st.st_mode) && + st.st_size)) { + + PFATAL("Couldn't find enqueued file: %s", afl_struct->queue_cur->fname); } @@ -204,7 +218,6 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t *data, if (pipe(pipefd) == -1) { - ck_free(fn); PFATAL( "Couldn't create a pipe for interacting with symqemu child process"); @@ -212,19 +225,12 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t *data, } - int fd = open(fn, O_RDONLY); - if (fd < 0) return 0; - ssize_t r = read(fd, data->mutator_buf, MAX_FILE); - DBG("fn=%s, fd=%d, size=%ld\n", fn, fd, r); - ck_free(fn); - close(fd); - if (data->input_file) { - fd = open(data->input_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); - ssize_t s = write(fd, data->mutator_buf, r); + int fd = open(data->input_file, O_WRONLY | O_CREAT | O_TRUNC, 0644); + ssize_t s = write(fd, buf, buf_size); close(fd); - DBG("wrote %zd/%zd to %s\n", s, r, data->input_file); + DBG("wrote %zd/%zd to %s\n", s, buf_size, data->input_file); } @@ -232,35 +238,20 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t *data, if (pid == -1) return 0; - if (pid) { + if (likely(pid)) { if (!data->input_file || afl_struct->fsrv.use_stdin) { close(pipefd[0]); - if (fd >= 0) { + if (fcntl(pipefd[1], F_GETPIPE_SZ)) { - if (r <= 0) { - - close(pipefd[1]); - return 0; - - } - - if (r > fcntl(pipefd[1], F_GETPIPE_SZ)) - fcntl(pipefd[1], F_SETPIPE_SZ, MAX_FILE); - ck_write(pipefd[1], data->mutator_buf, r, filename_new_queue); - - } else { - - ck_free(fn); - close(pipefd[1]); - PFATAL( - "Something happened to the enqueued file before sending its " - "contents to symqemu binary"); + fcntl(pipefd[1], F_SETPIPE_SZ, MAX_FILE); } + ck_write(pipefd[1], buf, buf_size, data->input_file); + close(pipefd[1]); } @@ -268,46 +259,6 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t *data, pid = waitpid(pid, NULL, 0); DBG("symqemu finished executing!\n"); - // At this point we need to transfer files to output dir, since their names - // collide and symqemu will just overwrite them - - struct dirent **nl; - int32_t items = scandir(data->out_dir, &nl, NULL, NULL); - u8 *origin_name = basename(filename_new_queue); - u8 source_name[4096], destination_name[4096]; - int32_t i; - - if (items > 0) { - - for (i = 0; i < (u32)items; ++i) { - - // symqemu output files start with a digit - if (!isdigit(nl[i]->d_name[0])) continue; - - struct stat st; - snprintf(source_name, sizeof(source_name), "%s/%s", data->out_dir, - nl[i]->d_name); - DBG("file=%s\n", source_name); - - if (stat(source_name, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { - - snprintf(destination_name, sizeof(destination_name), "%s/id:%06u,%s", - data->queue_dir, data->counter++, nl[i]->d_name); - DBG("src=%s dst=%s\n", source_name, destination_name); - rename(source_name, destination_name); - - } - - free(nl[i]); - - } - - free(nl); - - } - - DBG("Done!\n"); - } else /* (pid == 0) */ { // child if (afl_struct->fsrv.use_stdin) { @@ -338,33 +289,31 @@ uint8_t afl_custom_queue_new_entry(my_mutator_t *data, } - return 0; + /* back in mother process */ -} - -/* -uint32_t afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf, - size_t buf_size) { - - uint32_t count = 0, i; struct dirent **nl; - int32_t items = scandir(data->out_dir, &nl, NULL, NULL); + s32 i, items = scandir(data->out_dir, &nl, NULL, NULL); + found_items = 0; + char source_name[4096]; if (items > 0) { for (i = 0; i < (u32)items; ++i) { - struct stat st; - u8 * fn = alloc_printf("%s/%s", data->out_dir, nl[i]->d_name); - DBG("test=%s\n", fn); - if (stat(fn, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + // symqemu output files start with a digit + if (!isdigit(nl[i]->d_name[0])) continue; - DBG("found=%s\n", fn); - count++; + struct stat st; + snprintf(source_name, sizeof(source_name), "%s/%s", data->out_dir, + nl[i]->d_name); + DBG("file=%s\n", source_name); + + if (stat(source_name, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + + ++found_items; } - ck_free(fn); free(nl[i]); } @@ -373,65 +322,67 @@ uint32_t afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf, } - DBG("dir=%s, count=%u\n", data->out_dir, count); - return count; + DBG("Done, found %u items!\n", found_items); + + return found_items; } -*/ - -// here we actually just read the files generated from symqemu -/* -size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size, - u8 **out_buf, uint8_t *add_buf, size_t add_buf_size, +size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size, + u8 **out_buf, u8 *add_buf, size_t add_buf_size, size_t max_size) { struct dirent **nl; - int32_t i, done = 0, items = scandir(data->out_dir, &nl, NULL, NULL); - ssize_t size = 0; + s32 done = 0, i, items = scandir(data->out_dir, &nl, NULL, NULL); + char source_name[4096]; - if (items <= 0) return 0; + if (items > 0) { - for (i = 0; i < (u32)items; ++i) { + for (i = 0; i < (u32)items; ++i) { - struct stat st; - u8 * fn = alloc_printf("%s/%s", data->out_dir, nl[i]->d_name); + // symqemu output files start with a digit + if (!isdigit(nl[i]->d_name[0])) continue; - if (done == 0) { + struct stat st; + snprintf(source_name, sizeof(source_name), "%s/%s", data->out_dir, + nl[i]->d_name); + DBG("file=%s\n", source_name); - if (stat(fn, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { + if (stat(source_name, &st) == 0 && S_ISREG(st.st_mode) && st.st_size) { - int fd = open(fn, O_RDONLY); + int fd = open(source_name, O_RDONLY); + if (fd < 0) { goto got_an_issue; } - if (fd >= 0) { + ssize_t r = read(fd, data->mutator_buf, MAX_FILE); + close(fd); - size = read(fd, data->mutator_buf, max_size); - *out_buf = data->mutator_buf; + DBG("fn=%s, fd=%d, size=%ld\n", source_name, fd, r); - close(fd); - done = 1; + if (r < 1) { goto got_an_issue; } - } + done = 1; + --found_items; + unlink(source_name); + + *out_buf = data->mutator_buf; + return (u32)r; } - unlink(fn); + free(nl[i]); } - ck_free(fn); - free(nl[i]); + free(nl); } - free(nl); - DBG("FUZZ size=%lu\n", size); - return (uint32_t)size; +got_an_issue: + *out_buf = NULL; + return 0; } -*/ - /** * Deinitialize everything * diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 8fb7ecb1..beb2de2a 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -184,6 +184,7 @@ struct queue_entry { handicap, /* Number of queue cycles behind */ depth, /* Path depth */ exec_cksum, /* Checksum of the execution trace */ + custom, /* Marker for custom mutators */ stats_mutated; /* stats: # of mutations performed */ u8 *trace_mini; /* Trace bytes, if kept */ diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index c6e9a295..5c71fc59 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -1912,6 +1912,7 @@ custom_mutator_stage: afl->stage_name = "custom mutator"; afl->stage_short = "custom"; + afl->stage_cur = 0; afl->stage_val_type = STAGE_VAL_NONE; bool has_custom_fuzz = false; u32 shift = unlikely(afl->custom_only) ? 7 : 8; From 401d7617efbd2f38d9132eabfd1b1152abceda52 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 18 May 2023 10:50:10 +0200 Subject: [PATCH 123/149] symqemu mutator options --- custom_mutators/aflpp_tritondse/README.md | 7 ++-- custom_mutators/symqemu/README.md | 10 +++++- custom_mutators/symqemu/symqemu.c | 44 ++++++++++++++++++----- src/afl-common.c | 2 +- 4 files changed, 51 insertions(+), 12 deletions(-) diff --git a/custom_mutators/aflpp_tritondse/README.md b/custom_mutators/aflpp_tritondse/README.md index 8a5dd02b..608c2624 100644 --- a/custom_mutators/aflpp_tritondse/README.md +++ b/custom_mutators/aflpp_tritondse/README.md @@ -10,8 +10,11 @@ ../../afl-cc -o ../../test-instr ../../test-instr.c mkdir -p in echo aaaa > in/in -TRITON_DSE_TARGET=../../test-instr AFL_CUSTOM_MUTATOR_ONLY=1 AFL_SYNC_TIME=1 AFL_PYTHON_MODULE=aflpp_tritondse PYTHONPATH=. ../../afl-fuzz -i in -o out -- ../../test-instr +AFL_DISABLE_TRIM=1 AFL_CUSTOM_MUTATOR_ONLY=1 AFL_SYNC_TIME=1 AFL_PYTHON_MODULE=aflpp_tritondse PYTHONPATH=. ../../afl-fuzz -i in -o out -- ../../test-instr ``` Note that this custom mutator works differently, new finds are synced -after 10-60 seconds to the fuzzing instance. +after 10-60 seconds to the fuzzing instance. This is necessary because only +C/C++ mutators have access to the internal AFL++ state. + +Hence the symqemu customer mutator is more effective. diff --git a/custom_mutators/symqemu/README.md b/custom_mutators/symqemu/README.md index b7702c06..c3071afc 100644 --- a/custom_mutators/symqemu/README.md +++ b/custom_mutators/symqemu/README.md @@ -2,10 +2,18 @@ This uses the symcc to find new paths into the target. +## How to build and use + To use this custom mutator follow the steps in the symqemu repository [https://github.com/eurecom-s3/symqemu/](https://github.com/eurecom-s3/symqemu/) on how to build symqemu-x86_x64 and put it in your `PATH`. -just type `make` to build this custom mutator. +Just type `make` to build this custom mutator. ```AFL_CUSTOM_MUTATOR_LIBRARY=custom_mutators/symqemu/symqemu-mutator.so AFL_DISABLE_TRIM=1 afl-fuzz ...``` + +## Options + +`SYMQEMU_ALL=1` - use concolic solving on **all** queue items, not only interesting/favorite ones. + +`SYMQEMU_LATE=1` - use concolic solving only after there have been no finds for 5 minutes. diff --git a/custom_mutators/symqemu/symqemu.c b/custom_mutators/symqemu/symqemu.c index 163ae240..e2b07af6 100644 --- a/custom_mutators/symqemu/symqemu.c +++ b/custom_mutators/symqemu/symqemu.c @@ -23,6 +23,8 @@ static u32 found_items = 0; typedef struct my_mutator { afl_state_t *afl; + u32 all; + u32 late; u8 *mutator_buf; u8 *out_dir; u8 *target; @@ -156,18 +158,19 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) { data->argv[0] = data->symqemu; data->argv[1] = data->target; + data->afl = afl; + data->seed = seed; + afl_struct = afl; + + if (getenv("SYMQEMU_ALL")) { data->all = 1; } + if (getenv("SYMQEMU_LATE")) { data->late = 1; } + if (data->input_file) { setenv("SYMCC_INPUT_FILE", data->input_file, 1); } DBG("out_dir=%s, target=%s, input_file=%s, argc=%u\n", data->out_dir, data->target, data->input_file ? (char *)data->input_file : (char *)"", data->argc); - if (data->input_file) { setenv("SYMCC_INPUT_FILE", data->input_file, 1); } - - data->afl = afl; - data->seed = seed; - afl_struct = afl; - if (debug) { fprintf(stderr, "["); @@ -189,15 +192,40 @@ void afl_custom_splice_optout(void *data) { } +/* Get unix time in milliseconds */ + +inline u64 get_cur_time(void) { + + struct timeval tv; + struct timezone tz; + + gettimeofday(&tv, &tz); + + return (tv.tv_sec * 1000ULL) + (tv.tv_usec / 1000); + +} + u32 afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf, size_t buf_size) { - if (likely(!afl_struct->queue_cur->favored || - afl_struct->queue_cur->was_fuzzed)) { + if (likely((!afl_struct->queue_cur->favored || + afl_struct->queue_cur->was_fuzzed) && + !data->all)) { return 0; } + if (likely(data->late)) { + + if (unlikely(get_cur_time() - afl_struct->last_find_time <= + 10 * 60 * 1000)) { + + return 0; + + } + + } + int pipefd[2]; struct stat st; diff --git a/src/afl-common.c b/src/afl-common.c index a5c48e80..84ddefd8 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -949,7 +949,7 @@ void read_bitmap(u8 *fname, u8 *map, size_t len) { /* Get unix time in milliseconds */ -u64 get_cur_time(void) { +inline u64 get_cur_time(void) { struct timeval tv; struct timezone tz; From eec2c38a6891ea317a287498ce1da2f2c8f6f9ff Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 18 May 2023 12:29:43 +0200 Subject: [PATCH 124/149] symqemu fix --- custom_mutators/symqemu/symqemu.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/custom_mutators/symqemu/symqemu.c b/custom_mutators/symqemu/symqemu.c index e2b07af6..73a1640a 100644 --- a/custom_mutators/symqemu/symqemu.c +++ b/custom_mutators/symqemu/symqemu.c @@ -207,9 +207,8 @@ inline u64 get_cur_time(void) { u32 afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf, size_t buf_size) { - if (likely((!afl_struct->queue_cur->favored || - afl_struct->queue_cur->was_fuzzed) && - !data->all)) { + if (likely((!afl_struct->queue_cur->favored && !data->all) || + afl_struct->queue_cur->was_fuzzed)) { return 0; From 53a869b757287e8bebdfcbc96b8abe1729955171 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 18 May 2023 14:45:45 +0200 Subject: [PATCH 125/149] act on invalid AFL_CUSTOM_MUTATOR_ONLY usage --- src/afl-fuzz.c | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 4339ddd2..e2d8dea5 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1748,6 +1748,23 @@ int main(int argc, char **argv_orig, char **envp) { if (afl->afl_env.afl_custom_mutator_only) { + if (!afl->custom_mutators_count) { + + if (afl->shm.cmplog_mode) { + + WARNF( + "No custom mutator loaded, using AFL_CUSTOM_MUTATOR_ONLY is " + "pointless and only allowed now to allow experiments with CMPLOG."); + + } else { + + FATAL( + "No custom mutator loaded but AFL_CUSTOM_MUTATOR_ONLY specified."); + + } + + } + /* This ensures we don't proceed to havoc/splice */ afl->custom_only = 1; From 9a6c0ec0c0af42d33e4350ee2958b58fef1c39dd Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 21 May 2023 13:04:17 +0200 Subject: [PATCH 126/149] make AFL_CUSTOM_INFO overridable --- custom_mutators/symqemu/Makefile | 2 +- src/afl-fuzz.c | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/custom_mutators/symqemu/Makefile b/custom_mutators/symqemu/Makefile index 3361ab0f..958aec19 100644 --- a/custom_mutators/symqemu/Makefile +++ b/custom_mutators/symqemu/Makefile @@ -8,7 +8,7 @@ all: symqemu-mutator.so CFLAGS += -O3 -funroll-loops symqemu-mutator.so: symqemu.c - $(CC) $(CFLAGS) $(CPPFLAGS) -g -I../../include -shared -fPIC -o symqemu-mutator.so symqemu.c + $(CC) -g $(CFLAGS) $(CPPFLAGS) -g -I../../include -shared -fPIC -o symqemu-mutator.so symqemu.c clean: rm -f symqemu-mutator.so *.o *~ core diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index e2d8dea5..a61718a7 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1833,7 +1833,7 @@ int main(int argc, char **argv_orig, char **envp) { } - { + if (!getenv("AFL_CUSTOM_INFO_PROGRAM_ARGV")) { u8 envbuf[8096] = "", tmpbuf[8096] = ""; for (s32 i = optind + 1; i < argc; ++i) { @@ -1864,7 +1864,11 @@ int main(int argc, char **argv_orig, char **envp) { } - setenv("AFL_CUSTOM_INFO_OUT", afl->out_dir, 1); // same as __AFL_OUT_DIR + if (!getenv("AFL_CUSTOM_INFO_OUT") { + + setenv("AFL_CUSTOM_INFO_OUT", afl->out_dir, 1); // same as __AFL_OUT_DIR + + } setup_custom_mutators(afl); From d4085314c1c1d4e8bbe4159216f8cb83e0804ea7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 21 May 2023 13:44:07 +0200 Subject: [PATCH 127/149] fix --- src/afl-fuzz.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index a61718a7..559a7326 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1864,7 +1864,7 @@ int main(int argc, char **argv_orig, char **envp) { } - if (!getenv("AFL_CUSTOM_INFO_OUT") { + if (!getenv("AFL_CUSTOM_INFO_OUT")) { setenv("AFL_CUSTOM_INFO_OUT", afl->out_dir, 1); // same as __AFL_OUT_DIR From 1416fea1604a19408554678d7c9fb35b67da302b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 21 May 2023 14:49:24 +0200 Subject: [PATCH 128/149] cleaner tritondse --- custom_mutators/aflpp_tritondse/README.md | 6 ++++-- custom_mutators/aflpp_tritondse/aflpp_tritondse.py | 4 ++++ 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/custom_mutators/aflpp_tritondse/README.md b/custom_mutators/aflpp_tritondse/README.md index 608c2624..033655d2 100644 --- a/custom_mutators/aflpp_tritondse/README.md +++ b/custom_mutators/aflpp_tritondse/README.md @@ -15,6 +15,8 @@ AFL_DISABLE_TRIM=1 AFL_CUSTOM_MUTATOR_ONLY=1 AFL_SYNC_TIME=1 AFL_PYTHON_MODULE=a Note that this custom mutator works differently, new finds are synced after 10-60 seconds to the fuzzing instance. This is necessary because only -C/C++ mutators have access to the internal AFL++ state. +C/C++ custom mutators have access to the internal AFL++ state. -Hence the symqemu customer mutator is more effective. +Note that you should run first with `AFL_DEBUG` for 5-10 minutes and see if +all important libraries and syscalls are hooked (look at `WARNING` and `CRITICAL` +output during the run, best use with `AFL_NO_UI=1`) diff --git a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py index cef28f34..58b506b6 100644 --- a/custom_mutators/aflpp_tritondse/aflpp_tritondse.py +++ b/custom_mutators/aflpp_tritondse/aflpp_tritondse.py @@ -120,6 +120,10 @@ def init(seed): is_debug = True except KeyError: pass + if is_debug: + logging.basicConfig(level=logging.WARNING) + else: + logging.basicConfig(level=logging.CRITICAL) try: foo = os.environ['AFL_CUSTOM_INFO_OUT'] out_path = foo + '/../tritondse/queue' From 029e039cbcbf9e30f35eb255162679b8d609d25d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 21 May 2023 17:49:14 +0200 Subject: [PATCH 129/149] code format --- .custom-format.py | 2 +- frida_mode/src/lib/lib_apple.c | 4 +-- include/config.h | 2 +- instrumentation/SanitizerCoverageLTO.so.cc | 4 +-- .../SanitizerCoveragePCGUARD.so.cc | 4 +-- instrumentation/afl-compiler-rt.o.c | 14 ++++---- .../afl-llvm-lto-instrumentlist.so.cc | 2 +- instrumentation/cmplog-routines-pass.cc | 4 +-- instrumentation/compare-transform-pass.so.cc | 2 +- instrumentation/split-compares-pass.so.cc | 2 +- qemu_mode/libcompcov/libcompcov.so.c | 6 +++- src/afl-cc.c | 8 ++--- src/afl-fuzz-redqueen.c | 32 +++++++++---------- utils/afl_untracer/afl-untracer.c | 2 +- utils/libtokencap/libtokencap.so.c | 6 +++- utils/socket_fuzzing/socketfuzz.c | 3 +- 16 files changed, 53 insertions(+), 44 deletions(-) diff --git a/.custom-format.py b/.custom-format.py index 1295ce55..1d5c8839 100755 --- a/.custom-format.py +++ b/.custom-format.py @@ -24,7 +24,7 @@ import importlib.metadata # string_re = re.compile('(\\"(\\\\.|[^"\\\\])*\\")') # TODO: for future use -CURRENT_LLVM = os.getenv('LLVM_VERSION', 14) +CURRENT_LLVM = os.getenv('LLVM_VERSION', 15) CLANG_FORMAT_BIN = os.getenv("CLANG_FORMAT_BIN", "") diff --git a/frida_mode/src/lib/lib_apple.c b/frida_mode/src/lib/lib_apple.c index 634e0e30..d29d0303 100644 --- a/frida_mode/src/lib/lib_apple.c +++ b/frida_mode/src/lib/lib_apple.c @@ -17,8 +17,8 @@ static gboolean lib_get_main_module(const GumModuleDetails *details, GumDarwinModule **ret = (GumDarwinModule **)user_data; GumDarwinModule *module = gum_darwin_module_new_from_memory( - details->path, mach_task_self(), details->range->base_address, - GUM_DARWIN_MODULE_FLAGS_NONE, NULL); + details->path, mach_task_self(), details->range->base_address, + GUM_DARWIN_MODULE_FLAGS_NONE, NULL); FVERBOSE("Found main module: %s", module->name); diff --git a/include/config.h b/include/config.h index 764c29dc..194786f7 100644 --- a/include/config.h +++ b/include/config.h @@ -81,7 +81,7 @@ will be kept and written to the crash/ directory as RECORD:... files. Note that every crash will be written, not only unique ones! */ -//#define AFL_PERSISTENT_RECORD +// #define AFL_PERSISTENT_RECORD /* console output colors: There are three ways to configure its behavior * 1. default: colored outputs fixed on: defined USE_COLOR && defined diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index b3b0d2cd..d7b03634 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -1478,8 +1478,8 @@ GlobalVariable *ModuleSanitizerCoverageLTO::CreateFunctionLocalArrayInSection( ArrayType *ArrayTy = ArrayType::get(Ty, NumElements); auto Array = new GlobalVariable( - *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, - Constant::getNullValue(ArrayTy), "__sancov_gen_"); + *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, + Constant::getNullValue(ArrayTy), "__sancov_gen_"); #if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 41c38283..8fed2042 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -714,8 +714,8 @@ GlobalVariable *ModuleSanitizerCoverageAFL::CreateFunctionLocalArrayInSection( ArrayType *ArrayTy = ArrayType::get(Ty, NumElements); auto Array = new GlobalVariable( - *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, - Constant::getNullValue(ArrayTy), "__sancov_gen_"); + *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, + Constant::getNullValue(ArrayTy), "__sancov_gen_"); #if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 5372fae0..3f8b519b 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -544,12 +544,12 @@ static void __afl_map_shm(void) { if (__afl_map_size && __afl_map_size > MAP_SIZE) { - u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); - if (!map_env || atoi((char *)map_env) < MAP_SIZE) { + u8 *map_env = (u8 *)getenv("AFL_MAP_SIZE"); + if (!map_env || atoi((char *)map_env) < MAP_SIZE) { - fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); - send_forkserver_error(FS_ERROR_MAP_SIZE); - _exit(1); + fprintf(stderr, "FS_ERROR_MAP_SIZE\n"); + send_forkserver_error(FS_ERROR_MAP_SIZE); + _exit(1); } @@ -561,13 +561,13 @@ static void __afl_map_shm(void) { if (!__afl_area_ptr || __afl_area_ptr == (void *)-1) { - if (__afl_map_addr) + if (__afl_map_addr) send_forkserver_error(FS_ERROR_MAP_ADDR); else send_forkserver_error(FS_ERROR_SHMAT); perror("shmat for map"); - _exit(1); + _exit(1); } diff --git a/instrumentation/afl-llvm-lto-instrumentlist.so.cc b/instrumentation/afl-llvm-lto-instrumentlist.so.cc index db5bd55e..61f97d77 100644 --- a/instrumentation/afl-llvm-lto-instrumentlist.so.cc +++ b/instrumentation/afl-llvm-lto-instrumentlist.so.cc @@ -45,7 +45,7 @@ #include "llvm/IR/Module.h" #include "llvm/Pass.h" #include "llvm/Support/Debug.h" -//#include "llvm/Transforms/IPO/PassManagerBuilder.h" +// #include "llvm/Transforms/IPO/PassManagerBuilder.h" #include "llvm/Passes/PassPlugin.h" #include "llvm/Passes/PassBuilder.h" #include "llvm/IR/PassManager.h" diff --git a/instrumentation/cmplog-routines-pass.cc b/instrumentation/cmplog-routines-pass.cc index 39db5aa4..c3fbed8d 100644 --- a/instrumentation/cmplog-routines-pass.cc +++ b/instrumentation/cmplog-routines-pass.cc @@ -542,7 +542,7 @@ bool CmpLogRoutines::hookRtns(Module &M) { Value *v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); Value *v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); Value *v3Pbitcast = IRB.CreateBitCast( - v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); Value *v3Pcasted = IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); args.push_back(v1Pcasted); @@ -608,7 +608,7 @@ bool CmpLogRoutines::hookRtns(Module &M) { Value *v1Pcasted = IRB.CreatePointerCast(v1P, i8PtrTy); Value *v2Pcasted = IRB.CreatePointerCast(v2P, i8PtrTy); Value *v3Pbitcast = IRB.CreateBitCast( - v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); + v3P, IntegerType::get(C, v3P->getType()->getPrimitiveSizeInBits())); Value *v3Pcasted = IRB.CreateIntCast(v3Pbitcast, IntegerType::get(C, 64), false); args.push_back(v1Pcasted); diff --git a/instrumentation/compare-transform-pass.so.cc b/instrumentation/compare-transform-pass.so.cc index efc99d20..5dd705cf 100644 --- a/instrumentation/compare-transform-pass.so.cc +++ b/instrumentation/compare-transform-pass.so.cc @@ -623,7 +623,7 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, IRBuilder<> cur_lenchk_IRB(&*(cur_lenchk_bb->getFirstInsertionPt())); Value *icmp = cur_lenchk_IRB.CreateICmpEQ( - sizedValue, ConstantInt::get(sizedValue->getType(), i)); + sizedValue, ConstantInt::get(sizedValue->getType(), i)); cur_lenchk_IRB.CreateCondBr(icmp, end_bb, cur_cmp_bb); cur_lenchk_bb->getTerminator()->eraseFromParent(); diff --git a/instrumentation/split-compares-pass.so.cc b/instrumentation/split-compares-pass.so.cc index 8a07610c..aec6758e 100644 --- a/instrumentation/split-compares-pass.so.cc +++ b/instrumentation/split-compares-pass.so.cc @@ -60,7 +60,7 @@ using namespace llvm; // uncomment this toggle function verification at each step. horribly slow, but // helps to pinpoint a potential problem in the splitting code. -//#define VERIFY_TOO_MUCH 1 +// #define VERIFY_TOO_MUCH 1 namespace { diff --git a/qemu_mode/libcompcov/libcompcov.so.c b/qemu_mode/libcompcov/libcompcov.so.c index b6ee0019..b57e9701 100644 --- a/qemu_mode/libcompcov/libcompcov.so.c +++ b/qemu_mode/libcompcov/libcompcov.so.c @@ -68,7 +68,11 @@ static int debug_fd = -1; #define MAX_MAPPINGS 1024 -static struct mapping { void *st, *en; } __compcov_ro[MAX_MAPPINGS]; +static struct mapping { + + void *st, *en; + +} __compcov_ro[MAX_MAPPINGS]; static u32 __compcov_ro_cnt; diff --git a/src/afl-cc.c b/src/afl-cc.c index 972ac8cd..e3cc04dd 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -933,10 +933,10 @@ static void edit_params(u32 argc, char **argv, char **envp) { } - //#if LLVM_MAJOR >= 13 - // // Use the old pass manager in LLVM 14 which the AFL++ passes still - // use. cc_params[cc_par_cnt++] = "-flegacy-pass-manager"; - //#endif + // #if LLVM_MAJOR >= 13 + // // Use the old pass manager in LLVM 14 which the AFL++ passes still + // use. cc_params[cc_par_cnt++] = "-flegacy-pass-manager"; + // #endif if (lto_mode && !have_c) { diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 6e4a655b..d9dc50df 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -28,8 +28,8 @@ #include "afl-fuzz.h" #include "cmplog.h" -//#define _DEBUG -//#define CMPLOG_INTROSPECTION +// #define _DEBUG +// #define CMPLOG_INTROSPECTION // CMP attribute enum enum { @@ -571,7 +571,7 @@ static u8 its_fuzz(afl_state_t *afl, u8 *buf, u32 len, u8 *status) { } -//#ifdef CMPLOG_SOLVE_TRANSFORM +// #ifdef CMPLOG_SOLVE_TRANSFORM static int strntoll(const char *str, size_t sz, char **end, int base, long long *out) { @@ -771,7 +771,7 @@ static void to_base64(u8 *src, u8 *dst, u32 dst_len) { #endif -//#endif +// #endif static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u64 pattern, u64 repl, u64 o_pattern, @@ -803,8 +803,8 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, // o_pattern, pattern, repl, changed_val, idx, taint_len, // hshape, attr); - //#ifdef CMPLOG_SOLVE_TRANSFORM - // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 + // #ifdef CMPLOG_SOLVE_TRANSFORM + // reverse atoi()/strnu?toll() is expensive, so we only to it in lvl 3 if (afl->cmplog_enable_transform && (lvl & LVL3)) { u8 *endptr; @@ -1120,7 +1120,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - //#endif + // #endif // we only allow this for ascii2integer (above) so leave if this is the case if (unlikely(pattern == o_pattern)) { return 0; } @@ -1275,7 +1275,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, // 16 = modified float, 32 = modified integer (modified = wont match // in original buffer) - //#ifdef CMPLOG_SOLVE_ARITHMETIC + // #ifdef CMPLOG_SOLVE_ARITHMETIC if (!afl->cmplog_enable_arith || lvl < LVL3 || attr == IS_TRANSFORM) { return 0; @@ -1440,8 +1440,8 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, } - //#endif /* - // CMPLOG_SOLVE_ARITHMETIC + // #endif /* + // CMPLOG_SOLVE_ARITHMETIC return 0; @@ -1948,9 +1948,9 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, #ifndef CMPLOG_COMBINE (void)(cbuf); #endif - //#ifndef CMPLOG_SOLVE_TRANSFORM - // (void)(changed_val); - //#endif + // #ifndef CMPLOG_SOLVE_TRANSFORM + // (void)(changed_val); + // #endif if (afl->fsrv.total_execs - last_update > screen_update) { @@ -2418,7 +2418,7 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - //#endif + // #endif return 0; @@ -2818,9 +2818,9 @@ u8 input_to_state_stage(afl_state_t *afl, u8 *orig_buf, u8 *buf, u32 len) { } else if ((lvl & LVL1) - //#ifdef CMPLOG_SOLVE_TRANSFORM + // #ifdef CMPLOG_SOLVE_TRANSFORM || ((lvl & LVL3) && afl->cmplog_enable_transform) - //#endif + // #endif ) { if (unlikely(rtn_fuzz(afl, k, orig_buf, buf, cbuf, len, lvl, taint))) { diff --git a/utils/afl_untracer/afl-untracer.c b/utils/afl_untracer/afl-untracer.c index a18e314e..e1038212 100644 --- a/utils/afl_untracer/afl-untracer.c +++ b/utils/afl_untracer/afl-untracer.c @@ -288,7 +288,7 @@ library_list_t *find_library(char *name) { #pragma GCC optimize("O0") void breakpoint(void) { - if (debug) fprintf(stderr, "Breakpoint function \"breakpoint\" reached.\n"); + if (debug) fprintf(stderr, "Breakpoint function \"breakpoint\" reached.\n"); } diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index 299056ab..b21f3068 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -81,7 +81,11 @@ void *(*__libc_memmem)(const void *haystack, size_t haystack_len, #define MAX_MAPPINGS 1024 -static struct mapping { void *st, *en; } __tokencap_ro[MAX_MAPPINGS]; +static struct mapping { + + void *st, *en; + +} __tokencap_ro[MAX_MAPPINGS]; static u32 __tokencap_ro_cnt; static u8 __tokencap_ro_loaded; diff --git a/utils/socket_fuzzing/socketfuzz.c b/utils/socket_fuzzing/socketfuzz.c index 3ec8383b..7497519e 100644 --- a/utils/socket_fuzzing/socketfuzz.c +++ b/utils/socket_fuzzing/socketfuzz.c @@ -23,7 +23,8 @@ #include #include #include -//#include "logging.h" // switche from preeny_info() to fprintf(stderr, "Info: " +// #include "logging.h" // switched from preeny_info() to fprintf(stderr, "Info: +// " // // originals From d5e3223f0340181e74d352db3def2c98cf14d628 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 09:01:49 +0200 Subject: [PATCH 130/149] fix custom mutator only check --- src/afl-fuzz.c | 54 +++++++++++++++++++++++++------------------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 559a7326..4134b99e 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -1746,33 +1746,6 @@ int main(int argc, char **argv_orig, char **envp) { check_if_tty(afl); if (afl->afl_env.afl_force_ui) { afl->not_on_tty = 0; } - if (afl->afl_env.afl_custom_mutator_only) { - - if (!afl->custom_mutators_count) { - - if (afl->shm.cmplog_mode) { - - WARNF( - "No custom mutator loaded, using AFL_CUSTOM_MUTATOR_ONLY is " - "pointless and only allowed now to allow experiments with CMPLOG."); - - } else { - - FATAL( - "No custom mutator loaded but AFL_CUSTOM_MUTATOR_ONLY specified."); - - } - - } - - /* This ensures we don't proceed to havoc/splice */ - afl->custom_only = 1; - - /* Ensure we also skip all deterministic steps */ - afl->skip_deterministic = 1; - - } - get_core_count(afl); atexit(at_exit); @@ -1872,6 +1845,33 @@ int main(int argc, char **argv_orig, char **envp) { setup_custom_mutators(afl); + if (afl->afl_env.afl_custom_mutator_only) { + + if (!afl->custom_mutators_count) { + + if (afl->shm.cmplog_mode) { + + WARNF( + "No custom mutator loaded, using AFL_CUSTOM_MUTATOR_ONLY is " + "pointless and only allowed now to allow experiments with CMPLOG."); + + } else { + + FATAL( + "No custom mutator loaded but AFL_CUSTOM_MUTATOR_ONLY specified."); + + } + + } + + /* This ensures we don't proceed to havoc/splice */ + afl->custom_only = 1; + + /* Ensure we also skip all deterministic steps */ + afl->skip_deterministic = 1; + + } + if (afl->limit_time_sig > 0 && afl->custom_mutators_count) { if (afl->custom_only) { From dd736126dc9da78d6828cdf76f7eb8b389af7ed3 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 09:06:29 +0200 Subject: [PATCH 131/149] allow llvm_instrument native --- src/afl-cc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/afl-cc.c b/src/afl-cc.c index e3cc04dd..64c0ce66 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -1838,7 +1838,8 @@ int main(int argc, char **argv, char **envp) { } if (strncasecmp(ptr2, "llvmnative", strlen("llvmnative")) == 0 || - strncasecmp(ptr2, "llvm-native", strlen("llvm-native")) == 0) { + strncasecmp(ptr2, "llvm-native", strlen("llvm-native")) == 0 || + strncasecmp(ptr2, "native", strlen("native")) == 0) { if (!instrument_mode || instrument_mode == INSTRUMENT_LLVMNATIVE) instrument_mode = INSTRUMENT_LLVMNATIVE; From 22837b5ad2d1cc6313c706ac9cb7fcd005cd3c2c Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 12:14:58 +0200 Subject: [PATCH 132/149] response file fix --- src/afl-cc.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/src/afl-cc.c b/src/afl-cc.c index 64c0ce66..84fe70ec 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -574,14 +574,15 @@ static void process_params(u32 argc, char **argv) { } - u8 *tmpbuf = malloc(st.st_size + 1), *ptr; + u8 *tmpbuf = malloc(st.st_size + 2), *ptr; char **args = malloc(sizeof(char *) * (st.st_size >> 1)); int count = 1, cont = 0, cont_act = 0; - while (fgets(tmpbuf, st.st_size, f)) { + while (fgets(tmpbuf, st.st_size + 1, f)) { ptr = tmpbuf; - // no leading whitespace + // fprintf(stderr, "1: %s\n", ptr); + // no leading whitespace while (isspace(*ptr)) { ++ptr; @@ -603,6 +604,8 @@ static void process_params(u32 argc, char **argv) { } + // fprintf(stderr, "2: %s\n", ptr); + // remove whitespace at end while (*ptr && isspace(ptr[strlen(ptr) - 1])) { @@ -611,6 +614,7 @@ static void process_params(u32 argc, char **argv) { } + // fprintf(stderr, "3: %s\n", ptr); if (*ptr) { do { From 8985524d3a7e9991ededcd2e7f01a112b3107871 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 14:15:36 +0200 Subject: [PATCH 133/149] todo --- TODO.md | 1 + 1 file changed, 1 insertion(+) diff --git a/TODO.md b/TODO.md index 2efcefea..d6a2e6fd 100644 --- a/TODO.md +++ b/TODO.md @@ -2,6 +2,7 @@ ## Should + - redo PCGUARD + LTO for llvm 15+ - splicing selection weighted? - support persistent and deferred fork server in afl-showmap? - better autodetection of shifting runtime timeout values From 501226c992e5c47672907c5dde7f968f4e8fb001 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 14:41:59 +0200 Subject: [PATCH 134/149] correct rtn cmplog map size --- include/cmplog.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/cmplog.h b/include/cmplog.h index 6e16e6b0..e4821444 100644 --- a/include/cmplog.h +++ b/include/cmplog.h @@ -34,7 +34,7 @@ #define CMP_MAP_W 65536 #define CMP_MAP_H 32 -#define CMP_MAP_RTN_H (CMP_MAP_H / 4) +#define CMP_MAP_RTN_H (CMP_MAP_H / 2) #define SHAPE_BYTES(x) (x + 1) From eeed38c5f8335b5a2b69d85b950aa33682a2c079 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 18:31:34 +0200 Subject: [PATCH 135/149] fix gnumakefile for non-gcc --- GNUmakefile | 11 ++++++++--- src/afl-fuzz-redqueen.c | 12 ++++++------ 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 4ecdae52..6962d28a 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -100,8 +100,13 @@ else LDFLAGS += $(SDK_LD) endif +COMPILER_TYPE=$(shell $(CC) --version|grep "Free Software Foundation") +ifneq ($(COMPILER_TYPE), "") + # $(info gcc is being used) + CFLAGS_OPT += -Wno-error=format-truncation -Wno-format-truncation +endif + ifeq "$(SYS)" "SunOS" - CFLAGS_OPT += -Wno-format-truncation LDFLAGS = -lkstat -lrt -lsocket -lnsl endif @@ -139,13 +144,13 @@ endif ifdef DEBUG $(info Compiling DEBUG version of binaries) - override CFLAGS += -ggdb3 -O0 -Wall -Wextra -Werror -Wno-error=format-truncation= $(CFLAGS_OPT) + override CFLAGS += -ggdb3 -O0 -Wall -Wextra -Werror $(CFLAGS_OPT) else CFLAGS ?= -O2 $(CFLAGS_OPT) # -funroll-loops is slower on modern compilers endif override CFLAGS += -g -Wno-pointer-sign -Wno-variadic-macros -Wall -Wextra -Wno-pointer-arith \ - -fPIC -I include/ -DAFL_PATH=\"$(HELPER_PATH)\" -Wno-format-truncation \ + -fPIC -I include/ -DAFL_PATH=\"$(HELPER_PATH)\" \ -DBIN_PATH=\"$(BIN_PATH)\" -DDOC_PATH=\"$(DOC_PATH)\" # -fstack-protector diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index d9dc50df..7f42db3f 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -379,7 +379,7 @@ static u8 colorization(afl_state_t *afl, u8 *buf, u32 len, } - if (++afl->stage_cur % screen_update == 0) { show_stats(afl); }; + if (unlikely(++afl->stage_cur % screen_update == 0)) { show_stats(afl); }; } @@ -790,7 +790,7 @@ static u8 cmp_extend_encoding(afl_state_t *afl, struct cmp_header *h, u32 its_len = MIN(len - idx, taint_len); - if (afl->fsrv.total_execs - last_update > screen_update) { + if (unlikely(afl->fsrv.total_execs - last_update > screen_update)) { show_stats(afl); last_update = afl->fsrv.total_execs; @@ -1455,7 +1455,7 @@ static u8 cmp_extend_encodingN(afl_state_t *afl, struct cmp_header *h, u32 taint_len, u8 *orig_buf, u8 *buf, u8 *cbuf, u32 len, u8 do_reverse, u8 lvl, u8 *status) { - if (afl->fsrv.total_execs - last_update > screen_update) { + if (unlikely(afl->fsrv.total_execs - last_update > screen_update)) { show_stats(afl); last_update = afl->fsrv.total_execs; @@ -1952,7 +1952,7 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, // (void)(changed_val); // #endif - if (afl->fsrv.total_execs - last_update > screen_update) { + if (unlikely(afl->fsrv.total_execs - last_update > screen_update)) { show_stats(afl); last_update = afl->fsrv.total_execs; @@ -2002,10 +2002,10 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - u8 lmax = MAX(l0, ol0); + u8 lmin = MIN(l0, ol0); u8 save[40]; u32 saved_idx = idx, pre, from = 0, to = 0, i, j; - u32 its_len = MIN(MIN(lmax, hshape), len - idx); + u32 its_len = MIN(MIN(lmin, hshape), len - idx); its_len = MIN(its_len, taint_len); u32 saved_its_len = its_len; From b10a0914083911591a8ac816bd4bada6602bf8b5 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 18:48:03 +0200 Subject: [PATCH 136/149] real gcc gnumakefile fix --- GNUmakefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 6962d28a..715e7386 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -101,8 +101,8 @@ else endif COMPILER_TYPE=$(shell $(CC) --version|grep "Free Software Foundation") -ifneq ($(COMPILER_TYPE), "") - # $(info gcc is being used) +ifneq "$(COMPILER_TYPE)" "" + #$(info gcc is being used) CFLAGS_OPT += -Wno-error=format-truncation -Wno-format-truncation endif From c323e0dc63e97299da4a2f775f6f1639d5e13bf0 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 23 May 2023 19:46:35 +0200 Subject: [PATCH 137/149] revert fix --- src/afl-fuzz-redqueen.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/afl-fuzz-redqueen.c b/src/afl-fuzz-redqueen.c index 7f42db3f..41644cb9 100644 --- a/src/afl-fuzz-redqueen.c +++ b/src/afl-fuzz-redqueen.c @@ -2002,10 +2002,10 @@ static u8 rtn_extend_encoding(afl_state_t *afl, u8 entry, } - u8 lmin = MIN(l0, ol0); + u8 lmax = MAX(l0, ol0); u8 save[40]; u32 saved_idx = idx, pre, from = 0, to = 0, i, j; - u32 its_len = MIN(MIN(lmin, hshape), len - idx); + u32 its_len = MIN(MIN(lmax, hshape), len - idx); its_len = MIN(its_len, taint_len); u32 saved_its_len = its_len; From c9dfc279c78d9eb1b993cf9ee7ebe07d2d688822 Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 30 May 2023 14:47:34 +0100 Subject: [PATCH 138/149] doc: fix logo link in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 863c2fce..0208a9fe 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # American Fuzzy Lop plus plus (AFL++) -AFL++ logo +AFL++ logo Release version: [4.06c](https://github.com/AFLplusplus/AFLplusplus/releases) From b08e6bf8c6433adf7d34d0c0025c5cc5672f61ee Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 30 May 2023 15:57:24 +0100 Subject: [PATCH 139/149] doc: recommend llvm/clang-14 in docs Might as well recommend installing 14, as that's newer, and what's used in Docker. Also remove outdated Dockerfile versions, likely easier to remove versions here entirely, and anyone that wants to see what version is used, can look in the Dockerfile. --- docs/INSTALL.md | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 637e8658..9005a7eb 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -3,9 +3,8 @@ ## Linux on x86 An easy way to install AFL++ with everything compiled is available via docker: -You can use the [Dockerfile](../Dockerfile) (which has gcc-10 and clang-12 - -hence afl-clang-lto is available) or just pull directly from the Docker Hub -(for x86_64 and arm64): +You can use the [Dockerfile](../Dockerfile) or just pull directly from the +Docker Hub (for x86_64 and arm64): ```shell docker pull aflplusplus/aflplusplus: @@ -21,14 +20,14 @@ development state of AFL++. If you want to build AFL++ yourself, you have many options. The easiest choice is to build and install everything: -NOTE: depending on your Debian/Ubuntu/Kali/... release, replace `-12` with -whatever llvm version is available. We recommend llvm 12, 13 or 14. +NOTE: depending on your Debian/Ubuntu/Kali/... release, replace `-14` with +whatever llvm version is available. We recommend llvm 13, 14, 15 or 16. ```shell sudo apt-get update sudo apt-get install -y build-essential python3-dev automake cmake git flex bison libglib2.0-dev libpixman-1-dev python3-setuptools cargo libgtk-3-dev -# try to install llvm 12 and install the distro default if that fails -sudo apt-get install -y lld-12 llvm-12 llvm-12-dev clang-12 || sudo apt-get install -y lld llvm llvm-dev clang +# try to install llvm 14 and install the distro default if that fails +sudo apt-get install -y lld-14 llvm-14 llvm-14-dev clang-14 || sudo apt-get install -y lld llvm llvm-dev clang sudo apt-get install -y gcc-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-plugin-dev libstdc++-$(gcc --version|head -n1|sed 's/\..*//'|sed 's/.* //')-dev sudo apt-get install -y ninja-build # for QEMU mode git clone https://github.com/AFLplusplus/AFLplusplus From ad8f7d6eb3be245202ace23d4d1dd9152647a775 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 31 May 2023 11:40:48 +0200 Subject: [PATCH 140/149] switch user mailinglist reference to discord --- TODO.md | 1 + src/afl-forkserver.c | 8 ++++---- src/afl-fuzz-init.c | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/TODO.md b/TODO.md index d6a2e6fd..dc02a914 100644 --- a/TODO.md +++ b/TODO.md @@ -3,6 +3,7 @@ ## Should - redo PCGUARD + LTO for llvm 15+ + - test cmplog for less than 16bit - splicing selection weighted? - support persistent and deferred fork server in afl-showmap? - better autodetection of shifting runtime timeout values diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index 30c8901c..7322f1ad 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -1226,7 +1226,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " - Less likely, there is a horrible bug in the fuzzer. If other " "options\n" - " fail, poke for troubleshooting " + " fail, poke the Awesome Fuzzing Discord for troubleshooting " "tips.\n"); } else { @@ -1271,7 +1271,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " - Less likely, there is a horrible bug in the fuzzer. If other " "options\n" - " fail, poke for troubleshooting " + " fail, poke the Awesome Fuzzing Discord for troubleshooting " "tips.\n", stringify_mem_size(val_buf, sizeof(val_buf), fsrv->mem_limit << 20), fsrv->mem_limit - 1); @@ -1321,7 +1321,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " Retry with setting AFL_MAP_SIZE=10000000.\n\n" "Otherwise there is a horrible bug in the fuzzer.\n" - "Poke for troubleshooting tips.\n"); + "Poke the Awesome Fuzzing Discord for troubleshooting tips.\n"); } else { @@ -1370,7 +1370,7 @@ void afl_fsrv_start(afl_forkserver_t *fsrv, char **argv, " - Less likely, there is a horrible bug in the fuzzer. If other " "options\n" - " fail, poke for troubleshooting " + " fail, poke the Awesome Fuzzing Discord for troubleshooting " "tips.\n", getenv(DEFER_ENV_VAR) ? " - You are using deferred forkserver, but __AFL_INIT() is " diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index baf56a5f..13802f40 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -1012,7 +1012,7 @@ void perform_dry_run(afl_state_t *afl) { " - Least likely, there is a horrible bug in the fuzzer. If " "other options\n" - " fail, poke for " + " fail, poke the Awesome Fuzzing Discord for " "troubleshooting tips.\n", stringify_mem_size(val_buf, sizeof(val_buf), afl->fsrv.mem_limit << 20), @@ -1041,7 +1041,7 @@ void perform_dry_run(afl_state_t *afl) { " - Least likely, there is a horrible bug in the fuzzer. If " "other options\n" - " fail, poke for " + " fail, poke the Awesome Fuzzing Discord for " "troubleshooting tips.\n"); } From e596c9856b636d6c227115655aa74f316d8f27ca Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 31 May 2023 19:15:18 +0100 Subject: [PATCH 141/149] Support for instrumentation more than GB away from data structures --- frida_mode/src/instrument/instrument_arm64.c | 231 ++++++++++++++++--- frida_mode/test/testinstr/GNUmakefile | 5 + 2 files changed, 198 insertions(+), 38 deletions(-) diff --git a/frida_mode/src/instrument/instrument_arm64.c b/frida_mode/src/instrument/instrument_arm64.c index 4372861d..131eb4c5 100644 --- a/frida_mode/src/instrument/instrument_arm64.c +++ b/frida_mode/src/instrument/instrument_arm64.c @@ -76,6 +76,45 @@ typedef struct { } afl_log_code_asm_t; +typedef struct { + + uint32_t b_imm8; /* br #XX (end) */ + + uint32_t restoration_prolog; /* ldp x16, x17, [sp], #0x90 */ + + uint32_t stp_x0_x1; /* stp x0, x1, [sp, #-0xa0] */ + + uint32_t ldr_x0_p_prev_loc_1; /* ldr x0, #0xXXXX */ + uint32_t ldr_x1_ptr_x0; /* ldr x1, [x0] */ + + uint32_t ldr_x0_p_area_offset; /* ldr x0, #0xXXXX */ + uint32_t eor_x0_x1_x0; /* eor x0, x1, x0 */ + uint32_t ldr_x1_p_area_ptr; /* ldr x1, #0xXXXX */ + uint32_t add_x0_x1_x0; /* add x0, x1, x0 */ + + uint32_t ldrb_w1_x0; /* ldrb w1, [x0] */ + uint32_t add_w1_w1_1; /* add w1, w1, #1 */ + uint32_t add_w1_w1_w1_lsr_8; /* add x1, x1, x1, lsr #8 */ + + uint32_t strb_w1_ptr_x0; /* strb w1, [x0] */ + + uint32_t ldr_x0_p_prev_loc_2; /* ldr x0, #0xXXXX */ + uint32_t ldr_x1_p_area_offset_ror; /* ldr x1, #0xXXXX */ + uint32_t str_x1_ptr_x0; /* str x1, [x0] */ + + uint32_t ldp_x0_x1; /* ldp x0, x1, [sp, #-0xa0] */ + + uint32_t b_end; /* skip the data */ + + uint64_t area_ptr; + uint64_t prev_loc_ptr; + uint64_t area_offset; + uint64_t area_offset_ror; + + uint8_t end[0]; + +} afl_log_code_asm_long_t; + #pragma pack(pop) typedef union { @@ -85,6 +124,13 @@ typedef union { } afl_log_code; +typedef union { + + afl_log_code_asm_long_t code; + uint8_t bytes[0]; + +} afl_log_code_long; + static const afl_log_code_asm_t template = { @@ -119,6 +165,46 @@ static const afl_log_code_asm_t template = ; +static const afl_log_code_asm_long_t template_long = + {.b_imm8 = 0x1400001a, + + .restoration_prolog = 0xa8c947f0, /* ldp x16, x17, [sp], #0x90 */ + + .stp_x0_x1 = 0xa93607e0, /* stp x0, x1, [sp, #-0xa0] */ + + .ldr_x0_p_prev_loc_1 = 0x58000220, /* ldr x0, #0xXXXX */ + .ldr_x1_ptr_x0 = 0xf9400001, /* ldr x1, [x0] */ + + .ldr_x0_p_area_offset = 0x58000220, /* ldr x0, #0xXXXX */ + .eor_x0_x1_x0 = 0xca000020, /* eor x0, x1, x0 */ + .ldr_x1_p_area_ptr = 0x58000161, /* ldr x1, #0xXXXX */ + .add_x0_x1_x0 = 0x8b000020, /* add x0, x1, x0 */ + + .ldrb_w1_x0 = 0x39400001, /* ldrb w1, [x0] */ + .add_w1_w1_1 = 0x11000421, /* add w1, w1, #1 */ + .add_w1_w1_w1_lsr_8 = 0x8b412021, /* add x1, x1, x1, lsr #8 */ + + .strb_w1_ptr_x0 = 0x39000001, /* strb w1, [x0] */ + + .ldr_x0_p_prev_loc_2 = 0x580000e0, /* ldr x0, #0xXXXX */ + .ldr_x1_p_area_offset_ror = 0x58000141, /* ldr x1, #0xXXXX */ + .str_x1_ptr_x0 = 0xf9000001, /* str x1, [x0] */ + + .ldp_x0_x1 = 0xa97607e0, /* ldp x0, x1, [sp, #-0xa0] */ + + .b_end = 0x14000009, /* skip the data */ + + .area_ptr = 0x0, + .prev_loc_ptr = 0x0, + .area_offset = 0x0, + .area_offset_ror = 0x0, + + .end = {} + +} + +; + gboolean instrument_is_coverage_optimize_supported(void) { return true; @@ -266,16 +352,22 @@ static gboolean instrument_coverage_in_range(gssize offset) { } -static void instrument_patch_ardp(guint32 *patch, GumAddress insn, +static bool instrument_patch_ardp(guint32 *patch, GumAddress insn, GumAddress target) { - if (!PAGE_ALIGNED(target)) { FATAL("Target not page aligned"); } + if (!PAGE_ALIGNED(target)) { + + FWARNF("Target not page aligned"); + return false; + + } gssize distance = target - (GUM_ADDRESS(insn) & PAGE_MASK); if (!instrument_coverage_in_range(distance)) { - FATAL("Patch out of range 0x%016lX->0x%016lX = 0x%016lX", insn, target, - distance); + FVERBOSE("Patch out of range 0x%016lX->0x%016lX = 0x%016lX", insn, target, + distance); + return false; } @@ -283,6 +375,95 @@ static void instrument_patch_ardp(guint32 *patch, GumAddress insn, guint32 imm_high = ((distance >> 14) & 0x7FFFF) << 5; *patch |= imm_low; *patch |= imm_high; + return true; + +} + +bool instrument_write_inline(GumArm64Writer *cw, GumAddress code_addr, + guint64 area_offset, gsize area_offset_ror) { + + afl_log_code code = {0}; + code.code = template; + + /* + * Given our map is allocated on a 64KB boundary and our map is a multiple of + * 64KB in size, then it should also end on a 64 KB boundary. It is followed + * by our previous_pc, so this too should be 64KB aligned. + */ + g_assert(PAGE_ALIGNED(instrument_previous_pc_addr)); + g_assert(PAGE_ALIGNED(__afl_area_ptr)); + + if (!instrument_patch_ardp( + &code.code.adrp_x0_prev_loc1, + code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc1), + GUM_ADDRESS(instrument_previous_pc_addr))) { + + return false; + + } + + code.code.mov_x0_curr_loc |= area_offset << 5; + + if (!instrument_patch_ardp( + &code.code.adrp_x1_area_ptr, + code_addr + offsetof(afl_log_code, code.adrp_x1_area_ptr), + GUM_ADDRESS(__afl_area_ptr))) { + + return false; + + } + + if (!instrument_patch_ardp( + &code.code.adrp_x0_prev_loc2, + code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc2), + GUM_ADDRESS(instrument_previous_pc_addr))) { + + return false; + + } + + code.code.mov_x1_curr_loc_shr_1 |= (area_offset_ror << 5); + + if (instrument_suppress) { + + gum_arm64_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); + + } else { + + size_t offset = offsetof(afl_log_code, code.stp_x0_x1); + gum_arm64_writer_put_bytes(cw, &code.bytes[offset], + sizeof(afl_log_code) - offset); + + } + + return true; + +} + +bool instrument_write_inline_long(GumArm64Writer *cw, GumAddress code_addr, + guint64 area_offset, gsize area_offset_ror) { + + afl_log_code_long code = {0}; + code.code = template_long; + + code.code.area_ptr = GUM_ADDRESS(__afl_area_ptr); + code.code.prev_loc_ptr = GUM_ADDRESS(instrument_previous_pc_addr); + code.code.area_offset = area_offset; + code.code.area_offset_ror = GUM_ADDRESS(area_offset_ror); + + if (instrument_suppress) { + + gum_arm64_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code_long)); + + } else { + + size_t offset = offsetof(afl_log_code_long, code.stp_x0_x1); + gum_arm64_writer_put_bytes(cw, &code.bytes[offset], + sizeof(afl_log_code_long) - offset); + + } + + return true; } @@ -312,6 +493,8 @@ void instrument_coverage_optimize(const cs_insn *instr, } // gum_arm64_writer_put_brk_imm(cw, 0x0); + // uint32_t jmp_dot = 0x14000000; + // gum_arm64_writer_put_bytes(cw, (guint8 *)&jmp_dot, sizeof(jmp_dot)); if (instrument_suppress) { instrument_coverage_suppress_init(); } @@ -343,47 +526,19 @@ void instrument_coverage_optimize(const cs_insn *instr, } - code.code = template; - - /* - * Given our map is allocated on a 64KB boundary and our map is a multiple of - * 64KB in size, then it should also end on a 64 KB boundary. It is followed - * by our previous_pc, so this too should be 64KB aligned. - */ - g_assert(PAGE_ALIGNED(instrument_previous_pc_addr)); - g_assert(PAGE_ALIGNED(__afl_area_ptr)); - - instrument_patch_ardp( - &code.code.adrp_x0_prev_loc1, - code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc1), - GUM_ADDRESS(instrument_previous_pc_addr)); - - code.code.mov_x0_curr_loc |= area_offset << 5; - - instrument_patch_ardp( - &code.code.adrp_x1_area_ptr, - code_addr + offsetof(afl_log_code, code.adrp_x1_area_ptr), - GUM_ADDRESS(__afl_area_ptr)); - map_size_pow2 = util_log2(__afl_map_size); area_offset_ror = util_rotate(area_offset, 1, map_size_pow2); - instrument_patch_ardp( - &code.code.adrp_x0_prev_loc2, - code_addr + offsetof(afl_log_code, code.adrp_x0_prev_loc2), - GUM_ADDRESS(instrument_previous_pc_addr)); + code.code = template; - code.code.mov_x1_curr_loc_shr_1 |= (area_offset_ror << 5); + if (!instrument_write_inline(cw, code_addr, area_offset, area_offset_ror)) { - if (instrument_suppress) { + if (!instrument_write_inline_long(cw, code_addr, area_offset, + area_offset_ror)) { - gum_arm64_writer_put_bytes(cw, code.bytes, sizeof(afl_log_code)); + FATAL("Failed to write inline instrumentation"); - } else { - - size_t offset = offsetof(afl_log_code, code.stp_x0_x1); - gum_arm64_writer_put_bytes(cw, &code.bytes[offset], - sizeof(afl_log_code) - offset); + } } diff --git a/frida_mode/test/testinstr/GNUmakefile b/frida_mode/test/testinstr/GNUmakefile index ebc0b2dc..46d9b85f 100644 --- a/frida_mode/test/testinstr/GNUmakefile +++ b/frida_mode/test/testinstr/GNUmakefile @@ -67,3 +67,8 @@ debug: --ex 'set environment LD_PRELOAD=$(ROOT)afl-frida-trace.so' \ --ex 'set disassembly-flavor intel' \ --args $(TESTINSTBIN) $(TESTINSTR_DATA_FILE) + +lldb: + lldb \ + -O 'settings set target.env-vars DYLD_INSERT_LIBRARIES=$(ROOT)afl-frida-trace.so' \ + -- $(TESTINSTBIN) $(TESTINSTR_DATA_FILE) From 06e1c64745ed37bd826ff6f2c1a42340684998dc Mon Sep 17 00:00:00 2001 From: Your Name Date: Wed, 31 May 2023 17:34:36 +0100 Subject: [PATCH 142/149] Changes to support defered start --- frida_mode/include/entry.h | 2 -- frida_mode/src/entry.c | 26 +--------------- frida_mode/src/instrument/instrument.c | 1 - frida_mode/src/main.c | 41 ++++++++++++++++++++++---- 4 files changed, 37 insertions(+), 33 deletions(-) diff --git a/frida_mode/include/entry.h b/frida_mode/include/entry.h index edc41467..949fab71 100644 --- a/frida_mode/include/entry.h +++ b/frida_mode/include/entry.h @@ -14,8 +14,6 @@ void entry_init(void); void entry_start(void); -void entry_prologue(GumStalkerIterator *iterator, GumStalkerOutput *output); - void entry_on_fork(void); #endif diff --git a/frida_mode/src/entry.c b/frida_mode/src/entry.c index 05af7ebb..fc49b4d7 100644 --- a/frida_mode/src/entry.c +++ b/frida_mode/src/entry.c @@ -78,6 +78,7 @@ void entry_init(void) { void entry_start(void) { + FVERBOSE("AFL_ENTRYPOINT reached"); if (persistent_start == 0) { ranges_exclude(); @@ -85,32 +86,7 @@ void entry_start(void) { } - if (entry_point == 0) { entry_launch(); } - -} - -static void entry_callout(GumCpuContext *cpu_context, gpointer user_data) { - - UNUSED_PARAMETER(cpu_context); - UNUSED_PARAMETER(user_data); - entry_compiled = TRUE; entry_launch(); } -void entry_prologue(GumStalkerIterator *iterator, GumStalkerOutput *output) { - - UNUSED_PARAMETER(output); - FVERBOSE("AFL_ENTRYPOINT reached"); - - if (persistent_start == 0) { - - ranges_exclude(); - stalker_trust(); - - } - - gum_stalker_iterator_put_callout(iterator, entry_callout, NULL, NULL); - -} - diff --git a/frida_mode/src/instrument/instrument.c b/frida_mode/src/instrument/instrument.c index a6aac666..db73d845 100644 --- a/frida_mode/src/instrument/instrument.c +++ b/frida_mode/src/instrument/instrument.c @@ -169,7 +169,6 @@ static void instrument_basic_block(GumStalkerIterator *iterator, if (unlikely(begin)) { instrument_debug_start(instr->address, output); } - if (instr->address == entry_point) { entry_prologue(iterator, output); } if (instr->address == persistent_start) { persistent_prologue(output); } if (instr->address == persistent_ret) { persistent_epilogue(output); } diff --git a/frida_mode/src/main.c b/frida_mode/src/main.c index c8c50b37..f11c4b25 100644 --- a/frida_mode/src/main.c +++ b/frida_mode/src/main.c @@ -197,7 +197,7 @@ static void afl_print_env(void) { } -__attribute__((visibility("default"))) void afl_frida_start(void) { +void afl_frida_config(void) { FOKF(cRED "**********************"); FOKF(cRED "* " cYEL "******************" cRED " *"); @@ -225,9 +225,7 @@ __attribute__((visibility("default"))) void afl_frida_start(void) { js_start(); - /* Initialize */ output_init(); - embedded_init(); entry_init(); instrument_init(); @@ -240,12 +238,35 @@ __attribute__((visibility("default"))) void afl_frida_start(void) { ranges_init(); stats_init(); - /* Start */ +} + +void afl_frida_run(void) { + stalker_start(); entry_start(); } +__attribute__((visibility("default"))) void afl_frida_start(void) { + + afl_frida_config(); + afl_frida_run(); + +} + +typedef void *(*entry_func_t)(size_t a1, size_t a2, size_t a3, size_t a4, + size_t a5, size_t a6); + +static void *on_entry(size_t a1, size_t a2, size_t a3, size_t a4, size_t a5, + size_t a6) { + + intercept_unhook(GSIZE_TO_POINTER(entry_point)); + afl_frida_run(); + entry_func_t entry = (entry_func_t)entry_point; + return entry(a1, a2, a3, a4, a5, a6); + +} + static int on_main(int argc, char **argv, char **envp) { int ret; @@ -254,7 +275,17 @@ static int on_main(int argc, char **argv, char **envp) { intercept_unhook_self(); - afl_frida_start(); + afl_frida_config(); + + if (entry_point == 0) { + + afl_frida_run(); + + } else { + + intercept_hook(GSIZE_TO_POINTER(entry_point), on_entry, NULL); + + } if (js_main_hook != NULL) { From 9324f3f6289c62451e2add1f7553a7eda0d7d642 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 1 Jun 2023 12:19:45 +0200 Subject: [PATCH 143/149] rewrote PCGUARD --- GNUmakefile.llvm | 4 +- TODO.md | 2 - afl-cmin.bash | 4 +- docs/Changelog.md | 5 +- .../SanitizerCoveragePCGUARD.so.cc | 611 +++++++----------- src/afl-cc.c | 31 +- 6 files changed, 253 insertions(+), 404 deletions(-) diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index 2bb4e7f8..6c68f1f3 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -49,7 +49,7 @@ LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[ LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[5-9]' && echo 1 || echo 0 ) LLVM_NEW_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[0-9]' && echo 1 || echo 0 ) LLVM_NEWER_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[6-9]' && echo 1 || echo 0 ) -LLVM_10_OK = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[1-9]|^10\.[1-9]|^10\.0.[1-9]' && echo 1 || echo 0 ) +LLVM_13_OK = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[3-9]' && echo 1 || echo 0 ) LLVM_HAVE_LTO = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[1-9]' && echo 1 || echo 0 ) LLVM_BINDIR = $(shell $(LLVM_CONFIG) --bindir 2>/dev/null) LLVM_LIBDIR = $(shell $(LLVM_CONFIG) --libdir 2>/dev/null) @@ -422,7 +422,7 @@ endif $(CXX) $(CLANG_CPPFL) -Wdeprecated -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o ./SanitizerCoveragePCGUARD.so: instrumentation/SanitizerCoveragePCGUARD.so.cc instrumentation/afl-llvm-common.o | test_deps -ifeq "$(LLVM_10_OK)" "1" +ifeq "$(LLVM_13_OK)" "1" -$(CXX) $(CLANG_CPPFL) -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) -Wno-deprecated-copy-dtor -Wdeprecated instrumentation/afl-llvm-common.o endif diff --git a/TODO.md b/TODO.md index dc02a914..2b7e8fcf 100644 --- a/TODO.md +++ b/TODO.md @@ -2,9 +2,7 @@ ## Should - - redo PCGUARD + LTO for llvm 15+ - test cmplog for less than 16bit - - splicing selection weighted? - support persistent and deferred fork server in afl-showmap? - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt diff --git a/afl-cmin.bash b/afl-cmin.bash index d390ff65..dc6d5342 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -206,7 +206,7 @@ fi # Check for obvious errors. -if [ ! "$T_ARG" = "" -a ! "$F_ARG" = "" -a ! "$NYX_MODE" == 1 ]; then +if [ ! "$T_ARG" = "" -a -n "$F_ARG" -a ! "$NYX_MODE" == 1 ]; then echo "[-] Error: -T and -f can not be used together." 1>&2 exit 1 fi @@ -323,7 +323,7 @@ if [ ! "$T_ARG" = "" ]; then fi fi else - if [ "$F_ARG" = ""]; then + if [ -z "$F_ARG" ]; then echo "[*] Are you aware of the '-T all' parallelize option that massively improves the speed?" fi fi diff --git a/docs/Changelog.md b/docs/Changelog.md index e99747f6..facf2196 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -15,13 +15,16 @@ - new env `AFL_LLVM_LTO_SKIPINIT` to support the AFL++ based WASM (https://github.com/fgsect/WAFL) project - error and print help if afl-clan-lto is used with lto=thin + - rewrote our PCGUARD pass to be compatible with LLVM 15+ shenanigans, + requires LLVM 13+ now instead of 10.0.1+ + - fallback to native LLVM PCGUARD if our PCGUARD is unavailable - afl-showmap: - added custom mutator post_process and send support - add `-I filelist` option, an alternative to `-i in_dir` - afl-cmin + afl-cmin.bash: - `-T threads` parallel task support, can be a huge speedup! - qemu_mode: - - Persistent mode +QASAN support for ppc32 tragets by @worksbutnottested + - Persistent mode + QASAN support for ppc32 targets by @worksbutnottested - a new grammar custom mutator atnwalk was submitted by @voidptr127 ! - two new custom mutators are now available: - TritonDSE in custom_mutators/aflpp_tritondse diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 8fed2042..2abc58ec 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -13,42 +13,64 @@ #include "llvm/Transforms/Instrumentation/SanitizerCoverage.h" #include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" +#if LLVM_VERSION_MAJOR >= 15 + #if LLVM_VERSION_MAJOR < 17 + #include "llvm/ADT/Triple.h" + #endif +#endif #if LLVM_VERSION_MAJOR < 17 - #include "llvm/ADT/Triple.h" #include "llvm/Analysis/EHPersonalities.h" -#else - #include "llvm/IR/EHPersonalities.h" #endif #include "llvm/Analysis/PostDominators.h" -#include "llvm/IR/CFG.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/CFG.h" +#endif #include "llvm/IR/Constant.h" #include "llvm/IR/DataLayout.h" -#include "llvm/IR/DebugInfo.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/DebugInfo.h" +#endif #include "llvm/IR/Dominators.h" +#if LLVM_VERSION_MAJOR >= 17 + #include "llvm/Analysis/EHPersonalities.h" +#endif #include "llvm/IR/Function.h" -#include "llvm/IR/GlobalVariable.h" +#if LLVM_VERSION_MAJOR >= 16 + #include "llvm/IR/GlobalVariable.h" +#endif #include "llvm/IR/IRBuilder.h" -#include "llvm/IR/InlineAsm.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/InlineAsm.h" +#endif #include "llvm/IR/IntrinsicInst.h" #include "llvm/IR/Intrinsics.h" #include "llvm/IR/LLVMContext.h" -#include "llvm/IR/MDBuilder.h" -#include "llvm/IR/Mangler.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/IR/MDBuilder.h" + #include "llvm/IR/Mangler.h" +#endif #include "llvm/IR/Module.h" #include "llvm/IR/PassManager.h" +#include "llvm/Passes/PassBuilder.h" +#include "llvm/Passes/PassPlugin.h" #include "llvm/IR/Type.h" -#include "llvm/InitializePasses.h" +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/InitializePasses.h" +#endif #include "llvm/Support/CommandLine.h" #include "llvm/Support/Debug.h" #include "llvm/Support/SpecialCaseList.h" #include "llvm/Support/VirtualFileSystem.h" -#include "llvm/Support/raw_ostream.h" -#include "llvm/Transforms/Instrumentation.h" +#if LLVM_VERSION_MAJOR < 15 + #include "llvm/Support/raw_ostream.h" +#endif +#if LLVM_VERSION_MAJOR < 17 + #include "llvm/Transforms/Instrumentation.h" +#else + #include "llvm/TargetParser/Triple.h" +#endif #include "llvm/Transforms/Utils/BasicBlockUtils.h" #include "llvm/Transforms/Utils/ModuleUtils.h" -#include "llvm/Passes/PassPlugin.h" -#include "llvm/Passes/PassBuilder.h" -#include "llvm/IR/PassManager.h" #include "config.h" #include "debug.h" @@ -58,7 +80,8 @@ using namespace llvm; #define DEBUG_TYPE "sancov" -const char SanCovTracePCIndirName[] = "__sanitizer_cov_trace_pc_indir"; +static const uint64_t SanCtorAndDtorPriority = 2; + const char SanCovTracePCName[] = "__sanitizer_cov_trace_pc"; const char SanCovTraceCmp1[] = "__sanitizer_cov_trace_cmp1"; const char SanCovTraceCmp2[] = "__sanitizer_cov_trace_cmp2"; @@ -68,22 +91,13 @@ const char SanCovTraceConstCmp1[] = "__sanitizer_cov_trace_const_cmp1"; const char SanCovTraceConstCmp2[] = "__sanitizer_cov_trace_const_cmp2"; const char SanCovTraceConstCmp4[] = "__sanitizer_cov_trace_const_cmp4"; const char SanCovTraceConstCmp8[] = "__sanitizer_cov_trace_const_cmp8"; -const char SanCovTraceDiv4[] = "__sanitizer_cov_trace_div4"; -const char SanCovTraceDiv8[] = "__sanitizer_cov_trace_div8"; -const char SanCovTraceGep[] = "__sanitizer_cov_trace_gep"; const char SanCovTraceSwitchName[] = "__sanitizer_cov_trace_switch"; + const char SanCovModuleCtorTracePcGuardName[] = "sancov.module_ctor_trace_pc_guard"; -const char SanCovModuleCtor8bitCountersName[] = - "sancov.module_ctor_8bit_counters"; -const char SanCovModuleCtorBoolFlagName[] = "sancov.module_ctor_bool_flag"; -static const uint64_t SanCtorAndDtorPriority = 2; +const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init"; const char SanCovTracePCGuardName[] = "__sanitizer_cov_trace_pc_guard"; -const char SanCovTracePCGuardInitName[] = "__sanitizer_cov_trace_pc_guard_init"; -const char SanCov8bitCountersInitName[] = "__sanitizer_cov_8bit_counters_init"; -const char SanCovBoolFlagInitName[] = "__sanitizer_cov_bool_flag_init"; -const char SanCovPCsInitName[] = "__sanitizer_cov_pcs_init"; const char SanCovGuardsSectionName[] = "sancov_guards"; const char SanCovCountersSectionName[] = "sancov_cntrs"; @@ -99,27 +113,9 @@ namespace { SanitizerCoverageOptions OverrideFromCL(SanitizerCoverageOptions Options) { - // Sets CoverageType and IndirectCalls. - // SanitizerCoverageOptions CLOpts = getOptions(ClCoverageLevel); - Options.CoverageType = - SanitizerCoverageOptions::SCK_Edge; // std::max(Options.CoverageType, - // CLOpts.CoverageType); - Options.IndirectCalls = false; // CLOpts.IndirectCalls; - Options.TraceCmp = false; //|= ClCMPTracing; - Options.TraceDiv = false; //|= ClDIVTracing; - Options.TraceGep = false; //|= ClGEPTracing; - Options.TracePC = false; //|= ClTracePC; - Options.TracePCGuard = true; // |= ClTracePCGuard; - Options.Inline8bitCounters = 0; //|= ClInline8bitCounters; - // Options.InlineBoolFlag = 0; //|= ClInlineBoolFlag; - Options.PCTable = false; //|= ClCreatePCTable; - Options.NoPrune = false; //|= !ClPruneBlocks; - Options.StackDepth = false; //|= ClStackDepth; - if (!Options.TracePCGuard && !Options.TracePC && - !Options.Inline8bitCounters && !Options.StackDepth /*&& - !Options.InlineBoolFlag*/) - Options.TracePCGuard = true; // TracePCGuard is default. - + Options.CoverageType = SanitizerCoverageOptions::SCK_Edge; + // Options.NoPrune = true; + Options.TracePCGuard = true; // TracePCGuard is default. return Options; } @@ -139,20 +135,13 @@ class ModuleSanitizerCoverageAFL } PreservedAnalyses run(Module &M, ModuleAnalysisManager &MAM); - - bool instrumentModule(Module &M, DomTreeCallback DTCallback, - PostDomTreeCallback PDTCallback); + bool instrumentModule(Module &M, DomTreeCallback DTCallback, + PostDomTreeCallback PDTCallback); private: void instrumentFunction(Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback); - void InjectCoverageForIndirectCalls(Function &F, - ArrayRef IndirCalls); void InjectTraceForCmp(Function &F, ArrayRef CmpTraceTargets); - void InjectTraceForDiv(Function &F, - ArrayRef DivTraceTargets); - void InjectTraceForGep(Function &F, - ArrayRef GepTraceTargets); void InjectTraceForSwitch(Function &F, ArrayRef SwitchTraceTargets); bool InjectCoverage(Function &F, ArrayRef AllBlocks, @@ -173,20 +162,23 @@ class ModuleSanitizerCoverageAFL void SetNoSanitizeMetadata(Instruction *I) { +#if LLVM_VERSION_MAJOR == 15 + I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, None)); +#elif LLVM_VERSION_MAJOR >= 16 + I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, std::nullopt)); +#else I->setMetadata(I->getModule()->getMDKindID("nosanitize"), MDNode::get(*C, None)); +#endif } std::string getSectionName(const std::string &Section) const; std::string getSectionStart(const std::string &Section) const; std::string getSectionEnd(const std::string &Section) const; - FunctionCallee SanCovTracePCIndir; FunctionCallee SanCovTracePC, SanCovTracePCGuard; FunctionCallee SanCovTraceCmpFunction[4]; FunctionCallee SanCovTraceConstCmpFunction[4]; - FunctionCallee SanCovTraceDivFunction[2]; - FunctionCallee SanCovTraceGepFunction; FunctionCallee SanCovTraceSwitchFunction; GlobalVariable *SanCovLowestStack; Type *IntptrTy, *IntptrPtrTy, *Int64Ty, *Int64PtrTy, *Int32Ty, *Int32PtrTy, @@ -215,18 +207,16 @@ class ModuleSanitizerCoverageAFL } // namespace -#if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - extern "C" ::llvm::PassPluginLibraryInfo LLVM_ATTRIBUTE_WEAK llvmGetPassPluginInfo() { - return {LLVM_PLUGIN_API_VERSION, "SanitizerCoveragePCGUARD", "v0.1", + return {LLVM_PLUGIN_API_VERSION, "SanitizerCoveragePCGUARD", "v0.2", /* lambda to insert our pass into the pass pipeline. */ [](PassBuilder &PB) { - #if LLVM_VERSION_MAJOR <= 13 +#if LLVM_VERSION_MAJOR == 13 using OptimizationLevel = typename PassBuilder::OptimizationLevel; - #endif +#endif PB.registerOptimizerLastEPCallback( [](ModulePassManager &MPM, OptimizationLevel OL) { @@ -238,8 +228,7 @@ llvmGetPassPluginInfo() { } -#endif - +#if LLVM_VERSION_MAJOR == 1 PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, ModuleAnalysisManager &MAM) { @@ -257,34 +246,65 @@ PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, }; + if (!ModuleSancov.instrumentModule(M, DTCallback, PDTCallback)) + return PreservedAnalyses::all(); + + PreservedAnalyses PA = PreservedAnalyses::none(); + // GlobalsAA is considered stateless and does not get invalidated unless + // explicitly invalidated; PreservedAnalyses::none() is not enough. Sanitizers + // make changes that require GlobalsAA to be invalidated. + PA.abandon(); + return PA; + +} + +#else + #if LLVM_VERSION_MAJOR >= 16 +PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, + ModuleAnalysisManager &MAM) { + + #else +PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, + ModuleAnalysisManager &MAM) { + + #endif + ModuleSanitizerCoverageAFL ModuleSancov(Options); + auto &FAM = MAM.getResult(M).getManager(); + auto DTCallback = [&FAM](Function &F) -> const DominatorTree * { + + return &FAM.getResult(F); + + }; + + auto PDTCallback = [&FAM](Function &F) -> const PostDominatorTree * { + + return &FAM.getResult(F); + + }; + if (ModuleSancov.instrumentModule(M, DTCallback, PDTCallback)) return PreservedAnalyses::none(); return PreservedAnalyses::all(); } +#endif + std::pair ModuleSanitizerCoverageAFL::CreateSecStartEnd( Module &M, const char *Section, Type *Ty) { - GlobalVariable *SecStart = - new GlobalVariable(M, -#if LLVM_VERSION_MAJOR >= 15 - Ty, -#else - Ty->getPointerElementType(), -#endif - false, GlobalVariable::ExternalWeakLinkage, nullptr, - getSectionStart(Section)); + // Use ExternalWeak so that if all sections are discarded due to section + // garbage collection, the linker will not report undefined symbol errors. + // Windows defines the start/stop symbols in compiler-rt so no need for + // ExternalWeak. + GlobalValue::LinkageTypes Linkage = TargetTriple.isOSBinFormatCOFF() + ? GlobalVariable::ExternalLinkage + : GlobalVariable::ExternalWeakLinkage; + GlobalVariable *SecStart = new GlobalVariable(M, Ty, false, Linkage, nullptr, + getSectionStart(Section)); SecStart->setVisibility(GlobalValue::HiddenVisibility); - GlobalVariable *SecEnd = - new GlobalVariable(M, -#if LLVM_VERSION_MAJOR >= 15 - Ty, -#else - Ty->getPointerElementType(), -#endif - false, GlobalVariable::ExternalWeakLinkage, nullptr, - getSectionEnd(Section)); + GlobalVariable *SecEnd = new GlobalVariable(M, Ty, false, Linkage, nullptr, + getSectionEnd(Section)); SecEnd->setVisibility(GlobalValue::HiddenVisibility); IRBuilder<> IRB(M.getContext()); if (!TargetTriple.isOSBinFormatCOFF()) @@ -295,7 +315,8 @@ std::pair ModuleSanitizerCoverageAFL::CreateSecStartEnd( auto SecStartI8Ptr = IRB.CreatePointerCast(SecStart, Int8PtrTy); auto GEP = IRB.CreateGEP(Int8Ty, SecStartI8Ptr, ConstantInt::get(IntptrTy, sizeof(uint64_t))); - return std::make_pair(IRB.CreatePointerCast(GEP, Ty), SecEnd); + return std::make_pair(IRB.CreatePointerCast(GEP, PointerType::getUnqual(Ty)), + SecEnd); } @@ -307,8 +328,9 @@ Function *ModuleSanitizerCoverageAFL::CreateInitCallsForSections( auto SecStart = SecStartEnd.first; auto SecEnd = SecStartEnd.second; Function *CtorFunc; + Type *PtrTy = PointerType::getUnqual(Ty); std::tie(CtorFunc, std::ignore) = createSanitizerCtorAndInitFunctions( - M, CtorName, InitFunctionName, {Ty, Ty}, {SecStart, SecEnd}); + M, CtorName, InitFunctionName, {PtrTy, PtrTy}, {SecStart, SecEnd}); assert(CtorFunc->getName() == CtorName); if (TargetTriple.supportsCOMDAT()) { @@ -332,7 +354,6 @@ Function *ModuleSanitizerCoverageAFL::CreateInitCallsForSections( // to include the sancov constructor. This way the linker can deduplicate // the constructors but always leave one copy. CtorFunc->setLinkage(GlobalValue::WeakODRLinkage); - appendToUsed(M, CtorFunc); } @@ -344,37 +365,25 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( Module &M, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { setvbuf(stdout, NULL, _IONBF, 0); - if (getenv("AFL_DEBUG")) debug = 1; + + if (getenv("AFL_DEBUG")) { debug = 1; } if ((isatty(2) && !getenv("AFL_QUIET")) || debug) { SAYF(cCYA "SanitizerCoveragePCGUARD" VERSION cRST "\n"); - } else + } else { be_quiet = 1; + } + skip_nozero = getenv("AFL_LLVM_SKIP_NEVERZERO"); use_threadsafe_counters = getenv("AFL_LLVM_THREADSAFE_INST"); initInstrumentList(); scanForDangerousFunctions(&M); - if (debug) { - - fprintf(stderr, - "SANCOV: covtype:%u indirect:%d stack:%d noprune:%d " - "createtable:%d tracepcguard:%d tracepc:%d\n", - Options.CoverageType, Options.IndirectCalls == true ? 1 : 0, - Options.StackDepth == true ? 1 : 0, Options.NoPrune == true ? 1 : 0, - // Options.InlineBoolFlag == true ? 1 : 0, - Options.PCTable == true ? 1 : 0, - Options.TracePCGuard == true ? 1 : 0, - Options.TracePC == true ? 1 : 0); - - } - - if (Options.CoverageType == SanitizerCoverageOptions::SCK_None) return false; C = &(M.getContext()); DL = &M.getDataLayout(); CurModule = &M; @@ -397,16 +406,14 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( Int16Ty = IRB.getInt16Ty(); Int8Ty = IRB.getInt8Ty(); Int1Ty = IRB.getInt1Ty(); - LLVMContext &Ctx = M.getContext(); + LLVMContext &Ctx = M.getContext(); AFLMapPtr = new GlobalVariable(M, PointerType::get(Int8Ty, 0), false, GlobalValue::ExternalLinkage, 0, "__afl_area_ptr"); One = ConstantInt::get(IntegerType::getInt8Ty(Ctx), 1); Zero = ConstantInt::get(IntegerType::getInt8Ty(Ctx), 0); - SanCovTracePCIndir = - M.getOrInsertFunction(SanCovTracePCIndirName, VoidTy, IntptrTy); // Make sure smaller parameters are zero-extended to i64 if required by the // target ABI. AttributeList SanCovTraceCmpZeroExtAL; @@ -436,26 +443,13 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( SanCovTraceConstCmpFunction[3] = M.getOrInsertFunction(SanCovTraceConstCmp8, VoidTy, Int64Ty, Int64Ty); - { - - AttributeList AL; - AL = AL.addParamAttribute(*C, 0, Attribute::ZExt); - SanCovTraceDivFunction[0] = - M.getOrInsertFunction(SanCovTraceDiv4, AL, VoidTy, IRB.getInt32Ty()); - - } - - SanCovTraceDivFunction[1] = - M.getOrInsertFunction(SanCovTraceDiv8, VoidTy, Int64Ty); - SanCovTraceGepFunction = - M.getOrInsertFunction(SanCovTraceGep, VoidTy, IntptrTy); SanCovTraceSwitchFunction = M.getOrInsertFunction(SanCovTraceSwitchName, VoidTy, Int64Ty, Int64PtrTy); Constant *SanCovLowestStackConstant = M.getOrInsertGlobal(SanCovLowestStackName, IntptrTy); SanCovLowestStack = dyn_cast(SanCovLowestStackConstant); - if (!SanCovLowestStack) { + if (!SanCovLowestStack || SanCovLowestStack->getValueType() != IntptrTy) { C->emitError(StringRef("'") + SanCovLowestStackName + "' should not be declared by the user"); @@ -465,8 +459,6 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( SanCovLowestStack->setThreadLocalMode( GlobalValue::ThreadLocalMode::InitialExecTLSModel); - if (Options.StackDepth && !SanCovLowestStack->isDeclaration()) - SanCovLowestStack->setInitializer(Constant::getAllOnesValue(IntptrTy)); SanCovTracePC = M.getOrInsertFunction(SanCovTracePCName, VoidTy); SanCovTracePCGuard = @@ -481,40 +473,25 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( Ctor = CreateInitCallsForSections(M, SanCovModuleCtorTracePcGuardName, SanCovTracePCGuardInitName, Int32PtrTy, SanCovGuardsSectionName); - if (Function8bitCounterArray) - Ctor = CreateInitCallsForSections(M, SanCovModuleCtor8bitCountersName, - SanCov8bitCountersInitName, Int8PtrTy, - SanCovCountersSectionName); - if (FunctionBoolArray) { - Ctor = CreateInitCallsForSections(M, SanCovModuleCtorBoolFlagName, - SanCovBoolFlagInitName, Int1PtrTy, - SanCovBoolFlagSectionName); + if (Ctor && debug) { + + fprintf(stderr, "SANCOV: installed pcguard_init in ctor\n"); } - if (Ctor && Options.PCTable) { - - auto SecStartEnd = CreateSecStartEnd(M, SanCovPCsSectionName, IntptrPtrTy); - FunctionCallee InitFunction = declareSanitizerInitFunction( - M, SanCovPCsInitName, {IntptrPtrTy, IntptrPtrTy}); - IRBuilder<> IRBCtor(Ctor->getEntryBlock().getTerminator()); - IRBCtor.CreateCall(InitFunction, {SecStartEnd.first, SecStartEnd.second}); - - } - - // We don't reference these arrays directly in any of our runtime functions, - // so we need to prevent them from being dead stripped. - if (TargetTriple.isOSBinFormatMachO()) appendToUsed(M, GlobalsToAppendToUsed); + appendToUsed(M, GlobalsToAppendToUsed); appendToCompilerUsed(M, GlobalsToAppendToCompilerUsed); if (!be_quiet) { - if (!instr) - WARNF("No instrumentation targets found."); - else { + if (!instr) { - char modeline[100]; + WARNF("No instrumentation targets found."); + + } else { + + char modeline[128]; snprintf(modeline, sizeof(modeline), "%s%s%s%s%s%s", getenv("AFL_HARDEN") ? "hardened" : "non-hardened", getenv("AFL_USE_ASAN") ? ", ASAN" : "", @@ -535,39 +512,36 @@ bool ModuleSanitizerCoverageAFL::instrumentModule( } // True if block has successors and it dominates all of them. -bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) { +static bool isFullDominator(const BasicBlock *BB, const DominatorTree *DT) { - if (succ_begin(BB) == succ_end(BB)) return false; + if (succ_empty(BB)) return false; - for (const BasicBlock *SUCC : make_range(succ_begin(BB), succ_end(BB))) { + return llvm::all_of(successors(BB), [&](const BasicBlock *SUCC) { - if (!DT->dominates(BB, SUCC)) return false; + return DT->dominates(BB, SUCC); - } - - return true; + }); } // True if block has predecessors and it postdominates all of them. -bool isFullPostDominator(const BasicBlock *BB, const PostDominatorTree *PDT) { +static bool isFullPostDominator(const BasicBlock *BB, + const PostDominatorTree *PDT) { - if (pred_begin(BB) == pred_end(BB)) return false; + if (pred_empty(BB)) return false; - for (const BasicBlock *PRED : make_range(pred_begin(BB), pred_end(BB))) { + return llvm::all_of(predecessors(BB), [&](const BasicBlock *PRED) { - if (!PDT->dominates(BB, PRED)) return false; + return PDT->dominates(BB, PRED); - } - - return true; + }); } -bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, - const DominatorTree *DT, - const PostDominatorTree *PDT, - const SanitizerCoverageOptions &Options) { +static bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, + const DominatorTree *DT, + const PostDominatorTree *PDT, + const SanitizerCoverageOptions &Options) { // Don't insert coverage for blocks containing nothing but unreachable: we // will never call __sanitizer_cov() for them, so counting them in @@ -582,10 +556,6 @@ bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, if (Options.NoPrune || &F.getEntryBlock() == BB) return true; - if (Options.CoverageType == SanitizerCoverageOptions::SCK_Function && - &F.getEntryBlock() != BB) - return false; - // Do not instrument full dominators, or full post-dominators with multiple // predecessors. return !isFullDominator(BB, DT) && @@ -597,38 +567,47 @@ bool shouldInstrumentBlock(const Function &F, const BasicBlock *BB, // A twist here is that we treat From->To as a backedge if // * To dominates From or // * To->UniqueSuccessor dominates From -bool IsBackEdge(BasicBlock *From, BasicBlock *To, const DominatorTree *DT) { +#if 0 +static bool IsBackEdge(BasicBlock *From, BasicBlock *To, + const DominatorTree *DT) { - if (DT->dominates(To, From)) return true; + if (DT->dominates(To, From)) + return true; if (auto Next = To->getUniqueSuccessor()) - if (DT->dominates(Next, From)) return true; + if (DT->dominates(Next, From)) + return true; return false; } +#endif + // Prunes uninteresting Cmp instrumentation: // * CMP instructions that feed into loop backedge branch. // // Note that Cmp pruning is controlled by the same flag as the // BB pruning. -bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT, - const SanitizerCoverageOptions &Options) { +#if 0 +static bool IsInterestingCmp(ICmpInst *CMP, const DominatorTree *DT, + const SanitizerCoverageOptions &Options) { if (!Options.NoPrune) if (CMP->hasOneUse()) if (auto BR = dyn_cast(CMP->user_back())) for (BasicBlock *B : BR->successors()) - if (IsBackEdge(BR->getParent(), B, DT)) return false; + if (IsBackEdge(BR->getParent(), B, DT)) + return false; return true; } +#endif + void ModuleSanitizerCoverageAFL::instrumentFunction( Function &F, DomTreeCallback DTCallback, PostDomTreeCallback PDTCallback) { if (F.empty()) return; if (!isInInstrumentList(&F, FMNAME)) return; - if (F.getName().find(".module_ctor") != std::string::npos) return; // Should not instrument sanitizer init functions. if (F.getName().startswith("__sanitizer_")) @@ -647,15 +626,13 @@ void ModuleSanitizerCoverageAFL::instrumentFunction( if (F.hasPersonalityFn() && isAsynchronousEHPersonality(classifyEHPersonality(F.getPersonalityFn()))) return; + if (F.hasFnAttribute(Attribute::NoSanitizeCoverage)) return; if (Options.CoverageType >= SanitizerCoverageOptions::SCK_Edge) SplitAllCriticalEdges( F, CriticalEdgeSplittingOptions().setIgnoreUnreachableDests()); - SmallVector IndirCalls; - SmallVector BlocksToInstrument; - SmallVector CmpTraceTargets; - SmallVector SwitchTraceTargets; - SmallVector DivTraceTargets; - SmallVector GepTraceTargets; + SmallVector BlocksToInstrument; + SmallVector CmpTraceTargets; + SmallVector SwitchTraceTargets; const DominatorTree *DT = DTCallback(F); const PostDominatorTree *PDT = PDTCallback(F); @@ -665,47 +642,28 @@ void ModuleSanitizerCoverageAFL::instrumentFunction( if (shouldInstrumentBlock(F, &BB, DT, PDT, Options)) BlocksToInstrument.push_back(&BB); - for (auto &Inst : BB) { + /* + for (auto &Inst : BB) { - if (Options.IndirectCalls) { + if (Options.TraceCmp) { - CallBase *CB = dyn_cast(&Inst); - if (CB && !CB->getCalledFunction()) IndirCalls.push_back(&Inst); + if (ICmpInst *CMP = dyn_cast(&Inst)) + if (IsInterestingCmp(CMP, DT, Options)) + CmpTraceTargets.push_back(&Inst); + if (isa(&Inst)) + SwitchTraceTargets.push_back(&Inst); - } + } - if (Options.TraceCmp) { + } - if (ICmpInst *CMP = dyn_cast(&Inst)) - if (IsInterestingCmp(CMP, DT, Options)) - CmpTraceTargets.push_back(&Inst); - if (isa(&Inst)) SwitchTraceTargets.push_back(&Inst); - - } - - if (Options.TraceDiv) - if (BinaryOperator *BO = dyn_cast(&Inst)) - if (BO->getOpcode() == Instruction::SDiv || - BO->getOpcode() == Instruction::UDiv) - DivTraceTargets.push_back(BO); - if (Options.TraceGep) - if (GetElementPtrInst *GEP = dyn_cast(&Inst)) - GepTraceTargets.push_back(GEP); - if (Options.StackDepth) - if (isa(Inst) || - (isa(Inst) && !isa(Inst))) - IsLeafFunc = false; - - } + */ } InjectCoverage(F, BlocksToInstrument, IsLeafFunc); - InjectCoverageForIndirectCalls(F, IndirCalls); - InjectTraceForCmp(F, CmpTraceTargets); - InjectTraceForSwitch(F, SwitchTraceTargets); - InjectTraceForDiv(F, DivTraceTargets); - InjectTraceForGep(F, GepTraceTargets); + // InjectTraceForCmp(F, CmpTraceTargets); + // InjectTraceForSwitch(F, SwitchTraceTargets); } @@ -717,33 +675,30 @@ GlobalVariable *ModuleSanitizerCoverageAFL::CreateFunctionLocalArrayInSection( *CurModule, ArrayTy, false, GlobalVariable::PrivateLinkage, Constant::getNullValue(ArrayTy), "__sancov_gen_"); -#if LLVM_VERSION_MAJOR >= 13 if (TargetTriple.supportsCOMDAT() && (TargetTriple.isOSBinFormatELF() || !F.isInterposable())) if (auto Comdat = getOrCreateFunctionComdat(F, TargetTriple)) Array->setComdat(Comdat); + Array->setSection(getSectionName(Section)); +#if LLVM_VERSION_MAJOR >= 16 + Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue())); #else - if (TargetTriple.supportsCOMDAT() && !F.isInterposable()) - if (auto Comdat = - GetOrCreateFunctionComdat(F, TargetTriple, CurModuleUniqueId)) - Array->setComdat(Comdat); + Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedSize())); #endif - Array->setSection(getSectionName(Section)); -#if (LLVM_VERSION_MAJOR >= 11) || \ - (LLVM_VERSION_MAJOR == 10 && LLVM_VERSION_MINOR >= 1) - #if LLVM_VERSION_MAJOR >= 16 - Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedValue())); - #else - Array->setAlignment(Align(DL->getTypeStoreSize(Ty).getFixedSize())); - #endif -#else - Array->setAlignment(Align(4)); // cheating -#endif - GlobalsToAppendToUsed.push_back(Array); - GlobalsToAppendToCompilerUsed.push_back(Array); - MDNode *MD = MDNode::get(F.getContext(), ValueAsMetadata::get(&F)); - Array->addMetadata(LLVMContext::MD_associated, *MD); + // sancov_pcs parallels the other metadata section(s). Optimizers (e.g. + // GlobalOpt/ConstantMerge) may not discard sancov_pcs and the other + // section(s) as a unit, so we conservatively retain all unconditionally in + // the compiler. + // + // With comdat (COFF/ELF), the linker can guarantee the associated sections + // will be retained or discarded as a unit, so llvm.compiler.used is + // sufficient. Otherwise, conservatively make all of them retained by the + // linker. + if (Array->hasComdat()) + GlobalsToAppendToCompilerUsed.push_back(Array); + else + GlobalsToAppendToUsed.push_back(Array); return Array; @@ -768,8 +723,12 @@ GlobalVariable *ModuleSanitizerCoverageAFL::CreatePCArray( PCs.push_back((Constant *)IRB.CreatePointerCast( BlockAddress::get(AllBlocks[i]), IntptrPtrTy)); +#if LLVM_VERSION_MAJOR >= 16 + PCs.push_back(Constant::getNullValue(IntptrPtrTy)); +#else PCs.push_back((Constant *)IRB.CreateIntToPtr( ConstantInt::get(IntptrTy, 0), IntptrPtrTy)); +#endif } @@ -792,21 +751,13 @@ void ModuleSanitizerCoverageAFL::CreateFunctionLocalArrays( FunctionGuardArray = CreateFunctionLocalArrayInSection( AllBlocks.size() + special, F, Int32Ty, SanCovGuardsSectionName); - if (Options.Inline8bitCounters) - Function8bitCounterArray = CreateFunctionLocalArrayInSection( - AllBlocks.size(), F, Int8Ty, SanCovCountersSectionName); - /* - if (Options.InlineBoolFlag) - FunctionBoolArray = CreateFunctionLocalArrayInSection( - AllBlocks.size(), F, Int1Ty, SanCovBoolFlagSectionName); - */ - if (Options.PCTable) FunctionPCsArray = CreatePCArray(F, AllBlocks); - } bool ModuleSanitizerCoverageAFL::InjectCoverage( Function &F, ArrayRef AllBlocks, bool IsLeafFunc) { + if (AllBlocks.empty()) return false; + uint32_t cnt_cov = 0, cnt_sel = 0, cnt_sel_inc = 0; static uint32_t first = 1; @@ -855,7 +806,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } -#if (LLVM_VERSION_MAJOR >= 12) else if (t->getTypeID() == llvm::Type::FixedVectorTyID) { FixedVectorType *tt = dyn_cast(t); @@ -868,16 +818,14 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } -#endif - } } } - /* Create PCGUARD array */ CreateFunctionLocalArrays(F, AllBlocks, first + cnt_cov + cnt_sel_inc); + if (first) { first = 0; } selects += cnt_sel; @@ -889,12 +837,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( CallInst *callInst = nullptr; - /* - std::string errMsg; - raw_string_ostream os(errMsg); - IN.print(os); - fprintf(stderr, "X: %s\n", os.str().c_str()); - */ if ((callInst = dyn_cast(&IN))) { Function *Callee = callInst->getCalledFunction(); @@ -1033,12 +975,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } - /* - std::string errMsg; - raw_string_ostream os(errMsg); - x->print(os); - fprintf(stderr, "X: %s\n", os.str().c_str()); - */ result = IRB.CreateSelect(condition, x, y); } @@ -1063,13 +999,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( IRB.CreateLoad(PointerType::get(Int8Ty, 0), AFLMapPtr); ModuleSanitizerCoverageAFL::SetNoSanitizeMetadata(MapPtr); - /* - std::string errMsg; - raw_string_ostream os(errMsg); - result->print(os); - fprintf(stderr, "X: %s\n", os.str().c_str()); - */ - while (1) { /* Get CurLoc */ @@ -1159,29 +1088,6 @@ bool ModuleSanitizerCoverageAFL::InjectCoverage( } -// On every indirect call we call a run-time function -// __sanitizer_cov_indir_call* with two parameters: -// - callee address, -// - global cache array that contains CacheSize pointers (zero-initialized). -// The cache is used to speed up recording the caller-callee pairs. -// The address of the caller is passed implicitly via caller PC. -// CacheSize is encoded in the name of the run-time function. -void ModuleSanitizerCoverageAFL::InjectCoverageForIndirectCalls( - Function &F, ArrayRef IndirCalls) { - - if (IndirCalls.empty()) return; - for (auto I : IndirCalls) { - - IRBuilder<> IRB(I); - CallBase &CB = cast(*I); - Value *Callee = CB.getCalledOperand(); - if (isa(Callee)) continue; - IRB.CreateCall(SanCovTracePCIndir, IRB.CreatePointerCast(Callee, IntptrTy)); - - } - -} - // For every switch statement we insert a call: // __sanitizer_cov_trace_switch(CondValue, // {NumCases, ValueSizeInBits, Case0Value, Case1Value, Case2Value, ... }) @@ -1237,41 +1143,6 @@ void ModuleSanitizerCoverageAFL::InjectTraceForSwitch( } -void ModuleSanitizerCoverageAFL::InjectTraceForDiv( - Function &, ArrayRef DivTraceTargets) { - - for (auto BO : DivTraceTargets) { - - IRBuilder<> IRB(BO); - Value *A1 = BO->getOperand(1); - if (isa(A1)) continue; - if (!A1->getType()->isIntegerTy()) continue; - uint64_t TypeSize = DL->getTypeStoreSizeInBits(A1->getType()); - int CallbackIdx = TypeSize == 32 ? 0 : TypeSize == 64 ? 1 : -1; - if (CallbackIdx < 0) continue; - auto Ty = Type::getIntNTy(*C, TypeSize); - IRB.CreateCall(SanCovTraceDivFunction[CallbackIdx], - {IRB.CreateIntCast(A1, Ty, true)}); - - } - -} - -void ModuleSanitizerCoverageAFL::InjectTraceForGep( - Function &, ArrayRef GepTraceTargets) { - - for (auto GEP : GepTraceTargets) { - - IRBuilder<> IRB(GEP); - for (Use &Idx : GEP->indices()) - if (!isa(Idx) && Idx->getType()->isIntegerTy()) - IRB.CreateCall(SanCovTraceGepFunction, - {IRB.CreateIntCast(Idx, IntptrTy, true)}); - - } - -} - void ModuleSanitizerCoverageAFL::InjectTraceForCmp( Function &, ArrayRef CmpTraceTargets) { @@ -1321,27 +1192,44 @@ void ModuleSanitizerCoverageAFL::InjectCoverageAtBlock(Function &F, BasicBlock::iterator IP = BB.getFirstInsertionPt(); bool IsEntryBB = &BB == &F.getEntryBlock(); + DebugLoc EntryLoc; if (IsEntryBB) { - // Keep allocas and llvm.localescape calls in the entry block. Even + if (auto SP = F.getSubprogram()) + EntryLoc = DILocation::get(SP->getContext(), SP->getScopeLine(), 0, SP); + // Keep static allocas and llvm.localescape calls in the entry block. Even // if we aren't splitting the block, it's nice for allocas to be before // calls. IP = PrepareToSplitEntryBlock(BB, IP); +#if LLVM_VERSION_MAJOR < 15 + + } else { + + EntryLoc = IP->getDebugLoc(); + if (!EntryLoc) + if (auto *SP = F.getSubprogram()) + EntryLoc = DILocation::get(SP->getContext(), 0, 0, SP); +#endif } +#if LLVM_VERSION_MAJOR >= 15 + InstrumentationIRBuilder IRB(&*IP); +#else IRBuilder<> IRB(&*IP); - - if (Options.TracePC) { - - IRB.CreateCall(SanCovTracePC); - // ->setCannotMerge(); // gets the PC using GET_CALLER_PC. - - } - +#endif + if (EntryLoc) IRB.SetCurrentDebugLocation(EntryLoc); if (Options.TracePCGuard) { + /* + auto GuardPtr = IRB.CreateIntToPtr( + IRB.CreateAdd(IRB.CreatePointerCast(FunctionGuardArray, IntptrTy), + ConstantInt::get(IntptrTy, Idx * 4)), + Int32PtrTy); + IRB.CreateCall(SanCovTracePCGuard, GuardPtr)->setCannotMerge(); + */ + /* Get CurLoc */ Value *GuardPtr = IRB.CreateIntToPtr( @@ -1399,57 +1287,6 @@ void ModuleSanitizerCoverageAFL::InjectCoverageAtBlock(Function &F, } - if (Options.Inline8bitCounters) { - - auto CounterPtr = IRB.CreateGEP( - Function8bitCounterArray->getValueType(), Function8bitCounterArray, - {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)}); - auto Load = IRB.CreateLoad(Int8Ty, CounterPtr); - auto Inc = IRB.CreateAdd(Load, ConstantInt::get(Int8Ty, 1)); - auto Store = IRB.CreateStore(Inc, CounterPtr); - SetNoSanitizeMetadata(Load); - SetNoSanitizeMetadata(Store); - - } - - /* - if (Options.InlineBoolFlag) { - - auto FlagPtr = IRB.CreateGEP( - FunctionBoolArray->getValueType(), FunctionBoolArray, - {ConstantInt::get(IntptrTy, 0), ConstantInt::get(IntptrTy, Idx)}); - auto Load = IRB.CreateLoad(Int1Ty, FlagPtr); - auto ThenTerm = - SplitBlockAndInsertIfThen(IRB.CreateIsNull(Load), &*IP, false); - IRBuilder<> ThenIRB(ThenTerm); - auto Store = ThenIRB.CreateStore(ConstantInt::getTrue(Int1Ty), FlagPtr); - SetNoSanitizeMetadata(Load); - SetNoSanitizeMetadata(Store); - - } - - */ - - if (Options.StackDepth && IsEntryBB && !IsLeafFunc) { - - // Check stack depth. If it's the deepest so far, record it. - Module *M = F.getParent(); - Function *GetFrameAddr = Intrinsic::getDeclaration( - M, Intrinsic::frameaddress, - IRB.getInt8PtrTy(M->getDataLayout().getAllocaAddrSpace())); - auto FrameAddrPtr = - IRB.CreateCall(GetFrameAddr, {Constant::getNullValue(Int32Ty)}); - auto FrameAddrInt = IRB.CreatePtrToInt(FrameAddrPtr, IntptrTy); - auto LowestStack = IRB.CreateLoad(IntptrTy, SanCovLowestStack); - auto IsStackLower = IRB.CreateICmpULT(FrameAddrInt, LowestStack); - auto ThenTerm = SplitBlockAndInsertIfThen(IsStackLower, &*IP, false); - IRBuilder<> ThenIRB(ThenTerm); - auto Store = ThenIRB.CreateStore(FrameAddrInt, SanCovLowestStack); - SetNoSanitizeMetadata(LowestStack); - SetNoSanitizeMetadata(Store); - - } - } std::string ModuleSanitizerCoverageAFL::getSectionName( diff --git a/src/afl-cc.c b/src/afl-cc.c index 84fe70ec..9e56828c 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -997,7 +997,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (instrument_mode == INSTRUMENT_PCGUARD) { -#if LLVM_MAJOR >= 11 +#if LLVM_MAJOR >= 13 #if defined __ANDROID__ || ANDROID cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; instrument_mode = INSTRUMENT_LLVMNATIVE; @@ -1014,7 +1014,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } else { - #if LLVM_MAJOR >= 11 /* use new pass manager */ + #if LLVM_MAJOR >= 13 /* use new pass manager */ #if LLVM_MAJOR < 16 cc_params[cc_par_cnt++] = "-fexperimental-new-pass-manager"; #endif @@ -1035,12 +1035,12 @@ static void edit_params(u32 argc, char **argv, char **envp) { #if LLVM_MAJOR >= 4 if (!be_quiet) SAYF( - "Using unoptimized trace-pc-guard, upgrade to llvm 10.0.1+ for " + "Using unoptimized trace-pc-guard, upgrade to LLVM 13+ for " "enhanced version.\n"); cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard"; instrument_mode = INSTRUMENT_LLVMNATIVE; #else - FATAL("pcguard instrumentation requires llvm 4.0.1+"); + FATAL("pcguard instrumentation requires LLVM 4.0.1+"); #endif #endif @@ -1053,7 +1053,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { cc_params[cc_par_cnt++] = "-fsanitize-coverage=trace-pc-guard,bb,no-prune,pc-table"; #else - FATAL("pcguard instrumentation with pc-table requires llvm 6.0.1+"); + FATAL("pcguard instrumentation with pc-table requires LLVM 6.0.1+"); #endif } else { @@ -1063,7 +1063,7 @@ static void edit_params(u32 argc, char **argv, char **envp) { } #else - FATAL("pcguard instrumentation requires llvm 4.0.1+"); + FATAL("pcguard instrumentation requires LLVM 4.0.1+"); #endif } else { @@ -2031,7 +2031,7 @@ int main(int argc, char **argv, char **envp) { if (!compiler_mode) { // lto is not a default because outside of afl-cc RANLIB and AR have to - // be set to llvm versions so this would work + // be set to LLVM versions so this would work if (have_llvm) compiler_mode = LLVM; else if (have_gcc_plugin) @@ -2050,6 +2050,17 @@ int main(int argc, char **argv, char **envp) { } + /* if our PCGUARD implementation is not available then silently switch to + native LLVM PCGUARD */ + if (compiler_mode == CLANG && + (instrument_mode == INSTRUMENT_DEFAULT || + instrument_mode == INSTRUMENT_PCGUARD) && + find_object("SanitizerCoveragePCGUARD.so", argv[0]) == NULL) { + + instrument_mode = INSTRUMENT_LLVMNATIVE; + + } + if (compiler_mode == GCC) { if (clang_mode) { @@ -2096,12 +2107,12 @@ int main(int argc, char **argv, char **envp) { "-------------|\n" "MODES: NCC PERSIST DICT LAF " "CMPLOG SELECT\n" - " [LTO] llvm LTO: %s%s\n" + " [LTO] LLVM LTO: %s%s\n" " PCGUARD DEFAULT yes yes yes yes yes " " yes\n" " CLASSIC yes yes yes yes yes " " yes\n" - " [LLVM] llvm: %s%s\n" + " [LLVM] LLVM: %s%s\n" " PCGUARD %s yes yes module yes yes " "yes\n" " CLASSIC %s no yes module yes yes " @@ -2171,7 +2182,7 @@ int main(int argc, char **argv, char **envp) { " (instrumentation/README.lto.md)\n" " PERSIST: persistent mode support [code] (huge speed increase!)\n" " (instrumentation/README.persistent_mode.md)\n" - " DICT: dictionary in the target [yes=automatic or llvm module " + " DICT: dictionary in the target [yes=automatic or LLVM module " "pass]\n" " (instrumentation/README.lto.md + " "instrumentation/README.llvm.md)\n" From 2b500ce97ee50c4d237702e6121bbd38e56e8ec6 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 1 Jun 2023 12:27:34 +0200 Subject: [PATCH 144/149] llvm 15 fixes --- instrumentation/SanitizerCoverageLTO.so.cc | 2 +- instrumentation/SanitizerCoveragePCGUARD.so.cc | 4 +--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index d7b03634..2d17ffd4 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -331,7 +331,7 @@ llvmGetPassPluginInfo() { #if LLVM_VERSION_MAJOR <= 13 using OptimizationLevel = typename PassBuilder::OptimizationLevel; #endif -#if LLVM_VERSION_MAJOR >= 15 +#if LLVM_VERSION_MAJOR >= 16 PB.registerFullLinkTimeOptimizationLastEPCallback( #else PB.registerOptimizerLastEPCallback( diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 2abc58ec..29ab1427 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -162,9 +162,7 @@ class ModuleSanitizerCoverageAFL void SetNoSanitizeMetadata(Instruction *I) { -#if LLVM_VERSION_MAJOR == 15 - I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, None)); -#elif LLVM_VERSION_MAJOR >= 16 +#if LLVM_VERSION_MAJOR >= 16 I->setMetadata(LLVMContext::MD_nosanitize, MDNode::get(*C, std::nullopt)); #else I->setMetadata(I->getModule()->getMDKindID("nosanitize"), From b644e48f36485c645cbc0dadf0fddb2aa14cc079 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 1 Jun 2023 13:28:07 +0200 Subject: [PATCH 145/149] more llvm 15 specialities --- instrumentation/SanitizerCoveragePCGUARD.so.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 29ab1427..20f54b84 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -1212,7 +1212,7 @@ void ModuleSanitizerCoverageAFL::InjectCoverageAtBlock(Function &F, } -#if LLVM_VERSION_MAJOR >= 15 +#if LLVM_VERSION_MAJOR >= 16 InstrumentationIRBuilder IRB(&*IP); #else IRBuilder<> IRB(&*IP); From 28fd9716086f781f548423ea00e0e441e97037bc Mon Sep 17 00:00:00 2001 From: fanquake Date: Mon, 5 Jun 2023 16:54:23 +0100 Subject: [PATCH 146/149] build: fix compiler version in build output Currently, if I build like with Clang, I'll get: ```bash make LLVM_CONFIG=llvm-config-15 CC=clang-15 CXX=clang++-15 [+] Everything seems to be working, ready to compile. (gcc version 12.1.0 (Ubuntu 12.1.0-2ubuntu1~22.04) ) clang-15 -O2 -D_FORTIFY_SOURCE=1 .... ``` Which is somewhat confusing. Fix this, and in a way that still outputs the correct version info for Clang and GCC. Use `--version`, and pick the first line, as that is where they are consistent in output. `clang -v` gives the version first, whereas `gcc -v` gives the version on the last line. We switch to using $(CC), otherwise we also get incorrect output, and dropping CCVER altogether, given this is it's only use. --- GNUmakefile | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index 715e7386..55676d97 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -291,8 +291,6 @@ ifeq "$(shell command -v svn >/dev/null && svn proplist . 2>/dev/null && echo 1 IN_REPO=1 endif -CCVER=$(shell cc -v 2>&1|tail -n 1) - ifeq "$(shell echo 'int main() { return 0;}' | $(CC) $(CFLAGS) -fsanitize=address -x c - -o .test2 2>/dev/null && echo 1 || echo 0 ; rm -f .test2 )" "1" ASAN_CFLAGS=-fsanitize=address -fstack-protector-all -fno-omit-frame-pointer -DASAN_BUILD ASAN_LDFLAGS=-fsanitize=address -fstack-protector-all -fno-omit-frame-pointer @@ -439,7 +437,7 @@ endif .PHONY: ready ready: - @echo "[+] Everything seems to be working, ready to compile. ($(CCVER))" + @echo "[+] Everything seems to be working, ready to compile. ($(shell $(CC) --version 2>&1|head -n 1))" afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) src/$@.c -o $@ $(LDFLAGS) From abc26a932a187f4fb84ac178c44326c9e46efca5 Mon Sep 17 00:00:00 2001 From: cocochpie Date: Mon, 5 Jun 2023 20:33:33 +0000 Subject: [PATCH 147/149] Revive f567a89dae29afb2e421d649f0e750e77913f08c --- instrumentation/SanitizerCoveragePCGUARD.so.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 20f54b84..25851dda 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -20,6 +20,8 @@ #endif #if LLVM_VERSION_MAJOR < 17 #include "llvm/Analysis/EHPersonalities.h" +#else + #include "llvm/IR/EHPersonalities.h" #endif #include "llvm/Analysis/PostDominators.h" #if LLVM_VERSION_MAJOR < 15 @@ -31,8 +33,10 @@ #include "llvm/IR/DebugInfo.h" #endif #include "llvm/IR/Dominators.h" -#if LLVM_VERSION_MAJOR >= 17 +#if LLVM_VERSION_MAJOR < 17 #include "llvm/Analysis/EHPersonalities.h" +#else + #include "llvm/IR/EHPersonalities.h" #endif #include "llvm/IR/Function.h" #if LLVM_VERSION_MAJOR >= 16 From 9585f5cdfeb7b287ec8614a92f295127eba0a384 Mon Sep 17 00:00:00 2001 From: cocochpie Date: Tue, 6 Jun 2023 04:07:38 +0000 Subject: [PATCH 148/149] =?UTF-8?q?change=20the=20=E2=80=98#if=E2=80=99=20?= =?UTF-8?q?to=20>=3D=2017=20instead=20of=20<=2017?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- instrumentation/SanitizerCoveragePCGUARD.so.cc | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 25851dda..7171e7aa 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -18,11 +18,6 @@ #include "llvm/ADT/Triple.h" #endif #endif -#if LLVM_VERSION_MAJOR < 17 - #include "llvm/Analysis/EHPersonalities.h" -#else - #include "llvm/IR/EHPersonalities.h" -#endif #include "llvm/Analysis/PostDominators.h" #if LLVM_VERSION_MAJOR < 15 #include "llvm/IR/CFG.h" @@ -33,10 +28,10 @@ #include "llvm/IR/DebugInfo.h" #endif #include "llvm/IR/Dominators.h" -#if LLVM_VERSION_MAJOR < 17 - #include "llvm/Analysis/EHPersonalities.h" -#else +#if LLVM_VERSION_MAJOR >= 17 #include "llvm/IR/EHPersonalities.h" +#else + #include "llvm/Analysis/EHPersonalities.h" #endif #include "llvm/IR/Function.h" #if LLVM_VERSION_MAJOR >= 16 From 234d55ccd547b61839612cc068127dbceaf8a9ec Mon Sep 17 00:00:00 2001 From: fanquake Date: Tue, 6 Jun 2023 10:29:54 +0100 Subject: [PATCH 149/149] build: adjust LLVM development version check Adjust version check to only warn for LLVM 17.x and newer, which are the development versions. Otherwise we'll get: ```bash make LLVM_CONFIG=llvm-config-15 CC=clang-15 CXX=clang++-15 GNUmakefile.llvm:69: you are using an in-development llvm version - this might break llvm_mode! ``` for versions that are supported, and not in development. --- GNUmakefile.llvm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GNUmakefile.llvm b/GNUmakefile.llvm index 6c68f1f3..6ffac68f 100644 --- a/GNUmakefile.llvm +++ b/GNUmakefile.llvm @@ -46,7 +46,7 @@ LLVMVER = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/git//' | sed 's LLVM_MAJOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/\..*//' ) LLVM_MINOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/.*\.//' | sed 's/git//' | sed 's/svn//' | sed 's/ .*//' ) LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^[0-2]\.|^3.[0-7]\.' && echo 1 || echo 0 ) -LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[5-9]' && echo 1 || echo 0 ) +LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[7-9]' && echo 1 || echo 0 ) LLVM_NEW_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[0-9]' && echo 1 || echo 0 ) LLVM_NEWER_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[6-9]' && echo 1 || echo 0 ) LLVM_13_OK = $(shell $(LLVM_CONFIG) --version 2>/dev/null | grep -E -q '^1[3-9]' && echo 1 || echo 0 )