From b0492ba642e7afb9a1269c27a264d11e5c2e6ec5 Mon Sep 17 00:00:00 2001 From: Dominik Maier Date: Tue, 14 Jan 2020 13:58:36 +0100 Subject: [PATCH 01/44] Fixed description of unicorn harness.c --- unicorn_mode/samples/c/harness.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/unicorn_mode/samples/c/harness.c b/unicorn_mode/samples/c/harness.c index cc81ba7f..a987b8e1 100644 --- a/unicorn_mode/samples/c/harness.c +++ b/unicorn_mode/samples/c/harness.c @@ -1,11 +1,10 @@ /* Simple test harness for AFL++'s unicornafl c mode. - This loads the simple_target.bin binary (precompiled as MIPS code) into + This loads the simple_target_x86_64 binary into Unicorn's memory map for emulation, places the specified input into - simple_target's buffer (hardcoded to be at 0x300000), and executes 'main()'. - If any crashes occur during emulation, this script throws a matching signal - to tell AFL that a crash occurred. + argv[1], sets up argv, and argc and executes 'main()'. + If run inside AFL, afl_fuzz automatically does the "right thing" Run under AFL as follows: From d0ea8f84334a36133c289c711c233d1f5a1a8045 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 14 Jan 2020 14:45:55 +0100 Subject: [PATCH 02/44] todo update --- TODO | 3 +++ 1 file changed, 3 insertions(+) diff --git a/TODO b/TODO index 8a8fdd41..259fbd0a 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,9 @@ Roadmap 2.61+: ============== +afl-fuzz: + - sync_fuzzers(): only masters sync from all, slaves only sync from master + gcc_plugin: - laf-intel - better instrumentation From d1d5e7c02a4b17f42b6334f258899d6543c9142c Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 10:10:25 +0100 Subject: [PATCH 03/44] blacklist function support for llvm_mode --- TODO | 3 +++ docs/ChangeLog | 3 ++- llvm_mode/LLVMInsTrim.so.cc | 24 ++++++++++++++++++++++++ llvm_mode/afl-clang-fast.c | 3 ++- llvm_mode/afl-llvm-pass.so.cc | 27 ++++++++++++++++++++++++--- src/afl-fuzz-stats.c | 14 +++++++------- 6 files changed, 62 insertions(+), 12 deletions(-) diff --git a/TODO b/TODO index 259fbd0a..e935eafa 100644 --- a/TODO +++ b/TODO @@ -2,6 +2,9 @@ Roadmap 2.61+: ============== +Makefile: + - -march=native -Ofast -flto=full + afl-fuzz: - sync_fuzzers(): only masters sync from all, slaves only sync from master diff --git a/docs/ChangeLog b/docs/ChangeLog index f034d251..a94a2038 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -20,7 +20,8 @@ Version ++2.60d (develop): - afl-fuzz: - now prints the real python version support compiled in - afl-clang-fast now shows in the help output for which llvm version it - was compiled for. + was compiled for + - added blacklisted function check in llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 552cf580..11451b43 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -94,6 +94,28 @@ struct InsTrim : public ModulePass { } + // ripped from aflgo + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", + "llvm.", + "sancov.", + "__ubsan_handle_", + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + + } + bool runOnModule(Module &M) override { char be_quiet = 0; @@ -240,6 +262,8 @@ struct InsTrim : public ModulePass { } + if (isBlacklisted(&F)) continue; + std::unordered_set MS; if (!MarkSetOpt) { diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index 2b359cdf..b322b762 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -444,7 +444,8 @@ int main(int argc, char** argv) { "You can specify custom next-stage toolchain via AFL_CC and AFL_CXX. " "Setting\n" "AFL_HARDEN enables hardening optimizations in the compiled code.\n\n" - "afl-clang-fast was built for llvm %s with the llvm binary path of \"%s\".\n\n", + "afl-clang-fast was built for llvm %s with the llvm binary path of " + "\"%s\".\n\n", BIN_PATH, BIN_PATH, LLVM_VERSION, LLVM_BINDIR); exit(1); diff --git a/llvm_mode/afl-llvm-pass.so.cc b/llvm_mode/afl-llvm-pass.so.cc index 0b3db4ed..15cc6127 100644 --- a/llvm_mode/afl-llvm-pass.so.cc +++ b/llvm_mode/afl-llvm-pass.so.cc @@ -75,6 +75,28 @@ class AFLCoverage : public ModulePass { } + // ripped from aflgo + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", + "llvm.", + "sancov.", + "__ubsan_handle_", + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + + } + bool runOnModule(Module &M) override; // StringRef getPassName() const override { @@ -156,13 +178,11 @@ bool AFLCoverage::runOnModule(Module &M) { /* Instrument all the things! */ - const char *IntrinsicPrefix = "llvm."; int inst_blocks = 0; for (auto &F : M) { - auto Fname = F.getName(); - if (Fname.startswith(IntrinsicPrefix)) continue; + if (isBlacklisted(&F)) continue; for (auto &BB : F) { @@ -377,6 +397,7 @@ bool AFLCoverage::runOnModule(Module &M) { inst_blocks++; } + } /* Say something nice. */ diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 18f32ae8..7679403b 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -334,9 +334,9 @@ void show_stats(void) { /* Lord, forgive me this. */ - SAYF(SET_G1 bSTG bLT bH bSTOP cCYA + SAYF(SET_G1 bSTG bLT bH bSTOP cCYA " process timing " bSTG bH30 bH5 bH bHB bH bSTOP cCYA - " overall results " bSTG bH2 bH2 bRT "\n"); + " overall results " bSTG bH2 bH2 bRT "\n"); if (dumb_mode) { @@ -413,9 +413,9 @@ void show_stats(void) { " uniq hangs : " cRST "%-6s" bSTG bV "\n", DTD(cur_ms, last_hang_time), tmp); - SAYF(bVR bH bSTOP cCYA + SAYF(bVR bH bSTOP cCYA " cycle progress " bSTG bH10 bH5 bH2 bH2 bHB bH bSTOP cCYA - " map coverage " bSTG bH bHT bH20 bH2 bVL "\n"); + " map coverage " bSTG bH bHT bH20 bH2 bVL "\n"); /* This gets funny because we want to print several variable-length variables together, but then cram them into a fixed-width field - so we need to @@ -443,9 +443,9 @@ void show_stats(void) { SAYF(bSTOP " count coverage : " cRST "%-21s" bSTG bV "\n", tmp); - SAYF(bVR bH bSTOP cCYA + SAYF(bVR bH bSTOP cCYA " stage progress " bSTG bH10 bH5 bH2 bH2 bX bH bSTOP cCYA - " findings in depth " bSTG bH10 bH5 bH2 bH2 bVL "\n"); + " findings in depth " bSTG bH10 bH5 bH2 bH2 bVL "\n"); sprintf(tmp, "%s (%0.02f%%)", DI(queued_favored), ((double)queued_favored) * 100 / queued_paths); @@ -514,7 +514,7 @@ void show_stats(void) { /* Aaaalmost there... hold on! */ - SAYF(bVR bH cCYA bSTOP + SAYF(bVR bH cCYA bSTOP " fuzzing strategy yields " bSTG bH10 bHT bH10 bH5 bHB bH bSTOP cCYA " path geometry " bSTG bH5 bH2 bVL "\n"); From e673dc6dbebedc1374cd4fb9201cf784408c00f3 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 10:23:28 +0100 Subject: [PATCH 04/44] stronger afl-fuzz performance compilation --- Makefile | 22 +++++++++++++++++++--- docs/ChangeLog | 1 + src/afl-fuzz-bitmap.c | 2 +- 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/Makefile b/Makefile index 0c0b73c5..57d4c8c7 100644 --- a/Makefile +++ b/Makefile @@ -32,9 +32,25 @@ PROGS = afl-gcc afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) -CFLAGS ?= -O3 -funroll-loops -CFLAGS += -Wall -D_FORTIFY_SOURCE=2 -g -Wno-pointer-sign -I include/ \ - -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -march=native +endif + +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -flto=full +else + ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=thin -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -flto=thin + else + ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_PERFORMANCE += -flto + endif + endif +endif + +CFLAGS ?= -funroll-loops -Ofast $(CFLAGS_PERFORMANCE) +CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ + -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) diff --git a/docs/ChangeLog b/docs/ChangeLog index a94a2038..594f75e9 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -19,6 +19,7 @@ Version ++2.60d (develop): - afl-fuzz: - now prints the real python version support compiled in + - set stronger performance compile options and little tweaks - afl-clang-fast now shows in the help output for which llvm version it was compiled for - added blacklisted function check in llvm_mode diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 0f611cdf..515a7a79 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -524,7 +524,7 @@ u8 save_if_interesting(char** argv, void* mem, u32 len, u8 fault) { struct queue_entry* q = queue; while (q) { - if (q->exec_cksum == cksum) q->n_fuzz = q->n_fuzz + 1; + if (q->exec_cksum == cksum) { q->n_fuzz = q->n_fuzz + 1; break ; } q = q->next; From d5dff8960cf72f097a5d8e178ea28dfe9a72c0b3 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 12:00:25 +0100 Subject: [PATCH 05/44] O3 is faster than Ofast ... duh ... --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 57d4c8c7..893e51d1 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ else endif endif -CFLAGS ?= -funroll-loops -Ofast $(CFLAGS_PERFORMANCE) +CFLAGS ?= -O3 -funroll-loops $(CFLAGS_PERFORMANCE) CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function From b5c19a58f6a5badfea74f267a3c032cbd89b4bea Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 15 Jan 2020 15:54:54 +0100 Subject: [PATCH 06/44] fix for cc=clang and libradamsa optimization --- Makefile | 26 +++++++++++--------------- src/third_party/libradamsa/Makefile | 8 +++++--- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/Makefile b/Makefile index 893e51d1..56b77999 100644 --- a/Makefile +++ b/Makefile @@ -32,23 +32,19 @@ PROGS = afl-gcc afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) -ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -march=native -endif - ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -flto=full + CFLAGS_FLTO ?= -flto=full else ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=thin -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -flto=thin + CFLAGS_FLTO ?= -flto=thin else ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS_PERFORMANCE += -flto + CFLAGS_FLTO ?= -flto endif endif endif -CFLAGS ?= -O3 -funroll-loops $(CFLAGS_PERFORMANCE) +CFLAGS ?= -O3 -funroll-loops -march=native CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function @@ -245,13 +241,13 @@ afl-as: src/afl-as.c include/afl-as.h $(COMM_HDR) | test_x86 ln -sf afl-as as src/afl-common.o : src/afl-common.c include/common.h - $(CC) $(CFLAGS) -c src/afl-common.c -o src/afl-common.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-common.c -o src/afl-common.o src/afl-forkserver.o : src/afl-forkserver.c include/forkserver.h - $(CC) $(CFLAGS) -c src/afl-forkserver.c -o src/afl-forkserver.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-forkserver.c -o src/afl-forkserver.o src/afl-sharedmem.o : src/afl-sharedmem.c include/sharedmem.h - $(CC) $(CFLAGS) -c src/afl-sharedmem.c -o src/afl-sharedmem.o + $(CC) $(CFLAGS) $(CFLAGS_FLTO) -c src/afl-sharedmem.c -o src/afl-sharedmem.o radamsa: src/third_party/libradamsa/libradamsa.so cp src/third_party/libradamsa/libradamsa.so . @@ -260,16 +256,16 @@ src/third_party/libradamsa/libradamsa.so: src/third_party/libradamsa/libradamsa. $(MAKE) -C src/third_party/libradamsa/ afl-fuzz: include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(PYFLAGS) $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(PYFLAGS) $(LDFLAGS) afl-showmap: src/afl-showmap.c src/afl-common.o src/afl-sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) afl-tmin: src/afl-tmin.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(LDFLAGS) afl-analyze: src/afl-analyze.c src/afl-common.o src/afl-sharedmem.o $(COMM_HDR) | test_x86 - $(CC) $(CFLAGS) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) + $(CC) $(CFLAGS) $(CFLAGS_FLTO) src/$@.c src/afl-common.o src/afl-sharedmem.o -o $@ $(LDFLAGS) afl-gotcpu: src/afl-gotcpu.c $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) src/$@.c -o $@ $(LDFLAGS) diff --git a/src/third_party/libradamsa/Makefile b/src/third_party/libradamsa/Makefile index 0f56fa18..b061f8e1 100644 --- a/src/third_party/libradamsa/Makefile +++ b/src/third_party/libradamsa/Makefile @@ -2,17 +2,19 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) all: libradamsa.so +CFLAGS ?= -O3 -funroll-loops -march=native $(CFLAGS_FLTO) + libradamsa.so: libradamsa.a - $(CC) -O3 -shared libradamsa.a -o libradamsa.so + $(CC) $(CFLAGS) -shared libradamsa.a -o libradamsa.so libradamsa.a: libradamsa.c radamsa.h @echo " ***************************************************************" @echo " * Compiling libradamsa, wait some minutes (~3 on modern CPUs) *" @echo " ***************************************************************" - $(CC) -fPIC -O3 -I $(CUR_DIR) -o libradamsa.a -c libradamsa.c + $(CC) -fPIC $(CFLAGS) -I $(CUR_DIR) -o libradamsa.a -c libradamsa.c test: libradamsa.a libradamsa-test.c - cc -O3 -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a + cc $(CLFAGS) -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a ./libradamsa-test libradamsa-test.c | grep "library test passed" rm /tmp/libradamsa-*.fuzz From bd58094dbc87463680a54d99ffcff7ae2a591353 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 16 Jan 2020 13:38:04 +0100 Subject: [PATCH 07/44] fix libradamsa Makefile typo --- src/third_party/libradamsa/Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/third_party/libradamsa/Makefile b/src/third_party/libradamsa/Makefile index b061f8e1..c5a78ead 100644 --- a/src/third_party/libradamsa/Makefile +++ b/src/third_party/libradamsa/Makefile @@ -2,7 +2,11 @@ CUR_DIR := $(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) all: libradamsa.so -CFLAGS ?= -O3 -funroll-loops -march=native $(CFLAGS_FLTO) +# These can be overriden: +CFLAGS ?= -march=native $(CFLAGS_FLTO) + +# These are required: (otherwise radamsa gets very very slooooow) +CFLAGS += -O3 -funroll-loops libradamsa.so: libradamsa.a $(CC) $(CFLAGS) -shared libradamsa.a -o libradamsa.so @@ -14,7 +18,7 @@ libradamsa.a: libradamsa.c radamsa.h $(CC) -fPIC $(CFLAGS) -I $(CUR_DIR) -o libradamsa.a -c libradamsa.c test: libradamsa.a libradamsa-test.c - cc $(CLFAGS) -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a + $(CC) $(CFLAGS) -I $(CUR_DIR) -o libradamsa-test libradamsa-test.c libradamsa.a ./libradamsa-test libradamsa-test.c | grep "library test passed" rm /tmp/libradamsa-*.fuzz From 3671d7eb8a9dddc5be29ae5e3d4ba7669a898e53 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 16:00:11 +0100 Subject: [PATCH 08/44] fixes for gcc 5.5 (does not understand -march=native) on NetBSD --- Makefile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 56b77999..d208f509 100644 --- a/Makefile +++ b/Makefile @@ -44,11 +44,15 @@ else endif endif -CFLAGS ?= -O3 -funroll-loops -march=native +CFLAGS ?= -O3 -funroll-loops CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS += -march=native +endif + AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) ifneq "($filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null)" "" @@ -253,7 +257,7 @@ radamsa: src/third_party/libradamsa/libradamsa.so cp src/third_party/libradamsa/libradamsa.so . src/third_party/libradamsa/libradamsa.so: src/third_party/libradamsa/libradamsa.c src/third_party/libradamsa/radamsa.h - $(MAKE) -C src/third_party/libradamsa/ + $(MAKE) -C src/third_party/libradamsa/ CFLAGS="$(CFLAGS)" afl-fuzz: include/afl-fuzz.h $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o $(COMM_HDR) | test_x86 $(CC) $(CFLAGS) $(CFLAGS_FLTO) $(AFL_FUZZ_FILES) src/afl-common.o src/afl-sharedmem.o src/afl-forkserver.o -o $@ $(PYFLAGS) $(LDFLAGS) From 7c0704b30c85b5fdaba582bfb4299623c255f498 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 16:01:44 +0100 Subject: [PATCH 09/44] use default memory limit for afl-tmin (again) in test.sh, fix it in afl-tmin --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index c763e91f..8f40773c 100755 --- a/test/test.sh +++ b/test/test.sh @@ -158,7 +158,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { CODE=1 ;; esac - ../afl-tmin -m200 -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 + ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` test "$SIZE" = 1 && $ECHO "$GREEN[+] afl-tmin correctly minimized the testcase" test "$SIZE" = 1 || { From cb23fe2aba8487464755abe2debfb78980cc5eba Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 16:06:43 +0100 Subject: [PATCH 10/44] increased default memory limit (200megs) on NetBSD (due to kernel bug), fixes the afl-tmin test --- include/config.h | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/include/config.h b/include/config.h index c5139dbd..69380282 100644 --- a/include/config.h +++ b/include/config.h @@ -61,12 +61,15 @@ /* Default memory limit for child process (MB): */ -#ifndef WORD_SIZE_64 -#define MEM_LIMIT 25 +#ifndef __NetBSD__ +# ifndef WORD_SIZE_64 +# define MEM_LIMIT 25 +# else +# define MEM_LIMIT 50 +# endif /* ^!WORD_SIZE_64 */ #else -#define MEM_LIMIT 50 -#endif /* ^!WORD_SIZE_64 */ - +# define MEM_LIMIT 200 +#endif /* Default memory limit when running in QEMU mode (MB): */ #define MEM_LIMIT_QEMU 200 From 800d43b84666461cb682be20fdc5df48c13c252b Mon Sep 17 00:00:00 2001 From: van Hauser Date: Fri, 17 Jan 2020 16:17:08 +0100 Subject: [PATCH 11/44] small change for march=native --- Makefile | 10 +++++----- docs/ChangeLog | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index d208f509..703ed673 100644 --- a/Makefile +++ b/Makefile @@ -44,15 +44,15 @@ else endif endif -CFLAGS ?= -O3 -funroll-loops +ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" + CFLAGS_OPT = -march=native +endif + +CFLAGS ?= -O3 -funroll-loops $(CFLAGS_OPT) CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ -DDOC_PATH=\"$(DOC_PATH)\" -Wno-unused-function -ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" - CFLAGS += -march=native -endif - AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) ifneq "($filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null)" "" diff --git a/docs/ChangeLog b/docs/ChangeLog index 594f75e9..5347d244 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -17,6 +17,7 @@ sending a mail to . Version ++2.60d (develop): -------------------------- + - use -march=native if available - afl-fuzz: - now prints the real python version support compiled in - set stronger performance compile options and little tweaks From 1ac31361ca61f71b6a419064de5063aef80203e5 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 17 Jan 2020 20:41:30 +0100 Subject: [PATCH 12/44] as suggested, added a comment, why NetBSD needs a higher memory limit --- include/config.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/config.h b/include/config.h index 69380282..4eac82e0 100644 --- a/include/config.h +++ b/include/config.h @@ -67,7 +67,7 @@ # else # define MEM_LIMIT 50 # endif /* ^!WORD_SIZE_64 */ -#else +#else /* NetBSD's kernel needs more space for stack, see discussion for issue #165 */ # define MEM_LIMIT 200 #endif /* Default memory limit when running in QEMU mode (MB): */ From 858b5da24e3b060e2ebf6ab48ded22fbdd7d3ceb Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 18 Jan 2020 14:28:31 +0000 Subject: [PATCH 13/44] libdislocator: reallocarray API introduction --- libdislocator/libdislocator.so.c | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/libdislocator/libdislocator.so.c b/libdislocator/libdislocator.so.c index 20649470..b9ba8967 100644 --- a/libdislocator/libdislocator.so.c +++ b/libdislocator/libdislocator.so.c @@ -397,6 +397,28 @@ void* aligned_alloc(size_t align, size_t len) { } +/* specific BSD api mainly checking possible overflow for the size */ + +void* reallocarray(void* ptr, size_t elem_len, size_t elem_cnt) { + + const size_t elem_lim = 1UL << (sizeof(size_t) * 4); + const size_t elem_tot = elem_len * elem_cnt; + void* ret = NULL; + + if ((elem_len >= elem_lim || elem_cnt >= elem_lim) && elem_len > 0 && + elem_cnt > (SIZE_MAX / elem_len)) { + + DEBUGF("reallocarray size overflow (%zu)", elem_tot); + + } else { + + ret = realloc(ptr, elem_tot); + + } + + return ret; +} + __attribute__((constructor)) void __dislocator_init(void) { u8* tmp = (u8*)getenv("AFL_LD_LIMIT_MB"); From 6b0950b03d8a9fd0c21b4be71fd4a4bd6ab68547 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:13:57 +0100 Subject: [PATCH 14/44] fix some syntax errors regarding $(filter ...) --- Makefile | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Makefile b/Makefile index 703ed673..df5ad048 100644 --- a/Makefile +++ b/Makefile @@ -55,17 +55,17 @@ CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ AFL_FUZZ_FILES = $(wildcard src/afl-fuzz*.c) -ifneq "($filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null)" "" +ifneq "$(filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null))" "" PYTHON_INCLUDE ?= $(shell python3.7m-config --includes) PYTHON_LIB ?= $(shell python3.7m-config --ldflags) PYTHON_VERSION = 3.7m else - ifneq "($filter %3.7, $(shell python3.7-config --includes) 2> /dev/null" "" + ifneq "$(filter %3.7, $(shell python3.7-config --includes) 2> /dev/null)" "" PYTHON_INCLUDE ?= $(shell python3.7-config --includes) PYTHON_LIB ?= $(shell python3.7-config --ldflags) PYTHON_VERSION = 3.7 else - ifneq "($filter %2.7, $(shell python2.7-config --includes) 2> /dev/null" "" + ifneq "$(filter %2.7, $(shell python2.7-config --includes) 2> /dev/null)" "" PYTHON_INCLUDE ?= $(shell python2.7-config --includes) PYTHON_LIB ?= $(shell python2.7-config --ldflags) PYTHON_VERSION = 2.7 @@ -77,14 +77,14 @@ PYTHON_INCLUDE ?= $(shell test -e /usr/include/python3.7m && echo /usr/include/p PYTHON_INCLUDE ?= $(shell test -e /usr/include/python3.7 && echo /usr/include/python3.7) PYTHON_INCLUDE ?= $(shell test -e /usr/include/python2.7 && echo /usr/include/python2.7) -ifneq "($filter %3.7m, $(PYTHON_INCLUDE))" "" +ifneq "$(filter %3.7m, $(PYTHON_INCLUDE))" "" PYTHON_VERSION ?= 3.7m PYTHON_LIB ?= -lpython3.7m else - ifneq "($filter %3.7, $(PYTHON_INCLUDE))" "" + ifneq "$(filter %3.7, $(PYTHON_INCLUDE))" "" PYTHON_VERSION ?= 3.7 else - ifneq "($filter %2.7, $(PYTHON_INCLUDE))" "" + ifneq "$(filter %2.7, $(PYTHON_INCLUDE))" "" PYTHON_VERSION ?= 2.7 PYTHON_LIB ?= -lpython2.7 else From 00b1d16ac61e9f86cd0c1defec6299e0a5e3fdde Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:28:13 +0100 Subject: [PATCH 15/44] more fixes for python checks --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index df5ad048..dbb37feb 100644 --- a/Makefile +++ b/Makefile @@ -60,12 +60,12 @@ ifneq "$(filter %3.7m, $(shell python3.7m-config --includes 2>/dev/null))" "" PYTHON_LIB ?= $(shell python3.7m-config --ldflags) PYTHON_VERSION = 3.7m else - ifneq "$(filter %3.7, $(shell python3.7-config --includes) 2> /dev/null)" "" + ifneq "$(filter %3.7, $(shell python3.7-config --includes 2>/dev/null))" "" PYTHON_INCLUDE ?= $(shell python3.7-config --includes) PYTHON_LIB ?= $(shell python3.7-config --ldflags) PYTHON_VERSION = 3.7 else - ifneq "$(filter %2.7, $(shell python2.7-config --includes) 2> /dev/null)" "" + ifneq "$(filter %2.7, $(shell python2.7-config --includes 2>/dev/null))" "" PYTHON_INCLUDE ?= $(shell python2.7-config --includes) PYTHON_LIB ?= $(shell python2.7-config --ldflags) PYTHON_VERSION = 2.7 From db5d5017155a24cb04bef97a0cf97d45456e7901 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:46:14 +0100 Subject: [PATCH 16/44] set AFL_CC for libradamsa test (needed on FreeBSD) --- test/test.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 8f40773c..5bab0d7a 100755 --- a/test/test.sh +++ b/test/test.sh @@ -457,7 +457,13 @@ test -e ../libdislocator.so && { } rm -f test-compcov test -e ../libradamsa.so && { - test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 + # on FreeBSD need to set AFL_CC + if which clang >/dev/null; then + export AFL_CC=`which clang` + else + export AFL_CC=`$LLVM_CONFIG --bindir`/clang + fi + test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c test -e test-instr.plain || ../afl-gcc-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../${AFL_GCC} -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain && { From 08691fcc974a9fcf2df3e926959b21199df7e946 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:58:20 +0100 Subject: [PATCH 17/44] add forgotten stderr redirect --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 5bab0d7a..c770c1b7 100755 --- a/test/test.sh +++ b/test/test.sh @@ -463,7 +463,7 @@ test -e ../libradamsa.so && { else export AFL_CC=`$LLVM_CONFIG --bindir`/clang fi - test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c + test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../afl-gcc-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../${AFL_GCC} -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain && { From 0eec6221554c260b2d93de73e88c2279c4479753 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sat, 18 Jan 2020 16:35:21 +0100 Subject: [PATCH 18/44] Intel test taken from lto branch, extended (as in test.sh), and tested on RaspberryPi --- Makefile | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Makefile b/Makefile index dbb37feb..7260ee47 100644 --- a/Makefile +++ b/Makefile @@ -48,6 +48,14 @@ ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -march=native -o .te CFLAGS_OPT = -march=native endif +ifneq "$(shell uname -m)" "x86_64" + ifneq "$(shell uname -m)" "i386" + ifneq "$(shell uname -m)" "amd64" + AFL_NO_X86=1 + endif + endif +endif + CFLAGS ?= -O3 -funroll-loops $(CFLAGS_OPT) CFLAGS += -Wall -g -Wno-pointer-sign -I include/ \ -DAFL_PATH=\"$(HELPER_PATH)\" -DBIN_PATH=\"$(BIN_PATH)\" \ From e7770a70023381bc7ff96b1d346b0ff9741f62de Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sun, 19 Jan 2020 12:25:32 +0100 Subject: [PATCH 19/44] make exporting AFL_CC FreeBSD specific, since it seems to harm the libradamsa test on travis/arm64 --- test/test.sh | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/test/test.sh b/test/test.sh index c770c1b7..43b278b4 100755 --- a/test/test.sh +++ b/test/test.sh @@ -458,11 +458,14 @@ test -e ../libdislocator.so && { rm -f test-compcov test -e ../libradamsa.so && { # on FreeBSD need to set AFL_CC - if which clang >/dev/null; then - export AFL_CC=`which clang` - else - export AFL_CC=`$LLVM_CONFIG --bindir`/clang - fi + + test `uname -s` = 'FreeBSD' && { + if which clang >/dev/null; then + export AFL_CC=`which clang` + else + export AFL_CC=`$LLVM_CONFIG --bindir`/clang + fi + } test -e test-instr.plain || ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../afl-gcc-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 test -e test-instr.plain || ../${AFL_GCC} -o test-instr.plain ../test-instr.c > /dev/null 2>&1 From f706e210ec07d8797850781ed82d2279df9a88b9 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sun, 19 Jan 2020 21:20:51 +0100 Subject: [PATCH 20/44] add missing test cases for qemu_mode unsigaction library --- test/test.sh | 58 +++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 57 insertions(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 43b278b4..97cc1511 100755 --- a/test/test.sh +++ b/test/test.sh @@ -569,8 +569,64 @@ test -e ../afl-qemu-trace && { CODE=1 exit 1 } - $ECHO "$YELLOW[-] we need a test case for qemu_mode unsigaction library" rm -rf in out errors + test -e ../qemu_mode/unsigaction/unsigaction32.so && { + ${AFL_CC} -o test-unsigaction32 -m32 test-unsigaction.c >> errors 2>&1 && { + ./test-unsigaction32 + RETVAL_NORMAL32=$? + LD_PRELOAD=../qemu_mode/unsigaction/unsigaction32.so ./test-unsigaction32 + RETVAL_LIBUNSIGACTION32=$? + test $RETVAL_NORMAL32 = "2" -a $RETVAL_LIBUNSIGACTION32 = "0" && { + $ECHO "$GREEN[+] qemu_mode unsigaction library (32 bit) ignores signals" + } || { + test $RETVAL_NORMAL32 != "2" && { + $ECHO "$RED[!] cannot trigger signal in test program (32 bit)" + } + test $RETVAL_LIBUNSIGACTION32 != "0" && { + $ECHO "$RED[!] signal in test program (32 bit) is not ignored with unsigaction" + } + CODE=1 + } + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] cannot compile test program (32 bit) for unsigaction library" + CODE=1 + } + } || { + $ECHO "$YELLOW[-] we cannot test qemu_mode unsigaction library (32 bit) because it is not present" + INCOMPLETE=1 + } + test -e ../qemu_mode/unsigaction/unsigaction64.so && { + ${AFL_CC} -o test-unsigaction64 -m64 test-unsigaction.c >> errors 2>&1 && { + ./test-unsigaction64 + RETVAL_NORMAL64=$? + LD_PRELOAD=../qemu_mode/unsigaction/unsigaction64.so ./test-unsigaction64 + RETVAL_LIBUNSIGACTION64=$? + test $RETVAL_NORMAL64 = "2" -a $RETVAL_LIBUNSIGACTION64 = "0" && { + $ECHO "$GREEN[+] qemu_mode unsigaction library (64 bit) ignores signals" + } || { + test $RETVAL_NORMAL64 != "2" && { + $ECHO "$RED[!] cannot trigger signal in test program (64 bit)" + } + test $RETVAL_LIBUNSIGACTION64 != "0" && { + $ECHO "$RED[!] signal in test program (64 bit) is not ignored with unsigaction" + } + CODE=1 + } + } || { + echo CUT------------------------------------------------------------------CUT + cat errors + echo CUT------------------------------------------------------------------CUT + $ECHO "$RED[!] cannot compile test program (64 bit) for unsigaction library" + CODE=1 + } + } || { + $ECHO "$YELLOW[-] we cannot test qemu_mode unsigaction library (64 bit) because it is not present" + INCOMPLETE=1 + } + rm -rf errors test-unsigaction32 test-unsigaction64 } } || { $ECHO "$RED[!] gcc compilation of test targets failed - what is going on??" From 274c8d7d3cff7ad61f2a57c7f69914a3948711d2 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Sun, 19 Jan 2020 21:22:41 +0100 Subject: [PATCH 21/44] add missing test program (oops) --- test/test-unsigaction.c | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 test/test-unsigaction.c diff --git a/test/test-unsigaction.c b/test/test-unsigaction.c new file mode 100644 index 00000000..1a5e4b26 --- /dev/null +++ b/test/test-unsigaction.c @@ -0,0 +1,25 @@ +#include /* sigemptyset(), sigaction(), kill(), SIGUSR1 */ +#include /* exit() */ +#include /* getpid() */ +#include /* errno */ +#include /* fprintf() */ + +static void mysig_handler(int sig) +{ + exit(2); +} + +int main() +{ + /* setup sig handler */ + struct sigaction sa; + sa.sa_handler = mysig_handler; + sigemptyset(&sa.sa_mask); + sa.sa_flags = 0; + if (sigaction(SIGCHLD, &sa, NULL)) { + fprintf(stderr, "could not set signal handler %d, aborted\n", errno); + exit(1); + } + kill(getpid(), SIGCHLD); + return 0; +} From 72058fdcbcdc707824bd4211ce528237afc1140e Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 20 Jan 2020 12:56:55 +0100 Subject: [PATCH 22/44] another freebsd fix in test.sh --- test/test.sh | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/test/test.sh b/test/test.sh index 97cc1511..23d98278 100755 --- a/test/test.sh +++ b/test/test.sh @@ -179,11 +179,13 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { $ECHO "$BLUE[*] Testing: llvm_mode" test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { # on FreeBSD need to set AFL_CC - if which clang >/dev/null; then - export AFL_CC=`which clang` - else - export AFL_CC=`$LLVM_CONFIG --bindir`/clang - fi + test `uname -s` = 'FreeBSD' && { + if which clang >/dev/null; then + export AFL_CC=`which clang` + else + export AFL_CC=`$LLVM_CONFIG --bindir`/clang + fi + } ../afl-clang-fast -o test-instr.plain ../test-instr.c > /dev/null 2>&1 AFL_HARDEN=1 ../afl-clang-fast -o test-compcov.harden test-compcov.c > /dev/null 2>&1 test -e test-instr.plain && { From 0d5a8f69e9785cdaec4c9b62e186050112a7cb8f Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 20 Jan 2020 19:21:44 +0100 Subject: [PATCH 23/44] fixed Heiko's global search-replace :) --- include/afl-as.h | 2 +- include/afl-fuzz.h | 2 +- include/alloc-inl.h | 2 +- include/android-ashmem.h | 2 +- include/common.h | 2 +- include/config.h | 17 +++++++++-------- include/debug.h | 2 +- include/forkserver.h | 2 +- include/sharedmem.h | 2 +- include/types.h | 2 +- libdislocator/libdislocator.so.c | 1 + src/afl-analyze.c | 2 +- src/afl-as.c | 2 +- src/afl-common.c | 2 +- src/afl-forkserver.c | 2 +- src/afl-fuzz-bitmap.c | 9 +++++++-- src/afl-fuzz-extras.c | 2 +- src/afl-fuzz-globals.c | 2 +- src/afl-fuzz-init.c | 2 +- src/afl-fuzz-misc.c | 2 +- src/afl-fuzz-one.c | 2 +- src/afl-fuzz-python.c | 2 +- src/afl-fuzz-queue.c | 2 +- src/afl-fuzz-run.c | 2 +- src/afl-fuzz-stats.c | 2 +- src/afl-fuzz.c | 2 +- src/afl-gcc.c | 2 +- src/afl-gotcpu.c | 2 +- src/afl-sharedmem.c | 2 +- src/afl-showmap.c | 2 +- src/afl-tmin.c | 2 +- 31 files changed, 45 insertions(+), 38 deletions(-) diff --git a/include/afl-as.h b/include/afl-as.h index 3af42205..bd5e734a 100644 --- a/include/afl-as.h +++ b/include/afl-as.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 9ecf1f29..00d29f76 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/alloc-inl.h b/include/alloc-inl.h index 48598ed3..5592b295 100644 --- a/include/alloc-inl.h +++ b/include/alloc-inl.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/android-ashmem.h b/include/android-ashmem.h index 35a5ba5e..adddc05f 100755 --- a/include/android-ashmem.h +++ b/include/android-ashmem.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/common.h b/include/common.h index 8ab78b41..3b953470 100644 --- a/include/common.h +++ b/include/common.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/config.h b/include/config.h index 4eac82e0..83fcb8f9 100644 --- a/include/config.h +++ b/include/config.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi @@ -62,13 +62,14 @@ /* Default memory limit for child process (MB): */ #ifndef __NetBSD__ -# ifndef WORD_SIZE_64 -# define MEM_LIMIT 25 -# else -# define MEM_LIMIT 50 -# endif /* ^!WORD_SIZE_64 */ -#else /* NetBSD's kernel needs more space for stack, see discussion for issue #165 */ -# define MEM_LIMIT 200 +#ifndef WORD_SIZE_64 +#define MEM_LIMIT 25 +#else +#define MEM_LIMIT 50 +#endif /* ^!WORD_SIZE_64 */ +#else /* NetBSD's kernel needs more space for stack, see discussion for issue \ + #165 */ +#define MEM_LIMIT 200 #endif /* Default memory limit when running in QEMU mode (MB): */ diff --git a/include/debug.h b/include/debug.h index 68109927..d6c04935 100644 --- a/include/debug.h +++ b/include/debug.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/forkserver.h b/include/forkserver.h index 17bc65af..0fdcba48 100644 --- a/include/forkserver.h +++ b/include/forkserver.h @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/sharedmem.h b/include/sharedmem.h index 69291330..7604d64c 100644 --- a/include/sharedmem.h +++ b/include/sharedmem.h @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/include/types.h b/include/types.h index eba47be7..9e681e81 100644 --- a/include/types.h +++ b/include/types.h @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/libdislocator/libdislocator.so.c b/libdislocator/libdislocator.so.c index b9ba8967..221a629b 100644 --- a/libdislocator/libdislocator.so.c +++ b/libdislocator/libdislocator.so.c @@ -417,6 +417,7 @@ void* reallocarray(void* ptr, size_t elem_len, size_t elem_cnt) { } return ret; + } __attribute__((constructor)) void __dislocator_init(void) { diff --git a/src/afl-analyze.c b/src/afl-analyze.c index 3d4e636e..3de8c037 100644 --- a/src/afl-analyze.c +++ b/src/afl-analyze.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-as.c b/src/afl-as.c index 77ac2f97..8d689385 100644 --- a/src/afl-as.c +++ b/src/afl-as.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-common.c b/src/afl-common.c index 8c2f2b9a..6cb97cdf 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-forkserver.c b/src/afl-forkserver.c index de50c73c..77e1d648 100644 --- a/src/afl-forkserver.c +++ b/src/afl-forkserver.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-bitmap.c b/src/afl-fuzz-bitmap.c index 515a7a79..3ffda284 100644 --- a/src/afl-fuzz-bitmap.c +++ b/src/afl-fuzz-bitmap.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi @@ -524,7 +524,12 @@ u8 save_if_interesting(char** argv, void* mem, u32 len, u8 fault) { struct queue_entry* q = queue; while (q) { - if (q->exec_cksum == cksum) { q->n_fuzz = q->n_fuzz + 1; break ; } + if (q->exec_cksum == cksum) { + + q->n_fuzz = q->n_fuzz + 1; + break; + + } q = q->next; diff --git a/src/afl-fuzz-extras.c b/src/afl-fuzz-extras.c index fcc7749d..6c6dc28c 100644 --- a/src/afl-fuzz-extras.c +++ b/src/afl-fuzz-extras.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-globals.c b/src/afl-fuzz-globals.c index b3476778..f0d98192 100644 --- a/src/afl-fuzz-globals.c +++ b/src/afl-fuzz-globals.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 5fe3689e..219be822 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-misc.c b/src/afl-fuzz-misc.c index b8f376be..0da0cb0a 100644 --- a/src/afl-fuzz-misc.c +++ b/src/afl-fuzz-misc.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 74123300..b04683be 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-python.c b/src/afl-fuzz-python.c index f1cdecde..f06c8e25 100644 --- a/src/afl-fuzz-python.c +++ b/src/afl-fuzz-python.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 1b51e3aa..0880de75 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index fa7a872a..a006194d 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz-stats.c b/src/afl-fuzz-stats.c index 7679403b..f2afb295 100644 --- a/src/afl-fuzz-stats.c +++ b/src/afl-fuzz-stats.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 0af8b35f..9a7495ef 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-gcc.c b/src/afl-gcc.c index 301e2034..e46fe5cd 100644 --- a/src/afl-gcc.c +++ b/src/afl-gcc.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-gotcpu.c b/src/afl-gotcpu.c index 9a56159c..5be30238 100644 --- a/src/afl-gotcpu.c +++ b/src/afl-gotcpu.c @@ -4,7 +4,7 @@ Originally written by Michal Zalewski - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-sharedmem.c b/src/afl-sharedmem.c index 16eb14a7..04fcaa1c 100644 --- a/src/afl-sharedmem.c +++ b/src/afl-sharedmem.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 8c899c9d..b9da3208 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi diff --git a/src/afl-tmin.c b/src/afl-tmin.c index 3e33b72f..7ce0ccaa 100644 --- a/src/afl-tmin.c +++ b/src/afl-tmin.c @@ -6,7 +6,7 @@ Forkserver design by Jann Horn - Now maintained by by Marc Heuse , + Now maintained by Marc Heuse , Heiko Eißfeldt and Andrea Fioraldi From 00d086f816d6b517a6817d6093a83ed8a65b18fa Mon Sep 17 00:00:00 2001 From: van Hauser Date: Tue, 21 Jan 2020 12:53:36 +0100 Subject: [PATCH 24/44] USE_TRACE_PC unnecessary, set env AFL_LLVM_USE_TRACE_PC instead --- docs/ChangeLog | 8 +++++--- llvm_mode/README.md | 19 +++++++++---------- llvm_mode/afl-clang-fast.c | 33 ++++++++++++++++++++++++--------- 3 files changed, 38 insertions(+), 22 deletions(-) diff --git a/docs/ChangeLog b/docs/ChangeLog index 5347d244..bb3537dd 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -21,9 +21,11 @@ Version ++2.60d (develop): - afl-fuzz: - now prints the real python version support compiled in - set stronger performance compile options and little tweaks - - afl-clang-fast now shows in the help output for which llvm version it - was compiled for - - added blacklisted function check in llvm_mode + - afl-clang-fast: + - show in the help output for which llvm version it was compiled for + - now does not need to be recompiled between trace-pc and pass + instrumentation. compile normally and set AFL_LLVM_USE_TRACE_PC :) + - added blacklisted function check in all modules of llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/llvm_mode/README.md b/llvm_mode/README.md index 5afa4dfd..150d1a17 100644 --- a/llvm_mode/README.md +++ b/llvm_mode/README.md @@ -198,24 +198,23 @@ PS. Because there are task switches still involved, the mode isn't as fast as faster than the normal fork() model, and compared to in-process fuzzing, should be a lot more robust. -## 8) Bonus feature #3: new 'trace-pc-guard' mode +## 8) Bonus feature #3: 'trace-pc-guard' mode -Recent versions of LLVM are shipping with a built-in execution tracing feature +LLVM is shipping with a built-in execution tracing feature that provides AFL with the necessary tracing data without the need to post-process the assembly or install any compiler plugins. See: http://clang.llvm.org/docs/SanitizerCoverage.html#tracing-pcs-with-guards -If you have a sufficiently recent compiler and want to give it a try, build -afl-clang-fast this way: +If you have not an outdated compiler and want to give it a try, build +targets this way: ``` - AFL_TRACE_PC=1 make clean all + libtarget-1.0 $ AFL_LLVM_USE_TRACE_PC=1 make ``` -Note that this mode is currently about 20% slower than "vanilla" afl-clang-fast, +Note that this mode is about 20% slower than "vanilla" afl-clang-fast, and about 5-10% slower than afl-clang. This is likely because the -instrumentation is not inlined, and instead involves a function call. On systems -that support it, compiling your target with -flto should help. - - +instrumentation is not inlined, and instead involves a function call. +On systems that support it, compiling your target with -flto can help +a bit. diff --git a/llvm_mode/afl-clang-fast.c b/llvm_mode/afl-clang-fast.c index b322b762..7da7c5a3 100644 --- a/llvm_mode/afl-clang-fast.c +++ b/llvm_mode/afl-clang-fast.c @@ -204,13 +204,24 @@ static void edit_params(u32 argc, char** argv) { // "-fsanitize-coverage=trace-cmp,trace-div,trace-gep"; // cc_params[cc_par_cnt++] = "-sanitizer-coverage-block-threshold=0"; #else - cc_params[cc_par_cnt++] = "-Xclang"; - cc_params[cc_par_cnt++] = "-load"; - cc_params[cc_par_cnt++] = "-Xclang"; - if (getenv("AFL_LLVM_INSTRIM") != NULL || getenv("INSTRIM_LIB") != NULL) - cc_params[cc_par_cnt++] = alloc_printf("%s/libLLVMInsTrim.so", obj_path); - else - cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); + if (getenv("USE_TRACE_PC") || getenv("AFL_USE_TRACE_PC") || + getenv("AFL_LLVM_USE_TRACE_PC") || getenv("AFL_TRACE_PC")) { + + cc_params[cc_par_cnt++] = + "-fsanitize-coverage=trace-pc-guard"; // edge coverage by default + + } else { + + cc_params[cc_par_cnt++] = "-Xclang"; + cc_params[cc_par_cnt++] = "-load"; + cc_params[cc_par_cnt++] = "-Xclang"; + if (getenv("AFL_LLVM_INSTRIM") != NULL || getenv("INSTRIM_LIB") != NULL) + cc_params[cc_par_cnt++] = alloc_printf("%s/libLLVMInsTrim.so", obj_path); + else + cc_params[cc_par_cnt++] = alloc_printf("%s/afl-llvm-pass.so", obj_path); + + } + #endif /* ^USE_TRACE_PC */ cc_params[cc_par_cnt++] = "-Qunused-arguments"; @@ -282,8 +293,10 @@ static void edit_params(u32 argc, char** argv) { #ifdef USE_TRACE_PC - if (getenv("AFL_INST_RATIO")) - FATAL("AFL_INST_RATIO not available at compile time with 'trace-pc'."); + if (getenv("USE_TRACE_PC") || getenv("AFL_USE_TRACE_PC") || + getenv("AFL_LLVM_USE_TRACE_PC") || getenv("AFL_TRACE_PC")) + if (getenv("AFL_INST_RATIO")) + FATAL("AFL_INST_RATIO not available at compile time with 'trace-pc'."); #endif /* USE_TRACE_PC */ @@ -455,6 +468,8 @@ int main(int argc, char** argv) { #ifdef USE_TRACE_PC SAYF(cCYA "afl-clang-fast" VERSION cRST " [tpcg] by \n"); +#warning \ + "You do not need to specifically compile with USE_TRACE_PC anymore, setting the environment variable AFL_LLVM_USE_TRACE_PC is enough." #else SAYF(cCYA "afl-clang-fast" VERSION cRST " by \n"); #endif /* ^USE_TRACE_PC */ From fb221db8ae4d640aa6261633ca249a86305292c4 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 22 Jan 2020 08:35:41 +0100 Subject: [PATCH 25/44] clarify gcc plugin test case result --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 23d98278..0ae6fd09 100755 --- a/test/test.sh +++ b/test/test.sh @@ -336,7 +336,7 @@ test -e ../afl-gcc-fast -a -e ../afl-gcc-rt.o && { $ECHO "$GREEN[+] gcc_plugin run reported $TUPLES instrumented locations which is fine" } || { $ECHO "$RED[!] gcc_plugin instrumentation produces a weird number of instrumented locations: $TUPLES" - $ECHO "$YELLOW[-] the gcc_plugin instrumentation issue is not flagged as an error because travis builds would all fail otherwise :-(" + $ECHO "$YELLOW[-] this is a known issue in gcc, not afl++. It is not flagged as an error because travis builds would all fail otherwise :-(" #CODE=1 } } From 4fbcc37f8450136759913875b6234d2e3ab2f032 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 09:26:54 +0100 Subject: [PATCH 26/44] awk version for portability, tested on linux and FreeBSD so far --- afl-cmin.awk | 440 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 440 insertions(+) create mode 100755 afl-cmin.awk diff --git a/afl-cmin.awk b/afl-cmin.awk new file mode 100755 index 00000000..021f7059 --- /dev/null +++ b/afl-cmin.awk @@ -0,0 +1,440 @@ +#!/usr/bin/awk -f + +# getopt.awk --- Do C library getopt(3) function in awk + +# External variables: +# Optind -- index in ARGV of first nonoption argument +# Optarg -- string value of argument to current option +# Opterr -- if nonzero, print our own diagnostic +# Optopt -- current option letter + +# Returns: +# -1 at end of options +# "?" for unrecognized option +# a character representing the current option + +# Private Data: +# _opti -- index in multiflag option, e.g., -abc + +function getopt(argc, argv, options, thisopt, i) +{ + if (length(options) == 0) # no options given + return -1 + + if (argv[Optind] == "--") { # all done + Optind++ + _opti = 0 + return -1 + } else if (argv[Optind] !~ /^-[^:[:space:]]/) { + _opti = 0 + return -1 + } + if (_opti == 0) + _opti = 2 + thisopt = substr(argv[Optind], _opti, 1) + Optopt = thisopt + i = index(options, thisopt) + if (i == 0) { + if (Opterr) + printf("%c -- invalid option\n", thisopt) > "/dev/stderr" + if (_opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return "?" + } + if (substr(options, i + 1, 1) == ":") { + # get option argument + if (length(substr(argv[Optind], _opti + 1)) > 0) + Optarg = substr(argv[Optind], _opti + 1) + else + Optarg = argv[++Optind] + _opti = 0 + } else + Optarg = "" + if (_opti == 0 || _opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return thisopt +} + +BEGIN { + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + + # test program + if (_getopt_test) { + while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) + printf("c = <%c>, Optarg = <%s>\n", + _go_c, Optarg) + printf("non-option arguments:\n") + for (; Optind < ARGC; Optind++) + printf("\tARGV[%d] = <%s>\n", + Optind, ARGV[Optind]) + } +} + +function usage() { + print \ +"Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ +"\n" \ +"Required parameters:\n" \ +"\n" \ +" -i dir - input directory with starting corpus\n" \ +" -o dir - output directory for minimized files\n" \ +"\n" \ +"Execution control settings:\n" \ +"\n" \ +" -f file - location read by the fuzzed program (stdin)\n" \ +" -m megs - memory limit for child process ("mem_limit" MB)\n" \ +" -t msec - run time limit for child process (none)\n" \ +" -Q - use binary-only instrumentation (QEMU mode)\n" \ +" -U - use unicorn-based instrumentation (unicorn mode)\n" \ +"\n" \ +"Minimization settings:\n" \ +" -C - keep crashing inputs, reject everything else\n" \ +" -e - solve for edge coverage only, ignore hit counts\n" \ +"\n" \ +"For additional tips, please consult docs/README.md\n" \ +"\n" \ + > "/dev/stderr" + exit 1 +} + +function exists_and_is_executable(binarypath) { + return 0 == system("test -f "binarypath" -a -x "binarypath) +} + +BEGIN { + print "corpus minimization tool for afl-fuzz++ (awk version)\n" +print "PATH="ENVIRON["PATH"] + + # defaults + extra_par = "" + # process options + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eCQU?")) != -1) { + if (_go_c == "i") { + if (!Optarg) usage() + if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + in_dir = Optarg + continue + } else + if (_go_c == "o") { + if (!Optarg) usage() + if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + out_dir = Optarg + continue + } else + if (_go_c == "f") { + if (!Optarg) usage() + if (stdin_file) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + stdin_file = Optarg + continue + } else + if (_go_c == "m") { + if (!Optarg) usage() + if (mem_limit) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + mem_limit = Optarg + mem_limit_given = 1 + continue + } else + if (_go_c == "t") { + if (!Optarg) usage() + if (timeout) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + timeout = Optarg + continue + } else + if (_go_c == "C") { + ENVIRON["AFL_CMIN_CRASHES_ONLY"] = 1 + continue + } else + if (_go_c == "e") { + extra_par = extra_par " -e" + continue + } else + if (_go_c == "Q") { + if (qemu_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -Q" + if ( !mem_limit_given ) mem_limit = "250" + qemu_mode = 1 + continue + } else + if (_go_c == "U") { + if (unicorn_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -U" + if ( !mem_limit_given ) mem_limit = "250" + unicorn_mode = 1 + continue + } else + if (_go_c == "?") { + exit 1 + } else + usage() + } # while options + + if (!mem_limit) mem_limit = 200 + if (!timeout) timeout = "none" + + # get program args + i = 0 + prog_args_string = "" + for (; Optind < ARGC; Optind++) { + prog_args[i++] = ARGV[Optind] + if (i > 1) + prog_args_string = prog_args_string" "ARGV[Optind] + } + + # sanity checks + if (!prog_args[0] || !in_dir || !out_dir) usage() + + target_bin = prog_args[0] + + # Do a sanity check to discourage the use of /tmp, since we can't really + # handle this safely from an awk script. + + if (!ENVIRON["AFL_ALLOW_TMP"]) { + dirlist[0] = in_dir + dirlist[1] = target_bin + dirlist[2] = out_dir + dirlist[3] = stdin_file + "pwd" | getline dirlist[4] # current directory + for (dirind in dirlist) { + dir = dirlist[dirind] + + if (dir ~ /^(\/var)?\/tmp/) { + print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" + exit 1 + } + } + delete dirlist + } + + # If @@ is specified, but there's no -f, let's come up with a temporary input + # file name. + + trace_dir = out_dir "/.traces" + + if (!stdin_file) { + found_atat = 0 + for (prog_args_ind in prog_args) { + if ("@@" == prog_args[prog_args_ind]) { + found_atat = 1 + break + } + } + if (found_atat) { + stdin_file = trace_dir "/.cur_input" + } + } + + # Check for obvious errors. + + if (mem_limit && mem_limit != "none" && mem_limit < 5) { + print "[-] Error: dangerously low memory limit." > "/dev/stderr" + exit 1 + } + + if (timeout && timeout != "none" && timeout < 10) { + print "[-] Error: dangerously low timeout." > "/dev/stderr" + exit 1 + } + + if (target_bin && !exists_and_is_executable(target_bin)) { + + "which "target_bin" 2>/dev/null" | getline tnew + if (!tnew || !exists_and_is_executable(tnew)) { + print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr" + exit 1 + } + target_bin = tnew + } + + if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !unicorn_mode) { + if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { + print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" + exit 1 + } + } + + if (0 != system( "test -d "in_dir )) { + print "[-] Error: directory '"in_dir"' not found." > "/dev/stderr" + exit 1 + } + + if (0 == system( "test -d "in_dir"/queue" )) { + in_dir = in_dir "/queue" + } + + system("rm -rf "trace_dir" 2>/dev/null"); + system("rm "out_dir"/id[:_]* 2>/dev/null") + + if (0 == system( "test -d "out_dir" -a -e "out_dir"/*" )) { + print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr" + exit 1 + } + + if (stdin_file) { + # truncate input file + printf "" > stdin_file + close( stdin_file ) + } + + if (!ENVIRON["AFL_PATH"]) { + if (0 == system("test -f afl-cmin.awk")) { + path = "." + } else { + "which afl-showmap 2>/dev/null" | getline path + } + showmap = path + } else { + showmap = ENVIRON["AFL_PATH"] "/afl-showmap" + } + + if (!showmap || 0 != system("test -x "showmap )) { + print "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." > "/dev/stderr" + exit 1 + } + + # get list of input filenames sorted by size + i = 0 + while ("find "in_dir" -type f -exec stat -f '%z %N' \{\} \; | sort -n | cut -d' ' -f2-" | getline) { + infilesSmallToBig[i++] = $0 + } + in_count = i + + first_file = infilesSmallToBig[0] + + # Make sure that we're not dealing with a directory. + + if (0 == system("test -d "in_dir"/"first_file)) { + print "[-] Error: The input directory contains subdirectories - please fix." > "/dev/stderr" + exit 1 + } + + # Check for the more efficient way to copy files... + if (0 != system("mkdir -p -m 0700 "trace_dir)) { + print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" + exit 1 + } + + if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { + cp_tool = "ln" + } else { + cp_tool = "cp" + } + + # Make sure that we can actually get anything out of afl-showmap before we + # waste too much time. + + print "[*] Testing the target binary..." + + if (!stdin_file) { + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") + } else { + system("cp "in_dir"/"first_file" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + ++first_count + } + + if (first_count) { + print "[+] OK, "first_count" tuples recorded." + } else { + print "[-] Error: no instrumentation output detected (perhaps crash or timeout)." > "/dev/stderr" + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + exit 1 + } + + # Let's roll! + + ############################# + # STEP 1: Collecting traces # + ############################# + + print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." + + cur = 0; + if (!stdin_file) { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur; + printf "\r Processing file "cur"/"in_count + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"fn"\"") + } + } else { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur + printf "\r Processing file "cur"/"in_count + system("cp "in_dir"/"fn" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + key = line + if (!(key in key_count)) { + ++tuple_count + } + ++key_count[key] + if (! (key in best_file)) { + # this is the best file for this key + best_file[key] = fn + # copy file unless already done + if (! (fn in file_already_copied)) { + system(cp_tool" "in_dir"/"fn" "out_dir"/"fn) + file_already_copied[fn] = "" + ++out_count + } + } + } + close(tracefile_path) + } + + print "" + print "[+] Found "tuple_count" unique tuples across "in_count" files." + + if (out_count == 1) { + print "[!] WARNING: All test cases had the same traces, check syntax!" + } + print "[+] Narrowed down to "out_count" files, saved in '"out_dir"'." + + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + + exit 0 +} From 7ce627c92e9b0536e254422d5ef604c3f58e43ce Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 18:38:41 +0100 Subject: [PATCH 27/44] Oops, only this version works with FreeBSD, OpenBSD, NetBSD, MacOS, raspbian --- afl-cmin.awk | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/afl-cmin.awk b/afl-cmin.awk index 021f7059..28c460e8 100755 --- a/afl-cmin.awk +++ b/afl-cmin.awk @@ -110,7 +110,6 @@ function exists_and_is_executable(binarypath) { BEGIN { print "corpus minimization tool for afl-fuzz++ (awk version)\n" -print "PATH="ENVIRON["PATH"] # defaults extra_par = "" @@ -302,7 +301,13 @@ print "PATH="ENVIRON["PATH"] # get list of input filenames sorted by size i = 0 - while ("find "in_dir" -type f -exec stat -f '%z %N' \{\} \; | sort -n | cut -d' ' -f2-" | getline) { + # yuck, gnu stat is incompatible to bsd stat + if ("stat --version 2>/dev/null" !~ /GNU coreutils/) { + stat_format = "-f '%z %N'" + } else { + stat_format = "-c '%s %n'" + } + while ("cd "in_dir" && find . -type f -exec stat "stat_format" \{\} \\; | sort -n | cut -d' ' -f2-" | getline) { infilesSmallToBig[i++] = $0 } in_count = i From ce0b9dae5971f22cd0ae0b468322f78ee2a8a766 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 19:07:02 +0100 Subject: [PATCH 28/44] final step: rename afl-cmin to afl-cmin.bash and add a wrapper afl-cmin for afl-cmin.awk --- afl-cmin | 474 +------------------------------------------------- afl-cmin.bash | 470 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 474 insertions(+), 470 deletions(-) create mode 100755 afl-cmin.bash diff --git a/afl-cmin b/afl-cmin index 1dd782d8..75dc63a7 100755 --- a/afl-cmin +++ b/afl-cmin @@ -1,470 +1,4 @@ -#!/usr/bin/env bash -# -# american fuzzy lop++ - corpus minimization tool -# --------------------------------------------- -# -# Originally written by Michal Zalewski -# -# Copyright 2014, 2015 Google Inc. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at: -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# This tool tries to find the smallest subset of files in the input directory -# that still trigger the full range of instrumentation data points seen in -# the starting corpus. This has two uses: -# -# - Screening large corpora of input files before using them as a seed for -# afl-fuzz. The tool will remove functionally redundant files and likely -# leave you with a much smaller set. -# -# (In this case, you probably also want to consider running afl-tmin on -# the individual files later on to reduce their size.) -# -# - Minimizing the corpus generated organically by afl-fuzz, perhaps when -# planning to feed it to more resource-intensive tools. The tool achieves -# this by removing all entries that used to trigger unique behaviors in the -# past, but have been made obsolete by later finds. -# -# Note that the tool doesn't modify the files themselves. For that, you want -# afl-tmin. -# -# This script must use bash because other shells may have hardcoded limits on -# array sizes. -# - -echo "corpus minimization tool for afl-fuzz by Michal Zalewski" -echo - -######### -# SETUP # -######### - -# Process command-line options... - -MEM_LIMIT=200 -TIMEOUT=none - -unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN \ - AFL_CMIN_CRASHES_ONLY AFL_CMIN_ALLOW_ANY QEMU_MODE UNICORN_MODE - -while getopts "+i:o:f:m:t:eQUCh" opt; do - - case "$opt" in - - "h") - ;; - - "i") - IN_DIR="$OPTARG" - ;; - - "o") - OUT_DIR="$OPTARG" - ;; - "f") - STDIN_FILE="$OPTARG" - ;; - "m") - MEM_LIMIT="$OPTARG" - MEM_LIMIT_GIVEN=1 - ;; - "t") - TIMEOUT="$OPTARG" - ;; - "e") - EXTRA_PAR="$EXTRA_PAR -e" - ;; - "C") - export AFL_CMIN_CRASHES_ONLY=1 - ;; - "Q") - EXTRA_PAR="$EXTRA_PAR -Q" - test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 - QEMU_MODE=1 - ;; - "U") - EXTRA_PAR="$EXTRA_PAR -U" - test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 - UNICORN_MODE=1 - ;; - "?") - exit 1 - ;; - - esac - -done - -shift $((OPTIND-1)) - -TARGET_BIN="$1" - -if [ "$TARGET_BIN" = "" -o "$IN_DIR" = "" -o "$OUT_DIR" = "" ]; then - - cat 1>&2 <<_EOF_ -Usage: $0 [ options ] -- /path/to/target_app [ ... ] - -Required parameters: - - -i dir - input directory with the starting corpus - -o dir - output directory for minimized files - -Execution control settings: - - -f file - location read by the fuzzed program (stdin) - -m megs - memory limit for child process ($MEM_LIMIT MB) - -t msec - run time limit for child process (none) - -Q - use binary-only instrumentation (QEMU mode) - -U - use unicorn-based instrumentation (Unicorn mode) - -Minimization settings: - - -C - keep crashing inputs, reject everything else - -e - solve for edge coverage only, ignore hit counts - -For additional tips, please consult docs/README. - -_EOF_ - exit 1 -fi - -# Do a sanity check to discourage the use of /tmp, since we can't really -# handle this safely from a shell script. - -if [ "$AFL_ALLOW_TMP" = "" ]; then - - echo "$IN_DIR" | grep -qE '^(/var)?/tmp/' - T1="$?" - - echo "$TARGET_BIN" | grep -qE '^(/var)?/tmp/' - T2="$?" - - echo "$OUT_DIR" | grep -qE '^(/var)?/tmp/' - T3="$?" - - echo "$STDIN_FILE" | grep -qE '^(/var)?/tmp/' - T4="$?" - - echo "$PWD" | grep -qE '^(/var)?/tmp/' - T5="$?" - - if [ "$T1" = "0" -o "$T2" = "0" -o "$T3" = "0" -o "$T4" = "0" -o "$T5" = "0" ]; then - echo "[-] Error: do not use this script in /tmp or /var/tmp." 1>&2 - exit 1 - fi - -fi - -# If @@ is specified, but there's no -f, let's come up with a temporary input -# file name. - -TRACE_DIR="$OUT_DIR/.traces" - -if [ "$STDIN_FILE" = "" ]; then - - if echo "$*" | grep -qF '@@'; then - STDIN_FILE="$TRACE_DIR/.cur_input" - fi - -fi - -# Check for obvious errors. - -if [ ! "$MEM_LIMIT" = "none" ]; then - - if [ "$MEM_LIMIT" -lt "5" ]; then - echo "[-] Error: dangerously low memory limit." 1>&2 - exit 1 - fi - -fi - -if [ ! "$TIMEOUT" = "none" ]; then - - if [ "$TIMEOUT" -lt "10" ]; then - echo "[-] Error: dangerously low timeout." 1>&2 - exit 1 - fi - -fi - -if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then - - TNEW="`which "$TARGET_BIN" 2>/dev/null`" - - if [ ! -f "$TNEW" -o ! -x "$TNEW" ]; then - echo "[-] Error: binary '$TARGET_BIN' not found or not executable." 1>&2 - exit 1 - fi - - TARGET_BIN="$TNEW" - -fi - -if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$UNICORN_MODE" = "" ]; then - - if ! grep -qF "__AFL_SHM_ID" "$TARGET_BIN"; then - echo "[-] Error: binary '$TARGET_BIN' doesn't appear to be instrumented." 1>&2 - exit 1 - fi - -fi - -if [ ! -d "$IN_DIR" ]; then - echo "[-] Error: directory '$IN_DIR' not found." 1>&2 - exit 1 -fi - -test -d "$IN_DIR/queue" && IN_DIR="$IN_DIR/queue" - -find "$OUT_DIR" -name 'id[:_]*' -maxdepth 1 -exec rm -- {} \; 2>/dev/null -rm -rf "$TRACE_DIR" 2>/dev/null - -rmdir "$OUT_DIR" 2>/dev/null - -if [ -d "$OUT_DIR" ]; then - echo "[-] Error: directory '$OUT_DIR' exists and is not empty - delete it first." 1>&2 - exit 1 -fi - -mkdir -m 700 -p "$TRACE_DIR" || exit 1 - -if [ ! "$STDIN_FILE" = "" ]; then - rm -f "$STDIN_FILE" || exit 1 - touch "$STDIN_FILE" || exit 1 -fi - -if [ "$AFL_PATH" = "" ]; then - SHOWMAP="${0%/afl-cmin}/afl-showmap" -else - SHOWMAP="$AFL_PATH/afl-showmap" -fi - -if [ ! -x "$SHOWMAP" ]; then - echo "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." 1>&2 - rm -rf "$TRACE_DIR" - exit 1 -fi - -IN_COUNT=$((`ls -- "$IN_DIR" 2>/dev/null | wc -l`)) - -if [ "$IN_COUNT" = "0" ]; then - echo "[+] Hmm, no inputs in the target directory. Nothing to be done." - rm -rf "$TRACE_DIR" - exit 1 -fi - -FIRST_FILE=`ls "$IN_DIR" | head -1` - -# Make sure that we're not dealing with a directory. - -if [ -d "$IN_DIR/$FIRST_FILE" ]; then - echo "[-] Error: The target directory contains subdirectories - please fix." 1>&2 - rm -rf "$TRACE_DIR" - exit 1 -fi - -# Check for the more efficient way to copy files... - -if ln "$IN_DIR/$FIRST_FILE" "$TRACE_DIR/.link_test" 2>/dev/null; then - CP_TOOL=ln -else - CP_TOOL=cp -fi - -# Make sure that we can actually get anything out of afl-showmap before we -# waste too much time. - -echo "[*] Testing the target binary..." - -if [ "$STDIN_FILE" = "" ]; then - - AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$FIRST_FILE" - -else - - cp "$IN_DIR/$FIRST_FILE" "$STDIN_FILE" - AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" &2 - test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" - exit 1 - -fi - -# Let's roll! - -############################# -# STEP 1: COLLECTING TRACES # -############################# - -echo "[*] Obtaining traces for input files in '$IN_DIR'..." - -( - - CUR=0 - - if [ "$STDIN_FILE" = "" ]; then - - ls "$IN_DIR" | while read -r fn; do - - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " - - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" - - done - - else - - ls "$IN_DIR" | while read -r fn; do - - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " - - cp "$IN_DIR/$fn" "$STDIN_FILE" - - "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" "$TRACE_DIR/.all_uniq" - -TUPLE_COUNT=$((`grep -c . "$TRACE_DIR/.all_uniq"`)) - -echo "[+] Found $TUPLE_COUNT unique tuples across $IN_COUNT files." - -##################################### -# STEP 3: SELECTING CANDIDATE FILES # -##################################### - -# The next step is to find the best candidate for each tuple. The "best" -# part is understood simply as the smallest input that includes a particular -# tuple in its trace. Empirical evidence suggests that this produces smaller -# datasets than more involved algorithms that could be still pulled off in -# a shell script. - -echo "[*] Finding best candidates for each tuple..." - -CUR=0 - -ls -rS "$IN_DIR" | while read -r fn; do - - CUR=$((CUR+1)) - printf "\\r Processing file $CUR/$IN_COUNT... " - - sed "s#\$# $fn#" "$TRACE_DIR/$fn" >>"$TRACE_DIR/.candidate_list" - -done - -echo - -############################## -# STEP 4: LOADING CANDIDATES # -############################## - -# At this point, we have a file of tuple-file pairs, sorted by file size -# in ascending order (as a consequence of ls -rS). By doing sort keyed -# only by tuple (-k 1,1) and configured to output only the first line for -# every key (-s -u), we end up with the smallest file for each tuple. - -echo "[*] Sorting candidate list (be patient)..." - -sort -k1,1 -s -u "$TRACE_DIR/.candidate_list" | \ - sed 's/^/BEST_FILE[/;s/ /]="/;s/$/"/' >"$TRACE_DIR/.candidate_script" - -if [ ! -s "$TRACE_DIR/.candidate_script" ]; then - echo "[-] Error: no traces obtained from test cases, check syntax!" 1>&2 - test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" - exit 1 -fi - -# The sed command converted the sorted list to a shell script that populates -# BEST_FILE[tuple]="fname". Let's load that! - -. "$TRACE_DIR/.candidate_script" - -########################## -# STEP 5: WRITING OUTPUT # -########################## - -# The final trick is to grab the top pick for each tuple, unless said tuple is -# already set due to the inclusion of an earlier candidate; and then put all -# tuples associated with the newly-added file to the "already have" list. The -# loop works from least popular tuples and toward the most common ones. - -echo "[*] Processing candidates and writing output files..." - -CUR=0 - -touch "$TRACE_DIR/.already_have" - -while read -r cnt tuple; do - - CUR=$((CUR+1)) - printf "\\r Processing tuple $CUR/$TUPLE_COUNT... " - - # If we already have this tuple, skip it. - - grep -q "^$tuple\$" "$TRACE_DIR/.already_have" && continue - - FN=${BEST_FILE[tuple]} - - $CP_TOOL "$IN_DIR/$FN" "$OUT_DIR/$FN" - - if [ "$((CUR % 5))" = "0" ]; then - sort -u "$TRACE_DIR/$FN" "$TRACE_DIR/.already_have" >"$TRACE_DIR/.tmp" - mv -f "$TRACE_DIR/.tmp" "$TRACE_DIR/.already_have" - else - cat "$TRACE_DIR/$FN" >>"$TRACE_DIR/.already_have" - fi - -done <"$TRACE_DIR/.all_uniq" - -echo - -OUT_COUNT=`ls -- "$OUT_DIR" | wc -l` - -if [ "$OUT_COUNT" = "1" ]; then - echo "[!] WARNING: All test cases had the same traces, check syntax!" -fi - -echo "[+] Narrowed down to $OUT_COUNT files, saved in '$OUT_DIR'." -echo - -test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" - -exit 0 +#!/usr/bin/env sh +THISPATH=`dirname ${0}` +export PATH=${THISPATH}:$PATH +awk -f ${0}.awk -- ${@+"$@"} diff --git a/afl-cmin.bash b/afl-cmin.bash new file mode 100755 index 00000000..1dd782d8 --- /dev/null +++ b/afl-cmin.bash @@ -0,0 +1,470 @@ +#!/usr/bin/env bash +# +# american fuzzy lop++ - corpus minimization tool +# --------------------------------------------- +# +# Originally written by Michal Zalewski +# +# Copyright 2014, 2015 Google Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at: +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# This tool tries to find the smallest subset of files in the input directory +# that still trigger the full range of instrumentation data points seen in +# the starting corpus. This has two uses: +# +# - Screening large corpora of input files before using them as a seed for +# afl-fuzz. The tool will remove functionally redundant files and likely +# leave you with a much smaller set. +# +# (In this case, you probably also want to consider running afl-tmin on +# the individual files later on to reduce their size.) +# +# - Minimizing the corpus generated organically by afl-fuzz, perhaps when +# planning to feed it to more resource-intensive tools. The tool achieves +# this by removing all entries that used to trigger unique behaviors in the +# past, but have been made obsolete by later finds. +# +# Note that the tool doesn't modify the files themselves. For that, you want +# afl-tmin. +# +# This script must use bash because other shells may have hardcoded limits on +# array sizes. +# + +echo "corpus minimization tool for afl-fuzz by Michal Zalewski" +echo + +######### +# SETUP # +######### + +# Process command-line options... + +MEM_LIMIT=200 +TIMEOUT=none + +unset IN_DIR OUT_DIR STDIN_FILE EXTRA_PAR MEM_LIMIT_GIVEN \ + AFL_CMIN_CRASHES_ONLY AFL_CMIN_ALLOW_ANY QEMU_MODE UNICORN_MODE + +while getopts "+i:o:f:m:t:eQUCh" opt; do + + case "$opt" in + + "h") + ;; + + "i") + IN_DIR="$OPTARG" + ;; + + "o") + OUT_DIR="$OPTARG" + ;; + "f") + STDIN_FILE="$OPTARG" + ;; + "m") + MEM_LIMIT="$OPTARG" + MEM_LIMIT_GIVEN=1 + ;; + "t") + TIMEOUT="$OPTARG" + ;; + "e") + EXTRA_PAR="$EXTRA_PAR -e" + ;; + "C") + export AFL_CMIN_CRASHES_ONLY=1 + ;; + "Q") + EXTRA_PAR="$EXTRA_PAR -Q" + test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 + QEMU_MODE=1 + ;; + "U") + EXTRA_PAR="$EXTRA_PAR -U" + test "$MEM_LIMIT_GIVEN" = "" && MEM_LIMIT=250 + UNICORN_MODE=1 + ;; + "?") + exit 1 + ;; + + esac + +done + +shift $((OPTIND-1)) + +TARGET_BIN="$1" + +if [ "$TARGET_BIN" = "" -o "$IN_DIR" = "" -o "$OUT_DIR" = "" ]; then + + cat 1>&2 <<_EOF_ +Usage: $0 [ options ] -- /path/to/target_app [ ... ] + +Required parameters: + + -i dir - input directory with the starting corpus + -o dir - output directory for minimized files + +Execution control settings: + + -f file - location read by the fuzzed program (stdin) + -m megs - memory limit for child process ($MEM_LIMIT MB) + -t msec - run time limit for child process (none) + -Q - use binary-only instrumentation (QEMU mode) + -U - use unicorn-based instrumentation (Unicorn mode) + +Minimization settings: + + -C - keep crashing inputs, reject everything else + -e - solve for edge coverage only, ignore hit counts + +For additional tips, please consult docs/README. + +_EOF_ + exit 1 +fi + +# Do a sanity check to discourage the use of /tmp, since we can't really +# handle this safely from a shell script. + +if [ "$AFL_ALLOW_TMP" = "" ]; then + + echo "$IN_DIR" | grep -qE '^(/var)?/tmp/' + T1="$?" + + echo "$TARGET_BIN" | grep -qE '^(/var)?/tmp/' + T2="$?" + + echo "$OUT_DIR" | grep -qE '^(/var)?/tmp/' + T3="$?" + + echo "$STDIN_FILE" | grep -qE '^(/var)?/tmp/' + T4="$?" + + echo "$PWD" | grep -qE '^(/var)?/tmp/' + T5="$?" + + if [ "$T1" = "0" -o "$T2" = "0" -o "$T3" = "0" -o "$T4" = "0" -o "$T5" = "0" ]; then + echo "[-] Error: do not use this script in /tmp or /var/tmp." 1>&2 + exit 1 + fi + +fi + +# If @@ is specified, but there's no -f, let's come up with a temporary input +# file name. + +TRACE_DIR="$OUT_DIR/.traces" + +if [ "$STDIN_FILE" = "" ]; then + + if echo "$*" | grep -qF '@@'; then + STDIN_FILE="$TRACE_DIR/.cur_input" + fi + +fi + +# Check for obvious errors. + +if [ ! "$MEM_LIMIT" = "none" ]; then + + if [ "$MEM_LIMIT" -lt "5" ]; then + echo "[-] Error: dangerously low memory limit." 1>&2 + exit 1 + fi + +fi + +if [ ! "$TIMEOUT" = "none" ]; then + + if [ "$TIMEOUT" -lt "10" ]; then + echo "[-] Error: dangerously low timeout." 1>&2 + exit 1 + fi + +fi + +if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then + + TNEW="`which "$TARGET_BIN" 2>/dev/null`" + + if [ ! -f "$TNEW" -o ! -x "$TNEW" ]; then + echo "[-] Error: binary '$TARGET_BIN' not found or not executable." 1>&2 + exit 1 + fi + + TARGET_BIN="$TNEW" + +fi + +if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$UNICORN_MODE" = "" ]; then + + if ! grep -qF "__AFL_SHM_ID" "$TARGET_BIN"; then + echo "[-] Error: binary '$TARGET_BIN' doesn't appear to be instrumented." 1>&2 + exit 1 + fi + +fi + +if [ ! -d "$IN_DIR" ]; then + echo "[-] Error: directory '$IN_DIR' not found." 1>&2 + exit 1 +fi + +test -d "$IN_DIR/queue" && IN_DIR="$IN_DIR/queue" + +find "$OUT_DIR" -name 'id[:_]*' -maxdepth 1 -exec rm -- {} \; 2>/dev/null +rm -rf "$TRACE_DIR" 2>/dev/null + +rmdir "$OUT_DIR" 2>/dev/null + +if [ -d "$OUT_DIR" ]; then + echo "[-] Error: directory '$OUT_DIR' exists and is not empty - delete it first." 1>&2 + exit 1 +fi + +mkdir -m 700 -p "$TRACE_DIR" || exit 1 + +if [ ! "$STDIN_FILE" = "" ]; then + rm -f "$STDIN_FILE" || exit 1 + touch "$STDIN_FILE" || exit 1 +fi + +if [ "$AFL_PATH" = "" ]; then + SHOWMAP="${0%/afl-cmin}/afl-showmap" +else + SHOWMAP="$AFL_PATH/afl-showmap" +fi + +if [ ! -x "$SHOWMAP" ]; then + echo "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." 1>&2 + rm -rf "$TRACE_DIR" + exit 1 +fi + +IN_COUNT=$((`ls -- "$IN_DIR" 2>/dev/null | wc -l`)) + +if [ "$IN_COUNT" = "0" ]; then + echo "[+] Hmm, no inputs in the target directory. Nothing to be done." + rm -rf "$TRACE_DIR" + exit 1 +fi + +FIRST_FILE=`ls "$IN_DIR" | head -1` + +# Make sure that we're not dealing with a directory. + +if [ -d "$IN_DIR/$FIRST_FILE" ]; then + echo "[-] Error: The target directory contains subdirectories - please fix." 1>&2 + rm -rf "$TRACE_DIR" + exit 1 +fi + +# Check for the more efficient way to copy files... + +if ln "$IN_DIR/$FIRST_FILE" "$TRACE_DIR/.link_test" 2>/dev/null; then + CP_TOOL=ln +else + CP_TOOL=cp +fi + +# Make sure that we can actually get anything out of afl-showmap before we +# waste too much time. + +echo "[*] Testing the target binary..." + +if [ "$STDIN_FILE" = "" ]; then + + AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$FIRST_FILE" + +else + + cp "$IN_DIR/$FIRST_FILE" "$STDIN_FILE" + AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" &2 + test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" + exit 1 + +fi + +# Let's roll! + +############################# +# STEP 1: COLLECTING TRACES # +############################# + +echo "[*] Obtaining traces for input files in '$IN_DIR'..." + +( + + CUR=0 + + if [ "$STDIN_FILE" = "" ]; then + + ls "$IN_DIR" | while read -r fn; do + + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -- "$@" <"$IN_DIR/$fn" + + done + + else + + ls "$IN_DIR" | while read -r fn; do + + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + cp "$IN_DIR/$fn" "$STDIN_FILE" + + "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" "$TRACE_DIR/.all_uniq" + +TUPLE_COUNT=$((`grep -c . "$TRACE_DIR/.all_uniq"`)) + +echo "[+] Found $TUPLE_COUNT unique tuples across $IN_COUNT files." + +##################################### +# STEP 3: SELECTING CANDIDATE FILES # +##################################### + +# The next step is to find the best candidate for each tuple. The "best" +# part is understood simply as the smallest input that includes a particular +# tuple in its trace. Empirical evidence suggests that this produces smaller +# datasets than more involved algorithms that could be still pulled off in +# a shell script. + +echo "[*] Finding best candidates for each tuple..." + +CUR=0 + +ls -rS "$IN_DIR" | while read -r fn; do + + CUR=$((CUR+1)) + printf "\\r Processing file $CUR/$IN_COUNT... " + + sed "s#\$# $fn#" "$TRACE_DIR/$fn" >>"$TRACE_DIR/.candidate_list" + +done + +echo + +############################## +# STEP 4: LOADING CANDIDATES # +############################## + +# At this point, we have a file of tuple-file pairs, sorted by file size +# in ascending order (as a consequence of ls -rS). By doing sort keyed +# only by tuple (-k 1,1) and configured to output only the first line for +# every key (-s -u), we end up with the smallest file for each tuple. + +echo "[*] Sorting candidate list (be patient)..." + +sort -k1,1 -s -u "$TRACE_DIR/.candidate_list" | \ + sed 's/^/BEST_FILE[/;s/ /]="/;s/$/"/' >"$TRACE_DIR/.candidate_script" + +if [ ! -s "$TRACE_DIR/.candidate_script" ]; then + echo "[-] Error: no traces obtained from test cases, check syntax!" 1>&2 + test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" + exit 1 +fi + +# The sed command converted the sorted list to a shell script that populates +# BEST_FILE[tuple]="fname". Let's load that! + +. "$TRACE_DIR/.candidate_script" + +########################## +# STEP 5: WRITING OUTPUT # +########################## + +# The final trick is to grab the top pick for each tuple, unless said tuple is +# already set due to the inclusion of an earlier candidate; and then put all +# tuples associated with the newly-added file to the "already have" list. The +# loop works from least popular tuples and toward the most common ones. + +echo "[*] Processing candidates and writing output files..." + +CUR=0 + +touch "$TRACE_DIR/.already_have" + +while read -r cnt tuple; do + + CUR=$((CUR+1)) + printf "\\r Processing tuple $CUR/$TUPLE_COUNT... " + + # If we already have this tuple, skip it. + + grep -q "^$tuple\$" "$TRACE_DIR/.already_have" && continue + + FN=${BEST_FILE[tuple]} + + $CP_TOOL "$IN_DIR/$FN" "$OUT_DIR/$FN" + + if [ "$((CUR % 5))" = "0" ]; then + sort -u "$TRACE_DIR/$FN" "$TRACE_DIR/.already_have" >"$TRACE_DIR/.tmp" + mv -f "$TRACE_DIR/.tmp" "$TRACE_DIR/.already_have" + else + cat "$TRACE_DIR/$FN" >>"$TRACE_DIR/.already_have" + fi + +done <"$TRACE_DIR/.all_uniq" + +echo + +OUT_COUNT=`ls -- "$OUT_DIR" | wc -l` + +if [ "$OUT_COUNT" = "1" ]; then + echo "[!] WARNING: All test cases had the same traces, check syntax!" +fi + +echo "[+] Narrowed down to $OUT_COUNT files, saved in '$OUT_DIR'." +echo + +test "$AFL_KEEP_TRACES" = "" && rm -rf "$TRACE_DIR" + +exit 0 From 9da167dffdc14468d17ac3c1c942e483baf17433 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Wed, 22 Jan 2020 21:08:47 +0100 Subject: [PATCH 29/44] fix for modern linux --- afl-cmin.awk | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/afl-cmin.awk b/afl-cmin.awk index 28c460e8..fcdfb71f 100755 --- a/afl-cmin.awk +++ b/afl-cmin.awk @@ -109,7 +109,7 @@ function exists_and_is_executable(binarypath) { } BEGIN { - print "corpus minimization tool for afl-fuzz++ (awk version)\n" + print "corpus minimization tool for afl++ (awk version)\n" # defaults extra_par = "" @@ -289,7 +289,7 @@ BEGIN { } else { "which afl-showmap 2>/dev/null" | getline path } - showmap = path + showmap = path "/afl-showmap" } else { showmap = ENVIRON["AFL_PATH"] "/afl-showmap" } @@ -303,11 +303,12 @@ BEGIN { i = 0 # yuck, gnu stat is incompatible to bsd stat if ("stat --version 2>/dev/null" !~ /GNU coreutils/) { - stat_format = "-f '%z %N'" - } else { + # I dont get it why this does not work, output is "stat (GNU coreutils) 8.30" and still it goes here ... stat_format = "-c '%s %n'" + } else { + stat_format = "-f '%z %N'" } - while ("cd "in_dir" && find . -type f -exec stat "stat_format" \{\} \\; | sort -n | cut -d' ' -f2-" | getline) { + while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { infilesSmallToBig[i++] = $0 } in_count = i From c51f89b58e56338a5a430344548d1385432d173e Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Wed, 22 Jan 2020 21:50:35 +0100 Subject: [PATCH 30/44] rectification of vanhauser's fix, made it a bit more robust, enabled error output for travis debugging --- afl-cmin.awk | 40 ++++++++++++++++++++++++++++++++-------- test/test.sh | 2 +- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/afl-cmin.awk b/afl-cmin.awk index fcdfb71f..967c4e87 100755 --- a/afl-cmin.awk +++ b/afl-cmin.awk @@ -1,5 +1,28 @@ #!/usr/bin/awk -f +# awk script to minimize a test corpus of input files +# +# based on afl-cmin bash script written by Michal Zalewski +# rewritten by Heiko Eißfeldt (hexcoder-) +# +# uses getopt.awk package from Arnold Robbins +# +# external tools used by this script: +# test +# grep +# rm +# mkdir +# ln +# cp +# pwd +# which +# cd +# find +# stat +# sort +# cut +# and afl-showmap from this project :-) + # getopt.awk --- Do C library getopt(3) function in awk # External variables: @@ -285,11 +308,10 @@ BEGIN { if (!ENVIRON["AFL_PATH"]) { if (0 == system("test -f afl-cmin.awk")) { - path = "." + showmap = "./afl-showmap" } else { - "which afl-showmap 2>/dev/null" | getline path + "which afl-showmap 2>/dev/null" | getline showmap } - showmap = path "/afl-showmap" } else { showmap = ENVIRON["AFL_PATH"] "/afl-showmap" } @@ -301,12 +323,14 @@ BEGIN { # get list of input filenames sorted by size i = 0 - # yuck, gnu stat is incompatible to bsd stat - if ("stat --version 2>/dev/null" !~ /GNU coreutils/) { - # I dont get it why this does not work, output is "stat (GNU coreutils) 8.30" and still it goes here ... - stat_format = "-c '%s %n'" + # yuck, gnu stat is option incompatible to bsd stat + # we use a heuristic to differentiate between + # GNU stat and other stats + "stat --version 2>/dev/null" | getline statversion + if (statversion ~ /GNU coreutils/) { + stat_format = "-c '%s %n'" # GNU } else { - stat_format = "-f '%z %N'" + stat_format = "-f '%z %N'" # *BSD, MacOS } while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { infilesSmallToBig[i++] = $0 diff --git a/test/test.sh b/test/test.sh index 0ae6fd09..cc7fe224 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null 2>&1 + ../afl-cmin -i in -o in2 -- ./test-instr.plain CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; From 7e7ab8f5415409fd1bb643f4dfef44c5a3935006 Mon Sep 17 00:00:00 2001 From: hexcoder Date: Wed, 22 Jan 2020 22:24:00 +0100 Subject: [PATCH 31/44] Update binaryonly_fuzzing.txt --- docs/binaryonly_fuzzing.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/binaryonly_fuzzing.txt b/docs/binaryonly_fuzzing.txt index 239fb4b0..f8d68cd8 100644 --- a/docs/binaryonly_fuzzing.txt +++ b/docs/binaryonly_fuzzing.txt @@ -5,10 +5,10 @@ Fuzzing binary-only programs with afl++ afl++, libfuzzer and others are great if you have the source code, and it allows for very fast and coverage guided fuzzing. -However, if there is only the binary program and not source code available, -then standard afl++ (dumb mode) is not effective. +However, if there is only the binary program and no source code available, +then standard `afl-fuzz -n` (dumb mode) is not effective. -The following is a description of how these can be fuzzed with afl++ +The following is a description of how these binaries can be fuzzed with afl++ !!!!! TL;DR: try DYNINST with afl-dyninst. If it produces too many crashes then @@ -28,7 +28,7 @@ As it is included in afl++ this needs no URL. WINE+QEMU --------- -Wine mode can run Win32 PE with the QEMU instrumentation. +Wine mode can run Win32 PE binaries with the QEMU instrumentation. It needs Wine, python3 and the pefile python package installed. UNICORN @@ -37,7 +37,7 @@ Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. In contrast to QEMU, Unicorn does not offer a full system or even userland emulation. Runtime environment and/or loaders have to be written from scratch, if needed. On top, block chaining has been removed. This means the speed boost introduced in -to the patched QEMU Mode of afl++ cannot simply be ported over to Unicorn. +the patched QEMU Mode of afl++ cannot simply be ported over to Unicorn. For further information, check out ./unicorn_mode.txt. From c490b9aa3694ba9c33ba0657ddd5e19dd979f2ed Mon Sep 17 00:00:00 2001 From: hexcoder Date: Thu, 23 Jan 2020 09:11:35 +0100 Subject: [PATCH 32/44] afl-cmin debugging is done now, so suppress stdout messages again (but not stderr) --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index cc7fe224..3473155f 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; From 8b17cac71c5196bae11a5a7ee8f6a17bdb3917e0 Mon Sep 17 00:00:00 2001 From: hexcoder Date: Thu, 23 Jan 2020 09:46:07 +0100 Subject: [PATCH 33/44] add socket_fuzz description --- experimental/README.experiments | 3 +++ 1 file changed, 3 insertions(+) diff --git a/experimental/README.experiments b/experimental/README.experiments index af9739bd..5a505ad7 100644 --- a/experimental/README.experiments +++ b/experimental/README.experiments @@ -28,6 +28,9 @@ Here's a quick overview of the stuff you can find in this directory: mode to speed up certain fuzzing jobs. - post_library - an example of how to build postprocessors for AFL. + + - socketfuzz - a LD_PRELOAD library 'redirects' a socket to stdin + for fuzzing access with afl++ Note that the minimize_corpus.sh tool has graduated from the experimental/ directory and is now available as ../afl-cmin. The LLVM mode has likewise From a58800b90122f3d612a0badb243d2c1b6fc9c742 Mon Sep 17 00:00:00 2001 From: hexcoder Date: Thu, 23 Jan 2020 09:46:59 +0100 Subject: [PATCH 34/44] typo --- experimental/README.experiments | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/experimental/README.experiments b/experimental/README.experiments index 5a505ad7..543c078c 100644 --- a/experimental/README.experiments +++ b/experimental/README.experiments @@ -29,7 +29,7 @@ Here's a quick overview of the stuff you can find in this directory: - post_library - an example of how to build postprocessors for AFL. - - socketfuzz - a LD_PRELOAD library 'redirects' a socket to stdin + - socket_fuzzing - a LD_PRELOAD library 'redirects' a socket to stdin for fuzzing access with afl++ Note that the minimize_corpus.sh tool has graduated from the experimental/ From e7c95ebf5a4828b662252b10052a89923dd25030 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 23 Jan 2020 10:15:33 +0100 Subject: [PATCH 35/44] afl-cmin final touches --- Makefile | 2 +- afl-cmin | 473 ++++++++++++++++++++++++++++++++++++++++++++++++- afl-cmin.awk | 470 ------------------------------------------------ docs/ChangeLog | 2 + test/test.sh | 2 +- 5 files changed, 476 insertions(+), 473 deletions(-) delete mode 100755 afl-cmin.awk diff --git a/Makefile b/Makefile index 7260ee47..459cae5f 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ VERSION = $(shell grep '^\#define VERSION ' ../config.h | cut -d '"' -f2) # PROGS intentionally omit afl-as, which gets installed elsewhere. PROGS = afl-gcc afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze -SH_PROGS = afl-plot afl-cmin afl-whatsup afl-system-config +SH_PROGS = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-system-config MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) ifeq "$(shell echo 'int main() {return 0; }' | $(CC) -x c - -flto=full -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1" diff --git a/afl-cmin b/afl-cmin index 75dc63a7..a072a62a 100755 --- a/afl-cmin +++ b/afl-cmin @@ -1,4 +1,475 @@ #!/usr/bin/env sh THISPATH=`dirname ${0}` export PATH=${THISPATH}:$PATH -awk -f ${0}.awk -- ${@+"$@"} +awk -f - -- ${@+"$@"} <<'EOF' +#!/usr/bin/awk -f + +# awk script to minimize a test corpus of input files +# +# based on afl-cmin bash script written by Michal Zalewski +# rewritten by Heiko Eißfeldt (hexcoder-) +# +# uses getopt.awk package from Arnold Robbins +# +# external tools used by this script: +# test +# grep +# rm +# mkdir +# ln +# cp +# pwd +# which +# cd +# find +# stat +# sort +# cut +# and afl-showmap from this project :-) + +# getopt.awk --- Do C library getopt(3) function in awk + +# External variables: +# Optind -- index in ARGV of first nonoption argument +# Optarg -- string value of argument to current option +# Opterr -- if nonzero, print our own diagnostic +# Optopt -- current option letter + +# Returns: +# -1 at end of options +# "?" for unrecognized option +# a character representing the current option + +# Private Data: +# _opti -- index in multiflag option, e.g., -abc + +function getopt(argc, argv, options, thisopt, i) +{ + if (length(options) == 0) # no options given + return -1 + + if (argv[Optind] == "--") { # all done + Optind++ + _opti = 0 + return -1 + } else if (argv[Optind] !~ /^-[^:[:space:]]/) { + _opti = 0 + return -1 + } + if (_opti == 0) + _opti = 2 + thisopt = substr(argv[Optind], _opti, 1) + Optopt = thisopt + i = index(options, thisopt) + if (i == 0) { + if (Opterr) + printf("%c -- invalid option\n", thisopt) > "/dev/stderr" + if (_opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return "?" + } + if (substr(options, i + 1, 1) == ":") { + # get option argument + if (length(substr(argv[Optind], _opti + 1)) > 0) + Optarg = substr(argv[Optind], _opti + 1) + else + Optarg = argv[++Optind] + _opti = 0 + } else + Optarg = "" + if (_opti == 0 || _opti >= length(argv[Optind])) { + Optind++ + _opti = 0 + } else + _opti++ + return thisopt +} + +BEGIN { + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + + # test program + if (_getopt_test) { + while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) + printf("c = <%c>, Optarg = <%s>\n", + _go_c, Optarg) + printf("non-option arguments:\n") + for (; Optind < ARGC; Optind++) + printf("\tARGV[%d] = <%s>\n", + Optind, ARGV[Optind]) + } +} + +function usage() { + print \ +"Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ +"\n" \ +"Required parameters:\n" \ +"\n" \ +" -i dir - input directory with starting corpus\n" \ +" -o dir - output directory for minimized files\n" \ +"\n" \ +"Execution control settings:\n" \ +"\n" \ +" -f file - location read by the fuzzed program (stdin)\n" \ +" -m megs - memory limit for child process ("mem_limit" MB)\n" \ +" -t msec - run time limit for child process (none)\n" \ +" -Q - use binary-only instrumentation (QEMU mode)\n" \ +" -U - use unicorn-based instrumentation (unicorn mode)\n" \ +"\n" \ +"Minimization settings:\n" \ +" -C - keep crashing inputs, reject everything else\n" \ +" -e - solve for edge coverage only, ignore hit counts\n" \ +"\n" \ +"For additional tips, please consult docs/README.md\n" \ +"\n" \ + > "/dev/stderr" + exit 1 +} + +function exists_and_is_executable(binarypath) { + return 0 == system("test -f "binarypath" -a -x "binarypath) +} + +BEGIN { + print "corpus minimization tool for afl++ (awk version)\n" + + # defaults + extra_par = "" + # process options + Opterr = 1 # default is to diagnose + Optind = 1 # skip ARGV[0] + while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eCQU?")) != -1) { + if (_go_c == "i") { + if (!Optarg) usage() + if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + in_dir = Optarg + continue + } else + if (_go_c == "o") { + if (!Optarg) usage() + if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + out_dir = Optarg + continue + } else + if (_go_c == "f") { + if (!Optarg) usage() + if (stdin_file) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + stdin_file = Optarg + continue + } else + if (_go_c == "m") { + if (!Optarg) usage() + if (mem_limit) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + mem_limit = Optarg + mem_limit_given = 1 + continue + } else + if (_go_c == "t") { + if (!Optarg) usage() + if (timeout) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + timeout = Optarg + continue + } else + if (_go_c == "C") { + ENVIRON["AFL_CMIN_CRASHES_ONLY"] = 1 + continue + } else + if (_go_c == "e") { + extra_par = extra_par " -e" + continue + } else + if (_go_c == "Q") { + if (qemu_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -Q" + if ( !mem_limit_given ) mem_limit = "250" + qemu_mode = 1 + continue + } else + if (_go_c == "U") { + if (unicorn_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} + extra_par = extra_par " -U" + if ( !mem_limit_given ) mem_limit = "250" + unicorn_mode = 1 + continue + } else + if (_go_c == "?") { + exit 1 + } else + usage() + } # while options + + if (!mem_limit) mem_limit = 200 + if (!timeout) timeout = "none" + + # get program args + i = 0 + prog_args_string = "" + for (; Optind < ARGC; Optind++) { + prog_args[i++] = ARGV[Optind] + if (i > 1) + prog_args_string = prog_args_string" "ARGV[Optind] + } + + # sanity checks + if (!prog_args[0] || !in_dir || !out_dir) usage() + + target_bin = prog_args[0] + + # Do a sanity check to discourage the use of /tmp, since we can't really + # handle this safely from an awk script. + + if (!ENVIRON["AFL_ALLOW_TMP"]) { + dirlist[0] = in_dir + dirlist[1] = target_bin + dirlist[2] = out_dir + dirlist[3] = stdin_file + "pwd" | getline dirlist[4] # current directory + for (dirind in dirlist) { + dir = dirlist[dirind] + + if (dir ~ /^(\/var)?\/tmp/) { + print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" + exit 1 + } + } + delete dirlist + } + + # If @@ is specified, but there's no -f, let's come up with a temporary input + # file name. + + trace_dir = out_dir "/.traces" + + if (!stdin_file) { + found_atat = 0 + for (prog_args_ind in prog_args) { + if ("@@" == prog_args[prog_args_ind]) { + found_atat = 1 + break + } + } + if (found_atat) { + stdin_file = trace_dir "/.cur_input" + } + } + + # Check for obvious errors. + + if (mem_limit && mem_limit != "none" && mem_limit < 5) { + print "[-] Error: dangerously low memory limit." > "/dev/stderr" + exit 1 + } + + if (timeout && timeout != "none" && timeout < 10) { + print "[-] Error: dangerously low timeout." > "/dev/stderr" + exit 1 + } + + if (target_bin && !exists_and_is_executable(target_bin)) { + + "which "target_bin" 2>/dev/null" | getline tnew + if (!tnew || !exists_and_is_executable(tnew)) { + print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr" + exit 1 + } + target_bin = tnew + } + + if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !unicorn_mode) { + if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { + print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" + exit 1 + } + } + + if (0 != system( "test -d "in_dir )) { + print "[-] Error: directory '"in_dir"' not found." > "/dev/stderr" + exit 1 + } + + if (0 == system( "test -d "in_dir"/queue" )) { + in_dir = in_dir "/queue" + } + + system("rm -rf "trace_dir" 2>/dev/null"); + system("rm "out_dir"/id[:_]* 2>/dev/null") + + if (0 == system( "test -d "out_dir" -a -e "out_dir"/*" )) { + print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr" + exit 1 + } + + if (stdin_file) { + # truncate input file + printf "" > stdin_file + close( stdin_file ) + } + + if (!ENVIRON["AFL_PATH"]) { + if (0 == system("test -f afl-cmin.awk")) { + showmap = "./afl-showmap" + } else { + "which afl-showmap 2>/dev/null" | getline showmap + } + } else { + showmap = ENVIRON["AFL_PATH"] "/afl-showmap" + } + + if (!showmap || 0 != system("test -x "showmap )) { + print "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." > "/dev/stderr" + exit 1 + } + + # get list of input filenames sorted by size + i = 0 + # yuck, gnu stat is option incompatible to bsd stat + # we use a heuristic to differentiate between + # GNU stat and other stats + "stat --version 2>/dev/null" | getline statversion + if (statversion ~ /GNU coreutils/) { + stat_format = "-c '%s %n'" # GNU + } else { + stat_format = "-f '%z %N'" # *BSD, MacOS + } + while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { + infilesSmallToBig[i++] = $0 + } + in_count = i + + first_file = infilesSmallToBig[0] + + # Make sure that we're not dealing with a directory. + + if (0 == system("test -d "in_dir"/"first_file)) { + print "[-] Error: The input directory contains subdirectories - please fix." > "/dev/stderr" + exit 1 + } + + # Check for the more efficient way to copy files... + if (0 != system("mkdir -p -m 0700 "trace_dir)) { + print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" + exit 1 + } + + if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { + cp_tool = "ln" + } else { + cp_tool = "cp" + } + + # Make sure that we can actually get anything out of afl-showmap before we + # waste too much time. + + print "[*] Testing the target binary..." + + if (!stdin_file) { + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") + } else { + system("cp "in_dir"/"first_file" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + ++first_count + } + + if (first_count) { + print "[+] OK, "first_count" tuples recorded." + } else { + print "[-] Error: no instrumentation output detected (perhaps crash or timeout)." > "/dev/stderr" + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + exit 1 + } + + # Let's roll! + + ############################# + # STEP 1: Collecting traces # + ############################# + + print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." + + cur = 0; + if (!stdin_file) { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur; + printf "\r Processing file "cur"/"in_count + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"fn"\"") + } + } else { + while (cur < in_count) { + fn = infilesSmallToBig[cur] + ++cur + printf "\r Processing file "cur"/"in_count + system("cp "in_dir"/"fn" "stdin_file) + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { + key = line + if (!(key in key_count)) { + ++tuple_count + } + ++key_count[key] + if (! (key in best_file)) { + # this is the best file for this key + best_file[key] = fn + # copy file unless already done + if (! (fn in file_already_copied)) { + system(cp_tool" "in_dir"/"fn" "out_dir"/"fn) + file_already_copied[fn] = "" + ++out_count + } + } + } + close(tracefile_path) + } + + print "" + print "[+] Found "tuple_count" unique tuples across "in_count" files." + + if (out_count == 1) { + print "[!] WARNING: All test cases had the same traces, check syntax!" + } + print "[+] Narrowed down to "out_count" files, saved in '"out_dir"'." + + if (!ENVIRON["AFL_KEEP_TRACES"]) { + system("rm -rf "trace_dir" 2>/dev/null") + } + + exit 0 +} +EOF diff --git a/afl-cmin.awk b/afl-cmin.awk deleted file mode 100755 index 967c4e87..00000000 --- a/afl-cmin.awk +++ /dev/null @@ -1,470 +0,0 @@ -#!/usr/bin/awk -f - -# awk script to minimize a test corpus of input files -# -# based on afl-cmin bash script written by Michal Zalewski -# rewritten by Heiko Eißfeldt (hexcoder-) -# -# uses getopt.awk package from Arnold Robbins -# -# external tools used by this script: -# test -# grep -# rm -# mkdir -# ln -# cp -# pwd -# which -# cd -# find -# stat -# sort -# cut -# and afl-showmap from this project :-) - -# getopt.awk --- Do C library getopt(3) function in awk - -# External variables: -# Optind -- index in ARGV of first nonoption argument -# Optarg -- string value of argument to current option -# Opterr -- if nonzero, print our own diagnostic -# Optopt -- current option letter - -# Returns: -# -1 at end of options -# "?" for unrecognized option -# a character representing the current option - -# Private Data: -# _opti -- index in multiflag option, e.g., -abc - -function getopt(argc, argv, options, thisopt, i) -{ - if (length(options) == 0) # no options given - return -1 - - if (argv[Optind] == "--") { # all done - Optind++ - _opti = 0 - return -1 - } else if (argv[Optind] !~ /^-[^:[:space:]]/) { - _opti = 0 - return -1 - } - if (_opti == 0) - _opti = 2 - thisopt = substr(argv[Optind], _opti, 1) - Optopt = thisopt - i = index(options, thisopt) - if (i == 0) { - if (Opterr) - printf("%c -- invalid option\n", thisopt) > "/dev/stderr" - if (_opti >= length(argv[Optind])) { - Optind++ - _opti = 0 - } else - _opti++ - return "?" - } - if (substr(options, i + 1, 1) == ":") { - # get option argument - if (length(substr(argv[Optind], _opti + 1)) > 0) - Optarg = substr(argv[Optind], _opti + 1) - else - Optarg = argv[++Optind] - _opti = 0 - } else - Optarg = "" - if (_opti == 0 || _opti >= length(argv[Optind])) { - Optind++ - _opti = 0 - } else - _opti++ - return thisopt -} - -BEGIN { - Opterr = 1 # default is to diagnose - Optind = 1 # skip ARGV[0] - - # test program - if (_getopt_test) { - while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) - printf("c = <%c>, Optarg = <%s>\n", - _go_c, Optarg) - printf("non-option arguments:\n") - for (; Optind < ARGC; Optind++) - printf("\tARGV[%d] = <%s>\n", - Optind, ARGV[Optind]) - } -} - -function usage() { - print \ -"Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ -"\n" \ -"Required parameters:\n" \ -"\n" \ -" -i dir - input directory with starting corpus\n" \ -" -o dir - output directory for minimized files\n" \ -"\n" \ -"Execution control settings:\n" \ -"\n" \ -" -f file - location read by the fuzzed program (stdin)\n" \ -" -m megs - memory limit for child process ("mem_limit" MB)\n" \ -" -t msec - run time limit for child process (none)\n" \ -" -Q - use binary-only instrumentation (QEMU mode)\n" \ -" -U - use unicorn-based instrumentation (unicorn mode)\n" \ -"\n" \ -"Minimization settings:\n" \ -" -C - keep crashing inputs, reject everything else\n" \ -" -e - solve for edge coverage only, ignore hit counts\n" \ -"\n" \ -"For additional tips, please consult docs/README.md\n" \ -"\n" \ - > "/dev/stderr" - exit 1 -} - -function exists_and_is_executable(binarypath) { - return 0 == system("test -f "binarypath" -a -x "binarypath) -} - -BEGIN { - print "corpus minimization tool for afl++ (awk version)\n" - - # defaults - extra_par = "" - # process options - Opterr = 1 # default is to diagnose - Optind = 1 # skip ARGV[0] - while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eCQU?")) != -1) { - if (_go_c == "i") { - if (!Optarg) usage() - if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - in_dir = Optarg - continue - } else - if (_go_c == "o") { - if (!Optarg) usage() - if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - out_dir = Optarg - continue - } else - if (_go_c == "f") { - if (!Optarg) usage() - if (stdin_file) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - stdin_file = Optarg - continue - } else - if (_go_c == "m") { - if (!Optarg) usage() - if (mem_limit) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - mem_limit = Optarg - mem_limit_given = 1 - continue - } else - if (_go_c == "t") { - if (!Optarg) usage() - if (timeout) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - timeout = Optarg - continue - } else - if (_go_c == "C") { - ENVIRON["AFL_CMIN_CRASHES_ONLY"] = 1 - continue - } else - if (_go_c == "e") { - extra_par = extra_par " -e" - continue - } else - if (_go_c == "Q") { - if (qemu_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - extra_par = extra_par " -Q" - if ( !mem_limit_given ) mem_limit = "250" - qemu_mode = 1 - continue - } else - if (_go_c == "U") { - if (unicorn_mode) { print "Option "_go_c" is only allowed once" > "/dev/stderr"} - extra_par = extra_par " -U" - if ( !mem_limit_given ) mem_limit = "250" - unicorn_mode = 1 - continue - } else - if (_go_c == "?") { - exit 1 - } else - usage() - } # while options - - if (!mem_limit) mem_limit = 200 - if (!timeout) timeout = "none" - - # get program args - i = 0 - prog_args_string = "" - for (; Optind < ARGC; Optind++) { - prog_args[i++] = ARGV[Optind] - if (i > 1) - prog_args_string = prog_args_string" "ARGV[Optind] - } - - # sanity checks - if (!prog_args[0] || !in_dir || !out_dir) usage() - - target_bin = prog_args[0] - - # Do a sanity check to discourage the use of /tmp, since we can't really - # handle this safely from an awk script. - - if (!ENVIRON["AFL_ALLOW_TMP"]) { - dirlist[0] = in_dir - dirlist[1] = target_bin - dirlist[2] = out_dir - dirlist[3] = stdin_file - "pwd" | getline dirlist[4] # current directory - for (dirind in dirlist) { - dir = dirlist[dirind] - - if (dir ~ /^(\/var)?\/tmp/) { - print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr" - exit 1 - } - } - delete dirlist - } - - # If @@ is specified, but there's no -f, let's come up with a temporary input - # file name. - - trace_dir = out_dir "/.traces" - - if (!stdin_file) { - found_atat = 0 - for (prog_args_ind in prog_args) { - if ("@@" == prog_args[prog_args_ind]) { - found_atat = 1 - break - } - } - if (found_atat) { - stdin_file = trace_dir "/.cur_input" - } - } - - # Check for obvious errors. - - if (mem_limit && mem_limit != "none" && mem_limit < 5) { - print "[-] Error: dangerously low memory limit." > "/dev/stderr" - exit 1 - } - - if (timeout && timeout != "none" && timeout < 10) { - print "[-] Error: dangerously low timeout." > "/dev/stderr" - exit 1 - } - - if (target_bin && !exists_and_is_executable(target_bin)) { - - "which "target_bin" 2>/dev/null" | getline tnew - if (!tnew || !exists_and_is_executable(tnew)) { - print "[-] Error: binary '"target_bin"' not found or not executable." > "/dev/stderr" - exit 1 - } - target_bin = tnew - } - - if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !unicorn_mode) { - if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { - print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" - exit 1 - } - } - - if (0 != system( "test -d "in_dir )) { - print "[-] Error: directory '"in_dir"' not found." > "/dev/stderr" - exit 1 - } - - if (0 == system( "test -d "in_dir"/queue" )) { - in_dir = in_dir "/queue" - } - - system("rm -rf "trace_dir" 2>/dev/null"); - system("rm "out_dir"/id[:_]* 2>/dev/null") - - if (0 == system( "test -d "out_dir" -a -e "out_dir"/*" )) { - print "[-] Error: directory '"out_dir"' exists and is not empty - delete it first." > "/dev/stderr" - exit 1 - } - - if (stdin_file) { - # truncate input file - printf "" > stdin_file - close( stdin_file ) - } - - if (!ENVIRON["AFL_PATH"]) { - if (0 == system("test -f afl-cmin.awk")) { - showmap = "./afl-showmap" - } else { - "which afl-showmap 2>/dev/null" | getline showmap - } - } else { - showmap = ENVIRON["AFL_PATH"] "/afl-showmap" - } - - if (!showmap || 0 != system("test -x "showmap )) { - print "[-] Error: can't find 'afl-showmap' - please set AFL_PATH." > "/dev/stderr" - exit 1 - } - - # get list of input filenames sorted by size - i = 0 - # yuck, gnu stat is option incompatible to bsd stat - # we use a heuristic to differentiate between - # GNU stat and other stats - "stat --version 2>/dev/null" | getline statversion - if (statversion ~ /GNU coreutils/) { - stat_format = "-c '%s %n'" # GNU - } else { - stat_format = "-f '%z %N'" # *BSD, MacOS - } - while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { - infilesSmallToBig[i++] = $0 - } - in_count = i - - first_file = infilesSmallToBig[0] - - # Make sure that we're not dealing with a directory. - - if (0 == system("test -d "in_dir"/"first_file)) { - print "[-] Error: The input directory contains subdirectories - please fix." > "/dev/stderr" - exit 1 - } - - # Check for the more efficient way to copy files... - if (0 != system("mkdir -p -m 0700 "trace_dir)) { - print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" - exit 1 - } - - if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { - cp_tool = "ln" - } else { - cp_tool = "cp" - } - - # Make sure that we can actually get anything out of afl-showmap before we - # waste too much time. - - print "[*] Testing the target binary..." - - if (!stdin_file) { - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") - } else { - system("cp "in_dir"/"first_file" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { - ++first_count - } - - if (first_count) { - print "[+] OK, "first_count" tuples recorded." - } else { - print "[-] Error: no instrumentation output detected (perhaps crash or timeout)." > "/dev/stderr" - if (!ENVIRON["AFL_KEEP_TRACES"]) { - system("rm -rf "trace_dir" 2>/dev/null") - } - exit 1 - } - - # Let's roll! - - ############################# - # STEP 1: Collecting traces # - ############################# - - print "[*] Obtaining traces for "in_count" input files in '"in_dir"'." - - cur = 0; - if (!stdin_file) { - while (cur < in_count) { - fn = infilesSmallToBig[cur] - ++cur; - printf "\r Processing file "cur"/"in_count - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"fn"\"") - } - } else { - while (cur < in_count) { - fn = infilesSmallToBig[cur] - ++cur - printf "\r Processing file "cur"/"in_count - system("cp "in_dir"/"fn" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/"fn"\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" 0) { - key = line - if (!(key in key_count)) { - ++tuple_count - } - ++key_count[key] - if (! (key in best_file)) { - # this is the best file for this key - best_file[key] = fn - # copy file unless already done - if (! (fn in file_already_copied)) { - system(cp_tool" "in_dir"/"fn" "out_dir"/"fn) - file_already_copied[fn] = "" - ++out_count - } - } - } - close(tracefile_path) - } - - print "" - print "[+] Found "tuple_count" unique tuples across "in_count" files." - - if (out_count == 1) { - print "[!] WARNING: All test cases had the same traces, check syntax!" - } - print "[+] Narrowed down to "out_count" files, saved in '"out_dir"'." - - if (!ENVIRON["AFL_KEEP_TRACES"]) { - system("rm -rf "trace_dir" 2>/dev/null") - } - - exit 0 -} diff --git a/docs/ChangeLog b/docs/ChangeLog index bb3537dd..33c6f618 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -25,6 +25,8 @@ Version ++2.60d (develop): - show in the help output for which llvm version it was compiled for - now does not need to be recompiled between trace-pc and pass instrumentation. compile normally and set AFL_LLVM_USE_TRACE_PC :) + - afl-cmin is now a sh script (invoking awk) instead of bash for portability + the original script is still present as afl-cmin.bash - added blacklisted function check in all modules of llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/test/test.sh b/test/test.sh index 3473155f..0ae6fd09 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null 2>&1 CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; From 436873a19abe5858e56555db02095f4eb7e6febd Mon Sep 17 00:00:00 2001 From: van Hauser Date: Thu, 23 Jan 2020 11:55:53 +0100 Subject: [PATCH 36/44] show stderr on afl-cmin test.sh --- test/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test.sh b/test/test.sh index 0ae6fd09..3473155f 100755 --- a/test/test.sh +++ b/test/test.sh @@ -150,7 +150,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { } echo 000000000000000000000000 > in/in2 mkdir -p in2 - ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null 2>&1 + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in 1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; From 6abe33030396c8f15f00b4fe3d083f3841de3212 Mon Sep 17 00:00:00 2001 From: hexcoder- Date: Fri, 24 Jan 2020 20:58:15 +0100 Subject: [PATCH 37/44] afl-cmin more awk portability (mawk), add afl-cmin/afl-tmin tests for non-x86 platforms --- afl-cmin | 27 ++++++++------------------- test/test.sh | 26 +++++++++++++++++++++++--- 2 files changed, 31 insertions(+), 22 deletions(-) diff --git a/afl-cmin b/afl-cmin index a072a62a..de5a66ed 100755 --- a/afl-cmin +++ b/afl-cmin @@ -8,6 +8,10 @@ awk -f - -- ${@+"$@"} <<'EOF' # # based on afl-cmin bash script written by Michal Zalewski # rewritten by Heiko Eißfeldt (hexcoder-) +# tested with: +# gnu awk (x86 Linux) +# bsd awk (x86 *BSD) +# mawk (arm32 raspbian) # # uses getopt.awk package from Arnold Robbins # @@ -52,7 +56,7 @@ function getopt(argc, argv, options, thisopt, i) Optind++ _opti = 0 return -1 - } else if (argv[Optind] !~ /^-[^:[:space:]]/) { + } else if (argv[Optind] !~ /^-[^:\t ]/) { _opti = 0 return -1 } @@ -88,22 +92,6 @@ function getopt(argc, argv, options, thisopt, i) return thisopt } -BEGIN { - Opterr = 1 # default is to diagnose - Optind = 1 # skip ARGV[0] - - # test program - if (_getopt_test) { - while ((_go_c = getopt(ARGC, ARGV, "ab:cd")) != -1) - printf("c = <%c>, Optarg = <%s>\n", - _go_c, Optarg) - printf("non-option arguments:\n") - for (; Optind < ARGC; Optind++) - printf("\tARGV[%d] = <%s>\n", - Optind, ARGV[Optind]) - } -} - function usage() { print \ "Usage: afl-cmin [ options ] -- /path/to/target_app [ ... ]\n" \ @@ -311,7 +299,7 @@ BEGIN { } if (!ENVIRON["AFL_PATH"]) { - if (0 == system("test -f afl-cmin.awk")) { + if (0 == system("test -f afl-cmin")) { showmap = "./afl-showmap" } else { "which afl-showmap 2>/dev/null" | getline showmap @@ -336,7 +324,8 @@ BEGIN { } else { stat_format = "-f '%z %N'" # *BSD, MacOS } - while ("cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" | getline) { + cmdline = "cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" + while (cmdline | getline) { infilesSmallToBig[i++] = $0 } in_count = i diff --git a/test/test.sh b/test/test.sh index 3473155f..93a4e008 100755 --- a/test/test.sh +++ b/test/test.sh @@ -153,8 +153,8 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in -1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized testcase numbers" ;; -*) $ECHO "$RED[!] afl-cmin did not correctly minimize testcase numbers" +1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; +*) $ECHO "$RED[!] afl-cmin did not correctly minimizethe number of testcases" CODE=1 ;; esac @@ -176,7 +176,7 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { $ECHO "$YELLOW[-] not an intel platform, cannot test afl-gcc" } -$ECHO "$BLUE[*] Testing: llvm_mode" +$ECHO "$BLUE[*] Testing: llvm_mode, afl-showmap, afl-fuzz, afl-cmin and afl-tmin" test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { # on FreeBSD need to set AFL_CC test `uname -s` = 'FreeBSD' && { @@ -253,6 +253,26 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { $ECHO "$RED[!] afl-fuzz is not working correctly with llvm_mode" CODE=1 } + test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" || { + echo 000000000000000000000000 > in/in2 + mkdir -p in2 + ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null + CNT=`ls in2/ | wc -l` + case "$CNT" in +1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; +*) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" + CODE=1 + ;; + esac + ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 + SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` + test "$SIZE" = 1 && $ECHO "$GREEN[+] afl-tmin correctly minimized the testcase" + test "$SIZE" = 1 || { + $ECHO "$RED[!] afl-tmin did incorrectly minimize the testcase to $SIZE" + CODE=1 + } + rm -rf in2 + } rm -rf in out errors } rm -f test-instr.plain From 5d2330f04e45225588a11c64b26a7dbb1a2fbe1a Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 25 Jan 2020 05:27:10 +0100 Subject: [PATCH 38/44] nicer output for afl-system-config --- afl-system-config | 93 ++++++++++++++++++++++++++++------------------- 1 file changed, 55 insertions(+), 38 deletions(-) diff --git a/afl-system-config b/afl-system-config index 2a7df17f..1e180d8b 100755 --- a/afl-system-config +++ b/afl-system-config @@ -1,6 +1,6 @@ #!/bin/sh test "$1" = "-h" && { - echo afl-system-config by Marc Heuse + echo 'afl-system-config by Marc Heuse ' echo echo $0 echo @@ -12,55 +12,72 @@ test "$1" = "-h" && { exit 1 } +DONE= PLATFORM=`uname -s` -echo This reconfigures the system to have a better fuzzing performance +echo This reconfigures the system to have a better fuzzing performance. if [ '!' "$EUID" = 0 ] && [ '!' `id -u` = 0 ] ; then - echo Error you need to be root to run this - exit 1 + echo "Warning: you need to be root to run this!" + # we do not exit as other mechanisms exist that allows to do this than + # being root. let the errors speak for themselves. fi if [ "$PLATFORM" = "Linux" ] ; then -sysctl -w kernel.core_pattern=core -sysctl -w kernel.randomize_va_space=0 -sysctl -w kernel.sched_child_runs_first=1 -sysctl -w kernel.sched_autogroup_enabled=1 -sysctl -w kernel.sched_migration_cost_ns=50000000 -sysctl -w kernel.sched_latency_ns=250000000 -echo never > /sys/kernel/mm/transparent_hugepage/enabled -test -e /sys/devices/system/cpu/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/scaling_governor -test -e /sys/devices/system/cpu/cpufreq/policy0/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor -test -e /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor -test -e /sys/devices/system/cpu/intel_pstate/no_turbo && echo 0 > /sys/devices/system/cpu/intel_pstate/no_turbo -test -e /sys/devices/system/cpu/cpufreq/boost && echo 1 > /sys/devices/system/cpu/cpufreq/boost -echo -echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: -echo '/etc/default/grub:GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"' +{ + sysctl -w kernel.core_pattern=core + sysctl -w kernel.randomize_va_space=0 + sysctl -w kernel.sched_child_runs_first=1 + sysctl -w kernel.sched_autogroup_enabled=1 + sysctl -w kernel.sched_migration_cost_ns=50000000 + sysctl -w kernel.sched_latency_ns=250000000 + echo never > /sys/kernel/mm/transparent_hugepage/enabled + test -e /sys/devices/system/cpu/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/scaling_governor + test -e /sys/devices/system/cpu/cpufreq/policy0/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor + test -e /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor + test -e /sys/devices/system/cpu/intel_pstate/no_turbo && echo 0 > /sys/devices/system/cpu/intel_pstate/no_turbo + test -e /sys/devices/system/cpu/cpufreq/boost && echo 1 > /sys/devices/system/cpu/cpufreq/boost +} > /dev/null + echo Settings applied. + dmesg | egrep -q 'nospectre_v2|spectre_v2=off' || { + echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: + echo ' /etc/default/grub:GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"' + } + DONE=1 fi if [ "$PLATFORM" = "FreeBSD" ] ; then -sysctl kern.elf32.aslr.enable=0 -sysctl kern.elf64.aslr.enable=0 -echo -echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: -echo 'sysctl hw.ibrs_disable=1' -echo -echo 'Setting kern.pmap.pg_ps_enabled=0 into /boot/loader.conf might be helpful too.' +{ + sysctl kern.elf32.aslr.enable=0 + sysctl kern.elf64.aslr.enable=0 +} > /dev/null + echo Settings applied. + echo It is recommended to boot the kernel with lots of security off - if you are running a machine that is in a secured network - so set this: + echo ' sysctl hw.ibrs_disable=1' + echo 'Setting kern.pmap.pg_ps_enabled=0 into /boot/loader.conf might be helpful too.' + DONE=1 fi if [ "$PLATFORM" = "OpenBSD" ] ; then -echo -echo 'System security features cannot be disabled on OpenBSD.' + echo + echo 'System security features cannot be disabled on OpenBSD.' + DONE=1 fi if [ "$PLATFORM" = "NetBSD" ] ; then -echo -echo It is recommended to enable unprivileged users to set cpu affinity -echo to be able to use afl-gotcpu meaningfully. -/sbin/sysctl -w security.models.extensions.user_set_cpu_affinity=1 +{ + #echo It is recommended to enable unprivileged users to set cpu affinity + #echo to be able to use afl-gotcpu meaningfully. + /sbin/sysctl -w security.models.extensions.user_set_cpu_affinity=1 +} > /dev/null + echo Settings applied. + DONE=1 fi if [ "$PLATFORM" = "Darwin" ] ; then if [ $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ] ; then -echo We unload the default crash reporter here -SL=/System/Library; PL=com.apple.ReportCrash -launchctl unload -w ${SL}/LaunchAgents/${PL}.plist -sudo launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist + echo We unload the default crash reporter here + SL=/System/Library; PL=com.apple.ReportCrash + launchctl unload -w ${SL}/LaunchAgents/${PL}.plist + sudo launchctl unload -w ${SL}/LaunchDaemons/${PL}.Root.plist + echo Settings applied. + else + echo Nothing to do. fi + DONE=1 fi -echo -echo Also use AFL_TMPDIR to use a tmpfs for the input file +test -z "$DONE" && echo Error: Unknown platform: $PLATFORM +test -z "$AFL_TMPDIR" && echo Also use AFL_TMPDIR and point it to a tmpfs for the input file caching From 2c6847bfa0b57f3330b1aab9b91d935757db51b7 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Sat, 25 Jan 2020 16:11:42 +0100 Subject: [PATCH 39/44] added whitelist+blacklist to all llvm_mode passes --- docs/ChangeLog | 2 +- llvm_mode/LLVMInsTrim.so.cc | 29 +----- llvm_mode/MarkNodes.cc | 19 ++-- llvm_mode/compare-transform-pass.so.cc | 94 ++++++++++++++++++++ llvm_mode/split-compares-pass.so.cc | 118 +++++++++++++++++++++++++ llvm_mode/split-switches-pass.so.cc | 113 +++++++++++++++++++++++ test/test.sh | 16 ++-- 7 files changed, 343 insertions(+), 48 deletions(-) diff --git a/docs/ChangeLog b/docs/ChangeLog index 33c6f618..c1d53379 100644 --- a/docs/ChangeLog +++ b/docs/ChangeLog @@ -27,7 +27,7 @@ Version ++2.60d (develop): instrumentation. compile normally and set AFL_LLVM_USE_TRACE_PC :) - afl-cmin is now a sh script (invoking awk) instead of bash for portability the original script is still present as afl-cmin.bash - - added blacklisted function check in all modules of llvm_mode + - added blacklist and whitelisting function check in all modules of llvm_mode - added fix from Debian project to compile libdislocator and libtokencap diff --git a/llvm_mode/LLVMInsTrim.so.cc b/llvm_mode/LLVMInsTrim.so.cc index 11451b43..24df6d42 100644 --- a/llvm_mode/LLVMInsTrim.so.cc +++ b/llvm_mode/LLVMInsTrim.so.cc @@ -144,19 +144,6 @@ struct InsTrim : public ModulePass { // this is our default MarkSetOpt = true; - /* // I dont think this makes sense to port into LLVMInsTrim - char* inst_ratio_str = getenv("AFL_INST_RATIO"); - unsigned int inst_ratio = 100; - if (inst_ratio_str) { - - if (sscanf(inst_ratio_str, "%u", &inst_ratio) != 1 || !inst_ratio || - inst_ratio > 100) FATAL("Bad value of AFL_INST_RATIO (must be between 1 - and 100)"); - - } - - */ - LLVMContext &C = M.getContext(); IntegerType *Int8Ty = IntegerType::getInt8Ty(C); IntegerType *Int32Ty = IntegerType::getInt32Ty(C); @@ -203,8 +190,7 @@ struct InsTrim : public ModulePass { if (instFilename.str().empty()) { - /* If the original location is empty, try using the inlined location - */ + /* If the original location is empty, try using the inlined location */ DILocation *oDILoc = cDILoc->getInlinedAt(); if (oDILoc) { @@ -432,28 +418,19 @@ struct InsTrim : public ModulePass { IRB.CreateStore(Incr, MapPtrIdx) ->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, None)); - /* Set prev_loc to cur_loc >> 1 */ - /* - StoreInst *Store = IRB.CreateStore(ConstantInt::get(Int32Ty, L >> 1), - OldPrev); Store->setMetadata(M.getMDKindID("nosanitize"), MDNode::get(C, - None)); - */ - total_instr++; } } - OKF("Instrumented %u locations (%llu, %llu) (%s mode)\n" /*", ratio - %u%%)."*/ - , + OKF("Instrumented %u locations (%llu, %llu) (%s mode)\n", total_instr, total_rs, total_hs, getenv("AFL_HARDEN") ? "hardened" : ((getenv("AFL_USE_ASAN") || getenv("AFL_USE_MSAN")) ? "ASAN/MSAN" - : "non-hardened") /*, inst_ratio*/); + : "non-hardened")); return false; } diff --git a/llvm_mode/MarkNodes.cc b/llvm_mode/MarkNodes.cc index 2aeeda8d..caa8cede 100644 --- a/llvm_mode/MarkNodes.cc +++ b/llvm_mode/MarkNodes.cc @@ -65,16 +65,11 @@ void buildCFG(Function *F) { } - // uint32_t FakeID = 0; for (auto S = F->begin(), E = F->end(); S != E; ++S) { BasicBlock *BB = &*S; uint32_t MyID = LMap[BB]; - // if (succ_begin(BB) == succ_end(BB)) { - // Succs[MyID].push_back(FakeID); - // Marked.insert(MyID); - //} for (auto I = succ_begin(BB), E = succ_end(BB); I != E; ++I) { Succs[MyID].push_back(LMap[*I]); @@ -113,7 +108,7 @@ void DFStree(size_t now_id) { } -void turnCFGintoDAG(Function *F) { +void turnCFGintoDAG() { tSuccs = Succs; tag.resize(Blocks.size()); @@ -176,7 +171,7 @@ void DFS(uint32_t now) { } -void DominatorTree(Function *F) { +void DominatorTree() { if (Blocks.empty()) return; uint32_t s = start_point; @@ -390,7 +385,7 @@ void MarkSubGraph(uint32_t ss, uint32_t tt) { } -void MarkVertice(Function *F) { +void MarkVertice() { uint32_t s = start_point; @@ -411,8 +406,6 @@ void MarkVertice(Function *F) { timeStamp = 0; uint32_t t = 0; - // MarkSubGraph(s, t); - // return; while (s != t) { @@ -432,9 +425,9 @@ std::pair, std::vector > markNodes( reset(); labelEachBlock(F); buildCFG(F); - turnCFGintoDAG(F); - DominatorTree::DominatorTree(F); - MarkVertice(F); + turnCFGintoDAG(); + DominatorTree::DominatorTree(); + MarkVertice(); std::vector Result, ResultAbove; for (uint32_t x : Markabove) { diff --git a/llvm_mode/compare-transform-pass.so.cc b/llvm_mode/compare-transform-pass.so.cc index 0ccce875..5d924b63 100644 --- a/llvm_mode/compare-transform-pass.so.cc +++ b/llvm_mode/compare-transform-pass.so.cc @@ -18,7 +18,13 @@ #include #include +#include +#include +#include +#include + #include "llvm/ADT/Statistic.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" @@ -42,6 +48,23 @@ class CompareTransform : public ModulePass { static char ID; CompareTransform() : ModulePass(ID) { + char *instWhiteListFilename = getenv("AFL_LLVM_WHITELIST"); + if (instWhiteListFilename) { + + std::string line; + std::ifstream fileStream; + fileStream.open(instWhiteListFilename); + if (!fileStream) report_fatal_error("Unable to open AFL_LLVM_WHITELIST"); + getline(fileStream, line); + while (fileStream) { + + myWhitelist.push_back(line); + getline(fileStream, line); + + } + + } + } bool runOnModule(Module &M) override; @@ -57,6 +80,9 @@ class CompareTransform : public ModulePass { } + protected: + std::list myWhitelist; + private: bool transformCmps(Module &M, const bool processStrcmp, const bool processMemcmp, const bool processStrncmp, @@ -104,6 +130,74 @@ bool CompareTransform::transformCmps(Module &M, const bool processStrcmp, for (auto &BB : F) { + if (!myWhitelist.empty()) { + + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + + bool instrumentBlock = false; + + /* Get the current location using debug information. + * For now, just instrument the block if we are not able + * to determine our location. */ + DebugLoc Loc = IP->getDebugLoc(); + if (Loc) { + + DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); + + unsigned int instLine = cDILoc->getLine(); + StringRef instFilename = cDILoc->getFilename(); + + if (instFilename.str().empty()) { + + /* If the original location is empty, try using the inlined location + */ + DILocation *oDILoc = cDILoc->getInlinedAt(); + if (oDILoc) { + + instFilename = oDILoc->getFilename(); + instLine = oDILoc->getLine(); + + } + + } + + (void)instLine; + + /* Continue only if we know where we actually are */ + if (!instFilename.str().empty()) { + + for (std::list::iterator it = myWhitelist.begin(); + it != myWhitelist.end(); ++it) { + + /* We don't check for filename equality here because + * filenames might actually be full paths. Instead we + * check that the actual filename ends in the filename + * specified in the list. */ + if (instFilename.str().length() >= it->length()) { + + if (instFilename.str().compare( + instFilename.str().length() - it->length(), + it->length(), *it) == 0) { + + instrumentBlock = true; + break; + + } + + } + + } + + } + + } + + /* Either we couldn't figure out our location or the location is + * not whitelisted, so we skip instrumentation. */ + if (!instrumentBlock) continue; + + } + for (auto &IN : BB) { CallInst *callInst = nullptr; diff --git a/llvm_mode/split-compares-pass.so.cc b/llvm_mode/split-compares-pass.so.cc index eeac4a55..bc25b322 100644 --- a/llvm_mode/split-compares-pass.so.cc +++ b/llvm_mode/split-compares-pass.so.cc @@ -15,7 +15,17 @@ * limitations under the License. */ +#include +#include +#include + +#include +#include +#include +#include + #include "llvm/Pass.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/Support/raw_ostream.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/Transforms/IPO/PassManagerBuilder.h" @@ -35,6 +45,41 @@ class SplitComparesTransform : public ModulePass { static char ID; SplitComparesTransform() : ModulePass(ID) { + char *instWhiteListFilename = getenv("AFL_LLVM_WHITELIST"); + if (instWhiteListFilename) { + + std::string line; + std::ifstream fileStream; + fileStream.open(instWhiteListFilename); + if (!fileStream) report_fatal_error("Unable to open AFL_LLVM_WHITELIST"); + getline(fileStream, line); + while (fileStream) { + + myWhitelist.push_back(line); + getline(fileStream, line); + + } + + } + + } + + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", "llvm.", "sancov.", "__ubsan_handle_", "ign." + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + } bool runOnModule(Module &M) override; @@ -49,6 +94,9 @@ class SplitComparesTransform : public ModulePass { } + protected: + std::list myWhitelist; + private: int enableFPSplit; @@ -77,8 +125,78 @@ bool SplitComparesTransform::simplifyCompares(Module &M) { * all integer comparisons with >= and <= predicates to the icomps vector */ for (auto &F : M) { + if (isBlacklisted(&F)) continue; + for (auto &BB : F) { + if (!myWhitelist.empty()) { + + bool instrumentBlock = false; + + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + + /* Get the current location using debug information. + * For now, just instrument the block if we are not able + * to determine our location. */ + DebugLoc Loc = IP->getDebugLoc(); + if (Loc) { + + DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); + + unsigned int instLine = cDILoc->getLine(); + StringRef instFilename = cDILoc->getFilename(); + + if (instFilename.str().empty()) { + + /* If the original location is empty, try using the inlined location + */ + DILocation *oDILoc = cDILoc->getInlinedAt(); + if (oDILoc) { + + instFilename = oDILoc->getFilename(); + instLine = oDILoc->getLine(); + + } + + } + + (void)instLine; + + /* Continue only if we know where we actually are */ + if (!instFilename.str().empty()) { + + for (std::list::iterator it = myWhitelist.begin(); + it != myWhitelist.end(); ++it) { + + /* We don't check for filename equality here because + * filenames might actually be full paths. Instead we + * check that the actual filename ends in the filename + * specified in the list. */ + if (instFilename.str().length() >= it->length()) { + + if (instFilename.str().compare( + instFilename.str().length() - it->length(), + it->length(), *it) == 0) { + + instrumentBlock = true; + break; + + } + + } + + } + + } + + } + + /* Either we couldn't figure out our location or the location is + * not whitelisted, so we skip instrumentation. */ + if (!instrumentBlock) continue; + + } + for (auto &IN : BB) { CmpInst *selectcmpInst = nullptr; diff --git a/llvm_mode/split-switches-pass.so.cc b/llvm_mode/split-switches-pass.so.cc index 2743a71a..3a2838c0 100644 --- a/llvm_mode/split-switches-pass.so.cc +++ b/llvm_mode/split-switches-pass.so.cc @@ -18,7 +18,13 @@ #include #include +#include +#include +#include +#include + #include "llvm/ADT/Statistic.h" +#include "llvm/IR/DebugInfo.h" #include "llvm/IR/IRBuilder.h" #include "llvm/IR/LegacyPassManager.h" #include "llvm/IR/Module.h" @@ -42,6 +48,41 @@ class SplitSwitchesTransform : public ModulePass { static char ID; SplitSwitchesTransform() : ModulePass(ID) { + char *instWhiteListFilename = getenv("AFL_LLVM_WHITELIST"); + if (instWhiteListFilename) { + + std::string line; + std::ifstream fileStream; + fileStream.open(instWhiteListFilename); + if (!fileStream) report_fatal_error("Unable to open AFL_LLVM_WHITELIST"); + getline(fileStream, line); + while (fileStream) { + + myWhitelist.push_back(line); + getline(fileStream, line); + + } + + } + + } + + static bool isBlacklisted(const Function *F) { + + static const SmallVector Blacklist = { + + "asan.", "llvm.", "sancov.", "__ubsan_handle_", "ign." + + }; + + for (auto const &BlacklistFunc : Blacklist) { + + if (F->getName().startswith(BlacklistFunc)) { return true; } + + } + + return false; + } bool runOnModule(Module &M) override; @@ -71,6 +112,9 @@ class SplitSwitchesTransform : public ModulePass { typedef std::vector CaseVector; + protected: + std::list myWhitelist; + private: bool splitSwitches(Module &M); bool transformCmps(Module &M, const bool processStrcmp, @@ -268,10 +312,79 @@ bool SplitSwitchesTransform::splitSwitches(Module &M) { * all switches to switches vector for later processing */ for (auto &F : M) { + if (isBlacklisted(&F)) continue; + for (auto &BB : F) { SwitchInst *switchInst = nullptr; + if (!myWhitelist.empty()) { + + bool instrumentBlock = false; + BasicBlock::iterator IP = BB.getFirstInsertionPt(); + + /* Get the current location using debug information. + * For now, just instrument the block if we are not able + * to determine our location. */ + DebugLoc Loc = IP->getDebugLoc(); + if (Loc) { + + DILocation *cDILoc = dyn_cast(Loc.getAsMDNode()); + + unsigned int instLine = cDILoc->getLine(); + StringRef instFilename = cDILoc->getFilename(); + + if (instFilename.str().empty()) { + + /* If the original location is empty, try using the inlined location + */ + DILocation *oDILoc = cDILoc->getInlinedAt(); + if (oDILoc) { + + instFilename = oDILoc->getFilename(); + instLine = oDILoc->getLine(); + + } + + } + + (void)instLine; + + /* Continue only if we know where we actually are */ + if (!instFilename.str().empty()) { + + for (std::list::iterator it = myWhitelist.begin(); + it != myWhitelist.end(); ++it) { + + /* We don't check for filename equality here because + * filenames might actually be full paths. Instead we + * check that the actual filename ends in the filename + * specified in the list. */ + if (instFilename.str().length() >= it->length()) { + + if (instFilename.str().compare( + instFilename.str().length() - it->length(), + it->length(), *it) == 0) { + + instrumentBlock = true; + break; + + } + + } + + } + + } + + } + + /* Either we couldn't figure out our location or the location is + * not whitelisted, so we skip instrumentation. */ + if (!instrumentBlock) continue; + + } + if ((switchInst = dyn_cast(BB.getTerminator()))) { if (switchInst->getNumCases() < 1) continue; diff --git a/test/test.sh b/test/test.sh index 93a4e008..9676d22d 100755 --- a/test/test.sh +++ b/test/test.sh @@ -153,10 +153,10 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" && { ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in -1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; -*) $ECHO "$RED[!] afl-cmin did not correctly minimizethe number of testcases" - CODE=1 - ;; + *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" + CODE=1 + ;; esac ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` @@ -259,10 +259,10 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && { ../afl-cmin -i in -o in2 -- ./test-instr.plain > /dev/null CNT=`ls in2/ | wc -l` case "$CNT" in -1| *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; -*) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" - CODE=1 - ;; + *1) $ECHO "$GREEN[+] afl-cmin correctly minimized the number of testcases" ;; + *) $ECHO "$RED[!] afl-cmin did not correctly minimize the number of testcases" + CODE=1 + ;; esac ../afl-tmin -i in/in2 -o in2/in2 -- ./test-instr.plain > /dev/null 2>&1 SIZE=`ls -l in2/in2 2> /dev/null | awk '{print$5}'` From 3561a1b775989a0cf37221f810eec601cdb14bcf Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 27 Jan 2020 00:19:59 +0100 Subject: [PATCH 40/44] dockerfile update --- Dockerfile | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1947f211..7bb60610 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,9 @@ RUN apt-get update && apt-get install -y \ clang \ clang-9 \ flex \ + git \ + python3.7 \ + python3.7-dev \ gcc-9 \ gcc-9-plugin-dev \ gcc-9-multilib \ @@ -23,10 +26,12 @@ RUN apt-get update && apt-get install -y \ ca-certificates \ libpixman-1-dev \ && rm -rf /var/lib/apt/lists/* + ARG CC=gcc-9 ARG CXX=g++-9 ARG LLVM_CONFIG=llvm-config-9 -COPY . /app -RUN cd /app && make clean && make distrib && \ - make install && cd .. && rm -rf /app -WORKDIR /work + +RUN git clone https://github.com/vanhauser-thc/AFLplusplus + +RUN cd AFLplusplus && make clean && make distrib && \ + make install && cd .. && rm -rf AFLplusplus From fa64c0d4a5a6eb1eddd13071e3b326778bf6db5a Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 27 Jan 2020 10:40:13 +0100 Subject: [PATCH 41/44] important fixes for afl-cmin --- afl-cmin | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/afl-cmin b/afl-cmin index de5a66ed..865809e1 100755 --- a/afl-cmin +++ b/afl-cmin @@ -292,6 +292,24 @@ BEGIN { exit 1 } + # Check for the more efficient way to copy files... + if (0 != system("mkdir -p -m 0700 "trace_dir)) { + print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" + exit 1 + } + if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/auto_extras")) { + print "[-] Error: Cannot create directory "trace_dir"/.state/auto_extras" > "/dev/stderr" + exit 1 + } + if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/redundant_edges")) { + print "[-] Error: Cannot create directory "trace_dir"/.state/redundant_edges" > "/dev/stderr" + exit 1 + } + if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/deterministic_done")) { + print "[-] Error: Cannot create directory "trace_dir"/.state/deterministic_done" > "/dev/stderr" + exit 1 + } + if (stdin_file) { # truncate input file printf "" > stdin_file @@ -339,12 +357,6 @@ BEGIN { exit 1 } - # Check for the more efficient way to copy files... - if (0 != system("mkdir -p -m 0700 "trace_dir)) { - print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" - exit 1 - } - if (0 == system("ln "in_dir"/"first_file" "trace_dir"/.link_test")) { cp_tool = "ln" } else { @@ -360,7 +372,7 @@ BEGIN { system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") } else { system("cp "in_dir"/"first_file" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" Date: Mon, 27 Jan 2020 11:47:39 +0100 Subject: [PATCH 42/44] nearing afl-cmin perfection :-) --- afl-cmin | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/afl-cmin b/afl-cmin index 865809e1..f6e76263 100755 --- a/afl-cmin +++ b/afl-cmin @@ -297,18 +297,6 @@ BEGIN { print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr" exit 1 } - if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/auto_extras")) { - print "[-] Error: Cannot create directory "trace_dir"/.state/auto_extras" > "/dev/stderr" - exit 1 - } - if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/redundant_edges")) { - print "[-] Error: Cannot create directory "trace_dir"/.state/redundant_edges" > "/dev/stderr" - exit 1 - } - if (0 != system("mkdir -p -m 0700 "trace_dir"/.state/deterministic_done")) { - print "[-] Error: Cannot create directory "trace_dir"/.state/deterministic_done" > "/dev/stderr" - exit 1 - } if (stdin_file) { # truncate input file @@ -342,7 +330,7 @@ BEGIN { } else { stat_format = "-f '%z %N'" # *BSD, MacOS } - cmdline = "cd "in_dir" && find . -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" + cmdline = "cd "in_dir" && find . -maxdepth 1 -type f -exec stat "stat_format" \\{\\} \\; | sort -n | cut -d' ' -f2-" while (cmdline | getline) { infilesSmallToBig[i++] = $0 } From 3374ada561e5dcfe052c41837fc15bd29287b285 Mon Sep 17 00:00:00 2001 From: van Hauser Date: Mon, 27 Jan 2020 11:48:49 +0100 Subject: [PATCH 43/44] nearing afl-cmin perfection :-) --- afl-cmin | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/afl-cmin b/afl-cmin index f6e76263..e9d713aa 100755 --- a/afl-cmin +++ b/afl-cmin @@ -360,7 +360,7 @@ BEGIN { system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") } else { system("cp "in_dir"/"first_file" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" < /dev/null") + system( "AFL_CMIN_ALLOW_ANY=1 \""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" Date: Mon, 27 Jan 2020 13:06:00 +0100 Subject: [PATCH 44/44] updated binary_fuzzing document --- ...only_fuzzing.txt => binaryonly_fuzzing.md} | 103 ++++++++++-------- 1 file changed, 56 insertions(+), 47 deletions(-) rename docs/{binaryonly_fuzzing.txt => binaryonly_fuzzing.md} (57%) diff --git a/docs/binaryonly_fuzzing.txt b/docs/binaryonly_fuzzing.md similarity index 57% rename from docs/binaryonly_fuzzing.txt rename to docs/binaryonly_fuzzing.md index f8d68cd8..d22e4ce2 100644 --- a/docs/binaryonly_fuzzing.txt +++ b/docs/binaryonly_fuzzing.md @@ -1,6 +1,4 @@ - -Fuzzing binary-only programs with afl++ -======================================= +#Fuzzing binary-only programs with afl++ afl++, libfuzzer and others are great if you have the source code, and it allows for very fast and coverage guided fuzzing. @@ -16,36 +14,42 @@ TL;DR: try DYNINST with afl-dyninst. If it produces too many crashes then !!!!! -QEMU ----- +##QEMU Qemu is the "native" solution to the program. It is available in the ./qemu_mode/ directory and once compiled it can be accessed by the afl-fuzz -Q command line option. The speed decrease is at about 50% It is the easiest to use alternative and even works for cross-platform binaries. +Note that there is also honggfuzz: [https://github.com/google/honggfuzz](https://github.com/google/honggfuzz) +which now has a qemu_mode, but its performance is just 1.5%! + As it is included in afl++ this needs no URL. -WINE+QEMU ---------- + +##WINE+QEMU Wine mode can run Win32 PE binaries with the QEMU instrumentation. It needs Wine, python3 and the pefile python package installed. -UNICORN -------- +As it is included in afl++ this needs no URL. + + +##UNICORN Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. -In contrast to QEMU, Unicorn does not offer a full system or even userland emulation. -Runtime environment and/or loaders have to be written from scratch, if needed. -On top, block chaining has been removed. This means the speed boost introduced in -the patched QEMU Mode of afl++ cannot simply be ported over to Unicorn. -For further information, check out ./unicorn_mode.txt. +In contrast to QEMU, Unicorn does not offer a full system or even userland +emulation. Runtime environment and/or loaders have to be written from scratch, +if needed. On top, block chaining has been removed. This means the speed boost +introduced in the patched QEMU Mode of afl++ cannot simply be ported over to +Unicorn. For further information, check out ./unicorn_mode.txt. + +As it is included in afl++ this needs no URL. -DYNINST -------- +##DYNINST Dyninst is a binary instrumentation framework similar to Pintool and Dynamorio (see far below). However whereas Pintool and Dynamorio work at runtime, dyninst -instruments the target at load time, and then let it run. +instruments the target at load time, and then let it run - or save the +binary with the changes. This is great for some things, e.g. fuzzing, and not so effective for others, e.g. malware analysis. @@ -53,9 +57,9 @@ So what we can do with dyninst is taking every basic block, and put afl's instrumention code in there - and then save the binary. Afterwards we can just fuzz the newly saved target binary with afl-fuzz. Sounds great? It is. The issue though - it is a non-trivial problem to -insert instructions, which change addresses in the process space, so +insert instructions, which change addresses in the process space, so that everything is still working afterwards. Hence more often than not binaries -crash when they are run (because of instrumentation). +crash when they are run. The speed decrease is about 15-35%, depending on the optimization options used with afl-dyninst. @@ -63,11 +67,10 @@ used with afl-dyninst. So if dyninst works, it is the best option available. Otherwise it just doesn't work well. -https://github.com/vanhauser-thc/afl-dyninst +[https://github.com/vanhauser-thc/afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) -INTEL-PT --------- +##INTEL-PT If you have a newer Intel CPU, you can make use of Intels processor trace. The big issue with Intel's PT is the small buffer size and the complex encoding of the debug information collected through PT. @@ -77,30 +80,39 @@ the implementation and other factors). There are two afl intel-pt implementations: -1. https://github.com/junxzm1990/afl-pt +1. [https://github.com/junxzm1990/afl-pt](https://github.com/junxzm1990/afl-pt) => this needs Ubuntu 14.04.05 without any updates and the 4.4 kernel. -2. https://github.com/hunter-ht-2018/ptfuzzer +2. [https://github.com/hunter-ht-2018/ptfuzzer](https://github.com/hunter-ht-2018/ptfuzzer) => this needs a 4.14 or 4.15 kernel. the "nopti" kernel boot option must be used. This one is faster than the other. +Note that there is also honggfuzz: https://github.com/google/honggfuzz +But its IPT performance is just 6%! -CORESIGHT ---------- +##CORESIGHT Coresight is ARM's answer to Intel's PT. There is no implementation so far which handle coresight and getting it working on an ARM Linux is very difficult due to custom kernel building on embedded systems is difficult. And finding one that has coresight in the ARM chip is difficult too. My guess is that it is slower than Qemu, but faster than Intel PT. + If anyone finds any coresight implementation for afl please ping me: vh@thc.org -PIN & DYNAMORIO ---------------- +##FRIDA +Frida is a dynamic instrumentation engine like Pintool, Dyninst and Dynamorio. +What is special is that it is written Python, and scripted with Javascript. +It is mostly used to reverse binaries on mobile phones however can be used +everywhere. +There is a WIP fuzzer available at [https://github.com/andreafioraldi/frida-fuzzer](https://github.com/andreafioraldi/frida-fuzzer) + + +##PIN & DYNAMORIO Pintool and Dynamorio are dynamic instrumentation engines, and they can be used for getting basic block information at runtime. Pintool is only available for Intel x32/x64 on Linux, Mac OS and Windows @@ -115,30 +127,27 @@ Hence Dynamorio is the option to go for if everything fails, and Pintool only if Dynamorio fails too. Dynamorio solutions: - https://github.com/vanhauser-thc/afl-dynamorio - https://github.com/mxmssh/drAFL - https://github.com/googleprojectzero/winafl/ <= very good but windows only + * [https://github.com/vanhauser-thc/afl-dynamorio](https://github.com/vanhauser-thc/afl-dynamorio) + * [https://github.com/mxmssh/drAFL](https://github.com/mxmssh/drAFL) + * [https://github.com/googleprojectzero/winafl/](https://github.com/googleprojectzero/winafl/) <= very good but windows only Pintool solutions: - https://github.com/vanhauser-thc/afl-pin - https://github.com/mothran/aflpin - https://github.com/spinpx/afl_pin_mode <= only old Pintool version supported + * [https://github.com/vanhauser-thc/afl-pin](https://github.com/vanhauser-thc/afl-pin) + * [https://github.com/mothran/aflpin](https://github.com/mothran/aflpin) + * [https://github.com/spinpx/afl_pin_mode](https://github.com/spinpx/afl_pin_mode) <= only old Pintool version supported -Non-AFL solutions ------------------ +##Non-AFL solutions +There are many binary-only fuzzing frameworks. +Some are great for CTFs but don't work with large binaries, others are very +slow but have good path discovery, some are very hard to set-up ... -There are many binary-only fuzzing frameworks. Some are great for CTFs but don't -work with large binaries, others are very slow but have good path discovery, -some are very hard to set-up ... - -QSYM: https://github.com/sslab-gatech/qsym -Manticore: https://github.com/trailofbits/manticore -S2E: https://github.com/S2E - +* QSYM: [https://github.com/sslab-gatech/qsym](https://github.com/sslab-gatech/qsym) +* Manticore: [https://github.com/trailofbits/manticore](https://github.com/trailofbits/manticore) +* S2E: [https://github.com/S2E](https://github.com/S2E) +* +## Closing words -That's it! -News, corrections, updates? -Email vh@thc.org +That's it! News, corrections, updates? Send an email to vh@thc.org