From 79a69b14efd9cd3befceeddcc844a42bdbfdb47c Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 8 Aug 2022 15:30:48 +0200 Subject: [PATCH 01/25] 4.03a --- README.md | 2 +- docs/Changelog.md | 5 ++++- include/config.h | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 921fc0c6..a8b579d8 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Release version: [4.02c](https://github.com/AFLplusplus/AFLplusplus/releases) -GitHub version: 4.02c +GitHub version: 4.03a Repository: [https://github.com/AFLplusplus/AFLplusplus](https://github.com/AFLplusplus/AFLplusplus) diff --git a/docs/Changelog.md b/docs/Changelog.md index 957f6206..ec517104 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -8,6 +8,10 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to . +### Version ++4.03a (dev) + - ... your PR? :) + + ### Version ++4.02c (release) - afl-cc: - important fix for the default pcguard mode when LLVM IR vector @@ -22,7 +26,6 @@ sending a mail to . - change post_process hook to allow returning NULL and 0 length to tell afl-fuzz to skip this mutated input - ### Version ++4.01c (release) - fixed */build_...sh scripts to work outside of git - new custom_mutator: libafl with token fuzzing :) diff --git a/include/config.h b/include/config.h index d7a08428..1689e034 100644 --- a/include/config.h +++ b/include/config.h @@ -26,7 +26,7 @@ /* Version string: */ // c = release, a = volatile github dev, e = experimental branch -#define VERSION "++4.02c" +#define VERSION "++4.03a" /****************************************************** * * From 63b12c5f86398903fe69ccaf2847838acb8b6f8e Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 9 Aug 2022 08:25:00 +0100 Subject: [PATCH 02/25] Fix unstable test makefile --- frida_mode/test/unstable/GNUmakefile | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/frida_mode/test/unstable/GNUmakefile b/frida_mode/test/unstable/GNUmakefile index ed91da40..59b49449 100644 --- a/frida_mode/test/unstable/GNUmakefile +++ b/frida_mode/test/unstable/GNUmakefile @@ -4,7 +4,7 @@ BUILD_DIR:=$(PWD)build/ UNSTABLE_DATA_DIR:=$(BUILD_DIR)in/ UNSTABLE_DATA_FILE:=$(UNSTABLE_DATA_DIR)in -UNSTABLE_BIN:=$(BUILD_DIR)unstable +TEST_BIN:=$(BUILD_DIR)unstable UNSTABLE_SRC:=$(PWD)unstable.c QEMU_OUT:=$(BUILD_DIR)qemu-out @@ -42,7 +42,7 @@ endif .PHONY: all 32 clean qemu frida -all: $(UNSTABLE_BIN) +all: $(TEST_BIN) make -C $(ROOT)frida_mode/ 32: @@ -57,14 +57,14 @@ $(UNSTABLE_DATA_DIR): | $(BUILD_DIR) $(UNSTABLE_DATA_FILE): | $(UNSTABLE_DATA_DIR) echo -n "000" > $@ -$(UNSTABLE_BIN): $(UNSTABLE_SRC) | $(BUILD_DIR) +$(TEST_BIN): $(UNSTABLE_SRC) | $(BUILD_DIR) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ $< clean: rm -rf $(BUILD_DIR) -qemu: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) +qemu: $(TEST_BIN) $(UNSTABLE_DATA_FILE) AFL_QEMU_PERSISTENT_ADDR=$(AFL_QEMU_PERSISTENT_ADDR) \ $(ROOT)afl-fuzz \ -D \ @@ -72,9 +72,9 @@ qemu: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) -i $(UNSTABLE_DATA_DIR) \ -o $(QEMU_OUT) \ -- \ - $(UNSTABLE_BIN) @@ + $(TEST_BIN) @@ -frida: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) +frida: $(TEST_BIN) $(UNSTABLE_DATA_FILE) AFL_DEBUG=1 \ AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ AFL_FRIDA_INST_TRACE_UNIQUE=1 \ @@ -85,9 +85,9 @@ frida: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) -i $(UNSTABLE_DATA_DIR) \ -o $(FRIDA_OUT) \ -- \ - $(UNSTABLE_BIN) @@ + $(TEST_BIN) @@ -frida_coverage: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) +frida_coverage: $(TEST_BIN) $(UNSTABLE_DATA_FILE) AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ AFL_FRIDA_OUTPUT_STDOUT=/tmp/stdout.txt \ AFL_FRIDA_OUTPUT_STDERR=/tmp/stderr.txt \ @@ -98,9 +98,9 @@ frida_coverage: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) -i $(UNSTABLE_DATA_DIR) \ -o $(FRIDA_OUT) \ -- \ - $(UNSTABLE_BIN) @@ + $(TEST_BIN) @@ -frida_unstable: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) +frida_unstable: $(TEST_BIN) $(UNSTABLE_DATA_FILE) AFL_DEBUG=1 \ AFL_FRIDA_PERSISTENT_ADDR=$(AFL_FRIDA_PERSISTENT_ADDR) \ AFL_FRIDA_OUTPUT_STDOUT=/tmp/stdout.txt \ @@ -112,10 +112,10 @@ frida_unstable: $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) -i $(UNSTABLE_DATA_DIR) \ -o $(FRIDA_OUT) \ -- \ - $(UNSTABLE_BIN) @@ + $(TEST_BIN) @@ debug: gdb \ --ex 'set environment LD_PRELOAD=$(ROOT)afl-frida-trace.so' \ --ex 'set disassembly-flavor intel' \ - --args $(UNSTABLE_BIN) $(UNSTABLE_DATA_FILE) + --args $(TEST_BIN) $(UNSTABLE_DATA_FILE) From 3c811de9170650a1c379e87dda9656934b49ddbb Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 9 Aug 2022 10:19:16 +0200 Subject: [PATCH 03/25] update qemuafl --- qemu_mode/QEMUAFL_VERSION | 2 +- qemu_mode/qemuafl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index f39cb77e..ef4aa4c4 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -a120c3feb5 +d499b27d67 diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index a120c3fe..d499b27d 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit a120c3feb573d4cade292cdeb7c1f6b1ce109efe +Subproject commit d499b27d670c5bf5ba716c6c308dc7671561dae9 From 38fba715463cc56ac810cc033a8064458c8f39a7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 9 Aug 2022 10:49:05 +0200 Subject: [PATCH 04/25] add env var --- include/envs.h | 1 + 1 file changed, 1 insertion(+) diff --git a/include/envs.h b/include/envs.h index 52f2d09b..02bd2ece 100644 --- a/include/envs.h +++ b/include/envs.h @@ -197,6 +197,7 @@ static char *afl_environment_variables[] = { "AFL_QEMU_INST_RANGES", "AFL_QEMU_EXCLUDE_RANGES", "AFL_QEMU_SNAPSHOT", + "AFL_QEMU_TRACK_UNSTABLE", "AFL_QUIET", "AFL_RANDOM_ALLOC_CANARY", "AFL_REAL_PATH", From 39dadab065fef07ba9c94703edaeedc753d62182 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 10 Aug 2022 09:05:13 +0200 Subject: [PATCH 05/25] fix qemu_mode --- TODO.md | 1 + qemu_mode/QEMUAFL_VERSION | 2 +- qemu_mode/qemuafl | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/TODO.md b/TODO.md index c64c1236..93f22da4 100644 --- a/TODO.md +++ b/TODO.md @@ -2,6 +2,7 @@ ## Should + - env var to start fuzzing at once instead of calibrating everything first - makefiles should show provide a build summary success/failure - better documentation for custom mutators - better autodetection of shifting runtime timeout values diff --git a/qemu_mode/QEMUAFL_VERSION b/qemu_mode/QEMUAFL_VERSION index ef4aa4c4..d59a04e7 100644 --- a/qemu_mode/QEMUAFL_VERSION +++ b/qemu_mode/QEMUAFL_VERSION @@ -1 +1 @@ -d499b27d67 +12682ea816 diff --git a/qemu_mode/qemuafl b/qemu_mode/qemuafl index d499b27d..12682ea8 160000 --- a/qemu_mode/qemuafl +++ b/qemu_mode/qemuafl @@ -1 +1 @@ -Subproject commit d499b27d670c5bf5ba716c6c308dc7671561dae9 +Subproject commit 12682ea8169604a6c0f9b2b36eaa53ff7dcc7fd2 From ddea91de1d2e0a10e613acb8020b95220bcd789b Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 11 Aug 2022 17:55:23 +0100 Subject: [PATCH 06/25] Fixed makefile formatting --- frida_mode/GNUmakefile | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/frida_mode/GNUmakefile b/frida_mode/GNUmakefile index 43b8932a..39c96d5b 100644 --- a/frida_mode/GNUmakefile +++ b/frida_mode/GNUmakefile @@ -32,7 +32,7 @@ CFLAGS+=-fPIC \ -ffunction-sections \ ifdef IS_ANDROID -CFLAGS+=-DANDROID +CFLAGS+=-DANDROID endif AFL_CFLAGS:=-Wno-unused-parameter \ @@ -47,7 +47,7 @@ LDFLAGS+= -static-libstdc++ \ -DANDROID \ -llog \ -shared -else +else LDFLAGS+=-shared \ -lpthread \ -lresolv @@ -103,7 +103,7 @@ endif LDFLAGS+= -z noexecstack \ -Wl,--gc-sections \ -Wl,--exclude-libs,ALL \ - -ldl + -ldl LDSCRIPT:=-Wl,--version-script=$(PWD)frida.map endif @@ -118,7 +118,7 @@ endif ifdef IS_ANDROID OS:=android ifdef IS_x86 - ARCH:=x86 + ARCH:=x86 endif ifdef IS_x86 ARCH:=x86_64 @@ -247,17 +247,17 @@ else ifeq "$(ARCH)" "arm64" CFLAGS+=-I $(FRIDA_DIR)build/frida_thin-$(OS)-$(ARCH)/include/frida-1.0 \ -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/include/glib-2.0/ \ - -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/lib/glib-2.0/include/ \ + -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/lib/glib-2.0/include/ \ -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/include/capstone/ \ -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/include/json-glib-1.0/ \ ifeq "$(OS)" "android" - CFLAGS += -static-libstdc++ +CFLAGS += -static-libstdc++ endif else CFLAGS+=-I $(FRIDA_DIR)build/frida_thin-$(OS)-$(ARCH)/include/frida-1.0 \ -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/include/glib-2.0/ \ - -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/lib/glib-2.0/include/ \ + -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/lib/glib-2.0/include/ \ -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/include/capstone/ \ -I $(FRIDA_DIR)build/frida_thin-sdk-$(OS)-$(ARCH)/include/json-glib-1.0/ \ @@ -284,7 +284,7 @@ CFLAGS+=-I $(FRIDA_DIR)build/frida-$(OS)-$(ARCH)/include/frida-1.0 \ -I $(FRIDA_DIR)build/sdk-$(OS)-$(ARCH)/include/glib-2.0/ \ -I $(FRIDA_DIR)build/sdk-$(OS)-$(ARCH)/lib/glib-2.0/include/ \ -I $(FRIDA_DIR)build/sdk-$(OS)-$(ARCH)/include/capstone/ \ - -I $(FRIDA_DIR)build/sdk-$(OS)-$(ARCH)/include/json-glib-1.0/ + -I $(FRIDA_DIR)build/sdk-$(OS)-$(ARCH)/include/json-glib-1.0/ ifeq "$(OS)" "android" CFLAGS += -static-libstdc++ From b5002d74b4bd28d11119fd2aac121ab980ba5105 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 11 Aug 2022 17:55:23 +0100 Subject: [PATCH 07/25] Fixed warnings about format strings --- frida_mode/src/module.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/frida_mode/src/module.c b/frida_mode/src/module.c index 4b6b2b3f..2e4a4175 100644 --- a/frida_mode/src/module.c +++ b/frida_mode/src/module.c @@ -77,7 +77,9 @@ static int on_dlclose(void *handle) { range = &g_array_index(ranges, gum_range_t, i); base = range->range.base_address; limit = base + range->range.size; - FVERBOSE("Reserving range: 0x%016lx, 0x%016lX", base, limit); + FVERBOSE("Reserving range: 0x%016" G_GINT64_MODIFIER + "x, 0x%016" G_GINT64_MODIFIER "X", + base, limit); mem = gum_memory_allocate(GSIZE_TO_POINTER(base), range->range.size, page_size, GUM_PAGE_NO_ACCESS); if (mem == NULL) { FATAL("Failed to allocate %p (%d)", mem, errno); } From 2c8228dbe4aca2dd27acc7828440c477913d9708 Mon Sep 17 00:00:00 2001 From: Your Name Date: Thu, 11 Aug 2022 17:55:23 +0100 Subject: [PATCH 08/25] Changes to allow cross-compilation of pngtest --- frida_mode/test/png/GNUmakefile | 74 ++++++++++++++++++++++++++++++--- 1 file changed, 68 insertions(+), 6 deletions(-) diff --git a/frida_mode/test/png/GNUmakefile b/frida_mode/test/png/GNUmakefile index 0f591508..864265e0 100644 --- a/frida_mode/test/png/GNUmakefile +++ b/frida_mode/test/png/GNUmakefile @@ -2,10 +2,17 @@ PWD:=$(shell pwd)/ ROOT:=$(PWD)../../../ BUILD_DIR:=$(PWD)build/ +LIBZ_BUILD_DIR:=$(BUILD_DIR)libz/ LIBPNG_BUILD_DIR:=$(BUILD_DIR)libpng/ HARNESS_BUILD_DIR:=$(BUILD_DIR)harness/ PNGTEST_BUILD_DIR:=$(BUILD_DIR)pngtest/ +LIBZ_FILE:=$(LIBZ_BUILD_DIR)zlib-1.2.12.tar.gz +LIBZ_URL:=http://www.zlib.net/zlib-1.2.12.tar.gz +LIBZ_DIR:=$(LIBZ_BUILD_DIR)zlib-1.2.12/ +LIBZ_PC:=$(ZLIB_DIR)zlib.pc +LIBZ_LIB:=$(LIBZ_DIR)libz.a + LIBPNG_FILE:=$(LIBPNG_BUILD_DIR)libpng-1.2.56.tar.gz LIBPNG_URL:=https://downloads.sourceforge.net/project/libpng/libpng12/older-releases/1.2.56/libpng-1.2.56.tar.gz LIBPNG_DIR:=$(LIBPNG_BUILD_DIR)libpng-1.2.56/ @@ -32,11 +39,16 @@ FRIDA_OUT:=$(BUILD_DIR)frida-out .PHONY: all clean qemu frida +ARCH?="" + all: $(TEST_BIN) make -C $(ROOT)frida_mode/ 32: - CFLAGS="-m32" LDFLAGS="-m32" ARCH="x86" make all + CFLAGS="-m32" LDFLAGS="-m32" make $(TEST_BIN) + +arm: + ARCH="arm" CC="arm-linux-gnueabihf-gcc" CXX="arm-linux-gnueabihf-g++" make $(TEST_BIN) $(BUILD_DIR): mkdir -p $@ @@ -51,6 +63,8 @@ $(HARNESS_FILE): | $(HARNESS_BUILD_DIR) $(HARNESS_OBJ): $(HARNESS_FILE) $(CC) $(CFLAGS) $(LDFLAGS) -o $@ -c $< +harness: $(HARNESS_OBJ) + ######### PNGTEST ######## $(PNGTEST_BUILD_DIR): | $(BUILD_DIR) @@ -62,6 +76,34 @@ $(PNGTEST_FILE): | $(PNGTEST_BUILD_DIR) $(PNGTEST_OBJ): $(PNGTEST_FILE) | $(LIBPNG_DIR) $(CXX) $(CFLAGS) $(LDFLAGS) -std=c++11 -I $(LIBPNG_DIR) -o $@ -c $< +pngtest: $(PNGTEST_OBJ) + +######### LIBZ ######## + +$(LIBZ_BUILD_DIR): | $(BUILD_DIR) + mkdir -p $@ + +$(LIBZ_FILE): | $(LIBZ_BUILD_DIR) + wget -O $@ $(LIBZ_URL) + +$(LIBZ_DIR): $(LIBZ_FILE) + tar zxvf $(LIBZ_FILE) -C $(LIBZ_BUILD_DIR) + +$(LIBZ_PC): | $(LIBZ_DIR) + cd $(LIBZ_DIR) && \ + CFLAGS="$(CFLAGS) -fPIC" \ + ./configure \ + --static \ + --archs="$(ARCH)" + +$(LIBZ_LIB): $(LIBZ_PC) + CFLAGS="$(CFLAGS) -fPIC" \ + make \ + -C $(LIBZ_DIR) \ + -j + +libz: $(LIBZ_LIB) + ######### LIBPNG ######## $(LIBPNG_BUILD_DIR): | $(BUILD_DIR) @@ -73,11 +115,21 @@ $(LIBPNG_FILE): | $(LIBPNG_BUILD_DIR) $(LIBPNG_DIR): $(LIBPNG_FILE) tar zxvf $(LIBPNG_FILE) -C $(LIBPNG_BUILD_DIR) -$(LIBPNG_MAKEFILE): | $(LIBPNG_DIR) - cd $(LIBPNG_DIR) && ./configure +$(LIBPNG_MAKEFILE): $(LIBZ_LIB) | $(LIBPNG_DIR) + cd $(LIBPNG_DIR) && \ + CFLAGS="$(CFLAGS) -I$(LIBZ_DIR)" \ + LDFLAGS="-L$(LIBZ_DIR)" \ + ./configure \ + --host="$(ARCH)" $(LIBPNG_LIB): $(LIBPNG_MAKEFILE) - make -C $(LIBPNG_DIR) + CFLAGS="$(CFLAGS) -I$(LIBZ_DIR)" \ + LDFLAGS="-L$(LIBZ_DIR)" \ + make \ + -C $(LIBPNG_DIR) \ + -j + +png: $(LIBPNG_LIB) ######### TEST ######## @@ -86,11 +138,21 @@ $(TEST_BIN): $(HARNESS_OBJ) $(PNGTEST_OBJ) $(LIBPNG_LIB) $(CFLAGS) \ $(LDFLAGS) \ -o $@ \ - $(HARNESS_OBJ) $(PNGTEST_OBJ) $(LIBPNG_LIB) \ - -lz \ + $(HARNESS_OBJ) $(PNGTEST_OBJ) $(LIBPNG_LIB) $(LIBZ_LIB) \ $(TEST_BIN_LDFLAGS) \ +test_bin: $(TEST_BIN) + +dowload: $(LIBZ_FILE) $(LIBPNG_FILE) $(HARNESS_FILE) $(PNGTEST_FILE) + clean: + rm -rf $(LIBZ_DIR) + rm -rf $(LIBPNG_DIR) + rm -f $(HARNESS_OBJ) + rm -f $(PNGTEST_OBJ) + rm -f $(TEST_BIN) + +clean_all: rm -rf $(BUILD_DIR) qemu: $(TEST_BIN) From 130b1f78646c3a2288f4d0c96e9634612fbdb1e4 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Sat, 13 Aug 2022 07:23:34 +0100 Subject: [PATCH 09/25] libtokencap annotate overriden calls to pyt them together in a special elf (sub)section and improve their locality. --- utils/libtokencap/libtokencap.so.c | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/utils/libtokencap/libtokencap.so.c b/utils/libtokencap/libtokencap.so.c index c69812a7..5dcb8f4c 100644 --- a/utils/libtokencap/libtokencap.so.c +++ b/utils/libtokencap/libtokencap.so.c @@ -354,7 +354,7 @@ static void __tokencap_dump(const u8 *ptr, size_t len, u8 is_text) { #undef strcmp -int strcmp(const char *str1, const char *str2) { +__attribute__((hot)) int strcmp(const char *str1, const char *str2) { if (__tokencap_is_ro(str1)) __tokencap_dump(str1, strlen(str1), 1); if (__tokencap_is_ro(str2)) __tokencap_dump(str2, strlen(str2), 1); @@ -378,7 +378,7 @@ int strcmp(const char *str1, const char *str2) { #undef strncmp -int strncmp(const char *str1, const char *str2, size_t len) { +__attribute__((hot)) int strncmp(const char *str1, const char *str2, size_t len) { if (__tokencap_is_ro(str1)) __tokencap_dump(str1, len, 1); if (__tokencap_is_ro(str2)) __tokencap_dump(str2, len, 1); @@ -404,7 +404,7 @@ int strncmp(const char *str1, const char *str2, size_t len) { #undef strcasecmp -int strcasecmp(const char *str1, const char *str2) { +__attribute__((hot)) int strcasecmp(const char *str1, const char *str2) { if (__tokencap_is_ro(str1)) __tokencap_dump(str1, strlen(str1), 1); if (__tokencap_is_ro(str2)) __tokencap_dump(str2, strlen(str2), 1); @@ -428,7 +428,7 @@ int strcasecmp(const char *str1, const char *str2) { #undef strncasecmp -int strncasecmp(const char *str1, const char *str2, size_t len) { +__attribute__((hot)) int strncasecmp(const char *str1, const char *str2, size_t len) { if (__tokencap_is_ro(str1)) __tokencap_dump(str1, len, 1); if (__tokencap_is_ro(str2)) __tokencap_dump(str2, len, 1); @@ -454,7 +454,7 @@ int strncasecmp(const char *str1, const char *str2, size_t len) { #undef memcmp -int memcmp(const void *mem1, const void *mem2, size_t len) { +__attribute__((hot)) int memcmp(const void *mem1, const void *mem2, size_t len) { if (__tokencap_is_ro(mem1)) __tokencap_dump(mem1, len, 0); if (__tokencap_is_ro(mem2)) __tokencap_dump(mem2, len, 0); @@ -481,7 +481,7 @@ int memcmp(const void *mem1, const void *mem2, size_t len) { #undef bcmp -int bcmp(const void *mem1, const void *mem2, size_t len) { +__attribute__((hot)) int bcmp(const void *mem1, const void *mem2, size_t len) { if (__tokencap_is_ro(mem1)) __tokencap_dump(mem1, len, 0); if (__tokencap_is_ro(mem2)) __tokencap_dump(mem2, len, 0); @@ -508,7 +508,7 @@ int bcmp(const void *mem1, const void *mem2, size_t len) { #undef strstr -char *strstr(const char *haystack, const char *needle) { +__attribute__((hot)) char *strstr(const char *haystack, const char *needle) { if (__tokencap_is_ro(haystack)) __tokencap_dump(haystack, strlen(haystack), 1); @@ -537,7 +537,7 @@ char *strstr(const char *haystack, const char *needle) { #undef strcasestr -char *strcasestr(const char *haystack, const char *needle) { +__attribute__((hot)) char *strcasestr(const char *haystack, const char *needle) { if (__tokencap_is_ro(haystack)) __tokencap_dump(haystack, strlen(haystack), 1); @@ -566,7 +566,7 @@ char *strcasestr(const char *haystack, const char *needle) { #undef memmem -void *memmem(const void *haystack, size_t haystack_len, const void *needle, +__attribute__((hot)) void *memmem(const void *haystack, size_t haystack_len, const void *needle, size_t needle_len) { if (__tokencap_is_ro(haystack)) __tokencap_dump(haystack, haystack_len, 1); From 3200e6515b9cc988d0d8dccd27257baccc8df021 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 14 Aug 2022 12:24:42 +0200 Subject: [PATCH 10/25] add AFL_NO_STARTUP_CALIBRATION feature --- docs/Changelog.md | 9 ++++++++- docs/env_variables.md | 3 +++ docs/fuzzing_in_depth.md | 7 +++++++ include/afl-fuzz.h | 4 +++- include/envs.h | 1 + src/afl-fuzz-init.c | 24 ++++++++++++++++++++++++ src/afl-fuzz-queue.c | 10 ++++++++-- src/afl-fuzz-state.c | 7 +++++++ src/afl-fuzz.c | 13 ++++++++++++- 9 files changed, 73 insertions(+), 5 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index ec517104..f5847ade 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,7 +9,14 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to . ### Version ++4.03a (dev) - - ... your PR? :) + - afl-fuzz: + - added AFL_NO_STARTUP_CALIBRATION to start fuzzing at once instead + of calibrating all initial seeds first. Good for large queues + and long execution times, especially in CIs. + - qemu_mode: + - added AFL_QEMU_TRACK_UNSTABLE to log the addresses of unstable + edges (together with AFL_DEBUG=1 afl-fuzz). thanks to + worksbutnottested! ### Version ++4.02c (release) diff --git a/docs/env_variables.md b/docs/env_variables.md index 00948fc1..bb54357b 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -462,6 +462,9 @@ checks or alter some of the more exotic semantics of the tool: some basic stats. This behavior is also automatically triggered when the output from afl-fuzz is redirected to a file or to a pipe. + - Setting `AFL_NO_STARTUP_CALIBRATION` will skip the initial calibration + of all starting seeds, and start fuzzing at once. + - In QEMU mode (-Q) and FRIDA mode (-O), `AFL_PATH` will be searched for afl-qemu-trace and afl-frida-trace.so. diff --git a/docs/fuzzing_in_depth.md b/docs/fuzzing_in_depth.md index 37889137..92c9910b 100644 --- a/docs/fuzzing_in_depth.md +++ b/docs/fuzzing_in_depth.md @@ -626,6 +626,9 @@ from other fuzzers in the campaign first. If you have a large corpus, a corpus from a previous run or are fuzzing in a CI, then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`. +If the queue in the CI is huge and/or the execution time is slow then you can +also add `AFL_NO_STARTUP_CALIBRATION=1` to skip the initial queue calibration +phase and start fuzzing at once. You can also use different fuzzers. If you are using AFL spinoffs or AFL conforming fuzzers, then just use the same -o directory and give it a unique @@ -902,6 +905,10 @@ complex file formats. Some notes on continuous integration (CI) fuzzing - this fuzzing is different to normal fuzzing campaigns as these are much shorter runnings. +If the queue in the CI is huge and/or the execution time is slow then you can +also add `AFL_NO_STARTUP_CALIBRATION=1` to skip the initial queue calibration +phase and start fuzzing at once. + 1. Always: * LTO has a much longer compile time which is diametrical to short fuzzing - hence use afl-clang-fast instead. diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 061076ed..822096e8 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -386,7 +386,8 @@ typedef struct afl_env_vars { afl_bench_until_crash, afl_debug_child, afl_autoresume, afl_cal_fast, afl_cycle_schedules, afl_expand_havoc, afl_statsd, afl_cmplog_only_new, afl_exit_on_seed_issues, afl_try_affinity, afl_ignore_problems, - afl_keep_timeouts, afl_pizza_mode, afl_no_crash_readme; + afl_keep_timeouts, afl_pizza_mode, afl_no_crash_readme, + afl_no_startup_calibration; u8 *afl_tmpdir, *afl_custom_mutator_library, *afl_python_module, *afl_path, *afl_hang_tmout, *afl_forksrv_init_tmout, *afl_preload, @@ -1122,6 +1123,7 @@ void bind_to_free_cpu(afl_state_t *); void setup_post(afl_state_t *); void read_testcases(afl_state_t *, u8 *); void perform_dry_run(afl_state_t *); +void no_dry_run(afl_state_t *); void pivot_inputs(afl_state_t *); u32 find_start_position(afl_state_t *); void find_timeout(afl_state_t *); diff --git a/include/envs.h b/include/envs.h index 02bd2ece..2204a100 100644 --- a/include/envs.h +++ b/include/envs.h @@ -165,6 +165,7 @@ static char *afl_environment_variables[] = { "AFL_NO_FORKSRV", "AFL_NO_UI", "AFL_NO_PYTHON", + "AFL_NO_STARTUP_CALIBRATION", "AFL_UNTRACER_FILE", "AFL_LLVM_USE_TRACE_PC", "AFL_MAP_SIZE", diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 4ffcfd2b..32e2b7b8 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -850,6 +850,30 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } +/* In case no initial calibration is to be performed (e.g. huge queue and slow +execution time), then setting AFL_NO_STARTUP_CALIBRATION will help getting +initial data. For this to succeed, non-calibrated corpus entries have to look +especially juicy so they are more likely to be selected then a calibrated good +looking one. */ + +void no_dry_run(afl_state_t *afl) { + + struct queue_entry *q; + u32 idx; + + for (idx = 0; idx < afl->queued_items; idx++) { + + q = afl->queue_buf[idx]; + if (unlikely(!q || q->disabled)) { continue; } + + q->exec_us = 1; + q->bitmap_size = MAP_SIZE; + q->tc_ref = MAP_SIZE; + + } + +} + /* Perform dry run of all test cases to confirm that the app is working as expected. This is done only for the initial inputs, and only once. */ diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 02d697ab..d8dbdfbe 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -795,8 +795,14 @@ void cull_queue(afl_state_t *afl) { u32 calculate_score(afl_state_t *afl, struct queue_entry *q) { - u32 avg_exec_us = afl->total_cal_us / afl->total_cal_cycles; - u32 avg_bitmap_size = afl->total_bitmap_size / afl->total_bitmap_entries; + u32 cal_cycles = afl->total_cal_cycles; + u32 bitmap_entries = afl->total_bitmap_entries; + + if (unlikely(!cal_cycles)) { cal_cycles = 1; } + if (unlikely(!bitmap_entries)) { bitmap_entries = 1; } + + u32 avg_exec_us = afl->total_cal_us / cal_cycles; + u32 avg_bitmap_size = afl->total_bitmap_size / bitmap_entries; u32 perf_score = 100; /* Adjust score based on execution speed of this path, compared to the diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index ddfd4b31..6770839a 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -265,6 +265,13 @@ void read_afl_environment(afl_state_t *afl, char **envp) { afl->afl_env.afl_cmplog_only_new = get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_NO_STARTUP_CALIBRATION", + + afl_environment_variable_len)) { + + afl->afl_env.afl_no_startup_calibration = + get_afl_env(afl_environment_variables[i]) ? 1 : 0; + } else if (!strncmp(env, "AFL_NO_UI", afl_environment_variable_len)) { afl->afl_env.afl_no_ui = diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 2e151abb..e3851473 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -273,6 +273,7 @@ static void usage(u8 *argv0, int more_help) { "AFL_NO_CPU_RED: avoid red color for showing very high cpu usage\n" "AFL_NO_FORKSRV: run target via execve instead of using the forkserver\n" "AFL_NO_SNAPSHOT: do not use the snapshot feature (if the snapshot lkm is loaded)\n" + "AFL_NO_STARTUP_CALIBRATION: no initial seed calibration, start fuzzing at once\n" "AFL_NO_UI: switch status screen off\n" DYN_COLOR @@ -2150,7 +2151,17 @@ int main(int argc, char **argv_orig, char **envp) { memset(afl->virgin_tmout, 255, map_size); memset(afl->virgin_crash, 255, map_size); - perform_dry_run(afl); + if (likely(!afl->afl_env.afl_no_startup_calibration)) { + + perform_dry_run(afl); + + } else { + + ACTF("skipping initial seed calibration due option override"); + usleep(1000); + no_dry_run(afl); + + } if (afl->q_testcase_max_cache_entries) { From 1a3b463c4cceabc38b7de83f67813e841153b536 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 14 Aug 2022 12:55:04 +0200 Subject: [PATCH 11/25] add build report --- GNUmakefile | 52 +++++++++++++++++++++++++++++++++++++++++++++++ TODO.md | 2 -- docs/Changelog.md | 1 + 3 files changed, 53 insertions(+), 2 deletions(-) diff --git a/GNUmakefile b/GNUmakefile index a64d511f..70299fc3 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -309,6 +309,17 @@ endif .PHONY: all all: test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_build all_done -$(MAKE) -C utils/aflpp_driver + @echo + @echo + @echo Build Summary: + @test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler" + @test -e afl-llvm-pass.so && echo "[+] LLVM basic mode successfully built" || echo "[-] LLVM mode could not be build, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" + @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be build, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" + @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be build, it is optional, if you want it, please install LLVM 11-14. More information at instrumentation/README.lto.md on how to build it" +ifneq "$(SYS)" "Darwin" + @test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this" +endif + @echo .PHONY: llvm llvm: @@ -674,6 +685,31 @@ endif -cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh endif endif + @echo + @echo + @echo Build Summary: + @test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler" +ifneq "$(SYS)" "Darwin" +ifeq "$(ARCH)" "aarch64" + ifndef NO_CORESIGHT + @test -e afl-cs-proxy && echo "[+] coresight_mode successfully built" || echo "[-] coresight_mode could not be built, it is optional and experimental, see coresight_mode/README.md for what is needed" + endif +endif +ifeq "$(SYS)" "Linux" +ifndef NO_NYX + @test -e libnyx.so && echo "[+] nyx_mode successfully built" || echo "[-] nyx_mode could not be built, it is optional, see nyx_mode/README.md for what is needed" +endif +endif + @test -e afl-qemu-trace && echo "[+] qemu_mode successfully built" || echo "[-] qemu_mode could not be built, see docs/INSTALL.md for what is needed" + ifeq "$(ARCH)" "aarch64" + ifndef NO_UNICORN_ARM64 + @test -e unicorn_mode/unicornafl/build_python/libunicornafl.so && echo "[+] unicorn_mode successfully built" || echo "[-] unicorn_mode could not be built, it is optional, see unicorn_mode/README.md for what is needed" + endif + else + @test -e unicorn_mode/unicornafl/build_python/libunicornafl.so && echo "[+] unicorn_mode successfully built" || echo "[-] unicorn_mode could not be built, it is optional, see unicorn_mode/README.md for what is needed" + endif +endif + @echo .PHONY: source-only source-only: all @@ -689,6 +725,22 @@ ifndef NO_NYX -cd nyx_mode && ./build_nyx_support.sh endif endif + @echo + @echo + @echo Build Summary: + @test -e afl-fuzz && echo "[+] afl-fuzz and supporting tools successfully built" || echo "[-] afl-fuzz could not be built, please set CC to a working compiler" + @test -e afl-llvm-pass.so && echo "[+] LLVM basic mode successfully built" || echo "[-] LLVM mode could not be build, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" + @test -e SanitizerCoveragePCGUARD.so && echo "[+] LLVM mode successfully built" || echo "[-] LLVM mode could not be build, please install at least llvm-11 and clang-11 or newer, see docs/INSTALL.md" + @test -e SanitizerCoverageLTO.so && echo "[+] LLVM LTO mode successfully built" || echo "[-] LLVM LTO mode could not be build, it is optional, if you want it, please install LLVM 11-14. More information at instrumentation/README.lto.md on how to build it" +ifneq "$(SYS)" "Darwin" + test -e afl-gcc-pass.so && echo "[+] gcc_mode successfully built" || echo "[-] gcc_mode could not be built, it is optional, install gcc-VERSION-plugin-dev to enable this" +endif +ifeq "$(SYS)" "Linux" +ifndef NO_NYX + @test -e libnyx.so && echo "[+] nyx_mode successfully built" || echo "[-] nyx_mode could not be built, it is optional, see nyx_mode/README.md for what is needed" +endif +endif + @echo %.8: % @echo .TH $* 8 $(BUILD_DATE) "afl++" > $@ diff --git a/TODO.md b/TODO.md index 93f22da4..99d2c419 100644 --- a/TODO.md +++ b/TODO.md @@ -2,8 +2,6 @@ ## Should - - env var to start fuzzing at once instead of calibrating everything first - - makefiles should show provide a build summary success/failure - better documentation for custom mutators - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt diff --git a/docs/Changelog.md b/docs/Changelog.md index f5847ade..2ce1d85c 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -9,6 +9,7 @@ Want to stay in the loop on major new features? Join our mailing list by sending a mail to . ### Version ++4.03a (dev) + - Building now gives a build summary what succeeded and what not - afl-fuzz: - added AFL_NO_STARTUP_CALIBRATION to start fuzzing at once instead of calibrating all initial seeds first. Good for large queues From 4e980713851c522436b6a6813f27dd95dd4e5fae Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 14 Aug 2022 14:40:26 +0200 Subject: [PATCH 12/25] better handling of -fsanitize=..,...,.. lists --- docs/Changelog.md | 2 + src/afl-cc.c | 109 ++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 98 insertions(+), 13 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index 2ce1d85c..d07cef54 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -14,6 +14,8 @@ sending a mail to . - added AFL_NO_STARTUP_CALIBRATION to start fuzzing at once instead of calibrating all initial seeds first. Good for large queues and long execution times, especially in CIs. + - afl-cc: + - better handling of -fsanitize=..,...,.. lists - qemu_mode: - added AFL_QEMU_TRACK_UNSTABLE to log the addresses of unstable edges (together with AFL_DEBUG=1 afl-fuzz). thanks to diff --git a/src/afl-cc.c b/src/afl-cc.c index 6def3ee7..cae6d949 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -51,7 +51,7 @@ static u32 cc_par_cnt = 1; /* Param count, including argv0 */ static u8 clang_mode; /* Invoked as afl-clang*? */ static u8 llvm_fullpath[PATH_MAX]; static u8 instrument_mode, instrument_opt_mode, ngram_size, ctx_k, lto_mode; -static u8 compiler_mode, plusplus_mode, have_instr_env = 0; +static u8 compiler_mode, plusplus_mode, have_instr_env = 0, need_aflpplib = 0; static u8 have_gcc, have_llvm, have_gcc_plugin, have_lto, have_instr_list = 0; static u8 *lto_flag = AFL_CLANG_FLTO, *argvnull; static u8 debug; @@ -310,6 +310,69 @@ static u8 *find_object(u8 *obj, u8 *argv0) { } +void parse_fsanitize(char *string) { + + char *p, *ptr = string + strlen("-fsanitize="); + char *new = malloc(strlen(string) + 1); + char *tmp = malloc(strlen(ptr)); + u32 count = 0, len, ende = 0; + strcpy(new, "-fsanitize="); + + do { + + p = strchr(ptr, ','); + if (!p) { + + p = ptr + strlen(ptr) + 1; + ende = 1; + + } + + len = p - ptr; + if (len) { + + strncpy(tmp, ptr, len); + tmp[len] = 0; + // fprintf(stderr, "Found: %s\n", tmp); + ptr += len + 1; + if (*tmp) { + + u32 copy = 1; + if (!strcmp(tmp, "fuzzer")) { + + need_aflpplib = 1; + copy = 0; + + } else if (!strncmp(tmp, "fuzzer", 6)) { + + copy = 0; + + } + + if (copy) { + + if (count) { strcat(new, ","); } + strcat(new, tmp); + ++count; + + } + + } + + } else { + + ptr++; /*fprintf(stderr, "NO!\n"); */ + + } + + } while (!ende); + + strcpy(string, new); + // fprintf(stderr, "string: %s\n", string); + // fprintf(stderr, "new: %s\n", new); + +} + /* Copy argv to cc_params, making the necessary edits. */ static void edit_params(u32 argc, char **argv, char **envp) { @@ -779,20 +842,35 @@ static void edit_params(u32 argc, char **argv, char **envp) { } - if ((!strncmp(cur, "-fsanitize=fuzzer-", strlen("-fsanitize=fuzzer-")) || - !strncmp(cur, "-fsanitize-coverage", strlen("-fsanitize-coverage"))) && - (strncmp(cur, "sanitize-coverage-allow", - strlen("sanitize-coverage-allow")) && - strncmp(cur, "sanitize-coverage-deny", - strlen("sanitize-coverage-deny")) && - instrument_mode != INSTRUMENT_LLVMNATIVE)) { + if (!strncmp(cur, "-fsanitize-coverage-", 20) && strstr(cur, "list=")) { + + have_instr_list = 1; + + } + + if (!strncmp(cur, "-fsanitize=", strlen("-fsanitize=")) && + strchr(cur, ',')) { + + parse_fsanitize(cur); + if (!cur || strlen(cur) <= strlen("-fsanitize=")) { continue; } + + } else if ((!strncmp(cur, "-fsanitize=fuzzer-", + + strlen("-fsanitize=fuzzer-")) || + !strncmp(cur, "-fsanitize-coverage", + strlen("-fsanitize-coverage"))) && + (strncmp(cur, "sanitize-coverage-allow", + strlen("sanitize-coverage-allow")) && + strncmp(cur, "sanitize-coverage-deny", + strlen("sanitize-coverage-deny")) && + instrument_mode != INSTRUMENT_LLVMNATIVE)) { if (!be_quiet) { WARNF("Found '%s' - stripping!", cur); } continue; } - if (!strcmp(cur, "-fsanitize=fuzzer")) { + if (need_aflpplib || !strcmp(cur, "-fsanitize=fuzzer")) { u8 *afllib = find_object("libAFLDriver.a", argv[0]); @@ -823,7 +901,15 @@ static void edit_params(u32 argc, char **argv, char **envp) { } - continue; + if (need_aflpplib) { + + need_aflpplib = 0; + + } else { + + continue; + + } } @@ -831,9 +917,6 @@ static void edit_params(u32 argc, char **argv, char **envp) { if (!strcmp(cur, "armv7a-linux-androideabi")) bit_mode = 32; if (!strcmp(cur, "-m64")) bit_mode = 64; - if (!strncmp(cur, "-fsanitize-coverage-", 20) && strstr(cur, "list=")) - have_instr_list = 1; - if (!strcmp(cur, "-fsanitize=address") || !strcmp(cur, "-fsanitize=memory")) asan_set = 1; From f00d83afbc1b3e82c61ada6b1834eb0f7312e863 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 15 Aug 2022 17:38:53 +0200 Subject: [PATCH 13/25] filter pipe in gcc_mode --- src/afl-cc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/afl-cc.c b/src/afl-cc.c index cae6d949..4c377ad1 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -822,6 +822,8 @@ static void edit_params(u32 argc, char **argv, char **envp) { } + if (compiler_mode == GCC_PLUGIN && !strcmp(cur, "-pipe")) { continue; } + if (!strcmp(cur, "-z") || !strcmp(cur, "-Wl,-z")) { u8 *param = *(argv + 1); From ba14c353c07d19ad37916947708a9c26537c6d62 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 15 Aug 2022 18:31:45 +0200 Subject: [PATCH 14/25] get map size from binaries within afl-cmin* --- afl-cmin | 17 +++++++++++++---- afl-cmin.bash | 6 ++++++ docs/Changelog.md | 2 ++ instrumentation/afl-compiler-rt.o.c | 18 +++++++++++++++++- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/afl-cmin b/afl-cmin index 51835648..44716af1 100755 --- a/afl-cmin +++ b/afl-cmin @@ -291,6 +291,15 @@ BEGIN { target_bin = tnew } + get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin + get_map_size | getline mapsize + if (mapsize && mapsize > 65535) { + + AFL_MAP_SIZE = "AFL_MAP_SIZE="mapsize" " + print "[+] Setting "AFL_MAP_SIZE + + } + if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !frida_mode && !unicorn_mode) { if (0 != system( "grep -q __AFL_SHM_ID "target_bin )) { print "[-] Error: binary '"target_bin"' doesn't appear to be instrumented." > "/dev/stderr" @@ -399,10 +408,10 @@ BEGIN { print "[*] Testing the target binary..." if (!stdin_file) { - system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") + system(AFL_MAP_SIZE "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"") } else { system("cp \""in_dir"/"first_file"\" "stdin_file) - system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" /dev/null` +test -n "$MAPSIZE" && { + export AFL_MAP_SIZE=$MAPSIZE + echo "[+] Setting AFL_MAP_SIZE=$MAPSIZE" +} + if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$FRIDA_MODE" = "" -a "$UNICORN_MODE" = "" ]; then if ! grep -qF "__AFL_SHM_ID" "$TARGET_BIN"; then diff --git a/docs/Changelog.md b/docs/Changelog.md index d07cef54..c5934c4a 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -16,6 +16,8 @@ sending a mail to . and long execution times, especially in CIs. - afl-cc: - better handling of -fsanitize=..,...,.. lists + - obtain the map size of a target with setting AFL_DUMP_MAP_SIZE=1 + note that this will exit the target before main() - qemu_mode: - added AFL_QEMU_TRACK_UNSTABLE to log the addresses of unstable edges (together with AFL_DEBUG=1 afl-fuzz). thanks to diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 97974c4a..1759898e 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -288,11 +288,18 @@ static void __afl_map_shm(void) { __afl_map_size = ++__afl_final_loc; // as we count starting 0 + if (getenv("AFL_DUMP_MAP_SIZE")) { + + printf("%u\n", __afl_map_size); + exit(-1); + + } + if (__afl_final_loc > MAP_SIZE) { char *ptr; u32 val = 0; - if ((ptr = getenv("AFL_MAP_SIZE")) != NULL) val = atoi(ptr); + if ((ptr = getenv("AFL_MAP_SIZE")) != NULL) { val = atoi(ptr); } if (val < __afl_final_loc) { if (__afl_final_loc > FS_OPT_MAX_MAPSIZE) { @@ -325,6 +332,15 @@ static void __afl_map_shm(void) { } + } else { + + if (getenv("AFL_DUMP_MAP_SIZE")) { + + printf("%u\n", MAP_SIZE); + exit(-1); + + } + } if (!id_str && __afl_area_ptr_dummy == __afl_area_initial) { From 2462c61df9cffa5e29433913a73095d212b70403 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 15 Aug 2022 19:22:46 +0200 Subject: [PATCH 15/25] fix new map size extraction for afl-gcc --- afl-cmin | 15 ++++++++------- afl-cmin.bash | 11 +++++++---- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/afl-cmin b/afl-cmin index 44716af1..4e0d78df 100755 --- a/afl-cmin +++ b/afl-cmin @@ -291,13 +291,14 @@ BEGIN { target_bin = tnew } - get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin - get_map_size | getline mapsize - if (mapsize && mapsize > 65535) { - - AFL_MAP_SIZE = "AFL_MAP_SIZE="mapsize" " - print "[+] Setting "AFL_MAP_SIZE - + if (0 == system ( "grep -aq AFL_DUMP_MAP_SIZE" target_bin )) { + echo "[!] Trying to obtain the map size of the target ..." + get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin + get_map_size | getline mapsize + if (mapsize && mapsize > 65535 && mapsize < 100000000) { + AFL_MAP_SIZE = "AFL_MAP_SIZE="mapsize" " + print "[+] Setting "AFL_MAP_SIZE + } } if (!ENVIRON["AFL_SKIP_BIN_CHECK"] && !qemu_mode && !frida_mode && !unicorn_mode) { diff --git a/afl-cmin.bash b/afl-cmin.bash index db3e8ae5..d2218cd0 100755 --- a/afl-cmin.bash +++ b/afl-cmin.bash @@ -215,10 +215,13 @@ if [ ! -f "$TARGET_BIN" -o ! -x "$TARGET_BIN" ]; then fi -MAPSIZE=`AFL_DUMP_MAP_SIZE=1 "./$TARGET_BIN" 2>/dev/null` -test -n "$MAPSIZE" && { - export AFL_MAP_SIZE=$MAPSIZE - echo "[+] Setting AFL_MAP_SIZE=$MAPSIZE" +grep -aq AFL_DUMP_MAP_SIZE "./$TARGET_BIN" && { + echo "[!] Trying to obtain the map size of the target ..." + MAPSIZE=`AFL_DUMP_MAP_SIZE=1 "./$TARGET_BIN" 2>/dev/null` + test -n "$MAPSIZE" && { + export AFL_MAP_SIZE=$MAPSIZE + echo "[+] Setting AFL_MAP_SIZE=$MAPSIZE" + } } if [ "$AFL_SKIP_BIN_CHECK" = "" -a "$QEMU_MODE" = "" -a "$FRIDA_MODE" = "" -a "$UNICORN_MODE" = "" ]; then From d7abf6936c6b5797107779fc99339ac9fb78b22e Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 16 Aug 2022 09:46:11 +0200 Subject: [PATCH 16/25] fix afl-cmin --- afl-cmin | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/afl-cmin b/afl-cmin index 4e0d78df..b170667a 100755 --- a/afl-cmin +++ b/afl-cmin @@ -291,7 +291,7 @@ BEGIN { target_bin = tnew } - if (0 == system ( "grep -aq AFL_DUMP_MAP_SIZE" target_bin )) { + if (0 == system ( "grep -aq AFL_DUMP_MAP_SIZE " target_bin )) { echo "[!] Trying to obtain the map size of the target ..." get_map_size = "AFL_DUMP_MAP_SIZE=1 " target_bin get_map_size | getline mapsize From 843ef46b2128f95b5820a0fe89c7ae57f6fcf65b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 16 Aug 2022 10:05:52 +0200 Subject: [PATCH 17/25] fix docs --- docs/FAQ.md | 15 +++++++++++++++ docs/INSTALL.md | 4 ++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/docs/FAQ.md b/docs/FAQ.md index 1822e46b..4a9080f8 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -255,3 +255,18 @@ If you find an interesting or important question missing, submit it via Solution: `git pull ; make clean install` of AFL++.

+ +
+ AFL++ map size warning.

+ + When you run a large instrumented program stand-alone or via afl-showmap + you might see a warning like the following: + + ``` + Warning: AFL++ tools might need to set AFL_MAP_SIZE to 223723 to be able to run this instrumented program if this crashes! + ``` + + Depending how the target works it might also crash afterwards. + + Solution: just do an `export AFL_MAP_SIZE=(the value in the warning)`. +

diff --git a/docs/INSTALL.md b/docs/INSTALL.md index 4f2b7174..86ba916f 100644 --- a/docs/INSTALL.md +++ b/docs/INSTALL.md @@ -21,8 +21,8 @@ development state of AFL++. If you want to build AFL++ yourself, you have many options. The easiest choice is to build and install everything: -NOTE: depending on your Debian/Ubuntu/Kali/... version release `-12` with -whatever llvm version is available! +NOTE: depending on your Debian/Ubuntu/Kali/... release, replace `-12` with +whatever llvm version is available. We recommend llvm 12, 13 or 14. ```shell sudo apt-get update From eeab1afd572aacfd60784a09712211d8c0b7fbe5 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Wed, 17 Aug 2022 12:50:15 +0200 Subject: [PATCH 18/25] alt no cal --- src/afl-fuzz-init.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 32e2b7b8..fdd40794 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -858,6 +858,7 @@ looking one. */ void no_dry_run(afl_state_t *afl) { +/* struct queue_entry *q; u32 idx; @@ -871,6 +872,7 @@ void no_dry_run(afl_state_t *afl) { q->tc_ref = MAP_SIZE; } +*/ } From 361263b0f24a2172c4d4be09b1e247aa8d799e9b Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 18 Aug 2022 08:06:17 +0200 Subject: [PATCH 19/25] better no dry run --- include/afl-fuzz.h | 1 - src/afl-fuzz-init.c | 26 -------------------------- src/afl-fuzz.c | 1 - 3 files changed, 28 deletions(-) diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index 822096e8..23c20cc4 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -1123,7 +1123,6 @@ void bind_to_free_cpu(afl_state_t *); void setup_post(afl_state_t *); void read_testcases(afl_state_t *, u8 *); void perform_dry_run(afl_state_t *); -void no_dry_run(afl_state_t *); void pivot_inputs(afl_state_t *); u32 find_start_position(afl_state_t *); void find_timeout(afl_state_t *); diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index fdd40794..4ffcfd2b 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -850,32 +850,6 @@ void read_testcases(afl_state_t *afl, u8 *directory) { } -/* In case no initial calibration is to be performed (e.g. huge queue and slow -execution time), then setting AFL_NO_STARTUP_CALIBRATION will help getting -initial data. For this to succeed, non-calibrated corpus entries have to look -especially juicy so they are more likely to be selected then a calibrated good -looking one. */ - -void no_dry_run(afl_state_t *afl) { - -/* - struct queue_entry *q; - u32 idx; - - for (idx = 0; idx < afl->queued_items; idx++) { - - q = afl->queue_buf[idx]; - if (unlikely(!q || q->disabled)) { continue; } - - q->exec_us = 1; - q->bitmap_size = MAP_SIZE; - q->tc_ref = MAP_SIZE; - - } -*/ - -} - /* Perform dry run of all test cases to confirm that the app is working as expected. This is done only for the initial inputs, and only once. */ diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index e3851473..e705f187 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2159,7 +2159,6 @@ int main(int argc, char **argv_orig, char **envp) { ACTF("skipping initial seed calibration due option override"); usleep(1000); - no_dry_run(afl); } From 7b2145b914ba3c8443437c68ae29458d832b1e35 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 19 Aug 2022 09:16:17 +0200 Subject: [PATCH 20/25] shorter calibration --- docs/Changelog.md | 2 ++ include/config.h | 5 +++-- src/afl-fuzz-run.c | 2 +- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index c5934c4a..e4c59978 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -14,6 +14,8 @@ sending a mail to . - added AFL_NO_STARTUP_CALIBRATION to start fuzzing at once instead of calibrating all initial seeds first. Good for large queues and long execution times, especially in CIs. + - default calibration cycles set to 7 from 8, and only add 5 cycles + to variables queue items instead of 12. - afl-cc: - better handling of -fsanitize=..,...,.. lists - obtain the map size of a target with setting AFL_DUMP_MAP_SIZE=1 diff --git a/include/config.h b/include/config.h index 1689e034..1262668a 100644 --- a/include/config.h +++ b/include/config.h @@ -153,8 +153,9 @@ /* Number of calibration cycles per every new test case (and for test cases that show variable behavior): */ -#define CAL_CYCLES 8U -#define CAL_CYCLES_LONG 20U +#define CAL_CYCLES_FAST 3U +#define CAL_CYCLES 7U +#define CAL_CYCLES_LONG 12U /* Number of subsequent timeouts before abandoning an input file: */ diff --git a/src/afl-fuzz-run.c b/src/afl-fuzz-run.c index c0e72ae6..ee4a3298 100644 --- a/src/afl-fuzz-run.c +++ b/src/afl-fuzz-run.c @@ -363,7 +363,7 @@ u8 calibrate_case(afl_state_t *afl, struct queue_entry *q, u8 *use_mem, ++q->cal_failed; afl->stage_name = "calibration"; - afl->stage_max = afl->afl_env.afl_cal_fast ? 3 : CAL_CYCLES; + afl->stage_max = afl->afl_env.afl_cal_fast ? CAL_CYCLES_FAST : CAL_CYCLES; /* Make sure the forkserver is up before we do anything, and let's not count its spin-up time toward binary calibration. */ From b4cb3784a5135703444357876c62d19be4a58862 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Fri, 19 Aug 2022 13:17:01 +0200 Subject: [PATCH 21/25] add malloc check --- src/afl-cc.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/afl-cc.c b/src/afl-cc.c index 4c377ad1..ffd15476 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -316,6 +316,8 @@ void parse_fsanitize(char *string) { char *new = malloc(strlen(string) + 1); char *tmp = malloc(strlen(ptr)); u32 count = 0, len, ende = 0; + + if (!new || !tmp) { FATAL("could not aquire memory"); } strcpy(new, "-fsanitize="); do { From 4ebde72f28dc38c765c3d4cef396f7c70f05b5e2 Mon Sep 17 00:00:00 2001 From: Eli Kobrin Date: Fri, 19 Aug 2022 13:21:40 +0300 Subject: [PATCH 22/25] Change map size dummy value. --- include/types.h | 2 +- src/afl-showmap.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/include/types.h b/include/types.h index 4a68b1b0..96ce78f8 100644 --- a/include/types.h +++ b/include/types.h @@ -57,7 +57,7 @@ typedef uint128_t u128; #define FS_OPT_SHDMEM_FUZZ 0x01000000 #define FS_OPT_NEWCMPLOG 0x02000000 #define FS_OPT_OLD_AFLPP_WORKAROUND 0x0f000000 -// FS_OPT_MAX_MAPSIZE is 8388608 = 0x800000 = 2^23 = 1 << 22 +// FS_OPT_MAX_MAPSIZE is 8388608 = 0x800000 = 2^23 = 1 << 23 #define FS_OPT_MAX_MAPSIZE ((0x00fffffeU >> 1) + 1) #define FS_OPT_GET_MAPSIZE(x) (((x & 0x00fffffe) >> 1) + 1) #define FS_OPT_SET_MAPSIZE(x) \ diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 4bcd1d59..52d46ac9 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1240,7 +1240,7 @@ int main(int argc, char **argv_orig, char **envp) { u32 save_be_quiet = be_quiet; be_quiet = !debug; - fsrv->map_size = 4194304; // dummy temporary value + fsrv->map_size = FS_OPT_MAX_MAPSIZE; // dummy temporary value u32 new_map_size = afl_fsrv_get_mapsize(fsrv, use_argv, &stop_soon, (get_afl_env("AFL_DEBUG_CHILD") || From 47d5dbbead71bb2da8c025bb49a7b857328f9ee5 Mon Sep 17 00:00:00 2001 From: Eli Kobrin Date: Fri, 19 Aug 2022 14:24:38 +0300 Subject: [PATCH 23/25] Fix. --- src/afl-showmap.c | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/afl-showmap.c b/src/afl-showmap.c index 52d46ac9..07f30326 100644 --- a/src/afl-showmap.c +++ b/src/afl-showmap.c @@ -1240,7 +1240,12 @@ int main(int argc, char **argv_orig, char **envp) { u32 save_be_quiet = be_quiet; be_quiet = !debug; - fsrv->map_size = FS_OPT_MAX_MAPSIZE; // dummy temporary value + if (map_size > 4194304) { + fsrv->map_size = map_size; + } + else { + fsrv->map_size = 4194304; // dummy temporary value + } u32 new_map_size = afl_fsrv_get_mapsize(fsrv, use_argv, &stop_soon, (get_afl_env("AFL_DEBUG_CHILD") || From 88ff8aa81e41717abb3d72f8714fdc38591b81a7 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 25 Aug 2022 08:47:30 +0200 Subject: [PATCH 24/25] fix gcc_mode cmplog --- docs/Changelog.md | 1 + src/afl-cc.c | 7 ++----- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/docs/Changelog.md b/docs/Changelog.md index e4c59978..842b727b 100644 --- a/docs/Changelog.md +++ b/docs/Changelog.md @@ -18,6 +18,7 @@ sending a mail to . to variables queue items instead of 12. - afl-cc: - better handling of -fsanitize=..,...,.. lists + - fix gcc_mode cmplog - obtain the map size of a target with setting AFL_DUMP_MAP_SIZE=1 note that this will exit the target before main() - qemu_mode: diff --git a/src/afl-cc.c b/src/afl-cc.c index ffd15476..c0449e64 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -498,13 +498,10 @@ static void edit_params(u32 argc, char **argv, char **envp) { alloc_printf("-fplugin=%s/afl-gcc-cmptrs-pass.so", obj_path); cc_params[cc_par_cnt++] = fplugin_arg; - } else { - - fplugin_arg = alloc_printf("-fplugin=%s/afl-gcc-pass.so", obj_path); - cc_params[cc_par_cnt++] = fplugin_arg; - } + fplugin_arg = alloc_printf("-fplugin=%s/afl-gcc-pass.so", obj_path); + cc_params[cc_par_cnt++] = fplugin_arg; cc_params[cc_par_cnt++] = "-fno-if-conversion"; cc_params[cc_par_cnt++] = "-fno-if-conversion2"; From eb5a914ef670d43cc41ce130edb4e0586d97e278 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 25 Aug 2022 15:52:46 +0200 Subject: [PATCH 25/25] fix pizza mode --- src/afl-fuzz-state.c | 6 +----- src/afl-fuzz.c | 2 +- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/src/afl-fuzz-state.c b/src/afl-fuzz-state.c index 6770839a..0576f84f 100644 --- a/src/afl-fuzz-state.c +++ b/src/afl-fuzz-state.c @@ -604,11 +604,7 @@ void read_afl_environment(afl_state_t *afl, char **envp) { } - if (afl->afl_env.afl_pizza_mode == 0) { - - afl->afl_env.afl_pizza_mode = 1; - - } else { + if (afl->afl_env.afl_pizza_mode) { afl->pizza_is_served = 1; diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index e705f187..1f0fcab1 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2560,7 +2560,7 @@ stop_fuzzing: write_bitmap(afl); save_auto(afl); - if (afl->afl_env.afl_pizza_mode) { + if (afl->pizza_is_served) { SAYF(CURSOR_SHOW cLRD "\n\n+++ Baking aborted %s +++\n" cRST, afl->stop_soon == 2 ? "programmatically" : "by the chef");