Merge pull request #1551 from AFLplusplus/dev

push to stable

GNUmakefile

@@ -592,6 +592,7 @@ clean:
 	-$(MAKE) -C utils/argv_fuzzing clean
 	-$(MAKE) -C utils/plot_ui clean
 	-$(MAKE) -C qemu_mode/unsigaction clean
+	-$(MAKE) -C qemu_mode/fastexit clean
 	-$(MAKE) -C qemu_mode/libcompcov clean
 	-$(MAKE) -C qemu_mode/libqasan clean
 	-$(MAKE) -C frida_mode clean

GNUmakefile.llvm

@@ -214,6 +214,17 @@ ifeq "$(LLVM_LTO)" "1"
     ifeq "$(AFL_REAL_LD)" ""
       ifneq "$(shell readlink $(LLVM_BINDIR)/ld.lld 2>&1)" ""
         AFL_REAL_LD = $(LLVM_BINDIR)/ld.lld
+      else ifneq "$(shell command -v ld.lld 2>/dev/null)" ""
+        AFL_REAL_LD = $(shell command -v ld.lld)
+        TMP_LDLDD_VERSION = $(shell $(AFL_REAL_LD) --version | awk '{ print $$2 }')
+        ifeq "$(LLVMVER)" "$(TMP_LDLDD_VERSION)"
+          $(warning ld.lld found in a weird location ($(AFL_REAL_LD)), but its the same version as LLVM so we will allow it)
+        else
+          $(warning ld.lld found in a weird location ($(AFL_REAL_LD)) and its of a different version than LLMV ($(TMP_LDLDD_VERSION) vs. $(LLVMVER)) - cannot enable LTO mode)
+          AFL_REAL_LD=
+          LLVM_LTO = 0
+        endif
+        undefine TMP_LDLDD_VERSION
       else
         $(warning ld.lld not found, cannot enable LTO mode)
         LLVM_LTO = 0
@@ -229,7 +240,7 @@ AFL_CLANG_FUSELD=
 ifeq "$(LLVM_LTO)" "1"
   ifeq "$(shell echo 'int main() {return 0; }' | $(CLANG_BIN) -x c - -fuse-ld=`command -v ld` -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1"
     AFL_CLANG_FUSELD=1
-    ifeq "$(shell echo 'int main() {return 0; }' | $(CLANG_BIN) -x c - -fuse-ld=ld.lld --ld-path=$(LLVM_BINDIR)/ld.lld -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1"
+    ifeq "$(shell echo 'int main() {return 0; }' | $(CLANG_BIN) -x c - -fuse-ld=ld.lld --ld-path=$(AFL_REAL_LD) -o .test 2>/dev/null && echo 1 || echo 0 ; rm -f .test )" "1"
       AFL_CLANG_LDPATH=1
     endif
   else

README.md

@@ -2,7 +2,7 @@
 
 <img align="right" src="https://raw.githubusercontent.com/AFLplusplus/Website/master/static/aflpp_bg.svg" alt="AFL++ logo" width="250" heigh="250">
 
-Release version: [4.03c](https://github.com/AFLplusplus/AFLplusplus/releases)
+Release version: [4.04c](https://github.com/AFLplusplus/AFLplusplus/releases)
 
 GitHub version: 4.04a
 

docs/Changelog.md

@@ -4,12 +4,24 @@
   release of the tool. See README.md for the general instruction manual.
 
 
-### Version ++4.04a (dev)
-  - fix gramatron and grammar_mutatur build scripts
+### Version ++4.04c (release)
+  - fix gramatron and grammar_mutator build scripts
   - enhancements to the afl-persistent-config and afl-system-config
     scripts
+  - afl-fuzz:
+    - force writing all stats on exit
   - afl-cc:
     - make gcc_mode (afl-gcc-fast) work with gcc down to version 3.6
+  - qemu_mode:
+    - fixed 10x speed degredation in v4.03c, thanks to @ele7enxxh for
+      reporting!
+    - added qemu_mode/fastexit helper library
+  - unicorn_mode:
+    - Enabled tricore arch (by @jma-qb)
+    - Updated Capstone version in Rust bindings
+  - llvm-mode:
+    - AFL runtime will always pass inputs via shared memory, when possible,
+      ignoring the command line.
 
 
 ### Version ++4.03c (release)

include/config.h

@@ -26,7 +26,7 @@
 /* Version string: */
 
 // c = release, a = volatile github dev, e = experimental branch
-#define VERSION "++4.04a"
+#define VERSION "++4.04c"
 
 /******************************************************
  *                                                    *

instrumentation/README.llvm.md

@@ -116,7 +116,7 @@ PCGUARD analysis.
 Several options are present to make llvm_mode faster or help it rearrange the
 code to make afl-fuzz path discovery easier.
 
-If you need just to instrument specific parts of the code, you can the
+If you need just to instrument specific parts of the code, you can create the
 instrument file list which C/C++ files to actually instrument. See
 [README.instrument_list.md](README.instrument_list.md)
 
@@ -275,4 +275,4 @@ then this can give a small performance boost.
 Please note that the default counter implementations are not thread safe!
 
 Support for thread safe counters in mode LLVM CLASSIC can be activated with
-setting `AFL_LLVM_THREADSAFE_INST=1`.
+setting `AFL_LLVM_THREADSAFE_INST=1`.

instrumentation/afl-compiler-rt.o.c

@@ -97,6 +97,7 @@ u8        *__afl_dictionary;
 u8        *__afl_fuzz_ptr;
 static u32 __afl_fuzz_len_dummy;
 u32       *__afl_fuzz_len = &__afl_fuzz_len_dummy;
+int        __afl_sharedmem_fuzzing __attribute__((weak));
 
 u32 __afl_final_loc;
 u32 __afl_map_size = MAP_SIZE;
@@ -119,8 +120,6 @@ __thread PREV_LOC_T __afl_prev_caller[CTX_MAX_K];
 __thread u32        __afl_prev_ctx;
 #endif
 
-int __afl_sharedmem_fuzzing __attribute__((weak));
-
 struct cmp_map *__afl_cmp_map;
 struct cmp_map *__afl_cmp_map_backup;
 
@@ -347,6 +346,22 @@ static void __afl_map_shm(void) {
 
   }
 
+  if (__afl_sharedmem_fuzzing && (!id_str || !getenv(SHM_FUZZ_ENV_VAR) ||
+                                  fcntl(FORKSRV_FD, F_GETFD) == -1 ||
+                                  fcntl(FORKSRV_FD + 1, F_GETFD) == -1)) {
+
+    if (__afl_debug) {
+
+      fprintf(stderr,
+              "DEBUG: running not inside afl-fuzz, disabling shared memory "
+              "testcases\n");
+
+    }
+
+    __afl_sharedmem_fuzzing = 0;
+
+  }
+
   if (!id_str) {
 
     u32 val = 0;
@@ -543,7 +558,7 @@ static void __afl_map_shm(void) {
     if (!__afl_area_ptr_dummy) {
 
       fprintf(stderr,
-              "Error: AFL++ could not aquire %u bytes of memory, exiting!\n",
+              "Error: AFL++ could not acquire %u bytes of memory, exiting!\n",
               __afl_final_loc);
       exit(-1);
 
@@ -757,10 +772,10 @@ static void __afl_start_snapshots(void) {
      assume we're not running in forkserver mode and just execute program. */
 
   status |= (FS_OPT_ENABLED | FS_OPT_SNAPSHOT | FS_OPT_NEWCMPLOG);
-  if (__afl_sharedmem_fuzzing != 0) status |= FS_OPT_SHDMEM_FUZZ;
+  if (__afl_sharedmem_fuzzing) { status |= FS_OPT_SHDMEM_FUZZ; }
   if (__afl_map_size <= FS_OPT_MAX_MAPSIZE)
     status |= (FS_OPT_SET_MAPSIZE(__afl_map_size) | FS_OPT_MAPSIZE);
-  if (__afl_dictionary_len && __afl_dictionary) status |= FS_OPT_AUTODICT;
+  if (__afl_dictionary_len && __afl_dictionary) { status |= FS_OPT_AUTODICT; }
   memcpy(tmp, &status, 4);
 
   if (write(FORKSRV_FD + 1, tmp, 4) != 4) { return; }
@@ -1021,7 +1036,7 @@ static void __afl_start_forkserver(void) {
 
   }
 
-  if (__afl_sharedmem_fuzzing != 0) { status_for_fsrv |= FS_OPT_SHDMEM_FUZZ; }
+  if (__afl_sharedmem_fuzzing) { status_for_fsrv |= FS_OPT_SHDMEM_FUZZ; }
   if (status_for_fsrv) {
 
     status_for_fsrv |= (FS_OPT_ENABLED | FS_OPT_NEWCMPLOG);

instrumentation/afl-gcc-cmplog-pass.so.cc

@@ -245,7 +245,7 @@ struct afl_cmplog_pass : afl_base_pass {
 
       tree   s = make_ssa_name(t);
       gimple g = gimple_build_assign(s, VIEW_CONVERT_EXPR,
-                                      build1(VIEW_CONVERT_EXPR, t, lhs));
+                                     build1(VIEW_CONVERT_EXPR, t, lhs));
       lhs = s;
       gsi_insert_before(&gsi, g, GSI_SAME_STMT);
 
@@ -281,7 +281,7 @@ struct afl_cmplog_pass : afl_base_pass {
     }
 
     /* Insert the call.  */
-    tree    att = build_int_cst(t8u, attr);
+    tree   att = build_int_cst(t8u, attr);
     gimple call;
     if (pass_n)
       call = gimple_build_call(fn, 4, lhs, rhs, att,

instrumentation/afl-gcc-common.h

@@ -501,7 +501,8 @@ struct afl_base_pass : gimple_opt_pass {
 // compatibility for older gcc versions
 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) >= \
     60200                                               /* >= version 6.2.0 */
-#define gimple gimple *
+  #define gimple gimple *
 #else
-#define gimple gimple
+  #define gimple gimple
 #endif
+

instrumentation/afl-gcc-pass.so.cc

@@ -127,7 +127,7 @@
 #include "afl-gcc-common.h"
 #if (__GNUC__ * 10000 + __GNUC_MINOR__ * 100 + __GNUC_PATCHLEVEL__) >= \
     60200                                               /* >= version 6.2.0 */
-#include "memmodel.h"
+  #include "memmodel.h"
 #endif
 
 /* This plugin, being under the same license as GCC, satisfies the

qemu_mode/QEMUAFL_VERSION

@@ -1 +1 @@
-dc19175a0b
+ff9de4fbeb

qemu_mode/README.md

@@ -13,8 +13,8 @@ afl-cc.
 The usual performance cost is 2-5x, which is considerably better than seen so
 far in experiments with tools such as DynamoRIO and PIN.
 
-The idea and much of the initial implementation comes from Andrew Griffiths. The
-actual implementation on current QEMU (shipped as qemuafl) is from Andrea
+The idea and much of the initial implementation comes from Andrew Griffiths.
+The actual implementation on current QEMU (shipped as qemuafl) is from Andrea
 Fioraldi. Special thanks to abiondo that re-enabled TCG chaining.
 
 ## 2) How to use QEMU mode
@@ -30,17 +30,13 @@ glib2-devel).
 Once the binaries are compiled, you can leverage the QEMU tool by calling
 afl-fuzz and all the related utilities with `-Q` in the command line.
 
-Note that QEMU requires a generous memory limit to run; somewhere around 200 MB
-is a good starting point, but considerably more may be needed for more complex
-programs. The default `-m` limit will be automatically bumped up to 200 MB when
-specifying `-Q` to afl-fuzz; be careful when overriding this.
-
 In principle, if you set `CPU_TARGET` before calling ./build_qemu_support.sh,
 you should get a build capable of running non-native binaries (say, you can try
 `CPU_TARGET=arm`). This is also necessary for running 32-bit binaries on a
 64-bit system (`CPU_TARGET=i386`). If you're trying to run QEMU on a different
 architecture, you can also set `HOST` to the cross-compiler prefix to use (for
 example `HOST=arm-linux-gnueabi` to use arm-linux-gnueabi-gcc).
+Another common target is `CPU_TARGET=aarch64`.
 
 You can also compile statically-linked binaries by setting `STATIC=1`. This can
 be useful when compiling QEMU on a different system than the one you're planning
@@ -219,9 +215,6 @@ program may be utilizing. In particular, it does not appear to have full support
 for AVX2/FMA3. Using binaries for older CPUs or recompiling them with
 `-march=core2`, can help.
 
-Beyond that, this is an early-stage mechanism, so fields reports are welcome.
-You can send them to <afl-users@googlegroups.com>.
-
 ## 14) Alternatives: static rewriting
 
 Statically rewriting binaries just once, instead of attempting to translate them
@@ -230,4 +223,4 @@ with peril, because it depends on being able to properly and fully model program
 control flow without actually executing each and every code path.
 
 For more information and hints, check out
-[docs/fuzzing_binary-only_targets.md](../docs/fuzzing_binary-only_targets.md).
+[docs/fuzzing_binary-only_targets.md](../docs/fuzzing_binary-only_targets.md).

qemu_mode/README.persistent.md

@@ -27,11 +27,12 @@ function and will patch the return address (on stack or in the link register) to
 return to START (like WinAFL).
 
 *Note:* If the target is compiled with position independent code (PIE/PIC) qemu
-loads these to a specific base address. For 64 bit you have to add 0x4000000000
-(9 zeroes) and for 32 bit 0x40000000 (7 zeroes) to the address. On strange
-setups the base address set by QEMU for PIE executable may change. You can check
-it printing the process map using `AFL_QEMU_DEBUG_MAPS=1 afl-qemu-trace
-TARGET-BINARY`.
+loads these to a specific base address. For amd64 bit you have to add
+0x4000000000 (9 zeroes) and for 32 bit 0x40000000 (7 zeroes) to the address.
+For aarch64 it is usually 0x5500000000.
+On strange setups the base address set by QEMU for PIE executable may change.
+You can check it printing the process map using
+`AFL_QEMU_DEBUG_MAPS=1 afl-qemu-trace TARGET-BINARY`.
 
 If this address is not valid, afl-fuzz will error during startup with the
 message that the forkserver was not found.

qemu_mode/build_qemu_support.sh

@@ -360,8 +360,10 @@ if ! command -v "$CROSS" > /dev/null ; then
     make -C libcompcov && echo "[+] libcompcov ready"
     echo "[+] Building unsigaction ..."
     make -C unsigaction && echo "[+] unsigaction ready"
+    echo "[+] Building fastexit ..."
+    make -C fastexit && echo "[+] fastexit ready"
     echo "[+] Building libqasan ..."
-    make -C libqasan && echo "[+] unsigaction ready"
+    make -C libqasan && echo "[+] libqasan ready"
     echo "[+] Building qemu libfuzzer helpers ..."
     make -C ../utils/aflpp_driver
   else

qemu_mode/fastexit/Makefile

@@ -0,0 +1,30 @@
+#
+# american fuzzy lop++ - fastexit
+# --------------------------------
+#
+# Written by Andrea Fioraldi <andreafioraldi@gmail.com>
+#
+# Copyright 2019-2022 Andrea Fioraldi. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+
+TARGETS=fastexit.so fastexit32.so fastexit64.so
+
+all:  $(TARGETS)
+
+fastexit.so: fastexit.c
+	@if $(CC) -fPIC -shared fastexit.c -o fastexit.so 2>/dev/null ; then echo "fastexit build success"; else echo "fastexit build failure (that's fine)"; fi
+
+fastexit32.so: fastexit.c
+	@if $(CC) -fPIC -m32 -shared fastexit.c -o fastexit32.so 2>/dev/null ; then echo "fastexit32 build success"; else echo "fastexit32 build failure (that's fine)"; fi
+
+fastexit64.so: fastexit.c
+	@if $(CC) -fPIC -m64 -shared fastexit.c -o fastexit64.so 2>/dev/null ; then echo "fastexit64 build success"; else echo "fastexit64 build failure (that's fine)"; fi
+
+clean:
+	rm -f fastexit.so

qemu_mode/fastexit/README.md

@@ -0,0 +1,5 @@
+# fastexit
+
+This library forces _exit on exit when preloaded to gain speed.
+
+Gives speed on complex targets like Android or Wine.

qemu_mode/fastexit/fastexit.c

@@ -0,0 +1,6 @@
+#include <unistd.h>
+#include <stdlib.h>
+
+void exit(int status) {
+  _exit(status);
+}

qemu_mode/libqasan/malloc.c

@@ -306,9 +306,7 @@ int __libqasan_posix_memalign(void **ptr, size_t align, size_t len) {
 
   }
 
-  size_t rem = len % align;
-  size_t size = len;
-  if (rem) size += rem;
+  size_t size = len + align;
 
   int state = QASAN_SWAP(QASAN_DISABLED);  // disable qasan for this thread
 

src/afl-cc.c

@@ -317,7 +317,7 @@ void parse_fsanitize(char *string) {
   char *tmp = malloc(strlen(ptr));
   u32   count = 0, len, ende = 0;
 
-  if (!new || !tmp) { FATAL("could not aquire memory"); }
+  if (!new || !tmp) { FATAL("could not acquire memory"); }
   strcpy(new, "-fsanitize=");
 
   do {

src/afl-fuzz.c

@@ -2132,6 +2132,20 @@ int main(int argc, char **argv_orig, char **envp) {
 
   }
 
+  if (afl->fsrv.out_file && afl->fsrv.use_shmem_fuzz) {
+
+    afl->fsrv.out_file = NULL;
+    afl->fsrv.use_stdin = 0;
+    if (!afl->unicorn_mode && !afl->fsrv.use_stdin) {
+
+      WARNF(
+          "You specified -f or @@ on the command line but the target harness "
+          "specified fuzz cases via shmem, switching to shmem!");
+
+    }
+
+  }
+
   deunicode_extras(afl);
   dedup_extras(afl);
   if (afl->extras_cnt) { OKF("Loaded a total of %u extras.", afl->extras_cnt); }
@@ -2556,6 +2570,7 @@ int main(int argc, char **argv_orig, char **envp) {
 stop_fuzzing:
 
   afl->force_ui_update = 1;  // ensure the screen is reprinted
+  afl->stop_soon = 1;        // ensure everything is written
   show_stats(afl);           // print the screen one last time
   write_bitmap(afl);
   save_auto(afl);

src/afl-showmap.c

@@ -1268,7 +1268,7 @@ int main(int argc, char **argv_orig, char **envp) {
           (new_map_size > map_size && new_map_size - map_size > MAP_SIZE)) {
 
         if (!be_quiet)
-          ACTF("Aquired new map size for target: %u bytes\n", new_map_size);
+          ACTF("Acquired new map size for target: %u bytes\n", new_map_size);
 
         afl_shm_deinit(&shm);
         afl_fsrv_kill(fsrv);

src/afl-tmin.c

@@ -1252,7 +1252,7 @@ int main(int argc, char **argv_orig, char **envp) {
           (new_map_size > map_size && new_map_size - map_size > MAP_SIZE)) {
 
         if (!be_quiet)
-          ACTF("Aquired new map size for target: %u bytes\n", new_map_size);
+          ACTF("Acquired new map size for target: %u bytes\n", new_map_size);
 
         afl_shm_deinit(&shm);
         afl_fsrv_kill(fsrv);

unicorn_mode/UNICORNAFL_VERSION

@@ -1 +1 @@
-6e00ceac
+0a31c2b28bf7037fe8b0ff376521fdbdf28a9efe

unicorn_mode/samples/speedtest/rust/Cargo.toml

@@ -11,5 +11,5 @@ panic = "abort"
 
 [dependencies]
 unicornafl = { path = "../../../unicornafl/bindings/rust/", version="1.0.0" }
-capstone="0.10.0"
+capstone="0.11.0"
 libc="0.2.66"

utils/aflpp_driver/aflpp_driver.c

@@ -35,6 +35,7 @@ $AFL_HOME/afl-fuzz -i IN -o OUT ./a.out
 #include <assert.h>
 #include <errno.h>
 #include <stdarg.h>
+#include <stdbool.h>
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -68,7 +69,7 @@ __attribute__((weak)) int LLVMFuzzerInitialize(int *argc, char ***argv);
 int                       LLVMFuzzerRunDriver(int *argc, char ***argv,
                                               int (*callback)(const uint8_t *data, size_t size));
 
-// Default nop ASan hooks for manual posisoning when not linking the ASan
+// Default nop ASan hooks for manual poisoning when not linking the ASan
 // runtime
 // https://github.com/google/sanitizers/wiki/AddressSanitizerManualPoisoning
 __attribute__((weak)) void __asan_poison_memory_region(
@@ -290,6 +291,12 @@ int LLVMFuzzerRunDriver(int *argcp, char ***argvp,
 
   }
 
+  bool in_afl = !(!getenv(SHM_FUZZ_ENV_VAR) || !getenv(SHM_ENV_VAR) ||
+                  fcntl(FORKSRV_FD, F_GETFD) == -1 ||
+                  fcntl(FORKSRV_FD + 1, F_GETFD) == -1);
+
+  if (!in_afl) { __afl_sharedmem_fuzzing = 0; }
+
   output_file = stderr;
   maybe_duplicate_stderr();
   maybe_close_fd_mask();
@@ -310,23 +317,20 @@ int LLVMFuzzerRunDriver(int *argcp, char ***argvp,
 
   int N = INT_MAX;
 
-  if (argc == 2 && !strcmp(argv[1], "-")) {
+  if (!in_afl && argc == 2 && !strcmp(argv[1], "-")) {
 
-    __afl_sharedmem_fuzzing = 0;
     __afl_manual_init();
     return ExecuteFilesOnyByOne(argc, argv, callback);
 
-  } else if (argc == 2 && argv[1][0] == '-') {
+  } else if (argc == 2 && argv[1][0] == '-' && argv[1][1]) {
 
     N = atoi(argv[1] + 1);
 
-  } else if (argc == 2 && (N = atoi(argv[1])) > 0) {
+  } else if (argc == 2 && argv[1][0] != '-' && (N = atoi(argv[1])) > 0) {
 
     printf("WARNING: using the deprecated call style `%s %d`\n", argv[0], N);
 
-  } else if (argc > 1) {
-
-    __afl_sharedmem_fuzzing = 0;
+  } else if (!in_afl && argc > 1 && argv[1][0] != '-') {
 
     if (argc == 2) { __afl_manual_init(); }
 

utils/libdislocator/README.md

@@ -34,8 +34,8 @@ heap-related security bugs in several ways:
 
   - Size alignment to `max_align_t` can be enforced with `AFL_ALIGNED_ALLOC=1`. In
     this case, a tail canary is inserted in the padding bytes at the end of the
-    allocated zone. This reduce the ability of libdislocator to detect
-    off-by-one bugs but also it make slibdislocator compliant to the C standard.
+    allocated zone. This reduces the ability of libdislocator to detect
+    off-by-one bugs but also it makes libdislocator compliant to the C standard.
 
 Basically, it is inspired by some of the non-default options available for the
 OpenBSD allocator - see malloc.conf(5) on that platform for reference. It is

utils/libdislocator/libdislocator.so.c

@@ -510,6 +510,24 @@ __attribute__((alloc_size(2, 3))) void *reallocarray(void *ptr, size_t elem_len,
 
 }
 
+int reallocarr(void *ptr, size_t elem_len, size_t elem_cnt) {
+
+  void        *ret = NULL;
+  const size_t elem_tot = elem_len * elem_cnt;
+
+  if (elem_tot == 0) {
+
+    void **h = &ptr;
+    *h = ret;
+    return 0;
+
+  }
+
+  ret = reallocarray(ptr, elem_len, elem_cnt);
+  return ret ? 0 : -1;
+
+}
+
 #if defined(__APPLE__)
 size_t malloc_size(const void *ptr) {