Merge pull request #1301 from AFLplusplus/dev

v4.00c release

.custom-format.py

@@ -6,7 +6,7 @@
 # Written and maintaned by Andrea Fioraldi <andreafioraldi@gmail.com>
 #
 # Copyright 2015, 2016, 2017 Google Inc. All rights reserved.
-# Copyright 2019-2020 AFLplusplus Project. All rights reserved.
+# Copyright 2019-2022 AFLplusplus Project. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

.github/ISSUE_TEMPLATE/bug_report.md

@@ -8,10 +8,11 @@ assignees: ''
 ---
 
 **IMPORTANT**
-1. You have verified that the issue to be present in the current `dev` branch
-2. Please supply the command line options and relevant environment variables, e.g. a copy-paste of the contents of `out/default/fuzzer_setup`
+1. You have verified that the issue to be present in the current `dev` branch.
+2. Please supply the command line options and relevant environment variables,
+   e.g., a copy-paste of the contents of `out/default/fuzzer_setup`.
 
-Thank you for making afl++ better!
+Thank you for making AFL++ better!
 
 **Describe the bug**
 A clear and concise description of what the bug is.

.github/workflows/ci.yml

@@ -7,24 +7,47 @@ on:
     branches: [ stable, dev ]
 
 jobs:
-  build:
+  linux:
     runs-on: '${{ matrix.os }}'
     strategy:
       matrix:
         os: [ubuntu-20.04, ubuntu-18.04]
+    env:
+      AFL_SKIP_CPUFREQ: 1
+      AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1
     steps:
       - uses: actions/checkout@v2
       - name: debug
-        run: apt-cache search plugin-dev | grep gcc- ; echo ; apt-cache search clang-format- | grep clang-format-
+        run: apt-cache search plugin-dev | grep gcc-; echo; apt-cache search clang-format- | grep clang-format-
       - name: update
         run: sudo apt-get update && sudo apt-get upgrade -y
       - name: install packages
         run: sudo apt-get install -y -m -f --install-suggests build-essential git libtool libtool-bin automake bison libglib2.0-0 clang llvm-dev libc++-dev findutils libcmocka-dev python3-dev python3-setuptools ninja-build
       - name: compiler installed
-        run: gcc -v ; echo ; clang -v
+        run: gcc -v; echo; clang -v
       - name: install gcc plugin
         run: sudo apt-get install -y -m -f --install-suggests $(readlink /usr/bin/gcc)-plugin-dev
       - name: build afl++
         run: make distrib ASAN_BUILD=1
       - name: run tests
-        run: sudo -E ./afl-system-config ; export AFL_SKIP_CPUFREQ=1 ; make tests
+        run: sudo -E ./afl-system-config; make tests
+  macos:
+    runs-on: macOS-latest
+    env:
+      AFL_MAP_SIZE: 65536
+      AFL_SKIP_CPUFREQ: 1
+      AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES: 1
+    steps:
+      - uses: actions/checkout@v2
+      - name: install
+        run: brew install make gcc
+      - name: fix install
+        run: cd /usr/local/bin; ln -s gcc-11 gcc; ln -s g++-11 g++; which gcc; gcc -v
+      - name: build
+        run: export PATH=/usr/local/Cellar/llvm/*/":$PATH"; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; sudo -E ./afl-system-config; gmake ASAN_BUILD=1
+      - name: frida
+        run: export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; cd frida_mode; gmake
+      - name: run tests
+        run: sudo -E ./afl-system-config; export CC=/usr/local/Cellar/llvm/*/bin/clang; export CXX="$CC"++; export PATH=/usr/local/Cellar/llvm/*/":/usr/local/bin:$PATH"; export LLVM_CONFIG=/usr/local/Cellar/llvm/*/bin/llvm-config; gmake tests
+      - name: force frida test for MacOS
+        run: export AFL_PATH=`pwd`; /usr/local/bin/gcc -o test-instr test-instr.c; mkdir in; echo > in/in; AFL_NO_UI=1 ./afl-fuzz -O -i in -o out -V 5 -- ./test-instr

.gitignore

@@ -30,6 +30,7 @@ afl-g++-fast
 afl-gotcpu
 afl-ld
 afl-ld-lto
+afl-cs-proxy
 afl-qemu-trace
 afl-showmap
 afl-tmin
@@ -54,6 +55,7 @@ afl-showmap.8
 afl-system-config.8
 afl-tmin.8
 afl-whatsup.8
+afl-persistent-config.8
 afl-c++
 afl-cc
 afl-lto
@@ -85,4 +87,13 @@ gmon.out
 afl-frida-trace.so
 utils/afl_network_proxy/afl-network-client
 utils/afl_network_proxy/afl-network-server
+utils/plot_ui/afl-plot-ui
 *.o.tmp
+utils/afl_proxy/afl-proxy
+utils/optimin/build
+utils/optimin/optimin
+utils/persistent_mode/persistent_demo
+utils/persistent_mode/persistent_demo_new
+utils/persistent_mode/test-instr
+!coresight_mode
+!coresight_mode/coresight-trace

.gitmodules

@@ -7,3 +7,24 @@
 [submodule "qemu_mode/qemuafl"]
 	path = qemu_mode/qemuafl
 	url = https://github.com/AFLplusplus/qemuafl
+[submodule "custom_mutators/gramatron/json-c"]
+	path = custom_mutators/gramatron/json-c
+	url = https://github.com/json-c/json-c
+[submodule "utils/optimin/EvalMaxSAT"]
+	path = utils/optimin/EvalMaxSAT
+	url = https://github.com/FlorentAvellaneda/EvalMaxSAT
+[submodule "coresight_mode/patchelf"]
+	path = coresight_mode/patchelf
+	url = https://github.com/NixOS/patchelf.git
+[submodule "coresight_mode/coresight-trace"]
+	path = coresight_mode/coresight-trace
+	url = https://github.com/RICSecLab/coresight-trace.git
+[submodule "nyx_mode/libnyx"]
+	path = nyx_mode/libnyx
+	url = https://github.com/nyx-fuzz/libnyx.git
+[submodule "nyx_mode/QEMU-Nyx"]
+	path = nyx_mode/QEMU-Nyx
+	url = https://github.com/nyx-fuzz/qemu-nyx.git
+[submodule "nyx_mode/packer"]
+	path = nyx_mode/packer
+	url = https://github.com/nyx-fuzz/packer.git

Android.bp

@@ -1,3 +1,11 @@
+//
+// NOTE: This file is outdated. None of the AFL++ team uses Android hence
+//       we need users to keep this updated.
+//       In the current state it will likely fail, please send fixes!
+//       Also, this should build frida_mode.
+//
+
+
 cc_defaults {
   name: "afl-defaults",
 
@@ -175,7 +183,7 @@ cc_binary_host {
 }
 
 cc_library_static {
-  name: "afl-llvm-rt",
+  name: "afl-compiler-rt",
   compile_multilib: "64",
   vendor_available: true,
   host_supported: true,
@@ -225,6 +233,7 @@ cc_library_headers {
   ],
 }
 
+/*
 cc_prebuilt_library_static {
   name: "libfrida-gum",
   compile_multilib: "64",
@@ -272,7 +281,7 @@ cc_binary {
   ],
 
   static_libs: [
-    "afl-llvm-rt",
+    "afl-compiler-rt",
     "libfrida-gum",
   ],
 
@@ -290,6 +299,7 @@ cc_binary {
     "utils/afl_frida/android",
   ],
 }
+*/
 
 cc_binary {
   name: "afl-fuzz-32",
@@ -346,7 +356,7 @@ cc_binary_host {
 }
 
 cc_library_static {
-  name: "afl-llvm-rt-32",
+  name: "afl-compiler-rt-32",
   compile_multilib: "32",
   vendor_available: true,
   host_supported: true,
@@ -385,6 +395,7 @@ cc_library_static {
   ],
 }
 
+/*
 cc_prebuilt_library_static {
   name: "libfrida-gum-32",
   compile_multilib: "32",
@@ -400,6 +411,7 @@ cc_prebuilt_library_static {
     "utils/afl_frida/android/arm",
   ],
 }
+*/
 
 subdirs = [
   "custom_mutators",

CITATION.cff

@@ -0,0 +1,31 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+  - given-names: Marc
+    family-names: Heuse
+    email: mh@mh-sec.de
+  - given-names: Heiko
+    family-names: Eißfeldt
+    email: heiko.eissfeldt@hexco.de
+  - given-names: Andrea
+    family-names: Fioraldi
+    email: andreafioraldi@gmail.com
+  - given-names: Dominik
+    family-names: Meier
+    email: mail@dmnk.co
+title: "AFL++"
+version: 3.14
+type: software
+date-released: 2021-07-19
+url: "https://github.com/AFLplusplus/AFLplusplus"
+keywords:
+  - fuzzing
+  - fuzzer
+  - fuzz-testing
+  - instrumentation
+  - afl-fuzz
+  - qemu
+  - llvm
+  - unicorn-emulator
+  - securiy
+license: AGPL-3.0-or-later

CONTRIBUTING.md

@@ -1,4 +1,6 @@
-# How to submit a Pull Request to AFLplusplus
+# Contributing to AFL++
+
+## How to submit a pull request
 
 All contributions (pull requests) must be made against our `dev` branch.
 
@@ -15,10 +17,43 @@ project, or added a file in a directory we already format, otherwise run:
 ./.custom-format.py -i file-that-you-have-created.c
 ```
 
-Regarding the coding style, please follow the AFL style.
-No camel case at all and use AFL's macros wherever possible
-(e.g. WARNF, FATAL, MAP_SIZE, ...).
+Regarding the coding style, please follow the AFL style. No camel case at all
+and use AFL's macros wherever possible (e.g., WARNF, FATAL, MAP_SIZE, ...).
 
-Remember that AFLplusplus has to build and run on many platforms, so
-generalize your Makefiles/GNUmakefile (or your patches to our pre-existing
-Makefiles) to be as generic as possible.
+Remember that AFL++ has to build and run on many platforms, so generalize your
+Makefiles/GNUmakefile (or your patches to our pre-existing Makefiles) to be as
+generic as possible.
+
+## How to contribute to the docs
+
+We welcome contributions to our docs.
+
+Before creating a new file, please check if your content matches an existing
+file in one the following folders:
+
+* [docs/](docs/) (this is where you can find most of our docs content)
+* [frida_mode/](frida_mode/)
+* [instrumentation/](instrumentation/)
+* [qemu_mode/](qemu_mode/)
+* [unicorn_mode/](unicorn_mode/)
+
+When working on the docs, please keep the following guidelines in mind:
+
+* Edit or create Markdown files and use Markdown markup.
+  * Do: fuzzing_gui_program.md
+  * Don't: fuzzing_gui_program.txt
+* Use underscore in file names.
+  * Do: fuzzing_network_service.md
+  * Don't: fuzzing-network-service.md
+* Use a maximum of 80 characters per line to make reading in a console easier.
+* Make all pull requests against `dev`, see
+  [#how-to-submit-a-pull-request-to-afl](#how-to-submit-a-pull-request-to-afl).
+
+And finally, here are some best practices for writing docs content:
+
+* Use clear and simple language.
+* Structure your content with headings and paragraphs.
+* Use bulleted lists to present similar content in a way that makes it easy to
+  scan.
+* Use numbered lists for procedures or prioritizing.
+* Link to related content, for example, prerequisites or in-depth discussions.

Dockerfile

@@ -16,6 +16,8 @@ env NO_ARCH_OPT 1
 RUN apt-get update && \
     apt-get -y install --no-install-suggests --no-install-recommends \
     automake \
+    cmake \
+    meson \
     ninja-build \
     bison flex \
     build-essential \

GNUmakefile

@@ -10,7 +10,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 
 # For Heiko:
@@ -32,7 +32,7 @@ VERSION     = $(shell grep '^$(HASH)define VERSION ' ../config.h | cut -d '"' -f
 # PROGS intentionally omit afl-as, which gets installed elsewhere.
 
 PROGS       = afl-fuzz afl-showmap afl-tmin afl-gotcpu afl-analyze
-SH_PROGS    = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-system-config
+SH_PROGS    = afl-plot afl-cmin afl-cmin.bash afl-whatsup afl-system-config afl-persistent-config afl-cc
 MANPAGES=$(foreach p, $(PROGS) $(SH_PROGS), $(p).8) afl-as.8
 ASAN_OPTIONS=detect_leaks=0
 
@@ -42,7 +42,7 @@ ARCH = $(shell uname -m)
 $(info [*] Compiling afl++ for OS $(SYS) on ARCH $(ARCH))
 
 ifdef NO_SPLICING
-  override CFLAGS += -DNO_SPLICING
+  override CFLAGS_OPT += -DNO_SPLICING
 endif
 
 ifdef ASAN_BUILD
@@ -115,13 +115,13 @@ endif
 
 ifdef PROFILING
   $(info Compiling with profiling information, for analysis: gprof ./afl-fuzz gmon.out > prof.txt)
-  CFLAGS_OPT += -pg -DPROFILING=1
-  LDFLAGS += -pg
+  override CFLAGS_OPT += -pg -DPROFILING=1
+  override LDFLAGS += -pg
 endif
 
 ifdef INTROSPECTION
   $(info Compiling with introspection documentation)
-  CFLAGS_OPT += -DINTROSPECTION=1
+  override CFLAGS_OPT += -DINTROSPECTION=1
 endif
 
 ifneq "$(ARCH)" "x86_64"
@@ -136,7 +136,7 @@ endif
 
 ifdef DEBUG
   $(info Compiling DEBUG version of binaries)
-  CFLAGS += -ggdb3 -O0 -Wall -Wextra -Werror
+  override CFLAGS += -ggdb3 -O0 -Wall -Wextra -Werror $(CFLAGS_OPT)
 else
   CFLAGS ?= -O3 -funroll-loops $(CFLAGS_OPT)
 endif
@@ -147,28 +147,28 @@ override CFLAGS += -g -Wno-pointer-sign -Wno-variadic-macros -Wall -Wextra -Wpoi
 
 ifeq "$(SYS)" "FreeBSD"
   override CFLAGS  += -I /usr/local/include/
-  LDFLAGS += -L /usr/local/lib/
+  override LDFLAGS += -L /usr/local/lib/
 endif
 
 ifeq "$(SYS)" "DragonFly"
   override CFLAGS  += -I /usr/local/include/
-  LDFLAGS += -L /usr/local/lib/
+  override LDFLAGS += -L /usr/local/lib/
 endif
 
 ifeq "$(SYS)" "OpenBSD"
   override CFLAGS  += -I /usr/local/include/ -mno-retpoline
-  LDFLAGS += -Wl,-z,notext -L /usr/local/lib/
+  override LDFLAGS += -Wl,-z,notext -L /usr/local/lib/
 endif
 
 ifeq "$(SYS)" "NetBSD"
   override CFLAGS  += -I /usr/pkg/include/
-  LDFLAGS += -L /usr/pkg/lib/
+  override LDFLAGS += -L /usr/pkg/lib/
 endif
 
 ifeq "$(SYS)" "Haiku"
   SHMAT_OK=0
   override CFLAGS  += -DUSEMMAP=1 -Wno-error=format -fPIC
-  LDFLAGS += -Wno-deprecated-declarations -lgnu -lnetwork
+  override LDFLAGS += -Wno-deprecated-declarations -lgnu -lnetwork
   SPECIAL_PERFORMANCE += -DUSEMMAP=1
 endif
 
@@ -244,22 +244,22 @@ ifneq "$(filter Linux GNU%,$(SYS))" ""
  ifndef DEBUG
   override CFLAGS += -D_FORTIFY_SOURCE=2
  endif
-  LDFLAGS += -ldl -lrt -lm
+  override LDFLAGS += -ldl -lrt -lm
 endif
 
 ifneq "$(findstring FreeBSD, $(SYS))" ""
   override CFLAGS  += -pthread
-  LDFLAGS += -lpthread
+  override LDFLAGS += -lpthread
 endif
 
 ifneq "$(findstring NetBSD, $(SYS))" ""
   override CFLAGS  += -pthread
-  LDFLAGS += -lpthread
+  override LDFLAGS += -lpthread
 endif
 
 ifneq "$(findstring OpenBSD, $(SYS))" ""
   override CFLAGS  += -pthread
-  LDFLAGS += -lpthread
+  override LDFLAGS += -lpthread
 endif
 
 COMM_HDR    = include/alloc-inl.h include/config.h include/debug.h include/types.h
@@ -310,12 +310,14 @@ all:	test_x86 test_shm test_python ready $(PROGS) afl-as llvm gcc_plugin test_bu
 
 .PHONY: llvm
 llvm:
-	-$(MAKE) -j -f GNUmakefile.llvm
+	-$(MAKE) -j4 -f GNUmakefile.llvm
 	@test -e afl-cc || { echo "[-] Compiling afl-cc failed. You seem not to have a working compiler." ; exit 1; }
 
 .PHONY: gcc_plugin
 gcc_plugin:
+ifneq "$(SYS)" "Darwin"
 	-$(MAKE) -f GNUmakefile.gcc_plugin
+endif
 
 .PHONY: man
 man:    $(MANPAGES)
@@ -343,14 +345,15 @@ performance-test:	source-only
 help:
 	@echo "HELP --- the following make targets exist:"
 	@echo "=========================================="
-	@echo "all: just the main afl++ binaries"
-	@echo "binary-only: everything for binary-only fuzzing: qemu_mode, unicorn_mode, libdislocator, libtokencap"
-	@echo "source-only: everything for source code fuzzing: gcc_plugin, libdislocator, libtokencap"
+	@echo "all: the main afl++ binaries and llvm/gcc instrumentation"
+	@echo "binary-only: everything for binary-only fuzzing: frida_mode, nyx_mode, qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator, libtokencap"
+	@echo "source-only: everything for source code fuzzing: nyx_mode, libdislocator, libtokencap"
 	@echo "distrib: everything (for both binary-only and source code fuzzing)"
 	@echo "man: creates simple man pages from the help option of the programs"
 	@echo "install: installs everything you have compiled with the build option above"
 	@echo "clean: cleans everything compiled (not downloads when on a checkout)"
 	@echo "deepclean: cleans everything including downloads"
+	@echo "uninstall: uninstall afl++ from the system"
 	@echo "code-format: format the code, do this before you commit and send a PR please!"
 	@echo "tests: this runs the test framework. It is more catered for the developers, but if you run into problems this helps pinpointing the problem"
 	@echo "unit: perform unit tests (based on cmocka and GNU linker)"
@@ -538,7 +541,7 @@ test_build: afl-cc afl-gcc afl-as afl-showmap
 #	echo 1 | ASAN_OPTIONS=detect_leaks=0 ./afl-showmap -m none -q -o .test-instr1 ./test-instr
 #	@rm -f test-instr
 #	@cmp -s .test-instr0 .test-instr1; DR="$$?"; rm -f .test-instr0 .test-instr1; if [ "$$DR" = "0" ]; then echo; echo "Oops, the instrumentation of afl-gcc does not seem to be behaving correctly!"; \
-#		gcc -v 2>&1 | grep -q -- --with-as= && ( echo; echo "Gcc is configured not to use an external assembler with the -B option."; echo "See docs/INSTALL.md section 5 how to build a -B enabled gcc." ) || \
+#		gcc -v 2>&1 | grep -q -- --with-as= && ( echo; echo "Gcc is configured not to use an external assembler with the -B option." ) || \
 #		( echo; echo "Please post to https://github.com/AFLplusplus/AFLplusplus/issues to troubleshoot the issue." ); echo; exit 0; fi
 #	@echo
 #	@echo "[+] All right, the instrumentation of afl-gcc seems to be working!"
@@ -561,67 +564,103 @@ all_done: test_build
 
 .PHONY: clean
 clean:
-	rm -f $(PROGS) libradamsa.so afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-qemu-trace afl-gcc-fast afl-gcc-pass.so afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand
+	rm -rf $(PROGS) afl-fuzz-document afl-as as afl-g++ afl-clang afl-clang++ *.o src/*.o *~ a.out core core.[1-9][0-9]* *.stackdump .test .test1 .test2 test-instr .test-instr0 .test-instr1 afl-cs-proxy afl-qemu-trace afl-gcc-fast afl-g++-fast ld *.so *.8 test/unittests/*.o test/unittests/unit_maybe_alloc test/unittests/preallocable .afl-* afl-gcc afl-g++ afl-clang afl-clang++ test/unittests/unit_hash test/unittests/unit_rand *.dSYM
 	-$(MAKE) -f GNUmakefile.llvm clean
 	-$(MAKE) -f GNUmakefile.gcc_plugin clean
-	$(MAKE) -C utils/libdislocator clean
-	$(MAKE) -C utils/libtokencap clean
+	-$(MAKE) -C utils/libdislocator clean
+	-$(MAKE) -C utils/libtokencap clean
 	$(MAKE) -C utils/aflpp_driver clean
-	$(MAKE) -C utils/afl_network_proxy clean
-	$(MAKE) -C utils/socket_fuzzing clean
-	$(MAKE) -C utils/argv_fuzzing clean
-	$(MAKE) -C qemu_mode/unsigaction clean
-	$(MAKE) -C qemu_mode/libcompcov clean
-	$(MAKE) -C qemu_mode/libqasan clean
+	-$(MAKE) -C utils/afl_network_proxy clean
+	-$(MAKE) -C utils/socket_fuzzing clean
+	-$(MAKE) -C utils/argv_fuzzing clean
+	-$(MAKE) -C utils/plot_ui clean
+	-$(MAKE) -C qemu_mode/unsigaction clean
+	-$(MAKE) -C qemu_mode/libcompcov clean
+	-$(MAKE) -C qemu_mode/libqasan clean
 	-$(MAKE) -C frida_mode clean
+	rm -rf nyx_mode/packer/linux_initramfs/init.cpio.gz nyx_mode/libnyx/libnyx/target/release/* nyx_mode/QEMU-Nyx/x86_64-softmmu/qemu-system-x86_64
 ifeq "$(IN_REPO)" "1"
+	-test -e coresight_mode/coresight-trace/Makefile && $(MAKE) -C coresight_mode/coresight-trace clean || true
 	-test -e qemu_mode/qemuafl/Makefile && $(MAKE) -C qemu_mode/qemuafl clean || true
-	test -e unicorn_mode/unicornafl/Makefile && $(MAKE) -C unicorn_mode/unicornafl clean || true
+	-test -e unicorn_mode/unicornafl/Makefile && $(MAKE) -C unicorn_mode/unicornafl clean || true
+	-test -e nyx_mode/QEMU-Nyx/Makefile && $(MAKE) -C nyx_mode/QEMU-Nyx clean || true
 else
+	rm -rf coresight_mode/coresight_trace
 	rm -rf qemu_mode/qemuafl
 	rm -rf unicorn_mode/unicornafl
 endif
 
 .PHONY: deepclean
 deepclean:	clean
+	rm -rf coresight_mode/coresight-trace
 	rm -rf unicorn_mode/unicornafl
 	rm -rf qemu_mode/qemuafl
+	rm -rf nyx_mode/libnyx nyx_mode/packer nyx_mode/QEMU-Nyx
 ifeq "$(IN_REPO)" "1"
-# NEVER EVER ACTIVATE THAT!!!!! git reset --hard >/dev/null 2>&1 || true
+	git checkout coresight_mode/coresight-trace
 	git checkout unicorn_mode/unicornafl
 	git checkout qemu_mode/qemuafl
+	git checkout nyx_mode/libnyx
+	git checkout nyx_mode/packer
+	git checkout nyx_mode/QEMU-Nyx
 endif
 
 .PHONY: distrib
 distrib: all
-	-$(MAKE) -j -f GNUmakefile.llvm
+	-$(MAKE) -j4 -f GNUmakefile.llvm
+ifneq "$(SYS)" "Darwin"
 	-$(MAKE) -f GNUmakefile.gcc_plugin
-	$(MAKE) -C utils/libdislocator
-	$(MAKE) -C utils/libtokencap
-	$(MAKE) -C utils/afl_network_proxy
-	$(MAKE) -C utils/socket_fuzzing
-	$(MAKE) -C utils/argv_fuzzing
+endif
+	-$(MAKE) -C utils/libdislocator
+	-$(MAKE) -C utils/libtokencap
+	-$(MAKE) -C utils/afl_network_proxy
+	-$(MAKE) -C utils/socket_fuzzing
+	-$(MAKE) -C utils/argv_fuzzing
+	# -$(MAKE) -C utils/plot_ui
 	-$(MAKE) -C frida_mode
+ifneq "$(SYS)" "Darwin"
+ifeq "$(ARCH)" "aarch64"
+	-$(MAKE) -C coresight_mode
+endif
+ifeq "$(SYS)" "Linux"
+	-cd nyx_mode && ./build_nyx_support.sh
+endif
 	-cd qemu_mode && sh ./build_qemu_support.sh
 	-cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+endif
 
 .PHONY: binary-only
 binary-only: test_shm test_python ready $(PROGS)
-	$(MAKE) -C utils/libdislocator
-	$(MAKE) -C utils/libtokencap
-	$(MAKE) -C utils/afl_network_proxy
-	$(MAKE) -C utils/socket_fuzzing
-	$(MAKE) -C utils/argv_fuzzing
+	-$(MAKE) -C utils/libdislocator
+	-$(MAKE) -C utils/libtokencap
+	-$(MAKE) -C utils/afl_network_proxy
+	-$(MAKE) -C utils/socket_fuzzing
+	-$(MAKE) -C utils/argv_fuzzing
+	# -$(MAKE) -C utils/plot_ui
 	-$(MAKE) -C frida_mode
+ifneq "$(SYS)" "Darwin"
+ifeq "$(ARCH)" "aarch64"
+	-$(MAKE) -C coresight_mode
+endif
+ifeq "$(SYS)" "Linux"
+	-cd nyx_mode && ./build_nyx_support.sh
+endif
 	-cd qemu_mode && sh ./build_qemu_support.sh
 	-cd unicorn_mode && unset CFLAGS && sh ./build_unicorn_support.sh
+endif
 
 .PHONY: source-only
 source-only: all
-	-$(MAKE) -j -f GNUmakefile.llvm
+	-$(MAKE) -j4 -f GNUmakefile.llvm
+ifneq "$(SYS)" "Darwin"
 	-$(MAKE) -f GNUmakefile.gcc_plugin
-	$(MAKE) -C utils/libdislocator
-	$(MAKE) -C utils/libtokencap
+endif
+	-$(MAKE) -C utils/libdislocator
+	-$(MAKE) -C utils/libtokencap
+	# -$(MAKE) -C utils/plot_ui
+ifeq "$(SYS)" "Linux"
+	-cd nyx_mode && ./build_nyx_support.sh
+endif
 
 %.8:	%
 	@echo .TH $* 8 $(BUILD_DATE) "afl++" > $@
@@ -650,6 +689,7 @@ install: all $(MANPAGES)
 	@rm -f $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-32.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-64.o $${DESTDIR}$(HELPER_PATH)/afl-gcc-rt.o
 	install -m 755 $(PROGS) $(SH_PROGS) $${DESTDIR}$(BIN_PATH)
 	@if [ -f afl-qemu-trace ]; then install -m 755 afl-qemu-trace $${DESTDIR}$(BIN_PATH); fi
+	@if [ -f utils/plot_ui/afl-plot-ui ]; then install -m 755 utils/plot_ui/afl-plot-ui $${DESTDIR}$(BIN_PATH); fi
 	@if [ -f libdislocator.so ]; then set -e; install -m 755 libdislocator.so $${DESTDIR}$(HELPER_PATH); fi
 	@if [ -f libtokencap.so ]; then set -e; install -m 755 libtokencap.so $${DESTDIR}$(HELPER_PATH); fi
 	@if [ -f libcompcov.so ]; then set -e; install -m 755 libcompcov.so $${DESTDIR}$(HELPER_PATH); fi
@@ -658,11 +698,14 @@ install: all $(MANPAGES)
 	@if [ -f socketfuzz32.so -o -f socketfuzz64.so ]; then $(MAKE) -C utils/socket_fuzzing install; fi
 	@if [ -f argvfuzz32.so -o -f argvfuzz64.so ]; then $(MAKE) -C utils/argv_fuzzing install; fi
 	@if [ -f afl-frida-trace.so ]; then install -m 755 afl-frida-trace.so $${DESTDIR}$(HELPER_PATH); fi
+	@if [ -f libnyx.so ]; then install -m 755 libnyx.so $${DESTDIR}$(HELPER_PATH); fi
 	@if [ -f utils/afl_network_proxy/afl-network-server ]; then $(MAKE) -C utils/afl_network_proxy install; fi
 	@if [ -f utils/aflpp_driver/libAFLDriver.a ]; then set -e; install -m 644 utils/aflpp_driver/libAFLDriver.a $${DESTDIR}$(HELPER_PATH); fi
 	@if [ -f utils/aflpp_driver/libAFLQemuDriver.a ]; then set -e; install -m 644 utils/aflpp_driver/libAFLQemuDriver.a $${DESTDIR}$(HELPER_PATH); fi
 	-$(MAKE) -f GNUmakefile.llvm install
+ifneq "$(SYS)" "Darwin"
 	-$(MAKE) -f GNUmakefile.gcc_plugin install
+endif
 	ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-gcc
 	ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-g++
 	ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang
@@ -674,3 +717,16 @@ install: all $(MANPAGES)
 	install -m 644 docs/*.md $${DESTDIR}$(DOC_PATH)
 	cp -r testcases/ $${DESTDIR}$(MISC_PATH)
 	cp -r dictionaries/ $${DESTDIR}$(MISC_PATH)
+
+.PHONY: uninstall
+uninstall:
+	-cd $${DESTDIR}$(BIN_PATH) && rm -f $(PROGS) $(SH_PROGS) afl-cs-proxy afl-qemu-trace afl-plot-ui afl-fuzz-document afl-network-server afl-g* afl-plot.sh afl-as afl-ld-lto afl-c* afl-lto*
+	-cd $${DESTDIR}$(HELPER_PATH) && rm -f afl-g*.*o afl-llvm-*.*o afl-compiler-*.*o libdislocator.so libtokencap.so libcompcov.so libqasan.so afl-frida-trace.so libnyx.so socketfuzz*.so argvfuzz*.so libAFLDriver.a libAFLQemuDriver.a as afl-as SanitizerCoverage*.so compare-transform-pass.so cmplog-*-pass.so split-*-pass.so dynamic_list.txt
+	-rm -rf $${DESTDIR}$(MISC_PATH)/testcases $${DESTDIR}$(MISC_PATH)/dictionaries
+	-sh -c "ls docs/*.md | sed 's|^docs/|$${DESTDIR}$(DOC_PATH)/|' | xargs rm -f"
+	-cd $${DESTDIR}$(MAN_PATH) && rm -f $(MANPAGES)
+	-rmdir $${DESTDIR}$(BIN_PATH) 2>/dev/null
+	-rmdir $${DESTDIR}$(HELPER_PATH) 2>/dev/null
+	-rmdir $${DESTDIR}$(MISC_PATH) 2>/dev/null
+	-rmdir $${DESTDIR}$(DOC_PATH) 2>/dev/null
+	-rmdir $${DESTDIR}$(MAN_PATH) 2>/dev/null

GNUmakefile.gcc_plugin

@@ -11,13 +11,13 @@
 # from Laszlo Szekeres.
 #
 # Copyright 2015 Google Inc. All rights reserved.
-# Copyright 2019-2020 AFLplusplus Project. All rights reserved.
+# Copyright 2019-2022 AFLplusplus Project. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 #TEST_MMAP=1
 PREFIX      ?= /usr/local

GNUmakefile.llvm

@@ -12,7 +12,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 
 # For Heiko:
@@ -36,7 +36,7 @@ ifeq "$(SYS)" "OpenBSD"
   LLVM_CONFIG ?= $(BIN_PATH)/llvm-config
   HAS_OPT = $(shell test -x $(BIN_PATH)/opt && echo 0 || echo 1)
   ifeq "$(HAS_OPT)" "1"
-    $(warning llvm_mode needs a complete llvm installation (versions 6.0 up to 12) -> e.g. "pkg_add llvm-7.0.1p9")
+    $(warning llvm_mode needs a complete llvm installation (versions 6.0 up to 13) -> e.g. "pkg_add llvm-7.0.1p9")
   endif
 else
   LLVM_CONFIG ?= llvm-config
@@ -46,14 +46,14 @@ LLVMVER  = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/git//' | sed 's
 LLVM_MAJOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/\..*//' )
 LLVM_MINOR = $(shell $(LLVM_CONFIG) --version 2>/dev/null | sed 's/.*\.//' | sed 's/git//' | sed 's/svn//' | sed 's/ .*//' )
 LLVM_UNSUPPORTED = $(shell $(LLVM_CONFIG) --version 2>/dev/null | egrep -q '^[0-2]\.|^3.[0-7]\.' && echo 1 || echo 0 )
-LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | egrep -q '^1[3-9]' && echo 1 || echo 0 )
+LLVM_TOO_NEW = $(shell $(LLVM_CONFIG) --version 2>/dev/null | egrep -q '^1[4-9]' && echo 1 || echo 0 )
 LLVM_NEW_API = $(shell $(LLVM_CONFIG) --version 2>/dev/null | egrep -q '^1[0-9]' && echo 1 || echo 0 )
 LLVM_10_OK = $(shell $(LLVM_CONFIG) --version 2>/dev/null | egrep -q '^1[1-9]|^10\.[1-9]|^10\.0.[1-9]' && echo 1 || echo 0 )
 LLVM_HAVE_LTO = $(shell $(LLVM_CONFIG) --version 2>/dev/null | egrep -q '^1[1-9]' && echo 1 || echo 0 )
 LLVM_BINDIR = $(shell $(LLVM_CONFIG) --bindir 2>/dev/null)
 LLVM_LIBDIR = $(shell $(LLVM_CONFIG) --libdir 2>/dev/null)
 LLVM_STDCXX = gnu++11
-LLVM_APPLE_XCODE = $(shell clang -v 2>&1 | grep -q Apple && echo 1 || echo 0)
+LLVM_APPLE_XCODE = $(shell $(CC) -v 2>&1 | grep -q Apple && echo 1 || echo 0)
 LLVM_LTO   = 0
 
 ifeq "$(LLVMVER)" ""
@@ -279,6 +279,8 @@ CLANG_LFL    = `$(LLVM_CONFIG) --ldflags` $(LDFLAGS)
 # User teor2345 reports that this is required to make things work on MacOS X.
 ifeq "$(SYS)" "Darwin"
   CLANG_LFL += -Wl,-flat_namespace -Wl,-undefined,suppress
+  override LLVM_HAVE_LTO := 0
+  override LLVM_LTO := 0
 else
   CLANG_CPPFL += -Wl,-znodelete
 endif
@@ -306,7 +308,7 @@ ifeq "$(TEST_MMAP)" "1"
 endif
 
 PROGS_ALWAYS = ./afl-cc ./afl-compiler-rt.o ./afl-compiler-rt-32.o ./afl-compiler-rt-64.o 
-PROGS        = $(PROGS_ALWAYS) ./afl-llvm-pass.so ./SanitizerCoveragePCGUARD.so ./split-compares-pass.so ./split-switches-pass.so ./cmplog-routines-pass.so ./cmplog-instructions-pass.so ./cmplog-switches-pass.so ./afl-llvm-dict2file.so ./compare-transform-pass.so ./afl-ld-lto ./afl-llvm-lto-instrumentlist.so ./afl-llvm-lto-instrumentation.so ./SanitizerCoverageLTO.so
+PROGS        = $(PROGS_ALWAYS) ./afl-llvm-pass.so ./SanitizerCoveragePCGUARD.so ./split-compares-pass.so ./split-switches-pass.so ./cmplog-routines-pass.so ./cmplog-instructions-pass.so ./cmplog-switches-pass.so ./afl-llvm-dict2file.so ./compare-transform-pass.so ./afl-ld-lto ./afl-llvm-lto-instrumentlist.so ./SanitizerCoverageLTO.so
 
 # If prerequisites are not given, warn, do not build anything, and exit with code 0
 ifeq "$(LLVMVER)" ""
@@ -388,11 +390,11 @@ instrumentation/afl-llvm-common.o: instrumentation/afl-llvm-common.cc instrument
 ifeq "$(LLVM_MIN_4_0_1)" "0"
 	$(info [!] N-gram branch coverage instrumentation is not available for llvm version $(LLVMVER))
 endif
-	$(CXX) $(CLANG_CPPFL) -DLLVMInsTrim_EXPORTS -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o
+	$(CXX) $(CLANG_CPPFL) -Wdeprecated -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o
 
 ./SanitizerCoveragePCGUARD.so: instrumentation/SanitizerCoveragePCGUARD.so.cc instrumentation/afl-llvm-common.o | test_deps
 ifeq "$(LLVM_10_OK)" "1"
-	-$(CXX) $(CLANG_CPPFL) -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o
+	-$(CXX) $(CLANG_CPPFL) -Wdeprecated -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o
 endif
 
 ./afl-llvm-lto-instrumentlist.so: instrumentation/afl-llvm-lto-instrumentlist.so.cc instrumentation/afl-llvm-common.o
@@ -406,11 +408,6 @@ ifeq "$(LLVM_LTO)" "1"
 endif
 
 ./SanitizerCoverageLTO.so: instrumentation/SanitizerCoverageLTO.so.cc
-ifeq "$(LLVM_LTO)" "1"
-	$(CXX) $(CLANG_CPPFL) -Wno-writable-strings -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o
-endif
-
-./afl-llvm-lto-instrumentation.so: instrumentation/afl-llvm-lto-instrumentation.so.cc instrumentation/afl-llvm-common.o
 ifeq "$(LLVM_LTO)" "1"
 	$(CXX) $(CLANG_CPPFL) -Wno-writable-strings -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) instrumentation/afl-llvm-common.o
 	$(CLANG_BIN) $(CFLAGS_SAFE) $(CPPFLAGS) -Wno-unused-result -O0 $(AFL_CLANG_FLTO) -fPIC -c instrumentation/afl-llvm-rt-lto.o.c -o ./afl-llvm-rt-lto.o
@@ -478,7 +475,7 @@ install: all
 	@if [ -f ./afl-cc ]; then set -e; install -m 755 ./afl-cc $${DESTDIR}$(BIN_PATH); ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-c++; fi
 	@rm -f $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt*.o $${DESTDIR}$(HELPER_PATH)/afl-gcc-rt*.o
 	@if [ -f ./afl-compiler-rt.o ]; then set -e; install -m 755 ./afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt.o ;fi
-	@if [ -f ./afl-lto ]; then set -e; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto++; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ./afl-llvm-lto-instrumentation.so ./afl-llvm-rt-lto*.o ./afl-llvm-lto-instrumentlist.so $${DESTDIR}$(HELPER_PATH); fi
+	@if [ -f ./afl-lto ]; then set -e; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-lto++; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-cc $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ./afl-llvm-rt-lto*.o ./afl-llvm-lto-instrumentlist.so $${DESTDIR}$(HELPER_PATH); fi
 	@if [ -f ./afl-ld-lto ]; then set -e; install -m 755 ./afl-ld-lto $${DESTDIR}$(BIN_PATH); fi
 	@if [ -f ./afl-compiler-rt-32.o ]; then set -e; install -m 755 ./afl-compiler-rt-32.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt-32.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-32.o ;fi
 	@if [ -f ./afl-compiler-rt-64.o ]; then set -e; install -m 755 ./afl-compiler-rt-64.o $${DESTDIR}$(HELPER_PATH); ln -sf afl-compiler-rt-64.o $${DESTDIR}$(HELPER_PATH)/afl-llvm-rt-64.o ; fi

QuickStartGuide.md

@@ -1 +0,0 @@
-docs/QuickStartGuide.md

TODO.md

@@ -1,38 +1,32 @@
 # TODO list for AFL++
 
-## Roadmap 3.00+
+## Should
 
+ - better autodetection of shifting runtime timeout values
  - Update afl->pending_not_fuzzed for MOpt
- - put fuzz target in top line of UI
  - afl-plot to support multiple plot_data
+ - parallel builds for source-only targets
+ - get rid of check_binary, replace with more forkserver communication
+
+## Maybe
+
  - afl_custom_fuzz_splice_optin()
  - afl_custom_splice()
- - better autodetection of shifting runtime timeout values
- - cmplog: use colorization input for havoc?
- - parallel builds for source-only targets
-
+ - cmdline option from-to range for mutations
 
 ## Further down the road
 
-afl-fuzz:
- - setting min_len/max_len/start_offset/end_offset limits for mutation output
-
-qemu_mode:
+QEMU mode/FRIDA mode:
  - non colliding instrumentation
  - rename qemu specific envs to AFL_QEMU (AFL_ENTRYPOINT, AFL_CODE_START/END,
    AFL_COMPCOV_LEVEL?)
- - add AFL_QEMU_EXITPOINT (maybe multiple?), maybe pointless as we have
+ - add AFL_QEMU_EXITPOINT (maybe multiple?), maybe pointless as there is
    persistent mode
- - add/implement AFL_QEMU_INST_LIBLIST and AFL_QEMU_NOINST_PROGRAM
- - add/implement AFL_QEMU_INST_REGIONS as a list of _START/_END addresses
-
 
 ## Ideas
 
  - LTO/sancov: write current edge to prev_loc and use that information when
-   using cmplog or __sanitizer_cov_trace_cmp*. maybe we can deduct by follow
-   up edge numbers that both following cmp paths have been found and then
-   disable working on this edge id -> cmplog_intelligence branch
- - use cmplog colorization taint result for havoc locations?
- - new instrumentation option for a thread-safe variant of feedback to shared mem.
-   The user decides, if this is needed (eg the target is multithreaded).
+   using cmplog or __sanitizer_cov_trace_cmp*. maybe we can deduct by follow up
+   edge numbers that both following cmp paths have been found and then disable
+   working on this edge id -> cmplog_intelligence branch
+ - use cmplog colorization taint result for havoc locations?

afl-cmin

@@ -396,7 +396,7 @@ BEGIN {
       system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -- \""target_bin"\" "prog_args_string" <\""in_dir"/"first_file"\"")
     } else {
       system("cp \""in_dir"/"first_file"\" "stdin_file)
-      system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
+      system( "AFL_CMIN_ALLOW_ANY=1 "AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"/.run_test\" -Z "extra_par" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
     }
 
     first_count = 0
@@ -432,8 +432,8 @@ BEGIN {
     retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string)
   } else {
     print "    Processing "in_count" files (forkserver mode)..."
-#    print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null"
-    retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -A \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
+#    print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null"
+    retval = system( AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
   }
 
   if (retval && !AFL_CMIN_CRASHES_ONLY) {

afl-cmin.bash

@@ -11,7 +11,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 # This tool tries to find the smallest subset of files in the input directory
 # that still trigger the full range of instrumentation data points seen in
@@ -310,7 +310,7 @@ if [ "$STDIN_FILE" = "" ]; then
 else
 
   cp "$IN_DIR/$FIRST_FILE" "$STDIN_FILE"
-  AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" </dev/null
+  AFL_CMIN_ALLOW_ANY=1 "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/.run_test" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null
 
 fi
 
@@ -360,7 +360,7 @@ echo "[*] Obtaining traces for input files in '$IN_DIR'..."
 
       cp "$IN_DIR/$fn" "$STDIN_FILE"
 
-      "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -A "$STDIN_FILE" -- "$@" </dev/null
+      "$SHOWMAP" -m "$MEM_LIMIT" -t "$TIMEOUT" -o "$TRACE_DIR/$fn" -Z $EXTRA_PAR -H "$STDIN_FILE" -- "$@" </dev/null
 
     done
 

afl-persistent-config

@@ -0,0 +1,133 @@
+#!/bin/bash
+# written by jhertz
+# 
+
+test "$1" = "-h" -o "$1" = "-hh" && {
+  echo 'afl-persistent-config'
+  echo
+  echo $0
+  echo
+  echo afl-persistent-config has no command line options
+  echo
+  echo afl-persistent-config permanently reconfigures the system to a high performance fuzzing state.
+  echo "WARNING: this reduces the security of the system!"
+  echo
+  echo Note that there is also afl-system-config which sets additional runtime
+  echo configuration options.
+  exit 0
+}
+
+echo
+echo "WARNING: This scripts makes permanent configuration changes to the system to"
+echo "         increase the performance for fuzzing. As a result, the system also"
+echo "         becomes less secure against attacks! If you use this script, setup"
+echo "         strong firewall rules and only make SSH available as a network"
+echo "         service!"
+echo
+echo -n "Type \"YES\" to continue: "
+read ANSWER
+if [[ "$ANSWER" != "YES" ]]; then
+  echo Input was not YES, aborting ...
+  exit 1
+fi
+
+echo
+PLATFORM=`uname -s`
+
+# check that we're on Mac
+if [[ "$PLATFORM" = "Darwin" ]] ; then
+
+  # check if UID == 0
+  if [[ "$EUID" -ne 0 ]]; then
+    echo "You need to be root to do this. E.g. use \"sudo\""
+    exit 1
+  fi
+
+  # check if SIP is disabled
+  if [[ ! $(csrutil status | grep "disabled") ]]; then
+    echo "SIP needs to be disabled. Restart and press Command-R at reboot, Utilities => Terminal => enter \"csrutil disable\""
+    exit 1
+  fi
+
+  echo "Checks passed."
+
+  echo "Installing /Library/LaunchDaemons/shm_setup.plist"
+
+  cat << EOF > /Library/LaunchDaemons/shm_setup.plist
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+  <dict>
+    <key>Label</key>
+    <string>shmemsetup</string>
+    <key>UserName</key>
+    <string>root</string>
+    <key>GroupName</key>
+    <string>wheel</string>
+    <key>ProgramArguments</key>
+    <array>
+      <string>/usr/sbin/sysctl</string>
+      <string>-w</string>
+      <string>kern.sysv.shmmax=524288000</string>
+      <string>kern.sysv.shmmin=1</string>
+      <string>kern.sysv.shmmni=128</string>
+      <string>kern.sysv.shmseg=48</string>
+      <string>kern.sysv.shmall=131072000</string>
+    </array>
+    <key>KeepAlive</key>
+    <false/>
+    <key>RunAtLoad</key>
+    <true/>
+  </dict>
+</plist>
+EOF
+
+  echo
+  echo "Reboot and enjoy your fuzzing"
+  exit 0
+fi
+
+if [[ "$PLATFORM" = "Linux" ]] ; then
+
+  # check if UID == 0
+  if [[ "$EUID" -ne 0 ]]; then
+    echo "You need to be root to do this. E.g. use \"sudo\""
+    exit 1
+  fi
+
+  echo "Checks passed."
+
+  test -d /etc/sysctl.d || echo Error: /etc/sysctl.d directory not found, cannot install shmem config
+  test -d /etc/sysctl.d -a '!' -e /etc/sysctl.d/99-fuzzing && {
+    echo "Installing /etc/sysctl.d/99-fuzzing"
+    cat << EOF > /etc/sysctl.d/99-fuzzing
+kernel.core_uses_pid=0
+kernel.core_pattern=core
+kernel.randomize_va_space=0
+kernel.sched_child_runs_first=1
+kernel.sched_autogroup_enabled=1
+kernel.sched_migration_cost_ns=50000000
+kernel.sched_latency_ns=250000000
+EOF
+  }
+
+  egrep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub 2>/dev/null || echo Error: /etc/default/grub with GRUB_CMDLINE_LINUX_DEFAULT is not present, cannot set boot options
+  egrep -q '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub 2>/dev/null && {
+    egrep '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub | egrep -q hardened_usercopy=off || {
+      echo "Configuring performance boot options"
+      LINE=`egrep '^GRUB_CMDLINE_LINUX_DEFAULT=' /etc/default/grub | sed 's/^GRUB_CMDLINE_LINUX_DEFAULT=//' | tr -d '"'`
+      OPTIONS="$LINE ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off srbds=off noexec=off noexec32=off tsx=on tsx=on tsx_async_abort=off mitigations=off audit=0 hardened_usercopy=off ssbd=force-off"
+      echo Setting boot options in /etc/default/grub to GRUB_CMDLINE_LINUX_DEFAULT=\"$OPTIONS\"
+      sed -i "s|^GRUB_CMDLINE_LINUX_DEFAULT=.*|GRUB_CMDLINE_LINUX_DEFAULT=\"$OPTIONS\"|" /etc/default/grub
+    }
+  }
+
+  echo
+  echo "Reboot and enjoy your fuzzing"
+  exit 0
+fi
+
+
+
+echo "Error: Unknown platform \"$PLATFORM\", currently supported are Linux and MacOS."
+exit 1

afl-plot

@@ -12,7 +12,7 @@
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 
 get_abs_path() {
@@ -22,16 +22,28 @@ get_abs_path() {
 echo "progress plotting utility for afl-fuzz by Michal Zalewski"
 echo
 
-if [ ! "$#" = "2" ]; then
+GRAPHICAL="0"
+
+if [ "$1"  = "-g" ] || [ "$1" = "--graphical" ]; then
+GRAPHICAL="1"
+shift
+fi
+
+if [ "$#" != "2" ]; then
 
   cat 1>&2 <<_EOF_
-$0 afl_state_dir graph_output_dir
+$0 [ -g | --graphical ] afl_state_dir graph_output_dir
 
-This program generates gnuplot images from afl-fuzz output data. Usage:
+This program generates gnuplot images from afl-fuzz output data.
 
-The afl_state_dir parameter should point to an existing state directory for any
-active or stopped instance of afl-fuzz; while graph_output_dir should point to
-an empty directory where this tool can write the resulting plots to.
+Usage:
+
+    afl_state_dir       should point to an existing state directory for any
+                        active or stopped instance of afl-fuzz
+    graph_output_dir    should point to an empty directory where this
+                        tool can write the resulting plots to
+    -g, --graphical     (optional) display the plots in a graphical window
+                        (you should have built afl-plot-ui to use this option)
 
 The program will put index.html and three PNG images in the output directory;
 you should be able to view it with any web browser of your choice.
@@ -102,18 +114,10 @@ fi
 rm -f "$outputdir/high_freq.png" "$outputdir/low_freq.png" "$outputdir/exec_speed.png" "$outputdir/edges.png"
 mv -f "$outputdir/index.html" "$outputdir/index.html.orig" 2>/dev/null
 
-echo "[*] Generating plots..."
-
-(
-
-cat <<_EOF_
-set terminal png truecolor enhanced size 1000,300 butt
-
-set output '$outputdir/high_freq.png'
-
+GNUPLOT_SETUP="
 #set xdata time
 #set timefmt '%s'
-#set format x "%b %d\n%H:%M"
+#set format x \"%b %d\n%H:%M\"
 set tics font 'small'
 unset mxtics
 unset mytics
@@ -127,36 +131,167 @@ set key outside
 set autoscale xfixmin
 set autoscale xfixmax
 
-set xlabel "relative time in seconds" font "small"
+set xlabel \"relative time in seconds\" font \"small\"
+"
 
-plot '$inputdir/plot_data' using 1:4 with filledcurve x1 title 'total paths' linecolor rgb '#000000' fillstyle transparent solid 0.2 noborder, \\
-     '' using 1:3 with filledcurve x1 title 'current path' linecolor rgb '#f0f0f0' fillstyle transparent solid 0.5 noborder, \\
-     '' using 1:5 with lines title 'pending paths' linecolor rgb '#0090ff' linewidth 3, \\
+PLOT_HF="
+set terminal png truecolor enhanced size 1000,300 butt
+set output '$outputdir/high_freq.png'
+
+$GNUPLOT_SETUP
+
+plot '$inputdir/plot_data' using 1:4 with filledcurve x1 title 'corpus count' linecolor rgb '#000000' fillstyle transparent solid 0.2 noborder, \\
+     '' using 1:3 with filledcurve x1 title 'current fuzz item' linecolor rgb '#f0f0f0' fillstyle transparent solid 0.5 noborder, \\
+     '' using 1:5 with lines title 'pending items' linecolor rgb '#0090ff' linewidth 3, \\
      '' using 1:6 with lines title 'pending favs' linecolor rgb '#c00080' linewidth 3, \\
      '' using 1:2 with lines title 'cycles done' linecolor rgb '#c000f0' linewidth 3
+"
 
+PLOT_LF="
 set terminal png truecolor enhanced size 1000,200 butt
 set output '$outputdir/low_freq.png'
 
+$GNUPLOT_SETUP
+
 plot '$inputdir/plot_data' using 1:8 with filledcurve x1 title '' linecolor rgb '#c00080' fillstyle transparent solid 0.2 noborder, \\
      '' using 1:8 with lines title ' uniq crashes' linecolor rgb '#c00080' linewidth 3, \\
      '' using 1:9 with lines title 'uniq hangs' linecolor rgb '#c000f0' linewidth 3, \\
      '' using 1:10 with lines title 'levels' linecolor rgb '#0090ff' linewidth 3
+"
 
+PLOT_ES="
 set terminal png truecolor enhanced size 1000,200 butt
 set output '$outputdir/exec_speed.png'
 
+$GNUPLOT_SETUP
+
 plot '$inputdir/plot_data' using 1:11 with filledcurve x1 title '' linecolor rgb '#0090ff' fillstyle transparent solid 0.2 noborder, \\
      '$inputdir/plot_data' using 1:11 with lines title '    execs/sec' linecolor rgb '#0090ff' linewidth 3 smooth bezier;
+"
 
+PLOT_EG="
 set terminal png truecolor enhanced size 1000,300 butt
 set output '$outputdir/edges.png'
 
+$GNUPLOT_SETUP
+
 plot '$inputdir/plot_data' using 1:13 with lines title '        edges' linecolor rgb '#0090ff' linewidth 3
+"
+
+if [ "$#" = "2" ] && [ "$GRAPHICAL" = "1" ]; then
+
+afl-plot-ui -h > /dev/null 2>&1
+
+if [ "$?" != "0" ]; then
+
+cat 1>&2 <<_EOF_
+You do not seem to have the afl-plot-ui utility installed. If you have installed afl-plot-ui, make sure the afl-plot-ui executable is in your PATH.
+If you are still facing any problems, please open an issue at https://github.com/AFLplusplus/AFLplusplus/issues.
+
+No plots have been generated. Please rerun without the "-g" or "--graphical" flag to generate the plots.
+_EOF_
+
+exit 1
+
+fi
+
+rm -rf "$outputdir/.tmp"
+mkdir -p "$outputdir/.tmp"
+mkfifo "$outputdir/.tmp/win_ids" || exit 1
+
+afl-plot-ui > "$outputdir/.tmp/win_ids" &
+W_IDS=$(cat "$outputdir/.tmp/win_ids")
+
+rm -rf "$outputdir/.tmp"
+
+W_ID1=$(echo "$W_IDS" | head -n 1)
+W_ID2=$(echo "$W_IDS" | head -n 2 | tail -n 1)
+W_ID3=$(echo "$W_IDS" | head -n 3 | tail -n 1)
+W_ID4=$(echo "$W_IDS" | tail -n 1)
+
+echo "[*] Generating plots..."
+
+(
+
+cat << _EOF_
+
+$PLOT_HF
+set term x11 window "$W_ID3"
+set output
+replot
+pause mouse close
 
 _EOF_
 
-) | gnuplot 
+) | gnuplot 2> /dev/null &
+
+(
+
+cat << _EOF_
+
+$PLOT_LF
+set term x11 window "$W_ID4"
+set output
+replot
+pause mouse close
+
+_EOF_
+
+) | gnuplot 2> /dev/null &
+
+(
+
+cat << _EOF_
+
+$PLOT_ES
+set term x11 window "$W_ID2"
+set output
+replot
+pause mouse close
+
+_EOF_
+
+) | gnuplot 2> /dev/null &
+
+(
+
+cat << _EOF_
+
+$PLOT_EG
+set term x11 window "$W_ID1"
+set output
+replot
+pause mouse close
+
+_EOF_
+
+) | gnuplot 2> /dev/null &
+
+sleep 1
+
+else
+
+echo "[*] Generating plots..."
+
+(
+
+cat << _EOF_
+
+$PLOT_HF
+
+$PLOT_LF
+
+$PLOT_ES
+
+$PLOT_EG
+
+_EOF_
+
+) | gnuplot
+
+echo "[?] You can also use -g flag to view the plots in an GUI window, and interact with the plots (if you have built afl-plot-ui). Run \"afl-plot-h\" to know more."
+
+fi
 
 if [ ! -s "$outputdir/exec_speed.png" ]; then
 

afl-system-config

@@ -6,10 +6,12 @@ test "$1" = "-h" -o "$1" = "-hh" && {
   echo
   echo afl-system-config has no command line options
   echo
-  echo afl-system reconfigures the system to a high performance fuzzing state
+  echo afl-system-config reconfigures the system to a high performance fuzzing state.
   echo "WARNING: this reduces the security of the system!"
   echo
-  exit 1
+  echo Note that there is also afl-persistent-config which sets additional permanent
+  echo configuration options.
+  exit 0
 }
 
 DONE=
@@ -32,8 +34,8 @@ if [ "$PLATFORM" = "Linux" ] ; then
   sysctl -w kernel.randomize_va_space=0
   sysctl -w kernel.sched_child_runs_first=1
   sysctl -w kernel.sched_autogroup_enabled=1
-  sysctl -w kernel.sched_migration_cost_ns=50000000
-  sysctl -w kernel.sched_latency_ns=250000000
+  sysctl -w kernel.sched_migration_cost_ns=50000000 2>/dev/null
+  sysctl -w kernel.sched_latency_ns=250000000 2>/dev/null
   echo never > /sys/kernel/mm/transparent_hugepage/enabled
   test -e /sys/devices/system/cpu/cpufreq/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/scaling_governor
   test -e /sys/devices/system/cpu/cpufreq/policy0/scaling_governor && echo performance | tee /sys/devices/system/cpu/cpufreq/policy*/scaling_governor
@@ -50,7 +52,7 @@ if [ "$PLATFORM" = "Linux" ] ; then
     echo '  /etc/default/grub:GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=0 l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off srbds=off noexec=off noexec32=off tsx=on tsx_async_abort=off arm64.nopauth audit=0 hardened_usercopy=off ssbd=force-off"'
     echo
   }
-  echo If you run fuzzing instances in docker, run them with \"--security-opt seccomp=unconfined\" for more speed
+  echo If you run fuzzing instances in docker, run them with \"--security-opt seccomp=unconfined\" for more speed.
   echo
   DONE=1
 fi
@@ -99,9 +101,10 @@ if [ "$PLATFORM" = "NetBSD" ] ; then
   DONE=1
 fi
 if [ "$PLATFORM" = "Darwin" ] ; then
-  sysctl kern.sysv.shmmax=8388608
+  sysctl kern.sysv.shmmax=524288000
+  sysctl kern.sysv.shmmin=1
   sysctl kern.sysv.shmseg=48
-  sysctl kern.sysv.shmall=98304
+  sysctl kern.sysv.shmall=131072000
   echo Settings applied.
   echo
   if [ $(launchctl list 2>/dev/null | grep -q '\.ReportCrash$') ] ; then

afl-whatsup

@@ -6,13 +6,13 @@
 # Originally written by Michal Zalewski
 #
 # Copyright 2015 Google Inc. All rights reserved.
-# Copyright 2019-2020 AFLplusplus Project. All rights reserved.
+# Copyright 2019-2022 AFLplusplus Project. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at:
 #
-#   http://www.apache.org/licenses/LICENSE-2.0
+#   https://www.apache.org/licenses/LICENSE-2.0
 #
 # This tool summarizes the status of any locally-running synchronized
 # instances of afl-fuzz.
@@ -91,9 +91,9 @@ TOTAL_CRASHES=0
 TOTAL_PFAV=0
 TOTAL_PENDING=0
 
-# Time since last path / crash / hang, formatted as string
+# Time since last find / crash / hang, formatted as string
 FMT_TIME="0 days 0 hours"
-FMT_PATH="${RED}none seen yet${NC}"
+FMT_FIND="${RED}none seen yet${NC}"
 FMT_CRASH="none seen yet"
 FMT_HANG="none seen yet"
 
@@ -135,7 +135,7 @@ fmt_duration()
 
 FIRST=true
 TOTAL_WCOP=
-TOTAL_LAST_PATH=0
+TOTAL_LAST_FIND=0
 
 for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do
 
@@ -169,7 +169,7 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do
     fi
 
     DEAD_CNT=$((DEAD_CNT + 1))
-    last_path=0
+    last_find=0
 
     if [ "$PROCESS_DEAD" = "" ]; then
 
@@ -183,17 +183,17 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do
 
   EXEC_SEC=0
   test -z "$RUN_UNIX" -o "$RUN_UNIX" = 0 || EXEC_SEC=$((execs_done / RUN_UNIX))
-  PATH_PERC=$((cur_path * 100 / paths_total))
+  PATH_PERC=$((cur_item * 100 / corpus_count))
 
   TOTAL_TIME=$((TOTAL_TIME + RUN_UNIX))
   TOTAL_EPS=$((TOTAL_EPS + EXEC_SEC))
   TOTAL_EXECS=$((TOTAL_EXECS + execs_done))
-  TOTAL_CRASHES=$((TOTAL_CRASHES + unique_crashes))
+  TOTAL_CRASHES=$((TOTAL_CRASHES + saved_crashes))
   TOTAL_PENDING=$((TOTAL_PENDING + pending_total))
   TOTAL_PFAV=$((TOTAL_PFAV + pending_favs))
 
-  if [ "$last_path" -gt "$TOTAL_LAST_PATH" ]; then
-    TOTAL_LAST_PATH=$last_path
+  if [ "$last_find" -gt "$TOTAL_LAST_FIND" ]; then
+    TOTAL_LAST_FIND=$last_find
   fi
 
   if [ "$SUMMARY_ONLY" = "" ]; then
@@ -210,7 +210,7 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do
       echo "  ${RED}slow execution, $EXEC_SEC execs/sec${NC}"
     fi
 
-    fmt_duration $last_path && FMT_PATH=$DUR_STRING
+    fmt_duration $last_find && FMT_FIND=$DUR_STRING
     fmt_duration $last_crash && FMT_CRASH=$DUR_STRING
     fmt_duration $last_hang && FMT_HANG=$DUR_STRING
     FMT_CWOP="not available"
@@ -220,7 +220,7 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do
       test "$cycles_wo_finds" -gt 50 && FMT_CWOP="${RED}$cycles_wo_finds${NC}"
     }
 
-    echo "  last_path       : $FMT_PATH"
+    echo "  last_find       : $FMT_FIND"
     echo "  last_crash      : $FMT_CRASH"
     echo "  last_hang       : $FMT_HANG"
     echo "  cycles_wo_finds : $FMT_CWOP"
@@ -229,12 +229,12 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do
     MEM_USAGE=$(ps aux | grep $fuzzer_pid | grep -v grep | awk '{print $4}')
 
     echo "  cpu usage $CPU_USAGE%, memory usage $MEM_USAGE%"
-    echo "  cycle $((cycles_done + 1)), lifetime speed $EXEC_SEC execs/sec, path $cur_path/$paths_total (${PATH_PERC}%)"
+    echo "  cycles $((cycles_done + 1)), lifetime speed $EXEC_SEC execs/sec, items $cur_item/$corpus_count (${PATH_PERC}%)"
 
-    if [ "$unique_crashes" = "0" ]; then
+    if [ "$saved_crashes" = "0" ]; then
       echo "  pending $pending_favs/$pending_total, coverage $bitmap_cvg, no crashes yet"
     else
-      echo "  pending $pending_favs/$pending_total, coverage $bitmap_cvg, crash count $unique_crashes (!)"
+      echo "  pending $pending_favs/$pending_total, coverage $bitmap_cvg, crashes saved $saved_crashes (!)"
     fi
 
     echo
@@ -243,7 +243,7 @@ for i in `find . -maxdepth 2 -iname fuzzer_stats | sort`; do
 
 done
 
-# Formatting for total time, time since last path, crash, and hang
+# Formatting for total time, time since last find, crash, and hang
 fmt_duration $((CUR_TIME - TOTAL_TIME)) && FMT_TIME=$DUR_STRING
 # Formatting for total execution
 FMT_EXECS="0 millions"
@@ -263,7 +263,7 @@ TOTAL_DAYS=$((TOTAL_TIME / 60 / 60 / 24))
 TOTAL_HRS=$(((TOTAL_TIME / 60 / 60) % 24))
 
 test -z "$TOTAL_WCOP" && TOTAL_WCOP="not available"
-fmt_duration $TOTAL_LAST_PATH && TOTAL_LAST_PATH=$DUR_STRING
+fmt_duration $TOTAL_LAST_FIND && TOTAL_LAST_FIND=$DUR_STRING
 
 test "$TOTAL_TIME" = "0" && TOTAL_TIME=1
 
@@ -293,15 +293,15 @@ echo "    Cumulative speed : $TOTAL_EPS execs/sec"
 if [ "$ALIVE_CNT" -gt "0" ]; then
   echo "       Average speed : $((TOTAL_EPS / ALIVE_CNT)) execs/sec"
 fi
-echo "       Pending paths : $TOTAL_PFAV faves, $TOTAL_PENDING total"
+echo "       Pending items : $TOTAL_PFAV faves, $TOTAL_PENDING total"
 
 if [ "$ALIVE_CNT" -gt "1" ]; then
   echo "  Pending per fuzzer : $((TOTAL_PFAV/ALIVE_CNT)) faves, $((TOTAL_PENDING/ALIVE_CNT)) total (on average)"
 fi
 
-echo "       Crashes found : $TOTAL_CRASHES locally unique"
+echo "       Crashes saved : $TOTAL_CRASHES"
 echo "Cycles without finds : $TOTAL_WCOP"
-echo "  Time without finds : $TOTAL_LAST_PATH"
+echo "  Time without finds : $TOTAL_LAST_FIND"
 echo
 
 exit 0

coresight_mode/.gitignore

@@ -0,0 +1,2 @@
+.local
+glibc*

coresight_mode/GNUmakefile

@@ -0,0 +1,62 @@
+#!/usr/bin/env make
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2021 Ricerca Security, Inc. All rights reserved.
+
+SHELL:=bash
+PREFIX?=$(shell pwd)/.local
+
+CS_TRACE:=coresight-trace
+
+PATCHELF?=$(PREFIX)/bin/patchelf
+
+PATCH_DIR:=patches
+
+GLIBC_VER:=2.33
+GLIBC_NAME:=glibc-$(GLIBC_VER)
+GLIBC_URL_BASE:=http://ftp.gnu.org/gnu/glibc
+GLIBC_LDSO?=$(PREFIX)/lib/ld-linux-aarch64.so.1
+
+OUTPUT?="$(TARGET).patched"
+
+all: build
+
+build:
+	git submodule update --init --recursive $(CS_TRACE)
+	$(MAKE) -C $(CS_TRACE)
+	cp $(CS_TRACE)/cs-proxy ../afl-cs-proxy
+
+patch: | $(PATCHELF) $(GLIBC_LDSO)
+	@if test -z "$(TARGET)"; then echo "TARGET is not set"; exit 1; fi
+	$(PATCHELF) \
+	  --set-interpreter $(GLIBC_LDSO) \
+	  --set-rpath $(dir $(GLIBC_LDSO)) \
+	  --output $(OUTPUT) \
+	  $(TARGET)
+
+$(PATCHELF): patchelf
+	git submodule update --init $<
+	cd $< && \
+	  ./bootstrap.sh && \
+	  ./configure --prefix=$(PREFIX) && \
+	  $(MAKE) && \
+	  $(MAKE) check && \
+	  $(MAKE) install
+
+$(GLIBC_LDSO): | $(GLIBC_NAME).tar.xz
+	tar -xf $(GLIBC_NAME).tar.xz
+	for file in $(shell find $(PATCH_DIR) -maxdepth 1 -type f); do \
+	  patch -p1 < $$file ; \
+	done
+	mkdir -p $(GLIBC_NAME)/build
+	cd $(GLIBC_NAME)/build && \
+	  ../configure --prefix=$(PREFIX) && \
+	  $(MAKE) && \
+	  $(MAKE) install
+
+$(GLIBC_NAME).tar.xz:
+	wget -O $@ $(GLIBC_URL_BASE)/$@
+
+clean:
+	$(MAKE) -C $(CS_TRACE) clean
+
+.PHONY: all build patch clean

coresight_mode/Makefile

@@ -0,0 +1,21 @@
+#!/usr/bin/env make
+# SPDX-License-Identifier: Apache-2.0
+# Copyright 2021 Ricerca Security, Inc. All rights reserved.
+
+all:
+	@echo trying to use GNU make...
+	@gmake all || echo please install GNUmake
+
+build:
+	@echo trying to use GNU make...
+	@gmake build || echo please install GNUmake
+
+patch:
+	@echo trying to use GNU make...
+	@gmake patch || echo please install GNUmake
+
+clean:
+	@echo trying to use GNU make...
+	@gmake clean || echo please install GNUmake
+
+.PHONY: all build patch clean

coresight_mode/README.md

@@ -0,0 +1,70 @@
+# AFL++ CoreSight mode
+
+CoreSight mode enables binary-only fuzzing on ARM64 Linux using CoreSight (ARM's hardware tracing technology).
+
+NOTE: CoreSight mode is in the early development stage. Not applicable for production use.
+Currently the following hardware boards are supported:
+* NVIDIA Jetson TX2 (NVIDIA Parker)
+* NVIDIA Jetson Nano (NVIDIA Tegra X1)
+* GIGABYTE R181-T90 (Marvell ThunderX2 CN99XX)
+
+## Getting started
+
+Please read the [RICSec/coresight-trace README](https://github.com/RICSecLab/coresight-trace/blob/master/README.md) and check the prerequisites (capstone) before getting started.
+
+CoreSight mode supports the AFL++ fork server mode to reduce `exec` system call
+overhead. To support it for binary-only fuzzing, it needs to modify the target
+ELF binary to re-link to the patched glibc. We employ this design from
+[PTrix](https://github.com/junxzm1990/afl-pt).
+
+Check out all the git submodules in the `cs_mode` directory:
+
+```bash
+git submodule update --init --recursive
+```
+
+### Build coresight-trace
+
+There are some notes on building coresight-trace. Refer to the [README](https://github.com/RICSecLab/coresight-trace/blob/master/README.md) for the details. Run make in the `cs_mode` directory:
+
+```bash
+make build
+```
+
+Make sure `cs-proxy` is placed in the AFL++ root directory as `afl-cs-proxy`.
+
+### Patch COTS binary
+
+The fork server mode requires patchelf and the patched glibc. The dependency build can be done by just run make:
+
+```bash
+make patch TARGET=$BIN
+```
+
+The above make command builds and installs the dependencies to `$PREFIX` (default to `$PWD/.local`) at the first time. Then, it runs `patchelf` to `$BIN` with output `$OUTPUT` (`$BIN.patched` by default).
+
+### Run afl-fuzz
+
+Run `afl-fuzz` with `-A` option to use CoreSight mode.
+
+```bash
+sudo afl-fuzz -A -i input -o output -- $OUTPUT @@
+```
+
+## Environment Variables
+
+There are AFL++ CoreSight mode-specific environment variables for run-time configuration.
+
+* `AFL_CS_CUSTOM_BIN` overrides the proxy application path. `afl-cs-proxy` will be used if not defined.
+
+* `AFLCS_COV` specifies coverage type on CoreSight trace decoding. `edge` and `path` is supported. The default value is `edge`.
+* `AFLCS_UDMABUF` is the u-dma-buf device number used to store trace data in the DMA region. The default value is `0`.
+
+## TODO List
+
+* Eliminate modified glibc dependency
+* Support parallel fuzzing
+
+## Acknowledgements
+
+This project has received funding from the Acquisition, Technology & Logistics Agency (ATLA) under the National Security Technology Research Promotion Fund 2021 (JPJ004596).

coresight_mode/patches/0001-Add-AFL-forkserver.patch

@@ -0,0 +1,117 @@
+diff --git a/glibc-2.33/elf/rtld.c b/glibc-2.33/elf/rtld.c
+index 596b6ac3..2ee270d4 100644
+--- a/glibc-2.33/elf/rtld.c
++++ b/glibc-2.33/elf/rtld.c
+@@ -169,6 +169,99 @@ uintptr_t __pointer_chk_guard_local
+ strong_alias (__pointer_chk_guard_local, __pointer_chk_guard)
+ #endif
+ 
++#define AFLCS_RTLD 1
++
++#if AFLCS_RTLD
++
++#include <sys/shm.h>
++#include <sys/types.h>
++#include <sys/wait.h>
++#include <dlfcn.h>
++#include <signal.h>
++
++#include <asm/unistd.h>
++#include <unistd.h>
++
++#define FORKSRV_FD 198
++
++#define AFLCS_ENABLE "__AFLCS_ENABLE"
++
++/* We use this additional AFLCS_# AFLCS_#+1 pair to communicate with proxy */
++#define AFLCS_FORKSRV_FD (FORKSRV_FD - 3)
++#define AFLCS_RTLD_SNIPPET do { __cs_start_forkserver(); } while(0)
++
++/* Fork server logic, invoked before we return from _dl_start. */
++
++static void __cs_start_forkserver(void) {
++  int status;
++  pid_t child_pid;
++  static char tmp[4] = {0, 0, 0, 0};
++
++  if (!getenv(AFLCS_ENABLE)) {
++    return;
++  }
++
++  if (write(AFLCS_FORKSRV_FD + 1, tmp, 4) != 4) {
++    _exit(-1);
++  }
++
++  /* All right, let's await orders... */
++  while (1) {
++    /* Whoops, parent dead? */
++    if (read(AFLCS_FORKSRV_FD, tmp, 4) != 4) {
++      _exit(1);
++    }
++
++    child_pid = INLINE_SYSCALL(clone, 5,
++        CLONE_CHILD_SETTID | CLONE_CHILD_CLEARTID | SIGCHLD, 0,
++        NULL, NULL, &THREAD_SELF->tid);
++    if (child_pid < 0) {
++      _exit(4);
++    }
++    if (!child_pid) {
++      /* Child process. Wait for parent start tracing */
++      kill(getpid(), SIGSTOP);
++      /* Close descriptors and run free. */
++      close(AFLCS_FORKSRV_FD);
++      close(AFLCS_FORKSRV_FD + 1);
++      return;
++    }
++
++    /* Parent. */
++    if (write(AFLCS_FORKSRV_FD + 1, &child_pid, 4) != 4) {
++      _exit(5);
++    }
++
++    /* Wait until SIGCONT is signaled. */
++    if (waitpid(child_pid, &status, WCONTINUED) < 0) {
++      _exit(6);
++    }
++    if (!WIFCONTINUED(status)) {
++      /* Relay status to proxy. */
++      if (write(AFLCS_FORKSRV_FD + 1, &status, 4) != 4) {
++        _exit(7);
++      }
++      continue;
++    }
++    while (1) {
++      /* Get status. */
++      if (waitpid(child_pid, &status, WUNTRACED) < 0) {
++        _exit(8);
++      }
++      /* Relay status to proxy. */
++      if (write(AFLCS_FORKSRV_FD + 1, &status, 4) != 4) {
++        _exit(9);
++      }
++      if (!(WIFSTOPPED(status) && WSTOPSIG(status) == SIGSTOP)) {
++        /* The child process is exited. */
++        break;
++      }
++    }
++  }
++}
++
++#endif /* AFLCS_RTLD */
++
+ /* Check that AT_SECURE=0, or that the passed name does not contain
+    directories and is not overly long.  Reject empty names
+    unconditionally.  */
+@@ -588,6 +681,12 @@ _dl_start (void *arg)
+ # define ELF_MACHINE_START_ADDRESS(map, start) (start)
+ #endif
+ 
++    /* AFL-CS-START */
++#if AFLCS_RTLD
++    AFLCS_RTLD_SNIPPET;
++#endif
++    /* AFL-CS-END */
++
+     return ELF_MACHINE_START_ADDRESS (GL(dl_ns)[LM_ID_BASE]._ns_loaded, entry);
+   }
+ }

custom_mutators/README.md

@@ -1,6 +1,6 @@
 # Custom Mutators
 
-Custom mutators enhance and alter the mutation strategies of afl++.
+Custom mutators enhance and alter the mutation strategies of AFL++.
 For further information and documentation on how to write your own, read [the docs](../docs/custom_mutators.md).
 
 ## Examples
@@ -11,10 +11,11 @@ The `./examples` folder contains examples for custom mutators in python and C.
 
 In `./rust`, you will find rust bindings, including a simple example in `./rust/example` and an example for structured fuzzing, based on lain, in`./rust/example_lain`.
 
-## The afl++ Grammar Mutator
+## The AFL++ Grammar Mutator
 
-If you use git to clone afl++, then the following will incorporate our
+If you use git to clone AFL++, then the following will incorporate our
 excellent grammar custom mutator:
+
 ```sh
 git submodule update --init
 ```
@@ -40,7 +41,7 @@ Multiple custom mutators can be used by separating their paths with `:` in the e
 
 ### Superion Mutators
 
-Adrian Tiron ported the Superion grammar fuzzer to afl++, it is WIP and
+Adrian Tiron ported the Superion grammar fuzzer to AFL++, it is WIP and
 requires cmake (among other things):
 [https://github.com/adrian-rt/superion-mutator](https://github.com/adrian-rt/superion-mutator)
 
@@ -52,8 +53,8 @@ transforms protobuf raw:
 https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator
 
 has a transform function you need to fill for your protobuf format, however
-needs to be ported to the updated afl++ custom mutator API (not much work):
+needs to be ported to the updated AFL++ custom mutator API (not much work):
 https://github.com/thebabush/afl-libprotobuf-mutator
 
-same as above but is for current afl++:
+same as above but is for current AFL++:
 https://github.com/P1umer/AFLplusplus-protobuf-mutator

custom_mutators/examples/example.c

@@ -349,12 +349,15 @@ uint8_t afl_custom_queue_get(my_mutator_t *data, const uint8_t *filename) {
  * @param data pointer returned in afl_custom_init for this fuzz case
  * @param filename_new_queue File name of the new queue entry
  * @param filename_orig_queue File name of the original queue entry
+ * @return if the file contents was modified return 1 (True), 0 (False)
+ *         otherwise
  */
-void afl_custom_queue_new_entry(my_mutator_t * data,
-                                const uint8_t *filename_new_queue,
-                                const uint8_t *filename_orig_queue) {
+uint8_t afl_custom_queue_new_entry(my_mutator_t * data,
+                                   const uint8_t *filename_new_queue,
+                                   const uint8_t *filename_orig_queue) {
 
   /* Additional analysis on the original or new test case */
+  return 0;
 
 }
 

custom_mutators/gramatron/JSONC_VERSION

@@ -0,0 +1 @@
+af8dd4a307e7b837f9fa2959549548ace4afe08b

custom_mutators/gramatron/README.md

@@ -0,0 +1,49 @@
+# GramaTron
+
+GramaTron is a coverage-guided fuzzer that uses grammar automatons to perform
+grammar-aware fuzzing.  Technical details about our framework are available in
+the [ISSTA'21 paper](https://nebelwelt.net/files/21ISSTA.pdf). The artifact to
+reproduce the experiments presented in the paper are present in `artifact/`.
+Instructions to run a sample campaign and incorporate new grammars is presented
+below:
+
+## Compiling
+
+Execute `./build_gramatron_mutator.sh`.
+
+## Running
+
+You have to set the grammar file to use with `GRAMATRON_AUTOMATION`:
+
+```
+export AFL_DISABLE_TRIM=1
+export AFL_CUSTOM_MUTATOR_ONLY=1
+export AFL_CUSTOM_MUTATOR_LIBRARY=./gramatron.so
+export GRAMATRON_AUTOMATION=grammars/ruby/source_automata.json
+afl-fuzz -i in -o out -- ./target
+```
+
+## Adding and testing a new grammar
+
+- Specify in a JSON format for CFG. Examples are correspond `source.json` files.
+- Run the automaton generation script (in `src/gramfuzz-mutator/preprocess`)
+  which will place the generated automaton in the same folder.
+
+  ```
+  ./preprocess/prep_automaton.sh <grammar_file> <start_symbol> [stack_limit]
+
+  E.g., ./preprocess/prep_automaton.sh ~/grammars/ruby/source.json PROGRAM
+  ```
+
+- If the grammar has no self-embedding rules, then you do not need to pass the
+  stack limit parameter. However, if it does have self-embedding rules, then you
+  need to pass the stack limit parameter. We recommend starting with `5` and
+  then increasing it if you need more complexity.
+- To sanity-check that the automaton is generating inputs as expected, you can
+  use the `test` binary housed in `src/gramfuzz-mutator`.
+
+  ```
+  ./test SanityCheck <automaton_file>
+
+  E.g., ./test SanityCheck ~/grammars/ruby/source_automata.json
+  ```

custom_mutators/gramatron/build_gramatron_mutator.sh

@@ -0,0 +1,149 @@
+#!/bin/sh
+#
+# american fuzzy lop++ - gramatron build script
+# ------------------------------------------------
+#
+# Originally written by Nathan Voss <njvoss99@gmail.com>
+#
+# Adapted from code by Andrew Griffiths <agriffiths@google.com> and
+#                      Michal Zalewski
+#
+# Adapted for AFLplusplus by Dominik Maier <mail@dmnk.co>
+#
+# Copyright 2017 Battelle Memorial Institute. All rights reserved.
+# Copyright 2019-2022 AFLplusplus Project. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# This script downloads, patches, and builds a version of Unicorn with
+# minor tweaks to allow Unicorn-emulated binaries to be run under
+# afl-fuzz.
+#
+# The modifications reside in patches/*. The standalone Unicorn library
+# will be written to /usr/lib/libunicornafl.so, and the Python bindings
+# will be installed system-wide.
+#
+# You must make sure that Unicorn Engine is not already installed before
+# running this script. If it is, please uninstall it first.
+
+JSONC_VERSION="$(cat ./JSONC_VERSION)"
+JSONC_REPO="https://github.com/json-c/json-c"
+
+echo "================================================="
+echo "Gramatron Mutator build script"
+echo "================================================="
+echo
+
+echo "[*] Performing basic sanity checks..."
+
+PLT=`uname -s`
+
+if [ ! -f "../../config.h" ]; then
+
+  echo "[-] Error: key files not found - wrong working directory?"
+  exit 1
+
+fi
+
+if [ ! -f "../../src/afl-performance.o" ]; then
+
+  echo "[-] Error: you must build afl-fuzz first and not do a \"make clean\""
+  exit 1
+
+fi
+
+PYTHONBIN=`command -v python3 || command -v python || command -v python2 || echo python3`
+MAKECMD=make
+TARCMD=tar
+
+if [ "$PLT" = "Darwin" ]; then
+  CORES=`sysctl -n hw.ncpu`
+  TARCMD=tar
+fi
+
+if [ "$PLT" = "FreeBSD" ]; then
+  MAKECMD=gmake
+  CORES=`sysctl -n hw.ncpu`
+  TARCMD=gtar
+fi
+
+if [ "$PLT" = "NetBSD" ] || [ "$PLT" = "OpenBSD" ]; then
+  MAKECMD=gmake
+  CORES=`sysctl -n hw.ncpu`
+  TARCMD=gtar
+fi
+
+PREREQ_NOTFOUND=
+for i in git $MAKECMD $TARCMD; do
+
+  T=`command -v "$i" 2>/dev/null`
+
+  if [ "$T" = "" ]; then
+
+    echo "[-] Error: '$i' not found. Run 'sudo apt-get install $i' or similar."
+    PREREQ_NOTFOUND=1
+
+  fi
+
+done
+
+test -z "$CC" && export CC=cc
+
+if echo "$CC" | grep -qF /afl-; then
+
+  echo "[-] Error: do not use afl-gcc or afl-clang to compile this tool."
+  PREREQ_NOTFOUND=1
+
+fi
+
+if [ "$PREREQ_NOTFOUND" = "1" ]; then
+  exit 1
+fi
+
+echo "[+] All checks passed!"
+
+echo "[*] Making sure json-c is checked out"
+
+git status 1>/dev/null 2>/dev/null
+if [ $? -eq 0 ]; then
+  echo "[*] initializing json-c submodule"
+  git submodule init || exit 1
+  git submodule update ./json-c 2>/dev/null # ignore errors
+else
+  echo "[*] cloning json-c"
+  test -d json-c || {
+    CNT=1
+    while [ '!' -d json-c -a "$CNT" -lt 4 ]; do
+      echo "Trying to clone json-c (attempt $CNT/3)"
+      git clone "$JSONC_REPO" 
+      CNT=`expr "$CNT" + 1`
+    done
+  }
+fi
+
+test -d json-c || { echo "[-] not checked out, please install git or check your internet connection." ; exit 1 ; }
+echo "[+] Got json-c."
+
+test -e json-c/.libs/libjson-c.a || {
+  cd "json-c" || exit 1
+  echo "[*] Checking out $JSONC_VERSION"
+  sh -c 'git stash && git stash drop' 1>/dev/null 2>/dev/null
+  git checkout "$JSONC_VERSION" || exit 1
+  sh autogen.sh || exit 1
+  export CFLAGS=-fPIC
+  ./configure --disable-shared || exit 1
+  make || exit 1
+  cd ..
+}
+
+echo
+echo
+echo "[+] Json-c successfully prepared!"
+echo "[+] Builing gramatron now."
+$CC -O3 -g -fPIC -Wno-unused-result -Wl,--allow-multiple-definition -I../../include -o gramatron.so -shared -I. -I/prg/dev/include gramfuzz.c gramfuzz-helpers.c gramfuzz-mutators.c gramfuzz-util.c hashmap.c ../../src/afl-performance.o json-c/.libs/libjson-c.a || exit 1
+echo
+echo "[+] gramatron successfully built!"

custom_mutators/gramatron/gramfuzz-helpers.c

@@ -0,0 +1,336 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "afl-fuzz.h"
+#include "gramfuzz.h"
+
+/*Slices from beginning till idx*/
+Array *slice(Array *input, int idx) {
+
+  // printf("\nSlice idx:%d", idx);
+  terminal *origptr;
+  terminal *term_ptr;
+  Array *   sliced = (Array *)malloc(sizeof(Array));
+  initArray(sliced, input->size);
+  // Populate dynamic array members
+  if (idx == 0) { return sliced; }
+  for (int x = 0; x < idx; x++) {
+
+    origptr = &input->start[x];
+    insertArray(sliced, origptr->state, origptr->symbol, origptr->symbol_len,
+                origptr->trigger_idx);
+
+  }
+
+  return sliced;
+
+}
+
+/* Slices from idx till end*/
+Array *slice_inverse(Array *input, int idx) {
+
+  // printf("\nSlice idx:%d", idx);
+  terminal *origptr;
+  terminal *term_ptr;
+  Array *   sliced = (Array *)malloc(sizeof(Array));
+  initArray(sliced, input->size);
+  for (int x = idx; x < input->used; x++) {
+
+    origptr = &input->start[x];
+    insertArray(sliced, origptr->state, origptr->symbol, origptr->symbol_len,
+                origptr->trigger_idx);
+
+  }
+
+  return sliced;
+
+}
+
+/*Carves with `start` included and `end` excluded*/
+Array *carve(Array *input, int start, int end) {
+
+  terminal *origptr;
+  terminal *term_ptr;
+  Array *   sliced = (Array *)malloc(sizeof(Array));
+  initArray(sliced, input->size);
+  for (int x = start; x < end; x++) {
+
+    origptr = &input->start[x];
+    insertArray(sliced, origptr->state, origptr->symbol, origptr->symbol_len,
+                origptr->trigger_idx);
+
+  }
+
+  return sliced;
+
+}
+
+/*Concats prefix + feature *mult*/
+void concatPrefixFeature(Array *prefix, Array *feature) {
+
+  // XXX: Currently we have hardcoded the multiplication threshold for adding
+  // the recursive feature. Might want to fix it to choose a random number upper
+  // bounded by a static value instead.
+  terminal *featureptr;
+  int       len = rand_below(global_afl, RECUR_THRESHOLD);
+  for (int x = 0; x < len; x++) {
+
+    for (int y = 0; y < feature->used; y++) {
+
+      featureptr = &feature->start[y];
+      insertArray(prefix, featureptr->state, featureptr->symbol,
+                  featureptr->symbol_len, featureptr->trigger_idx);
+
+    }
+
+  }
+
+}
+
+void concatPrefixFeatureBench(Array *prefix, Array *feature) {
+
+  // XXX: Currently we have hardcoded the multiplication threshold for adding
+  // the recursive feature. Might want to fix it to choose a random number upper
+  // bounded by a static value instead.
+  terminal *featureptr;
+  int       len =
+      5;  // 5 is the number of times we compare performing random recursion.
+  for (int x = 0; x < len; x++) {
+
+    for (int y = 0; y < feature->used; y++) {
+
+      featureptr = &feature->start[y];
+      insertArray(prefix, featureptr->state, featureptr->symbol,
+                  featureptr->symbol_len, featureptr->trigger_idx);
+
+    }
+
+  }
+
+}
+
+Array *spliceGF(Array *orig, Array *toSplice, int idx) {
+
+  terminal *toSplicePtr;
+  terminal *tempPtr;
+  // Iterate through the splice candidate from the `idx` till end
+  for (int x = idx; x < toSplice->used; x++) {
+
+    toSplicePtr = &toSplice->start[x];
+    insertArray(orig, toSplicePtr->state, toSplicePtr->symbol,
+                toSplicePtr->symbol_len, toSplicePtr->trigger_idx);
+
+  }
+
+  return orig;
+
+}
+
+Array *gen_input(state *pda, Array *input) {
+
+  state *   state_ptr;
+  trigger * trigger_ptr;
+  terminal *term_ptr;
+  int       offset = 0;
+  int       randval, error;
+  // Generating an input for the first time
+  if (input == NULL) {
+
+    input = (Array *)calloc(1, sizeof(Array));
+    initArray(input, INIT_SIZE);
+    curr_state = init_state;
+
+  }
+
+  while (curr_state != final_state) {
+
+    // Retrieving the state from the pda
+    state_ptr = pda + curr_state;
+
+    // Get a random trigger
+    randval = rand_below(global_afl, state_ptr->trigger_len);
+    trigger_ptr = (state_ptr->ptr) + randval;
+
+    // Insert into the dynamic array
+    insertArray(input, curr_state, trigger_ptr->term, trigger_ptr->term_len,
+                randval);
+    curr_state = trigger_ptr->dest;
+    offset += 1;
+
+  }
+
+  return input;
+
+}
+
+Array *gen_input_count(state *pda, Array *input, int *mut_count) {
+
+  state *   state_ptr;
+  trigger * trigger_ptr;
+  terminal *term_ptr;
+  int       offset = 0;
+  int       randval, error;
+  // Generating an input for the first time
+  if (input == NULL) {
+
+    input = (Array *)calloc(1, sizeof(Array));
+    initArray(input, INIT_SIZE);
+    curr_state = init_state;
+
+  }
+
+  while (curr_state != final_state) {
+
+    *mut_count += 1;
+    // Retrieving the state from the pda
+    state_ptr = pda + curr_state;
+
+    // Get a random trigger
+    randval = rand_below(global_afl, state_ptr->trigger_len);
+    trigger_ptr = (state_ptr->ptr) + randval;
+
+    // Insert into the dynamic array
+    insertArray(input, curr_state, trigger_ptr->term, trigger_ptr->term_len,
+                randval);
+    curr_state = trigger_ptr->dest;
+    offset += 1;
+
+  }
+
+  return input;
+
+}
+
+/*Creates a candidate from walk with state hashmap and
+ * recursion hashmap
+ */
+
+Candidate *gen_candidate(Array *input) {
+
+  terminal *  term_ptr;
+  IdxMap_new *idxmapPtr;
+  // Declare the State Hash Table
+  IdxMap_new *idxmapStart =
+      (IdxMap_new *)malloc(sizeof(IdxMap_new) * numstates);
+  for (int x = 0; x < numstates; x++) {
+
+    idxmapPtr = &idxmapStart[x];
+    utarray_new(idxmapPtr->nums, &ut_int_icd);
+
+  }
+
+  char *     trigger;
+  int        state;
+  char *     key;
+  Candidate *candidate = (Candidate *)malloc(sizeof(Candidate));
+  candidate->walk = input;
+  int offset = 0, error;
+
+  // Generate statemap for splicing
+  while (offset < input->used) {
+
+    term_ptr = &input->start[offset];
+    state = term_ptr->state;
+    // char *statenum = state + 1;
+    // int num = atoi(statenum);
+    idxmapPtr = &idxmapStart[state];
+    utarray_push_back(idxmapPtr->nums, &offset);
+    offset += 1;
+
+  }
+
+  candidate->statemap = idxmapStart;
+  return candidate;
+
+}
+
+char *get_state(char *trigger) {
+
+  // Get the state from transition
+  int trigger_idx = 0;
+  printf("\nTrigger:%s", trigger);
+  char *state = (char *)malloc(sizeof(char) * 10);
+  while (trigger[trigger_idx] != '_') {
+
+    state[trigger_idx] = trigger[trigger_idx];
+    trigger_idx += 1;
+
+  }
+
+  printf("\nTrigger Idx:%d", trigger_idx);
+  state[trigger_idx] = '\0';
+  return state;
+
+}
+
+void print_repr(Array *input, char *prefix) {
+
+  size_t    offset = 0;
+  terminal *term_ptr;
+  char      geninput[input->used * 100];
+  if (!input->used) {
+
+    printf("\n=============");
+    printf("\n%s:%s", prefix, "");
+    printf("\n=============");
+    return;
+
+  }
+
+  // This is done to create a null-terminated initial string
+  term_ptr = &input->start[offset];
+  strcpy(geninput, term_ptr->symbol);
+  offset += 1;
+
+  while (offset < input->used) {
+
+    term_ptr = &input->start[offset];
+    strcat(geninput, term_ptr->symbol);
+    offset += 1;
+
+  }
+
+  printf("\n=============");
+  printf("\n%s:%s", prefix, geninput);
+  printf("\n=============");
+
+}
+
+// int main(int argc, char*argv[]) {
+
+//     char *mode;
+//     if (argc == 1) {
+
+//         printf("\nUsage: ./gramfuzzer <mode>");
+//         return -1;
+//     }
+//     if (argc >= 2) {
+
+//         mode = argv[1];
+//         printf("\nMode:%s", mode);
+//     }
+//     if (! strcmp(mode, "Generate")) {
+
+//         GenInputBenchmark();
+//     }
+//     else if (! strcmp(mode, "RandomMutation")) {
+
+//         RandomMutationBenchmark();
+//     }
+//     else if (! strcmp(mode, "Splice")) {
+
+//         SpliceMutationBenchmark();
+//     }
+//     else if (! strcmp(mode, "Recursive")) {
+
+//         RandomRecursiveBenchmark();
+//     }
+//     else {
+
+//         printf("\nUnrecognized mode");
+//         return -1;
+//     }
+//     return 0;
+// }
+

custom_mutators/gramatron/gramfuzz-mutators.c

@@ -0,0 +1,247 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "afl-fuzz.h"
+#include "gramfuzz.h"
+
+Array *performRandomMutation(state *pda, Array *input) {
+
+  terminal *term_ptr;
+  // terminal *prev_ptr;
+  Array *mutated;
+  Array *sliced;
+
+  // Get offset at which to generate new input and slice it
+  int idx = rand_below(global_afl, input->used);
+  sliced = slice(input, idx);
+  // print_repr(sliced, "Slice");
+
+  // prev_ptr = & input->start[idx - 1];
+  // printf("\nState:%s Symbol:%s", prev_ptr->state, prev_ptr->symbol);
+  // Reset current state to that of the slice's last member
+  term_ptr = &input->start[idx];
+  curr_state = term_ptr->state;
+  // printf("\nState:%s Symbol:%s", curr_state, term_ptr->symbol);
+
+  // Set the next available cell to the one adjacent to this chosen point
+  mutated = gen_input(pda, sliced);
+  return mutated;
+
+}
+
+// Tries to perform splice operation between two automaton walks
+UT_icd intpair_icd = {sizeof(intpair_t), NULL, NULL, NULL};
+
+Array *performSpliceOne(Array *originput, IdxMap_new *statemap_orig,
+                        Array *splicecand) {
+
+  UT_array * stateptr, *pairs;
+  intpair_t  ip;
+  intpair_t *cand;
+
+  terminal *term_ptr;
+  Array *   prefix;
+  int       state;
+
+  // Initialize the dynamic holding the splice indice pairs
+  utarray_new(pairs, &intpair_icd);
+  // print_repr(originput, "Orig");
+  // print_repr(splicecand, "SpliceCand");
+
+  // Iterate through the splice candidate identifying potential splice points
+  // and pushing pair (orig_idx, splice_idx) to a dynamic array
+  for (int x = 0; x < splicecand->used; x++) {
+
+    term_ptr = &splicecand->start[x];
+    stateptr = statemap_orig[term_ptr->state].nums;
+    int length = utarray_len(stateptr);
+    if (length) {
+
+      int *splice_idx = (int *)utarray_eltptr(stateptr, rand_below(global_afl, length));
+      ip.orig_idx = *splice_idx;
+      ip.splice_idx = x;
+      utarray_push_back(pairs, &ip);
+
+    }
+
+  }
+
+  // Pick a random pair
+  int length = utarray_len(pairs);
+  cand = (intpair_t *)utarray_eltptr(pairs, rand_below(global_afl, length));
+  // printf("\n Orig_idx:%d Splice_idx:%d", cand->orig_idx, cand->splice_idx);
+
+  // Perform the splicing
+  prefix = slice(originput, cand->orig_idx);
+  Array *spliced = spliceGF(prefix, splicecand, cand->splice_idx);
+  // print_repr(spliced, "Spliced");
+  //
+  utarray_free(pairs);
+
+  return spliced;
+
+}
+
+UT_array **get_dupes(Array *input, int *recur_len) {
+
+  // Variables related to finding duplicates
+  int         offset = 0;
+  int         state;
+  terminal *  term_ptr;
+  IdxMap_new *idxMapPtr;
+  UT_array ** recurIdx;
+
+  // Declare the Recursive Map Table
+  IdxMap_new *idxmapStart =
+      (IdxMap_new *)malloc(sizeof(IdxMap_new) * numstates);
+  //
+  // UT_array *(recurIdx[numstates]);
+  recurIdx = malloc(sizeof(UT_array *) * numstates);
+
+  for (int x = 0; x < numstates; x++) {
+
+    idxMapPtr = &idxmapStart[x];
+    utarray_new(idxMapPtr->nums, &ut_int_icd);
+
+  }
+
+  // Obtain frequency distribution of states
+  while (offset < input->used) {
+
+    term_ptr = &input->start[offset];
+    state = term_ptr->state;
+    // int num = atoi(state + 1);
+    idxMapPtr = &idxmapStart[state];
+    utarray_push_back(idxMapPtr->nums, &offset);
+    offset += 1;
+
+  }
+
+  // Retrieve the duplicated states
+  offset = 0;
+  while (offset < numstates) {
+
+    idxMapPtr = &idxmapStart[offset];
+    int length = utarray_len(idxMapPtr->nums);
+    if (length >= 2) {
+
+      recurIdx[*recur_len] = idxMapPtr->nums;
+      *recur_len += 1;
+
+    }
+
+    // else {
+
+    //     utarray_free(idxMapPtr->nums);
+    // }
+    offset += 1;
+
+  }
+
+  if (*recur_len) {
+
+    // Declare the return struct
+    // We use this struct so that we save the reference to IdxMap_new and free
+    // it after we have used it in doMult
+    // Get_Dupes_Ret* getdupesret =
+    // (Get_Dupes_Ret*)malloc(sizeof(Get_Dupes_Ret));
+    return recurIdx;
+    // getdupesret->idxmap = idxmapStart;
+    // getdupesret->recurIdx = recurIdx;
+    // return getdupesret;
+
+  } else {
+
+    return NULL;
+
+  }
+
+}
+
+Array *doMult(Array *input, UT_array **recur, int recurlen) {
+
+  int       offset = 0;
+  int       idx = rand_below(global_afl, recurlen);
+  UT_array *recurMap = recur[idx];
+  UT_array *recurPtr;
+  Array *   prefix;
+  Array *   postfix;
+  Array *   feature;
+
+  // Choose two indices to get the recursive feature
+  int recurIndices = utarray_len(recurMap);
+  int firstIdx = 0;
+  int secondIdx = 0;
+  getTwoIndices(recurMap, recurIndices, &firstIdx, &secondIdx);
+
+  // Perform the recursive mut
+  // print_repr(input, "Orig");
+  prefix = slice(input, firstIdx);
+  // print_repr(prefix, "Prefix");
+  if (firstIdx < secondIdx) {
+
+    feature = carve(input, firstIdx, secondIdx);
+
+  } else {
+
+    feature = carve(input, secondIdx, firstIdx);
+
+  }
+
+  // print_repr(feature, "Feature");
+  concatPrefixFeature(prefix, feature);
+
+  // GC allocated structures
+  free(feature->start);
+  free(feature);
+  // for(int x = 0; x < recurlen; x++) {
+
+  //     utarray_free(recur[x]);
+  // }
+  // free(recur);
+  // print_repr(prefix, "Concat");
+  return spliceGF(prefix, input, secondIdx);
+
+}
+
+void getTwoIndices(UT_array *recur, int recurlen, int *firstIdx,
+                   int *secondIdx) {
+
+  int ArrayRecurIndices[recurlen];
+  int offset = 0, *p;
+  // Unroll into an array
+  for (p = (int *)utarray_front(recur); p != NULL;
+       p = (int *)utarray_next(recur, p)) {
+
+    ArrayRecurIndices[offset] = *p;
+    offset += 1;
+
+  }
+
+  /*Source:
+   * https://www.geeksforgeeks.org/shuffle-a-given-array-using-fisher-yates-shuffle-algorithm/
+   */
+  for (int i = offset - 1; i > 0; i--) {
+
+    // Pick a random index from 0 to i
+    int j = rand_below(global_afl, i + 1);
+
+    // Swap arr[i] with the element at random index
+    swap(&ArrayRecurIndices[i], &ArrayRecurIndices[j]);
+
+  }
+
+  *firstIdx = ArrayRecurIndices[0];
+  *secondIdx = ArrayRecurIndices[1];
+
+}
+
+void swap(int *a, int *b) {
+
+  int temp = *a;
+  *a = *b;
+  *b = temp;
+
+}
+

custom_mutators/gramatron/gramfuzz-util.c

@@ -0,0 +1,268 @@
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+#include "afl-fuzz.h"
+#include "gramfuzz.h"
+#ifdef _GNU_SOURCE
+  #undef _GNU_SOURCE
+#endif
+#define _GNU_SOURCE
+#include <sys/mman.h>
+
+/* Dynamic Array for adding to the input repr
+ * */
+void initArray(Array *a, size_t initialSize) {
+
+  a->start = (terminal *)calloc(1, sizeof(terminal) * initialSize);
+  a->used = 0;
+  a->size = initialSize;
+  a->inputlen = 0;
+
+}
+
+void insertArray(Array *a, int state, char *symbol, size_t symbol_len,
+                 int trigger_idx) {
+
+  // a->used is the number of used entries, because a->array[a->used++] updates
+  // a->used only *after* the array has been accessed. Therefore a->used can go
+  // up to a->size
+  terminal *term_ptr;
+  if (a->used == a->size) {
+
+    a->size = a->size * sizeof(terminal);
+    a->start = (terminal *)realloc(a->start, a->size * sizeof(terminal));
+
+  }
+
+  // Add the element
+  term_ptr = &a->start[a->used];
+  term_ptr->state = state;
+  term_ptr->symbol = symbol;
+  term_ptr->symbol_len = symbol_len;
+  term_ptr->trigger_idx = trigger_idx;
+
+  // Increment the pointer
+  a->used += 1;
+  a->inputlen += symbol_len;
+
+}
+
+void freeArray(Array *a) {
+
+  terminal *ptr;
+  for (int x = 0; x < a->used; x++) {
+
+    ptr = &a->start[x];
+    free(ptr);
+
+  }
+
+  a->start = NULL;
+  a->used = a->size = 0;
+
+}
+
+/* Dynamic array for adding indices of states/recursive features
+ * Source:
+ * https://stackoverflow.com/questions/3536153/c-dynamically-growing-array
+ */
+void initArrayIdx(IdxMap *a, size_t initialSize) {
+
+  a->array = (int *)malloc(initialSize * sizeof(int));
+  a->used = 0;
+  a->size = initialSize;
+
+}
+
+void insertArrayIdx(IdxMap *a, int idx) {
+
+  // a->used is the number of used entries, because a->array[a->used++] updates
+  // a->used only *after* the array has been accessed. Therefore a->used can go
+  // up to a->size
+  if (a->used == a->size) {
+
+    a->size *= 2;
+    a->array = (int *)realloc(a->array, a->size * sizeof(int));
+
+  }
+
+  a->array[a->used++] = idx;
+
+}
+
+void freeArrayIdx(IdxMap *a) {
+
+  free(a->array);
+  a->array = NULL;
+  a->used = a->size = 0;
+
+}
+
+/* Dynamic array for adding potential splice points
+ */
+void initArraySplice(SpliceCandArray *a, size_t initialSize) {
+
+  a->start = (SpliceCand *)malloc(initialSize * sizeof(SpliceCand));
+  a->used = 0;
+  a->size = initialSize;
+
+}
+
+void insertArraySplice(SpliceCandArray *a, Candidate *candidate, int idx) {
+
+  // a->used is the number of used entries, because a->array[a->used++] updates
+  // a->used only *after* the array has been accessed. Therefore a->used can go
+  // up to a->size
+  SpliceCand *candptr;
+  if (a->used == a->size) {
+
+    a->size = a->size * sizeof(SpliceCand);
+    a->start = (SpliceCand *)realloc(a->start, a->size * sizeof(SpliceCand));
+
+  }
+
+  // Add the element
+  candptr = &a->start[a->used];
+  candptr->splice_cand = candidate;
+  candptr->idx = idx;
+  a->used += 1;
+
+}
+
+void freeArraySplice(IdxMap *a) {
+
+  free(a->array);
+  a->array = NULL;
+  a->used = a->size = 0;
+
+}
+
+int fact(int n) {
+
+  int i, f = 1;
+  for (i = 1; i <= n; i++) {
+
+    f *= i;
+
+  }
+
+  return f;
+
+}
+
+/* Uses the walk to create the input in-memory */
+u8 *unparse_walk(Array *input) {
+
+  terminal *term_ptr;
+  int       offset = 0;
+  u8 *      unparsed = (u8 *)malloc(input->inputlen + 1);
+  term_ptr = &input->start[offset];
+  strcpy(unparsed, term_ptr->symbol);
+  offset += 1;
+  while (offset < input->used) {
+
+    term_ptr = &input->start[offset];
+    strcat(unparsed, term_ptr->symbol);
+    offset += 1;
+
+  }
+
+  return unparsed;
+
+}
+
+/*Dump the input representation into a file*/
+void write_input(Array *input, u8 *fn) {
+
+  FILE *fp;
+  // If file already exists, then skip creating the file
+  if (access(fn, F_OK) != -1) { return; }
+
+  fp = fopen(fn, "wbx+");
+  // If the input has already been flushed, then skip silently
+  if (fp == NULL) {
+
+    fprintf(stderr, "\n File '%s' could not be open, exiting\n", fn);
+    exit(1);
+
+  }
+
+  // Write the length parameters
+  fwrite(&input->used, sizeof(size_t), 1, fp);
+  fwrite(&input->size, sizeof(size_t), 1, fp);
+  fwrite(&input->inputlen, sizeof(size_t), 1, fp);
+
+  // Write the dynamic array to file
+  fwrite(input->start, input->size * sizeof(terminal), 1, fp);
+  // printf("\nUsed:%zu Size:%zu Inputlen:%zu", input->used, input->size,
+  // input->inputlen);
+  fclose(fp);
+
+}
+
+Array *parse_input(state *pda, FILE *fp) {
+
+  terminal *term;
+  state *   state_ptr;
+  trigger * trigger;
+  int       trigger_idx;
+  Array *   input = (Array *)calloc(1, sizeof(Array));
+
+  // Read the length parameters
+  fread(&input->used, sizeof(size_t), 1, fp);
+  fread(&input->size, sizeof(size_t), 1, fp);
+  fread(&input->inputlen, sizeof(size_t), 1, fp);
+
+  terminal *start_ptr = (terminal *)calloc(input->size, sizeof(terminal));
+  if (!start_ptr) {
+
+    fprintf(stderr, "alloc failed!\n");
+    return NULL;
+
+  }
+
+  // Read the dynamic array to memory
+  fread(start_ptr, input->size * sizeof(terminal), 1, fp);
+  // Update the pointers to the terminals since they would have
+  // changed
+  int idx = 0;
+  while (idx < input->used) {
+
+    terminal *term = &start_ptr[idx];
+    // Find the state
+    state_ptr = pda + term->state;
+    // Find the trigger and update the terminal address
+    trigger_idx = term->trigger_idx;
+    trigger = (state_ptr->ptr) + trigger_idx;
+    term->symbol = trigger->term;
+    idx += 1;
+
+  }
+
+  input->start = start_ptr;
+  // printf("\nUsed:%zu Size:%zu Inputlen:%zu", input->used, input->size,
+  // input->inputlen);
+
+  return input;
+
+}
+
+// Read the input representation into memory
+Array *read_input(state *pda, u8 *fn) {
+
+  FILE *fp;
+  fp = fopen(fn, "rb");
+  if (fp == NULL) {
+
+    fprintf(stderr, "\n File '%s' does not exist, exiting\n", fn);
+    exit(1);
+
+  }
+
+  Array *res = parse_input(pda, fp);
+  fclose(fp);
+  return res;
+
+}
+

custom_mutators/gramatron/gramfuzz.c

@@ -0,0 +1,429 @@
+// This simple example just creates random buffer <= 100 filled with 'A'
+// needs -I /path/to/AFLplusplus/include
+//#include "custom_mutator_helpers.h"
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+
+#include "afl-fuzz.h"
+#include "gramfuzz.h"
+
+#define MUTATORS 4  // Specify the total number of mutators
+
+typedef struct my_mutator {
+
+  afl_state_t *afl;
+
+  u8 *   mutator_buf;
+  u8 *   unparsed_input;
+  Array *mutated_walk;
+  Array *orig_walk;
+
+  IdxMap_new *statemap;  // Keeps track of the statemap
+  UT_array ** recurIdx;
+  // Get_Dupes_Ret* getdupesret; // Recursive feature map
+  int recurlen;
+
+  int mut_alloced;
+  int orig_alloced;
+  int mut_idx;  // Signals the current mutator being used, used to cycle through
+                // each mutator
+
+  unsigned int seed;
+
+} my_mutator_t;
+
+state *create_pda(u8 *automaton_file) {
+
+  struct json_object *parsed_json;
+  state *             pda;
+  json_object *       source_obj, *attr;
+  int                 arraylen, ii, ii2, trigger_len, error;
+
+  printf("\n[GF] Automaton file passed:%s", automaton_file);
+  // parsed_json =
+  // json_object_from_file("./gramfuzz/php_gnf_processed_full.json");
+  parsed_json = json_object_from_file(automaton_file);
+
+  // Getting final state
+  source_obj = json_object_object_get(parsed_json, "final_state");
+  printf("\t\nFinal=%s\n", json_object_get_string(source_obj));
+  final_state = atoi(json_object_get_string(source_obj));
+
+  // Getting initial state
+  source_obj = json_object_object_get(parsed_json, "init_state");
+  init_state = atoi(json_object_get_string(source_obj));
+  printf("\tInit=%s\n", json_object_get_string(source_obj));
+
+  // Getting number of states
+  source_obj = json_object_object_get(parsed_json, "numstates");
+  numstates = atoi(json_object_get_string(source_obj)) + 1;
+  printf("\tNumStates=%d\n", numstates);
+
+  // Allocate state space for each pda state
+  pda = (state *)calloc(atoi(json_object_get_string(source_obj)) + 1,
+                        sizeof(state));
+
+  // Getting PDA representation
+  source_obj = json_object_object_get(parsed_json, "pda");
+  enum json_type type;
+  json_object_object_foreach(source_obj, key, val) {
+
+    state *  state_ptr;
+    trigger *trigger_ptr;
+    int      offset;
+
+    // Get the correct offset into the pda to store state information
+    state_ptr = pda;
+    offset = atoi(key);
+    state_ptr += offset;
+    // Store state string
+    state_ptr->state_name = offset;
+
+    // Create trigger array of structs
+    trigger_len = json_object_array_length(val);
+    state_ptr->trigger_len = trigger_len;
+    trigger_ptr = (trigger *)calloc(trigger_len, sizeof(trigger));
+    state_ptr->ptr = trigger_ptr;
+
+    for (ii = 0; ii < trigger_len; ii++) {
+
+      json_object *obj = json_object_array_get_idx(val, ii);
+      // Get all the trigger trigger attributes
+      attr = json_object_array_get_idx(obj, 0);
+      (trigger_ptr)->id = strdup(json_object_get_string(attr));
+
+      attr = json_object_array_get_idx(obj, 1);
+      trigger_ptr->dest = atoi(json_object_get_string(attr));
+
+      attr = json_object_array_get_idx(obj, 2);
+      if (!strcmp("\\n", json_object_get_string(attr))) {
+
+        trigger_ptr->term = strdup("\n");
+
+      } else {
+
+        trigger_ptr->term = strdup(json_object_get_string(attr));
+
+      }
+
+      trigger_ptr->term_len = strlen(trigger_ptr->term);
+      trigger_ptr++;
+
+    }
+
+  }
+
+  // Delete the JSON object
+  json_object_put(parsed_json);
+
+  return pda;
+
+}
+
+my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) {
+
+  my_mutator_t *data = calloc(1, sizeof(my_mutator_t));
+  if (!data) {
+
+    perror("afl_custom_init alloc");
+    return NULL;
+
+  }
+
+  if ((data->mutator_buf = malloc(MAX_FILE)) == NULL) {
+
+    perror("mutator_buf alloc");
+    return NULL;
+
+  }
+
+  data->afl = afl;
+  global_afl = afl;  // dirty
+  data->seed = seed;
+
+  data->mut_alloced = 0;
+  data->orig_alloced = 0;
+  data->mut_idx = 0;
+  data->recurlen = 0;
+
+  // data->mutator_buf = NULL;
+  // data->unparsed_input = NULL;
+  // data->mutated_walk = NULL;
+  // data->orig_walk = NULL;
+  //
+  // data->statemap = NULL;  // Keeps track of the statemap
+  // data->recur_idx = NULL; // Will keep track of recursive feature indices
+  // u32 recur_len = 0;       // The number of recursive features
+  // data->mutator_buf = NULL;
+
+  char *automaton_file = getenv("GRAMATRON_AUTOMATION");
+  if (automaton_file) {
+
+    pda = create_pda(automaton_file);
+
+  } else {
+
+    fprintf(stderr,
+            "\nError: GrammaTron needs an automation json file set in "
+            "GRAMATRON_AUTOMATION\n");
+    exit(-1);
+
+  }
+
+  return data;
+
+}
+
+size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size,
+                       u8 **out_buf, uint8_t *add_buf, size_t add_buf_size,
+                       size_t max_size) {
+
+  u8 *unparsed_input;
+
+  // Pick a mutator
+  // int choice = rand() % MUTATORS;
+  // data->mut_idx = 1;
+  // GC old mutant
+  if (data->mut_alloced) {
+
+    free(data->mutated_walk->start);
+    free(data->mutated_walk);
+    data->mut_alloced = 0;
+
+  };
+
+  // printf("\nChoice:%d", choice);
+
+  if (data->mut_idx == 0) {  // Perform random mutation
+    data->mutated_walk = performRandomMutation(pda, data->orig_walk);
+    data->mut_alloced = 1;
+
+  } else if (data->mut_idx == 1 &&
+
+             data->recurlen) {  // Perform recursive mutation
+    data->mutated_walk =
+        doMult(data->orig_walk, data->recurIdx, data->recurlen);
+    data->mut_alloced = 1;
+
+  } else if (data->mut_idx == 2) {  // Perform splice mutation
+
+    // we cannot use the supplied splice data so choose a new random file
+    u32                 tid = rand_below(global_afl, data->afl->queued_items);
+    struct queue_entry *q = data->afl->queue_buf[tid];
+
+    // Read the input representation for the splice candidate
+    u8 *   automaton_fn = alloc_printf("%s.aut", q->fname);
+    Array *spliceCandidate = read_input(pda, automaton_fn);
+
+    if (spliceCandidate) {
+
+      data->mutated_walk =
+          performSpliceOne(data->orig_walk, data->statemap, spliceCandidate);
+      data->mut_alloced = 1;
+      free(spliceCandidate->start);
+      free(spliceCandidate);
+
+    } else {
+
+      data->mutated_walk = gen_input(pda, NULL);
+      data->mut_alloced = 1;
+
+    }
+
+    ck_free(automaton_fn);
+
+  } else {  // Generate an input from scratch
+
+    data->mutated_walk = gen_input(pda, NULL);
+    data->mut_alloced = 1;
+
+  }
+
+  // Cycle to the next mutator
+  if (data->mut_idx == MUTATORS - 1)
+    data->mut_idx =
+        0;  // Wrap around if we have reached end of the mutator list
+  else
+    data->mut_idx += 1;
+
+  // Unparse the mutated automaton walk
+  if (data->unparsed_input) { free(data->unparsed_input); }
+  data->unparsed_input = unparse_walk(data->mutated_walk);
+  *out_buf = data->unparsed_input;
+
+  return data->mutated_walk->inputlen;
+
+}
+
+/**
+ * Create the automaton-based representation for the corresponding input
+ *
+ * @param data pointer returned in afl_custom_init for this fuzz case
+ * @param filename_new_queue File name of the new queue entry
+ * @param filename_orig_queue File name of the original queue entry
+ */
+u8 afl_custom_queue_new_entry(my_mutator_t * data,
+                              const uint8_t *filename_new_queue,
+                              const uint8_t *filename_orig_queue) {
+
+  // get the filename
+  u8 *   automaton_fn, *unparsed_input;
+  Array *new_input;
+  s32    fd;
+
+  automaton_fn = alloc_printf("%s.aut", filename_new_queue);
+  // Check if this method is being called during initialization
+
+  // fprintf(stderr, "new: %s, old: %s, auto: %s\n",
+  // filename_new_queue,filename_orig_queue,automaton_fn);
+
+  if (filename_orig_queue) {
+
+    write_input(data->mutated_walk, automaton_fn);
+
+  } else {
+
+    new_input = gen_input(pda, NULL);
+    write_input(new_input, automaton_fn);
+
+    // Update the placeholder file
+    if (unlink(filename_new_queue)) {
+
+      PFATAL("Unable to delete '%s'", filename_new_queue);
+
+    }
+
+    unparsed_input = unparse_walk(new_input);
+    fd = open(filename_new_queue, O_WRONLY | O_CREAT | O_TRUNC,
+              S_IRUSR | S_IWUSR);
+    if (fd < 0) { PFATAL("Failed to update file '%s'", filename_new_queue); }
+    int written = write(fd, unparsed_input, new_input->inputlen + 1);
+    close(fd);
+
+    free(new_input->start);
+    free(new_input);
+    free(unparsed_input);
+
+  }
+
+  ck_free(automaton_fn);
+
+  return 1;
+
+}
+
+/**
+ * Get the corresponding tree representation for the candidate that is to be
+ * mutated
+ *
+ * @param[in] data pointer returned in afl_custom_init for this fuzz case
+ * @param filename File name of the test case in the queue entry
+ * @return Return True(1) if the fuzzer will fuzz the queue entry, and
+ *     False(0) otherwise.
+ */
+uint8_t afl_custom_queue_get(my_mutator_t *data, const uint8_t *filename) {
+
+  // get the filename
+  u8 *        automaton_fn = alloc_printf("%s.aut", filename);
+  IdxMap_new *statemap_ptr;
+  terminal *  term_ptr;
+  int         state;
+
+  // TODO: I don't think we need to update pointers when reading back
+  // Probably build two different versions of read_input one for flushing
+  // inputs to disk and the other that
+  if (data->orig_alloced) {
+
+    free(data->orig_walk->start);
+    free(data->orig_walk);
+    data->orig_alloced = 0;
+
+  }
+
+  if (data->statemap) {
+
+    for (int x = 0; x < numstates; x++) {
+
+      utarray_free(data->statemap[x].nums);
+
+    }
+
+    free(data->statemap);
+
+  }
+
+  if (data->recurIdx) {
+
+    data->recurlen = 0;
+    free(data->recurIdx);
+
+  }
+
+  data->orig_walk = read_input(pda, automaton_fn);
+  data->orig_alloced = 1;
+
+  // Create statemap for the fuzz candidate
+  IdxMap_new *statemap_start =
+      (IdxMap_new *)malloc(sizeof(IdxMap_new) * numstates);
+  for (int x = 0; x < numstates; x++) {
+
+    statemap_ptr = &statemap_start[x];
+    utarray_new(statemap_ptr->nums, &ut_int_icd);
+
+  }
+
+  int offset = 0;
+  while (offset < data->orig_walk->used) {
+
+    term_ptr = &data->orig_walk->start[offset];
+    state = term_ptr->state;
+    statemap_ptr = &statemap_start[state];
+    utarray_push_back(statemap_ptr->nums, &offset);
+    offset += 1;
+
+  }
+
+  data->statemap = statemap_start;
+
+  // Create recursive feature map (if it exists)
+  data->recurIdx = malloc(sizeof(UT_array *) * numstates);
+  // Retrieve the duplicated states
+  offset = 0;
+  while (offset < numstates) {
+
+    statemap_ptr = &data->statemap[offset];
+    int length = utarray_len(statemap_ptr->nums);
+    if (length >= 2) {
+
+      data->recurIdx[data->recurlen] = statemap_ptr->nums;
+      data->recurlen += 1;
+
+    }
+
+    offset += 1;
+
+  }
+
+  // data->getdupesret = get_dupes(data->orig_walk, &data->recurlen);
+
+  ck_free(automaton_fn);
+  return 1;
+
+}
+
+/**
+ * Deinitialize everything
+ *
+ * @param data The data ptr from afl_custom_init
+ */
+
+void afl_custom_deinit(my_mutator_t *data) {
+
+  free(data->mutator_buf);
+  free(data);
+
+}
+

custom_mutators/gramatron/gramfuzz.h

@@ -0,0 +1,255 @@
+#ifndef _GRAMFUZZ_H
+
+#define _GRAMFUZZ_H
+
+#include <json-c/json.h>
+#include <unistd.h>
+#include "hashmap.h"
+#include "uthash.h"
+#include "utarray.h"
+
+#define INIT_INPUTS 100  // No. of initial inputs to be generated
+
+// Set this as `numstates` + 1 where `numstates` is retrieved from gen automata
+// json #define STATES 63
+
+#define INIT_SIZE 100  // Initial size of the dynamic array holding the input
+
+#define SPLICE_CORPUS 10000
+#define RECUR_THRESHOLD 6
+#define SIZE_THRESHOLD 2048
+
+#define FLUSH_INTERVAL \
+  3600  // Inputs that gave new coverage will be dumped every FLUSH_INTERVAL
+        // seconds
+
+afl_state_t *global_afl;
+
+typedef struct trigger {
+
+  char * id;
+  int    dest;
+  char * term;
+  size_t term_len;
+
+} trigger;
+
+typedef struct state {
+
+  int      state_name;   // Integer State name
+  int      trigger_len;  // Number of triggers associated with this state
+  trigger *ptr;          // Pointer to beginning of the list of triggers
+
+} state;
+
+typedef struct terminal {
+
+  int    state;
+  int    trigger_idx;
+  size_t symbol_len;
+  char * symbol;
+
+} terminal;
+
+typedef struct buckethash {
+
+  int freq;
+
+} buckethash;
+
+int init_state;
+int curr_state;
+int final_state;
+int numstates;
+
+/*****************
+/ DYNAMIC ARRAY FOR WALKS
+*****************/
+
+typedef struct {
+
+  size_t    used;
+  size_t    size;
+  size_t    inputlen;
+  terminal *start;
+
+} Array;
+
+/*****************
+/ DYNAMIC ARRAY FOR STATEMAPS/RECURSION MAPS
+*****************/
+
+typedef struct {
+
+  int *  array;
+  size_t used;
+  size_t size;
+
+} IdxMap;
+
+typedef struct {
+
+  UT_array *nums;
+
+} IdxMap_new;
+
+typedef struct {
+
+  IdxMap_new *idxmap;
+  UT_array ** recurIdx;
+
+} Get_Dupes_Ret;
+
+/* Candidate Struct */
+typedef struct {
+
+  Array *     walk;
+  IdxMap_new *statemap;
+
+} Candidate;
+
+/* Splice Mutation helpers*/
+typedef struct {
+
+  Candidate *splice_cand;
+  int        idx;
+
+} SpliceCand;
+
+typedef struct {
+
+  SpliceCand *start;
+  size_t      used;
+  size_t      size;
+
+} SpliceCandArray;
+
+// Initialize dynamic array for potential splice points
+SpliceCand potential[SPLICE_CORPUS];
+
+typedef struct {
+
+  int orig_idx;
+  int splice_idx;
+
+} intpair_t;
+
+// Initialize dynamic array for potential splice points
+// SpliceCand potential[SPLICE_CORPUS];
+// IdxMap_new* rcuridx[STATES];
+
+/* Prototypes*/
+Array *    slice(Array *, int);
+state *    create_pda(u8 *);
+Array *    gen_input(state *, Array *);
+Array *    gen_input_count(state *, Array *, int *);
+int        updatebucket(map_t, int);
+void       itoa(int, char *, int);
+void       strrreverse(char *, char *);
+void       dbg_hashmap(map_t);
+void       print_repr(Array *, char *);
+int        isSatisfied(map_t);
+char *     get_state(char *);
+Candidate *gen_candidate(Array *);
+
+Array *spliceGF(Array *, Array *, int);
+Array *performSpliceOne(Array *, IdxMap_new *, Array *);
+/* Mutation Methods*/
+Array *    performRandomMutation(state *, Array *);
+Array *    performRandomMutationCount(state *, Array *, int *);
+Array *    performSpliceMutationBench(state *, Array *, Candidate **);
+UT_array **get_dupes(Array *, int *);
+Array *    doMult(Array *, UT_array **, int);
+Array *    doMultBench(Array *, UT_array **, int);
+
+/* Benchmarks*/
+void SpaceBenchmark(char *);
+void GenInputBenchmark(char *, char *);
+void RandomMutationBenchmark(char *, char *);
+void MutationAggrBenchmark(char *, char *);
+void SpliceMutationBenchmark(char *, char *);
+void SpliceMutationBenchmarkOne(char *, char *);
+void RandomRecursiveBenchmark(char *, char *);
+
+/* Testers */
+void SanityCheck(char *);
+
+/*Helpers*/
+void   initArray(Array *, size_t);
+void   insertArray(Array *, int, char *, size_t, int);
+void   freeArray(Array *);
+void   initArrayIdx(IdxMap *, size_t);
+void   insertArrayIdx(IdxMap *, int);
+void   freeArrayIdx(IdxMap *);
+void   initArraySplice(SpliceCandArray *, size_t);
+void   insertArraySplice(SpliceCandArray *, Candidate *, int);
+void   freeArraySplice(IdxMap *);
+void   getTwoIndices(UT_array *, int, int *, int *);
+void   swap(int *, int *);
+Array *slice_inverse(Array *, int);
+void   concatPrefixFeature(Array *, Array *);
+void   concatPrefixFeatureBench(Array *, Array *);
+Array *carve(Array *, int, int);
+int    fact(int);
+
+void                add_to_corpus(struct json_object *, Array *);
+struct json_object *term_to_json(terminal *);
+
+/* Gramatron specific prototypes */
+u8 *   unparse_walk(Array *);
+Array *performSpliceGF(state *, Array *, afl_state_t *);
+void   dump_input(u8 *, char *, int *);
+void   write_input(Array *, u8 *);
+Array *read_input(state *, u8 *);
+state *pda;
+
+// // AFL-specific struct
+// typedef uint8_t  u8;
+// typedef uint16_t u16;
+// typedef uint32_t u32;
+// #ifdef __x86_64__
+// typedef unsigned long long u64;
+// #else
+// typedef uint64_t u64;
+// #endif                                                    /* ^__x86_64__ */
+//
+// struct queue_entry {
+
+//   Array* walk;                           /* Pointer to the automaton walk*/
+//   u32 walk_len;                          /* Number of tokens in the input*/
+//   Candidate* cand;                     /* Preprocessed info about the
+//   candidate to allow for faster mutations*/
+//
+//   u8* fname;                          /* File name for the test case      */
+//   u32 len;                            /* Input length                     */
+//   UT_array** recur_idx;               /* Keeps track of recursive feature
+//   indices*/
+//
+//   u32 recur_len;                      /* The number of recursive features*/
+//
+//   u8  cal_failed,                     /* Calibration failed?              */
+//       trim_done,                      /* Trimmed?                         */
+//       was_fuzzed,                     /* Had any fuzzing done yet?        */
+//       passed_det,                     /* Deterministic stages passed?     */
+//       has_new_cov,                    /* Triggers new coverage?           */
+//       var_behavior,                   /* Variable behavior?               */
+//       favored,                        /* Currently favored?               */
+//       fs_redundant;                   /* Marked as redundant in the fs?   */
+//
+//   u32 bitmap_size,                    /* Number of bits set in bitmap     */
+//       exec_cksum;                     /* Checksum of the execution trace  */
+//
+//   u64 exec_us,                        /* Execution time (us)              */
+//       handicap,                       /* Number of queue cycles behind    */
+//       depth;                          /* Path depth                       */
+//
+//   u8* trace_mini;                     /* Trace bytes, if kept             */
+//   u32 tc_ref;                         /* Trace bytes ref count            */
+//
+//   struct queue_entry *next,           /* Next element, if any             */
+//                      *next_100;       /* 100 elements ahead               */
+//
+// };
+
+#endif
+

custom_mutators/gramatron/grammars/js/source.json

@@ -0,0 +1,606 @@
+{
+    "ARGLIST": [
+        "EXPR ',' ARGLIST",
+        "EXPR",
+        "EXPR ',' ARGLIST",
+        "EXPR"
+    ],
+    "ARGS": [
+        "'()'",
+        "'(' ARGLIST ')'",
+        "'()'",
+        "'(' ARGLIST ')'"
+    ],
+    "ARITHMETICOPERATION": [
+        "EXPR '/' EXPR",
+        "EXPR '*' EXPR",
+        "EXPR '+' EXPR",
+        "EXPR '-' EXPR",
+        "EXPR '%' EXPR",
+        "EXPR '**' EXPR",
+        "EXPR '++'"
+    ],
+    "ARRAY": [
+        "'[' ARRAYCONTENT ']'",
+        "'[]'"
+    ],
+    "ARRAYCONTENT": [
+        "EXPR ',' ARRAYCONTENT",
+        "EXPR"
+    ],
+    "BOOLEAN": [
+        "'true'",
+        "'false'"
+    ],
+    "BYTEWISEOPERATION": [
+        "EXPR '&' EXPR",
+        "EXPR '|' EXPR"
+    ],
+    "COMPARISONOPERATION": [
+        "EXPR '<' EXPR"
+    ],
+    "DECIMALDIGITS": [
+        "'20'",
+        "'1234'",
+        "'66'",
+        "'234_9'",
+        "'99999999999999999999'"
+    ],
+    "DECIMALNUMBER": [
+        "DECIMALDIGITS"
+    ],
+    "EXPR": [
+        "'(' EXPR ')'",
+        "VAR",
+        "'delete' SP EXPR",
+        "'new' SP IDENTIFIER ARGS",
+        "LITERAL",
+        "IDENTIFIER",
+        "METHODCALL",
+        "'(' ARITHMETICOPERATION ')'",
+        "'(' COMPARISONOPERATION ')'",
+        "'(' BYTEWISEOPERATION ')'",
+        "'(' LOGICALOPERATION ')'"
+    ],
+    "IDENTIFIER": [
+        "'Object'",
+        "VAR",
+        "'Function'",
+        "'main'",
+        "'opt'",
+        "'Boolean'",
+        "'Symbol'",
+        "'JSON'",
+        "'Error'",
+        "'EvalError'",
+        "'RangeError'",
+        "'ReferenceError'",
+        "'SyntaxError'",
+        "'TypeError'",
+        "'URIError'",
+        "'this'",
+        "'Number'",
+        "'Math'",
+        "'Date'",
+        "'String'",
+        "'RegExp'",
+        "'Array'",
+        "'Int8Array'",
+        "'Uint8Array'",
+        "'Uint8ClampedArray'",
+        "'Int16Array'",
+        "'Uint16Array'",
+        "'Int32Array'",
+        "'Uint32Array'",
+        "'Float32Array'",
+        "'Float64Array'",
+        "'DataView'",
+        "'ArrayBuffer'",
+        "'Map'",
+        "'Set'",
+        "'WeakMap'",
+        "'WeakSet'",
+        "'Promise'",
+        "'AsyncFunction'",
+        "'asyncGenerator'",
+        "'Reflect'",
+        "'Proxy'",
+        "'Intl'",
+        "'Intl.Collator'",
+        "'Intl.DateTimeFormat'",
+        "'Intl.NumberFormat'",
+        "'Intl.PluralRules'",
+        "'WebAssembly'",
+        "'WebAssembly.Module'",
+        "'WebAssembly.Instance'",
+        "'WebAssembly.Memory'",
+        "'WebAssembly.Table'",
+        "'WebAssembly.CompileError'",
+        "'WebAssembly.LinkError'",
+        "'WebAssembly.RuntimeError'",
+        "'arguments'",
+        "'Infinity'",
+        "'NaN'",
+        "'undefined'",
+        "'null'",
+        "'console'",
+        "' '"
+    ],
+    "IDENTIFIERLIST": [
+        "IDENTIFIER ',' IDENTIFIERLIST",
+        "'(' IDENTIFIERLIST '),' IDENTIFIERLIST",
+        "IDENTIFIER"
+    ],
+    "JSBLOCK": [
+        "JSSTATEMENT",
+        "JSSTATEMENT JSBLOCK"
+    ],
+    "JSSTATEMENT": [
+        "STATEMENT NEWLINE"
+    ],
+    "LITERAL": [
+        "'null'",
+        "BOOLEAN",
+        "NUMBER",
+        "ARRAY"
+    ],
+    "LOGICALOPERATION": [
+        "EXPR '&&' EXPR",
+        "EXPR '||' EXPR"
+    ],
+    "METHODCALL": [
+        "OBJECT PROPERTY METHODCALL1"
+    ],
+    "METHODCALL1": [
+        "'.' METHOD_NAME ARGS METHODCALL1",
+        "' '"
+    ],
+    "METHOD_NAME": [
+        "IDENTIFIER",
+        "'print'",
+        "'eval'",
+        "'uneval'",
+        "'isFinite'",
+        "'isNaN'",
+        "'parseFloat'",
+        "'parseInt'",
+        "'decodeURI'",
+        "'decodeURIComponent'",
+        "'encodeURI'",
+        "'encodeURIComponent'",
+        "'escape'",
+        "'unescape'",
+        "'assign'",
+        "'create'",
+        "'defineProperty'",
+        "'defineProperties'",
+        "'entries'",
+        "'freeze'",
+        "'getOwnPropertyDescriptor'",
+        "'getOwnPropertyDescriptors'",
+        "'getOwnPropertyNames'",
+        "'getOwnPropertySymbols'",
+        "'getPrototypeOf'",
+        "'is'",
+        "'isExtensible'",
+        "'isFrozen'",
+        "'isSealed'",
+        "'keys'",
+        "'preventExtensions'",
+        "'seal'",
+        "'setPrototypeOf'",
+        "'values'",
+        "'__defineGetter__'",
+        "'__defineSetter__'",
+        "'__lookupGetter__'",
+        "'__lookupSetter__'",
+        "'hasOwnProperty'",
+        "'isPrototypeOf'",
+        "'propertyIsEnumerable'",
+        "'toSource'",
+        "'toLocaleString'",
+        "'toString'",
+        "'unwatch'",
+        "'valueOf'",
+        "'watch'",
+        "'apply'",
+        "'bind'",
+        "'call'",
+        "'isGenerator'",
+        "'valueOf'",
+        "'for'",
+        "'keyFor'",
+        "'stringify'",
+        "'isInteger'",
+        "'isSafeInteger'",
+        "'toInteger'",
+        "'toExponential'",
+        "'toFixed'",
+        "'toLocaleString'",
+        "'toPrecision'",
+        "'abs'",
+        "'acos'",
+        "'acosh'",
+        "'asin'",
+        "'asinh'",
+        "'atan'",
+        "'atanh'",
+        "'atan2'",
+        "'cbrt'",
+        "'ceil'",
+        "'clz32'",
+        "'cos'",
+        "'cosh'",
+        "'exp'",
+        "'expm1'",
+        "'floor'",
+        "'fround'",
+        "'hypot'",
+        "'imul'",
+        "'log'",
+        "'log1p'",
+        "'log10'",
+        "'log2'",
+        "'max'",
+        "'min'",
+        "'pow'",
+        "'random'",
+        "'round'",
+        "'sign'",
+        "'sin'",
+        "'sinh'",
+        "'sqrt'",
+        "'tan'",
+        "'tanh'",
+        "'trunc'",
+        "'now'",
+        "'parse'",
+        "'UTC'",
+        "'getDate'",
+        "'getDay'",
+        "'getFullYear'",
+        "'getHours'",
+        "'getMilliseconds'",
+        "'getMinutes'",
+        "'getMonth'",
+        "'getSeconds'",
+        "'getTime'",
+        "'getTimezoneOffset'",
+        "'getUTCDate'",
+        "'getUTCDay'",
+        "'getUTCFullYear'",
+        "'getUTCHours'",
+        "'getUTCMilliseconds'",
+        "'getUTCMinutes'",
+        "'getUTCMonth'",
+        "'getUTCSeconds'",
+        "'getYear'",
+        "'setDate'",
+        "'setFullYear'",
+        "'setHours'",
+        "'setMilliseconds'",
+        "'setMinutes'",
+        "'setMonth'",
+        "'setSeconds'",
+        "'setTime'",
+        "'setUTCDate'",
+        "'setUTCFullYear'",
+        "'setUTCHours'",
+        "'setUTCMilliseconds'",
+        "'setUTCMinutes'",
+        "'setUTCMonth'",
+        "'setUTCSeconds'",
+        "'setYear'",
+        "'toDateString'",
+        "'toISOString'",
+        "'toJSON'",
+        "'toGMTString'",
+        "'toLocaleDateString'",
+        "'toLocaleFormat'",
+        "'toLocaleString'",
+        "'toLocaleTimeString'",
+        "'toTimeString'",
+        "'toUTCString'",
+        "'indexOf'",
+        "'substring'",
+        "'charAt'",
+        "'strcmp'",
+        "'fromCharCode'",
+        "'fromCodePoint'",
+        "'raw'",
+        "'charCodeAt'",
+        "'slice'",
+        "'codePointAt'",
+        "'concat'",
+        "'includes'",
+        "'endsWith'",
+        "'lastIndexOf'",
+        "'localeCompare'",
+        "'match'",
+        "'normalize'",
+        "'padEnd'",
+        "'padStart'",
+        "'quote'",
+        "'repeat'",
+        "'replace'",
+        "'search'",
+        "'split'",
+        "'startsWith'",
+        "'substr'",
+        "'toLocaleLowerCase'",
+        "'toLocaleUpperCase'",
+        "'toLowerCase'",
+        "'toUpperCase'",
+        "'trim'",
+        "'trimleft'",
+        "'trimright'",
+        "'anchor'",
+        "'big'",
+        "'blink'",
+        "'bold'",
+        "'fixed'",
+        "'fontcolor'",
+        "'fontsize'",
+        "'italics'",
+        "'link'",
+        "'small'",
+        "'strike'",
+        "'sub'",
+        "'sup'",
+        "'compile'",
+        "'exec'",
+        "'test'",
+        "'from'",
+        "'isArray'",
+        "'of'",
+        "'copyWithin'",
+        "'fill'",
+        "'pop'",
+        "'push'",
+        "'reverse'",
+        "'shift'",
+        "'sort'",
+        "'splice'",
+        "'unshift'",
+        "'concat'",
+        "'join'",
+        "'every'",
+        "'filter'",
+        "'findIndex'",
+        "'forEach'",
+        "'map'",
+        "'reduce'",
+        "'reduceRight'",
+        "'some'",
+        "'move'",
+        "'getInt8'",
+        "'getUint8'",
+        "'getInt16'",
+        "'getUint16'",
+        "'getInt32'",
+        "'getUint32'",
+        "'getFloat32'",
+        "'getFloat64'",
+        "'setInt8'",
+        "'setUint8'",
+        "'setInt16'",
+        "'setUint16'",
+        "'setInt32'",
+        "'setUint32'",
+        "'setFloat32'",
+        "'setFloat64'",
+        "'isView'",
+        "'transfer'",
+        "'clear'",
+        "'get'",
+        "'has'",
+        "'set'",
+        "'add'",
+        "'splat'",
+        "'check'",
+        "'extractLane'",
+        "'replaceLane'",
+        "'load'",
+        "'load1'",
+        "'load2'",
+        "'load3'",
+        "'store'",
+        "'store1'",
+        "'store2'",
+        "'store3'",
+        "'addSaturate'",
+        "'div'",
+        "'mul'",
+        "'neg'",
+        "'reciprocalApproximation'",
+        "'reciprocalSqrtApproximation'",
+        "'subSaturate'",
+        "'shuffle'",
+        "'swizzle'",
+        "'maxNum'",
+        "'minNum'",
+        "'select'",
+        "'equal'",
+        "'notEqual'",
+        "'lessThan'",
+        "'lessThanOrEqual'",
+        "'greaterThan'",
+        "'greaterThanOrEqual'",
+        "'and'",
+        "'or'",
+        "'xor'",
+        "'not'",
+        "'shiftLeftByScalar'",
+        "'shiftRightByScalar'",
+        "'allTrue'",
+        "'anyTrue'",
+        "'fromFloat32x4'",
+        "'fromFloat32x4Bits'",
+        "'fromFloat64x2Bits'",
+        "'fromInt32x4'",
+        "'fromInt32x4Bits'",
+        "'fromInt16x8Bits'",
+        "'fromInt8x16Bits'",
+        "'fromUint32x4'",
+        "'fromUint32x4Bits'",
+        "'fromUint16x8Bits'",
+        "'fromUint8x16Bits'",
+        "'neg'",
+        "'compareExchange'",
+        "'exchange'",
+        "'wait'",
+        "'wake'",
+        "'isLockFree'",
+        "'all'",
+        "'race'",
+        "'reject'",
+        "'resolve'",
+        "'catch'",
+        "'then'",
+        "'finally'",
+        "'next'",
+        "'throw'",
+        "'close'",
+        "'send'",
+        "'apply'",
+        "'construct'",
+        "'deleteProperty'",
+        "'ownKeys'",
+        "'getCanonicalLocales'",
+        "'supportedLocalesOf'",
+        "'resolvedOptions'",
+        "'formatToParts'",
+        "'resolvedOptions'",
+        "'instantiate'",
+        "'instantiateStreaming'",
+        "'compileStreaming'",
+        "'validate'",
+        "'customSections'",
+        "'exports'",
+        "'imports'",
+        "'grow'",
+        "'super'",
+        "'in'",
+        "'instanceof'",
+        "' '"
+    ],
+    "NEWLINE": [
+        "'\\n'"
+    ],
+    "NUMBER": [
+        "'1/2'",
+        "'1E2'",
+        "'1E02'",
+        "'1E+02'",
+        "'-1'",
+        "'-1.00'",
+        "'-1/2'",
+        "'-1E2'",
+        "'-1E02'",
+        "'-1E+02'",
+        "'1/0'",
+        "'0/0'",
+        "'-2147483648/-1'",
+        "'-9223372036854775808/-1'",
+        "'-0'",
+        "'-0.0'",
+        "'+0'"
+    ],
+    "OBJECT": [
+        "IDENTIFIER"
+    ],
+    "PROGRAM": [
+        "JSBLOCK"
+    ],
+    "PROPERTY": [
+        "'.length' PROPERTY",
+        "'.prototype' PROPERTY",
+        "'.constructor' PROPERTY",
+        "'.__proto__' PROPERTY",
+        "'.__noSuchMethod__' PROPERTY",
+        "'.__count__' PROPERTY",
+        "'.__parent__' PROPERTY",
+        "'.arguments' PROPERTY",
+        "'.arity' PROPERTY",
+        "'.caller' PROPERTY",
+        "'.name' PROPERTY",
+        "'.displayName' PROPERTY",
+        "'.iterator' PROPERTY",
+        "'.asyncIterator' PROPERTY",
+        "'.match' PROPERTY",
+        "'.replace' PROPERTY",
+        "'.search' PROPERTY",
+        "'.split' PROPERTY",
+        "'.hasInstance' PROPERTY",
+        "'.isConcatSpreadable' PROPERTY",
+        "'.unscopables' PROPERTY",
+        "'.species' PROPERTY",
+        "'.toPrimitive' PROPERTY",
+        "'.toStringTag' PROPERTY",
+        "'.fileName' PROPERTY",
+        "'.lineNumber' PROPERTY",
+        "'.columnNumber' PROPERTY",
+        "'.message' PROPERTY",
+        "'.name' PROPERTY",
+        "'.EPSILON' PROPERTY",
+        "'.MAX_SAFE_INTEGER' PROPERTY",
+        "'.MAX_VALUE' PROPERTY",
+        "'.MIN_SAFE_INTEGER' PROPERTY",
+        "'.MIN_VALUE' PROPERTY",
+        "'.NaN' PROPERTY",
+        "'.NEGATIVE_INFINITY' PROPERTY",
+        "'.POSITIVE_INFINITY' PROPERTY",
+        "'.E' PROPERTY",
+        "'.LN2' PROPERTY",
+        "'.LN10' PROPERTY",
+        "'.LOG2E' PROPERTY",
+        "'.LOG10E' PROPERTY",
+        "'.PI' PROPERTY",
+        "'.SQRT1_2' PROPERTY",
+        "'.SQRT2' PROPERTY",
+        "'.flags' PROPERTY",
+        "'.global' PROPERTY",
+        "'.ignoreCase' PROPERTY",
+        "'.multiline' PROPERTY",
+        "'.source' PROPERTY",
+        "'.sticky' PROPERTY",
+        "'.unicode' PROPERTY",
+        "'.buffer' PROPERTY",
+        "'.byteLength' PROPERTY",
+        "'.byteOffset' PROPERTY",
+        "'.BYTES_PER_ELEMENT' PROPERTY",
+        "'.compare' PROPERTY",
+        "'.format' PROPERTY",
+        "'.callee' PROPERTY",
+        "'.caller' PROPERTY",
+        "'.memory' PROPERTY",
+        "'.exports' PROPERTY",
+        "' '"
+    ],
+    "SP": [
+        "' '"
+    ],
+    "STATEMENT": [
+        "EXPR ';'",
+        "'var' SP VAR '=' EXPR ';'",
+        "'let' SP VAR '=' EXPR ';'",
+        "VAR '=' EXPR ';'",
+        "VAR PROPERTY '=' EXPR ';'",
+        "VAR '[' DECIMALNUMBER ']' '=' EXPR ';'",
+        "'const' SP VAR '=' EXPR ';'",
+        "'typeof' SP EXPR ';'",
+        "'void' SP EXPR ';'",
+        "'return' SP EXPR ';'",
+        "VAR ':'"
+    ],
+    "VAR": [
+        "'a'",
+        "'b'",
+        "'c'",
+        "'d'",
+        "'e'",
+        "'f'",
+        "'g'",
+        "'h'"
+    ]
+}

custom_mutators/gramatron/hashmap.c

@@ -0,0 +1,434 @@
+/*
+ * Generic map implementation.
+ */
+#include "hashmap.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+
+#define INITIAL_SIZE (256)
+#define MAX_CHAIN_LENGTH (8)
+
+/* We need to keep keys and values */
+typedef struct _hashmap_element {
+
+  char *key;
+  int   in_use;
+  any_t data;
+
+} hashmap_element;
+
+/* A hashmap has some maximum size and current size,
+ * as well as the data to hold. */
+typedef struct _hashmap_map {
+
+  int              table_size;
+  int              size;
+  hashmap_element *data;
+
+} hashmap_map;
+
+/*
+ * Return an empty hashmap, or NULL on failure.
+ */
+map_t hashmap_new() {
+
+  hashmap_map *m = (hashmap_map *)malloc(sizeof(hashmap_map));
+  if (!m) goto err;
+
+  m->data = (hashmap_element *)calloc(INITIAL_SIZE, sizeof(hashmap_element));
+  if (!m->data) goto err;
+
+  m->table_size = INITIAL_SIZE;
+  m->size = 0;
+
+  return m;
+err:
+  if (m) hashmap_free(m);
+  return NULL;
+
+}
+
+/* The implementation here was originally done by Gary S. Brown.  I have
+   borrowed the tables directly, and made some minor changes to the
+   crc32-function (including changing the interface). //ylo */
+
+/* ============================================================= */
+/*  COPYRIGHT (C) 1986 Gary S. Brown.  You may use this program, or       */
+/*  code or tables extracted from it, as desired without restriction.     */
+/*                                                                        */
+/*  First, the polynomial itself and its table of feedback terms.  The    */
+/*  polynomial is                                                         */
+/*  X^32+X^26+X^23+X^22+X^16+X^12+X^11+X^10+X^8+X^7+X^5+X^4+X^2+X^1+X^0   */
+/*                                                                        */
+/*  Note that we take it "backwards" and put the highest-order term in    */
+/*  the lowest-order bit.  The X^32 term is "implied"; the LSB is the     */
+/*  X^31 term, etc.  The X^0 term (usually shown as "+1") results in      */
+/*  the MSB being 1.                                                      */
+/*                                                                        */
+/*  Note that the usual hardware shift register implementation, which     */
+/*  is what we're using (we're merely optimizing it by doing eight-bit    */
+/*  chunks at a time) shifts bits into the lowest-order term.  In our     */
+/*  implementation, that means shifting towards the right.  Why do we     */
+/*  do it this way?  Because the calculated CRC must be transmitted in    */
+/*  order from highest-order term to lowest-order term.  UARTs transmit   */
+/*  characters in order from LSB to MSB.  By storing the CRC this way,    */
+/*  we hand it to the UART in the order low-byte to high-byte; the UART   */
+/*  sends each low-bit to hight-bit; and the result is transmission bit   */
+/*  by bit from highest- to lowest-order term without requiring any bit   */
+/*  shuffling on our part.  Reception works similarly.                    */
+/*                                                                        */
+/*  The feedback terms table consists of 256, 32-bit entries.  Notes:     */
+/*                                                                        */
+/*      The table can be generated at runtime if desired; code to do so   */
+/*      is shown later.  It might not be obvious, but the feedback        */
+/*      terms simply represent the results of eight shift/xor opera-      */
+/*      tions for all combinations of data and CRC register values.       */
+/*                                                                        */
+/*      The values must be right-shifted by eight bits by the "updcrc"    */
+/*      logic; the shift must be unsigned (bring in zeroes).  On some     */
+/*      hardware you could probably optimize the shift in assembler by    */
+/*      using byte-swap instructions.                                     */
+/*      polynomial $edb88320                                              */
+/*                                                                        */
+/*  --------------------------------------------------------------------  */
+
+static unsigned long crc32_tab[] = {
+
+    0x00000000L, 0x77073096L, 0xee0e612cL, 0x990951baL, 0x076dc419L,
+    0x706af48fL, 0xe963a535L, 0x9e6495a3L, 0x0edb8832L, 0x79dcb8a4L,
+    0xe0d5e91eL, 0x97d2d988L, 0x09b64c2bL, 0x7eb17cbdL, 0xe7b82d07L,
+    0x90bf1d91L, 0x1db71064L, 0x6ab020f2L, 0xf3b97148L, 0x84be41deL,
+    0x1adad47dL, 0x6ddde4ebL, 0xf4d4b551L, 0x83d385c7L, 0x136c9856L,
+    0x646ba8c0L, 0xfd62f97aL, 0x8a65c9ecL, 0x14015c4fL, 0x63066cd9L,
+    0xfa0f3d63L, 0x8d080df5L, 0x3b6e20c8L, 0x4c69105eL, 0xd56041e4L,
+    0xa2677172L, 0x3c03e4d1L, 0x4b04d447L, 0xd20d85fdL, 0xa50ab56bL,
+    0x35b5a8faL, 0x42b2986cL, 0xdbbbc9d6L, 0xacbcf940L, 0x32d86ce3L,
+    0x45df5c75L, 0xdcd60dcfL, 0xabd13d59L, 0x26d930acL, 0x51de003aL,
+    0xc8d75180L, 0xbfd06116L, 0x21b4f4b5L, 0x56b3c423L, 0xcfba9599L,
+    0xb8bda50fL, 0x2802b89eL, 0x5f058808L, 0xc60cd9b2L, 0xb10be924L,
+    0x2f6f7c87L, 0x58684c11L, 0xc1611dabL, 0xb6662d3dL, 0x76dc4190L,
+    0x01db7106L, 0x98d220bcL, 0xefd5102aL, 0x71b18589L, 0x06b6b51fL,
+    0x9fbfe4a5L, 0xe8b8d433L, 0x7807c9a2L, 0x0f00f934L, 0x9609a88eL,
+    0xe10e9818L, 0x7f6a0dbbL, 0x086d3d2dL, 0x91646c97L, 0xe6635c01L,
+    0x6b6b51f4L, 0x1c6c6162L, 0x856530d8L, 0xf262004eL, 0x6c0695edL,
+    0x1b01a57bL, 0x8208f4c1L, 0xf50fc457L, 0x65b0d9c6L, 0x12b7e950L,
+    0x8bbeb8eaL, 0xfcb9887cL, 0x62dd1ddfL, 0x15da2d49L, 0x8cd37cf3L,
+    0xfbd44c65L, 0x4db26158L, 0x3ab551ceL, 0xa3bc0074L, 0xd4bb30e2L,
+    0x4adfa541L, 0x3dd895d7L, 0xa4d1c46dL, 0xd3d6f4fbL, 0x4369e96aL,
+    0x346ed9fcL, 0xad678846L, 0xda60b8d0L, 0x44042d73L, 0x33031de5L,
+    0xaa0a4c5fL, 0xdd0d7cc9L, 0x5005713cL, 0x270241aaL, 0xbe0b1010L,
+    0xc90c2086L, 0x5768b525L, 0x206f85b3L, 0xb966d409L, 0xce61e49fL,
+    0x5edef90eL, 0x29d9c998L, 0xb0d09822L, 0xc7d7a8b4L, 0x59b33d17L,
+    0x2eb40d81L, 0xb7bd5c3bL, 0xc0ba6cadL, 0xedb88320L, 0x9abfb3b6L,
+    0x03b6e20cL, 0x74b1d29aL, 0xead54739L, 0x9dd277afL, 0x04db2615L,
+    0x73dc1683L, 0xe3630b12L, 0x94643b84L, 0x0d6d6a3eL, 0x7a6a5aa8L,
+    0xe40ecf0bL, 0x9309ff9dL, 0x0a00ae27L, 0x7d079eb1L, 0xf00f9344L,
+    0x8708a3d2L, 0x1e01f268L, 0x6906c2feL, 0xf762575dL, 0x806567cbL,
+    0x196c3671L, 0x6e6b06e7L, 0xfed41b76L, 0x89d32be0L, 0x10da7a5aL,
+    0x67dd4accL, 0xf9b9df6fL, 0x8ebeeff9L, 0x17b7be43L, 0x60b08ed5L,
+    0xd6d6a3e8L, 0xa1d1937eL, 0x38d8c2c4L, 0x4fdff252L, 0xd1bb67f1L,
+    0xa6bc5767L, 0x3fb506ddL, 0x48b2364bL, 0xd80d2bdaL, 0xaf0a1b4cL,
+    0x36034af6L, 0x41047a60L, 0xdf60efc3L, 0xa867df55L, 0x316e8eefL,
+    0x4669be79L, 0xcb61b38cL, 0xbc66831aL, 0x256fd2a0L, 0x5268e236L,
+    0xcc0c7795L, 0xbb0b4703L, 0x220216b9L, 0x5505262fL, 0xc5ba3bbeL,
+    0xb2bd0b28L, 0x2bb45a92L, 0x5cb36a04L, 0xc2d7ffa7L, 0xb5d0cf31L,
+    0x2cd99e8bL, 0x5bdeae1dL, 0x9b64c2b0L, 0xec63f226L, 0x756aa39cL,
+    0x026d930aL, 0x9c0906a9L, 0xeb0e363fL, 0x72076785L, 0x05005713L,
+    0x95bf4a82L, 0xe2b87a14L, 0x7bb12baeL, 0x0cb61b38L, 0x92d28e9bL,
+    0xe5d5be0dL, 0x7cdcefb7L, 0x0bdbdf21L, 0x86d3d2d4L, 0xf1d4e242L,
+    0x68ddb3f8L, 0x1fda836eL, 0x81be16cdL, 0xf6b9265bL, 0x6fb077e1L,
+    0x18b74777L, 0x88085ae6L, 0xff0f6a70L, 0x66063bcaL, 0x11010b5cL,
+    0x8f659effL, 0xf862ae69L, 0x616bffd3L, 0x166ccf45L, 0xa00ae278L,
+    0xd70dd2eeL, 0x4e048354L, 0x3903b3c2L, 0xa7672661L, 0xd06016f7L,
+    0x4969474dL, 0x3e6e77dbL, 0xaed16a4aL, 0xd9d65adcL, 0x40df0b66L,
+    0x37d83bf0L, 0xa9bcae53L, 0xdebb9ec5L, 0x47b2cf7fL, 0x30b5ffe9L,
+    0xbdbdf21cL, 0xcabac28aL, 0x53b39330L, 0x24b4a3a6L, 0xbad03605L,
+    0xcdd70693L, 0x54de5729L, 0x23d967bfL, 0xb3667a2eL, 0xc4614ab8L,
+    0x5d681b02L, 0x2a6f2b94L, 0xb40bbe37L, 0xc30c8ea1L, 0x5a05df1bL,
+    0x2d02ef8dL};
+
+/* Return a 32-bit CRC of the contents of the buffer. */
+
+unsigned long crc32(const unsigned char *s, unsigned int len) {
+
+  unsigned int  i;
+  unsigned long crc32val;
+
+  crc32val = 0;
+  for (i = 0; i < len; i++) {
+
+    crc32val = crc32_tab[(crc32val ^ s[i]) & 0xff] ^ (crc32val >> 8);
+
+  }
+
+  return crc32val;
+
+}
+
+/*
+ * Hashing function for a string
+ */
+unsigned int hashmap_hash_int(hashmap_map *m, char *keystring) {
+
+  unsigned long key = crc32((unsigned char *)(keystring), strlen(keystring));
+
+  /* Robert Jenkins' 32 bit Mix Function */
+  key += (key << 12);
+  key ^= (key >> 22);
+  key += (key << 4);
+  key ^= (key >> 9);
+  key += (key << 10);
+  key ^= (key >> 2);
+  key += (key << 7);
+  key ^= (key >> 12);
+
+  /* Knuth's Multiplicative Method */
+  key = (key >> 3) * 2654435761;
+
+  return key % m->table_size;
+
+}
+
+/*
+ * Return the integer of the location in data
+ * to store the point to the item, or MAP_FULL.
+ */
+int hashmap_hash(map_t in, char *key) {
+
+  int curr;
+  int i;
+
+  /* Cast the hashmap */
+  hashmap_map *m = (hashmap_map *)in;
+
+  /* If full, return immediately */
+  if (m->size >= (m->table_size / 2)) return MAP_FULL;
+
+  /* Find the best index */
+  curr = hashmap_hash_int(m, key);
+
+  /* Linear probing */
+  for (i = 0; i < MAX_CHAIN_LENGTH; i++) {
+
+    if (m->data[curr].in_use == 0) return curr;
+
+    if (m->data[curr].in_use == 1 && (strcmp(m->data[curr].key, key) == 0))
+      return curr;
+
+    curr = (curr + 1) % m->table_size;
+
+  }
+
+  return MAP_FULL;
+
+}
+
+/*
+ * Doubles the size of the hashmap, and rehashes all the elements
+ */
+int hashmap_rehash(map_t in) {
+
+  int              i;
+  int              old_size;
+  hashmap_element *curr;
+
+  /* Setup the new elements */
+  hashmap_map *    m = (hashmap_map *)in;
+  hashmap_element *temp =
+      (hashmap_element *)calloc(2 * m->table_size, sizeof(hashmap_element));
+  if (!temp) return MAP_OMEM;
+
+  /* Update the array */
+  curr = m->data;
+  m->data = temp;
+
+  /* Update the size */
+  old_size = m->table_size;
+  m->table_size = 2 * m->table_size;
+  m->size = 0;
+
+  /* Rehash the elements */
+  for (i = 0; i < old_size; i++) {
+
+    int status;
+
+    if (curr[i].in_use == 0) continue;
+
+    status = hashmap_put(m, curr[i].key, curr[i].data);
+    if (status != MAP_OK) return status;
+
+  }
+
+  free(curr);
+
+  return MAP_OK;
+
+}
+
+/*
+ * Add a pointer to the hashmap with some key
+ */
+int hashmap_put(map_t in, char *key, any_t value) {
+
+  int          index;
+  hashmap_map *m;
+
+  /* Cast the hashmap */
+  m = (hashmap_map *)in;
+
+  /* Find a place to put our value */
+  index = hashmap_hash(in, key);
+  while (index == MAP_FULL) {
+
+    if (hashmap_rehash(in) == MAP_OMEM) { return MAP_OMEM; }
+    index = hashmap_hash(in, key);
+
+  }
+
+  /* Set the data */
+  m->data[index].data = value;
+  m->data[index].key = key;
+  m->data[index].in_use = 1;
+  m->size++;
+
+  return MAP_OK;
+
+}
+
+/*
+ * Get your pointer out of the hashmap with a key
+ */
+int hashmap_get(map_t in, char *key, any_t *arg) {
+
+  int          curr;
+  int          i;
+  hashmap_map *m;
+
+  /* Cast the hashmap */
+  m = (hashmap_map *)in;
+
+  /* Find data location */
+  curr = hashmap_hash_int(m, key);
+
+  /* Linear probing, if necessary */
+  for (i = 0; i < MAX_CHAIN_LENGTH; i++) {
+
+    int in_use = m->data[curr].in_use;
+    if (in_use == 1) {
+
+      if (strcmp(m->data[curr].key, key) == 0) {
+
+        *arg = (m->data[curr].data);
+        return MAP_OK;
+
+      }
+
+    }
+
+    curr = (curr + 1) % m->table_size;
+
+  }
+
+  *arg = NULL;
+
+  /* Not found */
+  return MAP_MISSING;
+
+}
+
+/*
+ * Iterate the function parameter over each element in the hashmap.  The
+ * additional any_t argument is passed to the function as its first
+ * argument and the hashmap element is the second.
+ */
+int hashmap_iterate(map_t in, PFany f, any_t item) {
+
+  int i;
+
+  /* Cast the hashmap */
+  hashmap_map *m = (hashmap_map *)in;
+
+  /* On empty hashmap, return immediately */
+  if (hashmap_length(m) <= 0) return MAP_MISSING;
+
+  /* Linear probing */
+  for (i = 0; i < m->table_size; i++)
+    if (m->data[i].in_use != 0) {
+
+      any_t data = (any_t)(m->data[i].data);
+      int   status = f(item, data);
+      if (status != MAP_OK) { return status; }
+
+    }
+
+  return MAP_OK;
+
+}
+
+/*
+ * Remove an element with that key from the map
+ */
+int hashmap_remove(map_t in, char *key) {
+
+  int          i;
+  int          curr;
+  hashmap_map *m;
+
+  /* Cast the hashmap */
+  m = (hashmap_map *)in;
+
+  /* Find key */
+  curr = hashmap_hash_int(m, key);
+
+  /* Linear probing, if necessary */
+  for (i = 0; i < MAX_CHAIN_LENGTH; i++) {
+
+    int in_use = m->data[curr].in_use;
+    if (in_use == 1) {
+
+      if (strcmp(m->data[curr].key, key) == 0) {
+
+        /* Blank out the fields */
+        m->data[curr].in_use = 0;
+        m->data[curr].data = NULL;
+        m->data[curr].key = NULL;
+
+        /* Reduce the size */
+        m->size--;
+        return MAP_OK;
+
+      }
+
+    }
+
+    curr = (curr + 1) % m->table_size;
+
+  }
+
+  /* Data not found */
+  return MAP_MISSING;
+
+}
+
+/* Deallocate the hashmap */
+void hashmap_free(map_t in) {
+
+  hashmap_map *m = (hashmap_map *)in;
+  free(m->data);
+  free(m);
+
+}
+
+/* Return the length of the hashmap */
+int hashmap_length(map_t in) {
+
+  hashmap_map *m = (hashmap_map *)in;
+  if (m != NULL)
+    return m->size;
+  else
+    return 0;
+
+}
+

custom_mutators/gramatron/hashmap.h

@@ -0,0 +1,83 @@
+/*
+ * Generic hashmap manipulation functions
+ *
+ * Originally by Elliot C Back -
+ * http://elliottback.com/wp/hashmap-implementation-in-c/
+ *
+ * Modified by Pete Warden to fix a serious performance problem, support strings
+ * as keys and removed thread synchronization - http://petewarden.typepad.com
+ */
+#ifndef __HASHMAP_H__
+#define __HASHMAP_H__
+
+#define MAP_MISSING -3                                   /* No such element */
+#define MAP_FULL -2                                      /* Hashmap is full */
+#define MAP_OMEM -1                                        /* Out of Memory */
+#define MAP_OK 0                                                      /* OK */
+
+/*
+ * any_t is a pointer.  This allows you to put arbitrary structures in
+ * the hashmap.
+ */
+typedef void *any_t;
+
+/*
+ * PFany is a pointer to a function that can take two any_t arguments
+ * and return an integer. Returns status code..
+ */
+typedef int (*PFany)(any_t, any_t);
+
+/*
+ * map_t is a pointer to an internally maintained data structure.
+ * Clients of this package do not need to know how hashmaps are
+ * represented.  They see and manipulate only map_t's.
+ */
+typedef any_t map_t;
+
+/*
+ * Return an empty hashmap. Returns NULL if empty.
+ */
+extern map_t hashmap_new();
+
+/*
+ * Iteratively call f with argument (item, data) for
+ * each element data in the hashmap. The function must
+ * return a map status code. If it returns anything other
+ * than MAP_OK the traversal is terminated. f must
+ * not reenter any hashmap functions, or deadlock may arise.
+ */
+extern int hashmap_iterate(map_t in, PFany f, any_t item);
+
+/*
+ * Add an element to the hashmap. Return MAP_OK or MAP_OMEM.
+ */
+extern int hashmap_put(map_t in, char *key, any_t value);
+
+/*
+ * Get an element from the hashmap. Return MAP_OK or MAP_MISSING.
+ */
+extern int hashmap_get(map_t in, char *key, any_t *arg);
+
+/*
+ * Remove an element from the hashmap. Return MAP_OK or MAP_MISSING.
+ */
+extern int hashmap_remove(map_t in, char *key);
+
+/*
+ * Get any element. Return MAP_OK or MAP_MISSING.
+ * remove - should the element be removed from the hashmap
+ */
+extern int hashmap_get_one(map_t in, any_t *arg, int remove);
+
+/*
+ * Free the hashmap
+ */
+extern void hashmap_free(map_t in);
+
+/*
+ * Get the current size of a hashmap
+ */
+extern int hashmap_length(map_t in);
+
+#endif
+

custom_mutators/gramatron/preprocess/construct_automata.py

@@ -0,0 +1,275 @@
+import sys
+import json
+import re
+from collections import defaultdict
+# import pygraphviz as pgv
+
+gram_data = None
+state_count = 1
+pda = []
+worklist = []
+state_stacks = {} 
+
+# === If user provides upper bound on the stack size during FSA creation ===
+# Specifies the upper bound to which the stack is allowed to grow
+# If for any generated state, the stack size is >= stack_limit then this
+# state is not expanded further.
+stack_limit = None 
+# Holds the set of unexpanded rules owing to the user-passed stack constraint limit
+unexpanded_rules = set()
+
+def main(grammar, limit):
+    global worklist, gram_data, stack_limit
+    current = '0'
+    stack_limit = limit
+    if stack_limit:
+        print ('[X] Operating in bounded stack mode')
+
+    with open(grammar, 'r') as fd:
+        gram_data = json.load(fd)
+    start_symbol = gram_data["Start"][0]
+    worklist.append([current, [start_symbol]])
+    # print (grammar)
+    filename = (grammar.split('/')[-1]).split('.')[0]
+    
+
+    while worklist:
+        # Take an element from the worklist
+        # print ('================')
+        # print ('Worklist:', worklist)
+        element = worklist.pop(0)
+        prep_transitions(element)
+    
+    pda_file = filename + '_transition.json'
+    graph_file = filename + '.png'
+    # print ('XXXXXXXXXXXXXXXX')
+    # print ('PDA file:%s Png graph file:%s' % (pda_file, graph_file))
+    # XXX Commented out because visualization of current version of PHP causes segfault
+    # Create the graph and dump the transitions to a file
+    # create_graph(filename)
+    transformed = postprocess()
+    with open(filename + '_automata.json', 'w+') as fd:
+        json.dump(transformed, fd)
+    with open(filename + '_transition.json', 'w+') as fd:
+        json.dump(pda, fd)
+    if not unexpanded_rules:
+        print ('[X] No unexpanded rules, absolute FSA formed')
+        exit(0)
+    else:
+        print ('[X] Certain rules were not expanded due to stack size limit. Inexact approximation has been created and the disallowed rules have been put in {}_disallowed.json'.format(filename))
+        print ('[X] Number of unexpanded rules:', len(unexpanded_rules))
+        with open(filename + '_disallowed.json', 'w+') as fd:
+            json.dump(list(unexpanded_rules), fd)
+
+def create_graph(filename):
+    '''
+    Creates a DOT representation of the PDA
+    '''
+    global pda
+    G = pgv.AGraph(strict = False, directed = True)
+    for transition in pda:
+        print ('Transition:', transition)
+        G.add_edge(transition['source'], transition['dest'], 
+                label = 'Term:{}'.format(transition['terminal']))
+    G.layout(prog = 'dot')
+    print ('Do it up 2')
+    G.draw(filename + '.png')
+
+def prep_transitions(element):
+    '''
+    Generates transitions
+    '''
+    global gram_data, state_count, pda, worklist, state_stacks, stack_limit, unexpanded_rules
+    state = element[0]
+    try:
+        nonterminal = element[1][0] 
+    except IndexError:
+        # Final state was encountered, pop from worklist without doing anything
+        return
+    rules = gram_data[nonterminal]
+    count = 1
+    for rule in rules:
+        isRecursive  = False
+        # print ('Current state:', state)
+        terminal, ss, termIsRegex = tokenize(rule)
+        transition = get_template()
+        transition['trigger'] = '_'.join([state, str(count)])
+        transition['source'] = state
+        transition['dest'] = str(state_count) 
+        transition['ss'] = ss 
+        transition['terminal'] = terminal
+        transition['rule'] = "{} -> {}".format(nonterminal, rule )
+        if termIsRegex:
+            transition['termIsRegex'] = True
+        
+        # Creating a state stack for the new state
+        try:
+            state_stack = state_stacks[state][:]
+        except:
+            state_stack = []
+        if len(state_stack):
+            state_stack.pop(0)
+        if ss:
+            for symbol in ss[::-1]:
+                state_stack.insert(0, symbol)
+        transition['stack'] = state_stack 
+
+        # Check if a recursive transition state being created, if so make a backward
+        # edge and don't add anything to the worklist
+        # print (state_stacks)
+        if state_stacks:
+            for state_element, stack in state_stacks.items():
+                # print ('Stack:', sorted(stack))
+                # print ('State stack:', sorted(state_stack))
+                if sorted(stack) == sorted(state_stack):
+                    transition['dest'] = state_element
+                    # print ('Recursive:', transition)
+                    pda.append(transition)
+                    count += 1
+                    isRecursive = True
+                    break 
+        # If a recursive transition exercised don't add the same transition as a new
+        # edge, continue onto the next transitions
+        if isRecursive:
+            continue
+            
+        # If the generated state has a stack size > stack_limit then that state is abandoned
+        # and not added to the FSA or the worklist for further expansion
+        if stack_limit:
+            if (len(transition['stack']) > stack_limit):
+                unexpanded_rules.add(transition['rule'])
+                continue
+
+        # Create transitions for the non-recursive relations and add to the worklist
+        # print ('Normal:', transition)
+        # print ('State2:', state)
+        pda.append(transition)
+        worklist.append([transition['dest'], transition['stack']])
+        state_stacks[transition['dest']] = state_stack
+        state_count += 1
+        count += 1
+
+def tokenize(rule):
+    '''
+    Gets the terminal and the corresponding stack symbols from a rule in GNF form
+    '''
+    pattern = re.compile("([r])*\'([\s\S]+)\'([\s\S]*)")
+    terminal = None
+    ss = None
+    termIsRegex = False
+    match = pattern.match(rule)
+    if match.group(1):
+        termIsRegex = True
+    if match.group(2):
+        terminal = match.group(2)
+    else:
+        raise AssertionError("Rule is not in GNF form")
+
+    if match.group(3):
+        ss = (match.group(3)).split()
+
+    return terminal, ss, termIsRegex
+
+def get_template():
+    transition_template = {
+            'trigger':None,
+            'source': None,
+            'dest': None,
+            'termIsRegex': False,
+            'terminal' : None,
+            'stack': []
+            }
+    return transition_template
+
+def postprocess():
+    '''
+    Creates a representation to be passed on to the C-module
+    '''
+    global pda
+    final_struct = {}
+    memoized = defaultdict(list)
+    # Supporting data structures for if stack limit is imposed
+    culled_pda = []
+    culled_final = []
+    num_transitions = 0 # Keep track of number of transitions
+
+
+    states, final, initial = _get_states()
+
+    print (initial)
+    assert len(initial) == 1, 'More than one init state found'
+
+    # Cull transitions to states which were not expanded owing to the stack limit
+    if stack_limit:
+
+        blocklist = []
+        for final_state in final:
+            for transition in pda:
+                if (transition["dest"] == final_state) and (len(transition["stack"]) > 0):
+                    blocklist.append(transition["dest"])
+                    continue
+                else:
+                    culled_pda.append(transition)
+        
+        culled_final = [state for state in final if state not in blocklist]
+
+        assert len(culled_final) == 1, 'More than one final state found'
+
+        for transition in culled_pda:
+            state = transition["source"]
+            if transition["dest"] in blocklist:
+                    continue 
+            num_transitions += 1
+            memoized[state].append([transition["trigger"], transition["dest"], 
+                transition["terminal"]])
+        final_struct["init_state"] = initial
+        final_struct["final_state"] = culled_final[0]
+        # The reason we do this is because when states are culled, the indexing is
+        # still relative to the actual number of states hence we keep numstates recorded
+        # as the original number of states
+        print ('[X] Actual Number of states:', len(memoized.keys()))
+        print ('[X] Number of transitions:', num_transitions)
+        print ('[X] Original Number of states:', len(states))
+        final_struct["numstates"] = len(states) 
+        final_struct["pda"] = memoized
+        return final_struct
+    
+    # Running FSA construction in exact approximation mode and postprocessing it like so
+    for transition in pda:
+       state = transition["source"]
+       memoized[state].append([transition["trigger"], transition["dest"], 
+           transition["terminal"]])
+
+    final_struct["init_state"] = initial
+    final_struct["final_state"] = final[0]
+    print ('[X] Actual Number of states:', len(memoized.keys()))
+    final_struct["numstates"] = len(memoized.keys()) 
+    final_struct["pda"] = memoized
+    return final_struct
+
+
+def _get_states():
+    source = set()
+    dest = set()
+    global pda
+    for transition in pda:
+        source.add(transition["source"])
+        dest.add(transition["dest"])
+    source_copy = source.copy()
+    source_copy.update(dest)
+    return list(source_copy), list(dest.difference(source)), str(''.join(list(source.difference(dest))))
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description = 'Script to convert GNF grammar to PDA')
+    parser.add_argument(
+            '--gf',
+            type = str,
+            help = 'Location of GNF grammar')
+    parser.add_argument(
+            '--limit',
+            type = int,
+            default = None,
+            help = 'Specify the upper bound for the stack size')
+    args = parser.parse_args()
+    main(args.gf, args.limit)

custom_mutators/gramatron/preprocess/gnf_converter.py

@@ -0,0 +1,289 @@
+import sys
+import re
+import copy
+import json
+from string import ascii_uppercase
+from itertools import combinations
+from collections import defaultdict
+
+NONTERMINALSET = []
+COUNT = 1
+
+def main(grammar_file, out, start):
+    grammar = None
+    # If grammar file is a preprocessed NT file, then skip preprocessing
+    if '.json' in grammar_file:
+        with open(grammar_file, 'r') as fd:
+            grammar = json.load(fd)
+    elif '.g4' in grammar_file:
+        with open(grammar_file, 'r') as fd:
+            data = fd.readlines()
+        grammar = preprocess(data)
+    else:
+        raise('Unknwown file format passed. Accepts (.g4/.json)')
+
+    with open('debug_preprocess.json', 'w+') as fd:
+        json.dump(grammar, fd)
+    grammar = remove_unit(grammar) # eliminates unit productions
+    with open('debug_unit.json', 'w+') as fd:
+        json.dump(grammar, fd)
+    grammar = remove_mixed(grammar) # eliminate terminals existing with non-terminals
+    with open('debug_mixed.json', 'w+') as fd:
+        json.dump(grammar, fd)
+    grammar = break_rules(grammar) # eliminate rules with more than two non-terminals
+    with open('debug_break.json', 'w+') as fd:
+        json.dump(grammar, fd)
+    grammar = gnf(grammar)
+
+    # Dump GNF form of the grammar with only reachable rules 
+    # reachable_grammar = get_reachable(grammar, start)
+    # with open('debug_gnf_reachable.json', 'w+') as fd:
+    #     json.dump(reachable_grammar, fd)
+    with open('debug_gnf.json', 'w+') as fd:
+        json.dump(grammar, fd)
+
+    grammar["Start"] = [start]
+    with open(out, 'w+') as fd:
+        json.dump(grammar, fd)
+
+def get_reachable(grammar, start):
+    '''
+    Returns a grammar without dead rules
+    '''
+    reachable_nt = set()
+    worklist = list()
+    processed = set()
+    reachable_grammar = dict()
+    worklist.append(start)
+
+    while worklist:
+        nt = worklist.pop(0)
+        processed.add(nt)
+        reachable_grammar[nt] = grammar[nt]
+        rules = grammar[nt]
+        for rule in rules:
+            tokens = gettokens(rule)
+            for token in tokens:
+                if not isTerminal(token):
+                    if token not in processed:
+                        worklist.append(token)
+    return reachable_grammar
+
+
+def gettokens(rule):
+    pattern = re.compile("([^\s\"\']+)|\"([^\"]*)\"|\'([^\']*)\'")
+    return [matched.group(0) for matched in pattern.finditer(rule)]
+
+def gnf(grammar):
+    old_grammar = copy.deepcopy(grammar)
+    new_grammar = defaultdict(list)
+    isgnf = False
+    while not isgnf:
+        for lhs, rules in old_grammar.items():
+            for rule in rules:
+                tokens = gettokens(rule) 
+                if len(tokens) == 1 and isTerminal(rule):
+                    new_grammar[lhs].append(rule)
+                    continue
+                startoken = tokens[0]
+                endrule = tokens[1:]
+                if not isTerminal(startoken):
+                    newrules = []
+                    extendrules = old_grammar[startoken]
+                    for extension in extendrules:
+                        temprule = endrule[:]
+                        temprule.insert(0, extension)
+                        newrules.append(temprule)
+                    for newnew in newrules:
+                        new_grammar[lhs].append(' '.join(newnew))
+                else:
+                    new_grammar[lhs].append(rule)
+        isgnf = True
+        for lhs, rules in new_grammar.items():
+            for rule in rules:
+                # if "\' \'" or isTerminal(rule):
+                tokens = gettokens(rule)
+                if len(tokens) == 1 and isTerminal(rule):
+                    continue
+                startoken = tokens[0]
+                if not isTerminal(startoken):
+                    isgnf = False
+                    break
+        if not isgnf:
+            old_grammar = copy.deepcopy(new_grammar)
+            new_grammar = defaultdict(list)
+    return new_grammar
+                
+
+def preprocess(data):
+    productions = []
+    production = []
+    for line in data:
+        if line != '\n': 
+            production.append(line)
+        else:
+            productions.append(production)
+            production = []
+    final_rule_set = {}
+    for production in productions:
+        rules = []
+        init = production[0]
+        nonterminal = init.split(':')[0]
+        rules.append(strip_chars(init.split(':')[1]).strip('| '))
+        for production_rule in production[1:]:
+            rules.append(strip_chars(production_rule.split('|')[0]))
+        final_rule_set[nonterminal] = rules
+    # for line in data:
+    #     if line != '\n':
+    #         production.append(line)
+    return final_rule_set
+
+def remove_unit(grammar):
+    nounitproductions = False 
+    old_grammar = copy.deepcopy(grammar)
+    new_grammar = defaultdict(list)
+    while not nounitproductions:
+        for lhs, rules in old_grammar.items():
+            for rhs in rules:
+                # Checking if the rule is a unit production rule
+                if len(gettokens(rhs)) == 1:
+                    if not isTerminal(rhs):
+                        new_grammar[lhs].extend([rule for rule in old_grammar[rhs]])
+                    else:
+                        new_grammar[lhs].append(rhs)
+                else:
+                    new_grammar[lhs].append(rhs)
+        # Checking there are no unit productions left in the grammar 
+        nounitproductions = True
+        for lhs, rules in new_grammar.items():
+            for rhs in rules:
+                if len(gettokens(rhs)) == 1:
+                    if not isTerminal(rhs):
+                        nounitproductions = False
+                        break
+            if not nounitproductions:
+                break
+        # Unit productions are still there in the grammar -- repeat the process
+        if not nounitproductions:
+            old_grammar = copy.deepcopy(new_grammar)
+            new_grammar = defaultdict(list)
+    return new_grammar
+
+def isTerminal(rule):
+    # pattern = re.compile("([r]*\'[\s\S]+\')")
+    pattern = re.compile("\'(.*?)\'")
+    match = pattern.match(rule)
+    if match:
+        return True
+    else:
+        return False
+
+def remove_mixed(grammar):
+    '''
+    Remove rules where there are terminals mixed in with non-terminals
+    '''
+    new_grammar = defaultdict(list)
+    for lhs, rules in grammar.items():
+        for rhs in rules:
+            # tokens = rhs.split(' ')
+            regen_rule = []
+            tokens = gettokens(rhs)
+            if len(gettokens(rhs)) == 1:
+                new_grammar[lhs].append(rhs)
+                continue
+            for token in tokens:
+                # Identify if there is a terminal in the RHS
+                if isTerminal(token):
+                    # Check if a corresponding nonterminal already exists
+                    nonterminal = terminal_exist(token, new_grammar)
+                    if nonterminal:
+                        regen_rule.append(nonterminal)
+                    else:
+                        new_nonterm = get_nonterminal()
+                        new_grammar[new_nonterm].append(token)
+                        regen_rule.append(new_nonterm)
+                else:
+                    regen_rule.append(token)
+            new_grammar[lhs].append(' '.join(regen_rule))
+    return new_grammar
+
+def break_rules(grammar):
+    new_grammar = defaultdict(list)
+    old_grammar = copy.deepcopy(grammar)
+    nomulti = False
+    while not nomulti:
+        for lhs, rules in old_grammar.items():
+            for rhs in rules:
+                tokens = gettokens(rhs)
+                if len(tokens) > 2 and (not isTerminal(rhs)):
+                    split = tokens[:-1] 
+                    nonterminal = terminal_exist(' '.join(split), new_grammar)
+                    if nonterminal:
+                        newrule = ' '.join([nonterminal, tokens[-1]])
+                        new_grammar[lhs].append(newrule)
+                    else:
+                        nonterminal = get_nonterminal()
+                        new_grammar[nonterminal].append(' '.join(split))
+                        newrule = ' '.join([nonterminal, tokens[-1]])
+                        new_grammar[lhs].append(newrule)
+                else:
+                    new_grammar[lhs].append(rhs)
+        nomulti = True
+        for lhs, rules in new_grammar.items():
+            for rhs in rules:
+                # tokens = rhs.split(' ')
+                tokens = gettokens(rhs)
+                if len(tokens) > 2 and (not isTerminal(rhs)):
+                    nomulti = False
+                    break
+        if not nomulti:
+            old_grammar = copy.deepcopy(new_grammar)
+            new_grammar = defaultdict(list)
+    return new_grammar
+
+def strip_chars(rule):
+    return rule.strip('\n\t ')
+
+def get_nonterminal():
+    global NONTERMINALSET
+    if NONTERMINALSET:
+        return NONTERMINALSET.pop(0)
+    else:
+        _repopulate()
+        return NONTERMINALSET.pop(0)
+
+def _repopulate():
+    global COUNT
+    global NONTERMINALSET
+    NONTERMINALSET = [''.join(x) for x in list(combinations(ascii_uppercase, COUNT))]
+    COUNT += 1
+
+def terminal_exist(token, grammar):
+    for nonterminal, rules in grammar.items():
+        if token in rules:
+            return nonterminal
+    return None
+
+
+
+if __name__ == '__main__':
+    import argparse
+    parser = argparse.ArgumentParser(description = 'Script to convert grammar to GNF form')
+    parser.add_argument(
+            '--gf',
+            type = str,
+            required = True,
+            help = 'Location of grammar file')
+    parser.add_argument(
+            '--out',
+            type = str,
+            required = True,
+            help = 'Location of output file')
+    parser.add_argument(
+            '--start',
+            type = str,
+            required = True,
+            help = 'Start token')
+    args = parser.parse_args()
+
+    main(args.gf, args.out, args.start)

custom_mutators/gramatron/preprocess/prep_automaton.sh

@@ -0,0 +1,38 @@
+#!/bin/bash
+
+# This script creates a FSA describing the input grammar *.g4
+
+if [ ! "$#" -lt 4 ]; then
+  echo "Usage: ./prep_pda.sh <grammar_file> <start> [stack_limit]"         
+  exit 1
+fi
+
+GRAMMAR_FILE=$1
+GRAMMAR_DIR="$(dirname $GRAMMAR_FILE)"
+START="$2"
+STACK_LIMIT="$3"
+
+# Get filename
+FILE=$(basename -- "$GRAMMAR_FILE")
+echo "File:$FILE"
+FILENAME="${FILE%.*}"
+echo "Name:$FILENAME"
+
+
+# Create the GNF form of the grammar
+CMD="python gnf_converter.py --gf $GRAMMAR_FILE --out ${FILENAME}.json --start $START"
+$CMD
+
+# Generate grammar automaton 
+# Check if user provided a stack limit
+if [ -z "${STACK_LIMIT}" ]; then
+CMD="python3 construct_automata.py --gf ${FILENAME}.json" 
+else
+CMD="python construct_automata.py --gf ${FILENAME}.json --limit ${STACK_LIMIT}" 
+fi
+echo $CMD
+$CMD
+
+# Move PDA to the source dir of the grammar
+echo "Copying ${FILENAME}_automata.json to $GRAMMAR_DIR"
+mv "${FILENAME}_automata.json" $GRAMMAR_DIR/  

custom_mutators/gramatron/test.c

@@ -0,0 +1,154 @@
+/* This is the testing module for Gramatron
+ */
+#include "afl-fuzz.h"
+#include "gramfuzz.h"
+
+#define NUMINPUTS 50
+
+state *create_pda(u8 *automaton_file) {
+
+  struct json_object *parsed_json;
+  state *             pda;
+  json_object *       source_obj, *attr;
+  int                 arraylen, ii, ii2, trigger_len, error;
+
+  printf("\n[GF] Automaton file passed:%s", automaton_file);
+  // parsed_json =
+  // json_object_from_file("./gramfuzz/php_gnf_processed_full.json");
+  parsed_json = json_object_from_file(automaton_file);
+
+  // Getting final state
+  source_obj = json_object_object_get(parsed_json, "final_state");
+  printf("\t\nFinal=%s\n", json_object_get_string(source_obj));
+  final_state = atoi(json_object_get_string(source_obj));
+
+  // Getting initial state
+  source_obj = json_object_object_get(parsed_json, "init_state");
+  init_state = atoi(json_object_get_string(source_obj));
+  printf("\tInit=%s\n", json_object_get_string(source_obj));
+
+  // Getting number of states
+  source_obj = json_object_object_get(parsed_json, "numstates");
+  numstates = atoi(json_object_get_string(source_obj)) + 1;
+  printf("\tNumStates=%d\n", numstates);
+
+  // Allocate state space for each pda state
+  pda = (state *)calloc(atoi(json_object_get_string(source_obj)) + 1,
+                        sizeof(state));
+
+  // Getting PDA representation
+  source_obj = json_object_object_get(parsed_json, "pda");
+  enum json_type type;
+  json_object_object_foreach(source_obj, key, val) {
+
+    state *  state_ptr;
+    trigger *trigger_ptr;
+    int      offset;
+
+    // Get the correct offset into the pda to store state information
+    state_ptr = pda;
+    offset = atoi(key);
+    state_ptr += offset;
+
+    // Store state string
+    state_ptr->state_name = offset;
+
+    // Create trigger array of structs
+    trigger_len = json_object_array_length(val);
+    state_ptr->trigger_len = trigger_len;
+    trigger_ptr = (trigger *)calloc(trigger_len, sizeof(trigger));
+    state_ptr->ptr = trigger_ptr;
+    printf("\nName:%d Trigger:%d", offset, trigger_len);
+
+    for (ii = 0; ii < trigger_len; ii++) {
+
+      json_object *obj = json_object_array_get_idx(val, ii);
+      // Get all the trigger trigger attributes
+      attr = json_object_array_get_idx(obj, 0);
+      (trigger_ptr)->id = strdup(json_object_get_string(attr));
+
+      attr = json_object_array_get_idx(obj, 1);
+      trigger_ptr->dest = atoi(json_object_get_string(attr));
+
+      attr = json_object_array_get_idx(obj, 2);
+      if (!strcmp("\\n", json_object_get_string(attr))) {
+
+        trigger_ptr->term = strdup("\n");
+
+      } else {
+
+        trigger_ptr->term = strdup(json_object_get_string(attr));
+
+      }
+
+      trigger_ptr->term_len = strlen(trigger_ptr->term);
+      trigger_ptr++;
+
+    }
+
+  }
+
+  // Delete the JSON object
+  json_object_put(parsed_json);
+
+  return pda;
+
+}
+
+void SanityCheck(char *automaton_path) {
+
+  state *        pda = create_pda(automaton_path);
+  int            count = 0, state;
+  Get_Dupes_Ret *getdupesret;
+  IdxMap_new *   statemap;
+  IdxMap_new *   statemap_ptr;
+  terminal *     term_ptr;
+
+  while (count < NUMINPUTS) {
+
+    // Perform input generation
+    Array *generated = gen_input(pda, NULL);
+    print_repr(generated, "Gen");
+    count += 1;
+
+  }
+
+}
+
+int main(int argc, char *argv[]) {
+
+  char *         mode;
+  char *         automaton_path;
+  char *         output_dir = NULL;
+  struct timeval tv;
+  struct timeval tz;
+  // gettimeofday(&tv, &tz);
+  srand(1337);
+  if (argc == 3) {
+
+    mode = argv[1];
+    automaton_path = strdup(argv[2]);
+    printf("\nMode:%s Path:%s", mode, automaton_path);
+
+  } else {
+
+    printf("\nUsage: ./test <mode> <automaton_path>");
+    return -1;
+
+  }
+
+  if (!strcmp(mode, "SanityCheck")) {
+
+    SanityCheck(automaton_path);
+
+  } else {
+
+    printf("\nUnrecognized mode");
+    return -1;
+
+  }
+
+  return 0;
+
+}
+

custom_mutators/gramatron/test.h

@@ -0,0 +1,57 @@
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <json-c/json.h>
+#include <unistd.h>
+#include "hashmap.h"
+#include "uthash.h"
+#include "utarray.h"
+
+#define INIT_SIZE 100  // Initial size of the dynamic array holding the input
+
+typedef struct terminal {
+
+  int    state;
+  int    trigger_idx;
+  size_t symbol_len;
+  char * symbol;
+
+} terminal;
+
+typedef struct trigger {
+
+  char * id;
+  int    dest;
+  char * term;
+  size_t term_len;
+
+} trigger;
+
+typedef struct state {
+
+  int      state_name;   // Integer State name
+  int      trigger_len;  // Number of triggers associated with this state
+  trigger *ptr;          // Pointer to beginning of the list of triggers
+
+} state;
+
+typedef struct {
+
+  size_t    used;
+  size_t    size;
+  size_t    inputlen;
+  terminal *start;
+
+} Array;
+
+int init_state;
+int curr_state;
+int final_state;
+
+state *create_pda(char *);
+Array *gen_input(state *, Array *);
+void   print_repr(Array *, char *);
+void   initArray(Array *, size_t);
+void   insertArray(Array *, int, char *, size_t, int);
+

custom_mutators/gramatron/utarray.h

@@ -0,0 +1,392 @@
+/*
+Copyright (c) 2008-2018, Troy D. Hanson   http://troydhanson.github.com/uthash/
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
+IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
+PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER
+OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+
+/* a dynamic array implementation using macros
+ */
+#ifndef UTARRAY_H
+#define UTARRAY_H
+
+#define UTARRAY_VERSION 2.1.0
+
+#include <stddef.h>                                               /* size_t */
+#include <string.h>                                          /* memset, etc */
+#include <stdlib.h>                                                 /* exit */
+
+#ifdef __GNUC__
+  #define UTARRAY_UNUSED __attribute__((__unused__))
+#else
+  #define UTARRAY_UNUSED
+#endif
+
+#ifdef oom
+  #error \
+      "The name of macro 'oom' has been changed to 'utarray_oom'. Please update your code."
+  #define utarray_oom() oom()
+#endif
+
+#ifndef utarray_oom
+  #define utarray_oom() exit(-1)
+#endif
+
+typedef void(ctor_f)(void *dst, const void *src);
+typedef void(dtor_f)(void *elt);
+typedef void(init_f)(void *elt);
+typedef struct {
+
+  size_t  sz;
+  init_f *init;
+  ctor_f *copy;
+  dtor_f *dtor;
+
+} UT_icd;
+
+typedef struct {
+
+  unsigned i, n;           /* i: index of next available slot, n: num slots */
+  UT_icd   icd;               /* initializer, copy and destructor functions */
+  char *   d;                                     /* n slots of size icd->sz*/
+
+} UT_array;
+
+#define utarray_init(a, _icd)       \
+  do {                              \
+                                    \
+    memset(a, 0, sizeof(UT_array)); \
+    (a)->icd = *(_icd);             \
+                                    \
+  } while (0)
+
+#define utarray_done(a)                            \
+  do {                                             \
+                                                   \
+    if ((a)->n) {                                  \
+                                                   \
+      if ((a)->icd.dtor) {                         \
+                                                   \
+        unsigned _ut_i;                            \
+        for (_ut_i = 0; _ut_i < (a)->i; _ut_i++) { \
+                                                   \
+          (a)->icd.dtor(utarray_eltptr(a, _ut_i)); \
+                                                   \
+        }                                          \
+                                                   \
+      }                                            \
+      free((a)->d);                                \
+                                                   \
+    }                                              \
+    (a)->n = 0;                                    \
+                                                   \
+  } while (0)
+
+#define utarray_new(a, _icd)                    \
+  do {                                          \
+                                                \
+    (a) = (UT_array *)malloc(sizeof(UT_array)); \
+    if ((a) == NULL) { utarray_oom(); }         \
+    utarray_init(a, _icd);                      \
+                                                \
+  } while (0)
+
+#define utarray_free(a) \
+  do {                  \
+                        \
+    utarray_done(a);    \
+    free(a);            \
+                        \
+  } while (0)
+
+#define utarray_reserve(a, by)                                     \
+  do {                                                             \
+                                                                   \
+    if (((a)->i + (by)) > (a)->n) {                                \
+                                                                   \
+      char *utarray_tmp;                                           \
+      while (((a)->i + (by)) > (a)->n) {                           \
+                                                                   \
+        (a)->n = ((a)->n ? (2 * (a)->n) : 8);                      \
+                                                                   \
+      }                                                            \
+      utarray_tmp = (char *)realloc((a)->d, (a)->n * (a)->icd.sz); \
+      if (utarray_tmp == NULL) { utarray_oom(); }                  \
+      (a)->d = utarray_tmp;                                        \
+                                                                   \
+    }                                                              \
+                                                                   \
+  } while (0)
+
+#define utarray_push_back(a, p)                             \
+  do {                                                      \
+                                                            \
+    utarray_reserve(a, 1);                                  \
+    if ((a)->icd.copy) {                                    \
+                                                            \
+      (a)->icd.copy(_utarray_eltptr(a, (a)->i++), p);       \
+                                                            \
+    } else {                                                \
+                                                            \
+      memcpy(_utarray_eltptr(a, (a)->i++), p, (a)->icd.sz); \
+                                                            \
+    };                                                      \
+                                                            \
+  } while (0)
+
+#define utarray_pop_back(a)                          \
+  do {                                               \
+                                                     \
+    if ((a)->icd.dtor) {                             \
+                                                     \
+      (a)->icd.dtor(_utarray_eltptr(a, --((a)->i))); \
+                                                     \
+    } else {                                         \
+                                                     \
+      (a)->i--;                                      \
+                                                     \
+    }                                                \
+                                                     \
+  } while (0)
+
+#define utarray_extend_back(a)                            \
+  do {                                                    \
+                                                          \
+    utarray_reserve(a, 1);                                \
+    if ((a)->icd.init) {                                  \
+                                                          \
+      (a)->icd.init(_utarray_eltptr(a, (a)->i));          \
+                                                          \
+    } else {                                              \
+                                                          \
+      memset(_utarray_eltptr(a, (a)->i), 0, (a)->icd.sz); \
+                                                          \
+    }                                                     \
+    (a)->i++;                                             \
+                                                          \
+  } while (0)
+
+#define utarray_len(a) ((a)->i)
+
+#define utarray_eltptr(a, j) (((j) < (a)->i) ? _utarray_eltptr(a, j) : NULL)
+#define _utarray_eltptr(a, j) ((a)->d + ((a)->icd.sz * (j)))
+
+#define utarray_insert(a, p, j)                                   \
+  do {                                                            \
+                                                                  \
+    if ((j) > (a)->i) utarray_resize(a, j);                       \
+    utarray_reserve(a, 1);                                        \
+    if ((j) < (a)->i) {                                           \
+                                                                  \
+      memmove(_utarray_eltptr(a, (j) + 1), _utarray_eltptr(a, j), \
+              ((a)->i - (j)) * ((a)->icd.sz));                    \
+                                                                  \
+    }                                                             \
+    if ((a)->icd.copy) {                                          \
+                                                                  \
+      (a)->icd.copy(_utarray_eltptr(a, j), p);                    \
+                                                                  \
+    } else {                                                      \
+                                                                  \
+      memcpy(_utarray_eltptr(a, j), p, (a)->icd.sz);              \
+                                                                  \
+    };                                                            \
+    (a)->i++;                                                     \
+                                                                  \
+  } while (0)
+
+#define utarray_inserta(a, w, j)                                               \
+  do {                                                                         \
+                                                                               \
+    if (utarray_len(w) == 0) break;                                            \
+    if ((j) > (a)->i) utarray_resize(a, j);                                    \
+    utarray_reserve(a, utarray_len(w));                                        \
+    if ((j) < (a)->i) {                                                        \
+                                                                               \
+      memmove(_utarray_eltptr(a, (j) + utarray_len(w)), _utarray_eltptr(a, j), \
+              ((a)->i - (j)) * ((a)->icd.sz));                                 \
+                                                                               \
+    }                                                                          \
+    if ((a)->icd.copy) {                                                       \
+                                                                               \
+      unsigned _ut_i;                                                          \
+      for (_ut_i = 0; _ut_i < (w)->i; _ut_i++) {                               \
+                                                                               \
+        (a)->icd.copy(_utarray_eltptr(a, (j) + _ut_i),                         \
+                      _utarray_eltptr(w, _ut_i));                              \
+                                                                               \
+      }                                                                        \
+                                                                               \
+    } else {                                                                   \
+                                                                               \
+      memcpy(_utarray_eltptr(a, j), _utarray_eltptr(w, 0),                     \
+             utarray_len(w) * ((a)->icd.sz));                                  \
+                                                                               \
+    }                                                                          \
+    (a)->i += utarray_len(w);                                                  \
+                                                                               \
+  } while (0)
+
+#define utarray_resize(dst, num)                                   \
+  do {                                                             \
+                                                                   \
+    unsigned _ut_i;                                                \
+    if ((dst)->i > (unsigned)(num)) {                              \
+                                                                   \
+      if ((dst)->icd.dtor) {                                       \
+                                                                   \
+        for (_ut_i = (num); _ut_i < (dst)->i; ++_ut_i) {           \
+                                                                   \
+          (dst)->icd.dtor(_utarray_eltptr(dst, _ut_i));            \
+                                                                   \
+        }                                                          \
+                                                                   \
+      }                                                            \
+                                                                   \
+    } else if ((dst)->i < (unsigned)(num)) {                       \
+                                                                   \
+      utarray_reserve(dst, (num) - (dst)->i);                      \
+      if ((dst)->icd.init) {                                       \
+                                                                   \
+        for (_ut_i = (dst)->i; _ut_i < (unsigned)(num); ++_ut_i) { \
+                                                                   \
+          (dst)->icd.init(_utarray_eltptr(dst, _ut_i));            \
+                                                                   \
+        }                                                          \
+                                                                   \
+      } else {                                                     \
+                                                                   \
+        memset(_utarray_eltptr(dst, (dst)->i), 0,                  \
+               (dst)->icd.sz *((num) - (dst)->i));                 \
+                                                                   \
+      }                                                            \
+                                                                   \
+    }                                                              \
+    (dst)->i = (num);                                              \
+                                                                   \
+  } while (0)
+
+#define utarray_concat(dst, src)                 \
+  do {                                           \
+                                                 \
+    utarray_inserta(dst, src, utarray_len(dst)); \
+                                                 \
+  } while (0)
+
+#define utarray_erase(a, pos, len)                                        \
+  do {                                                                    \
+                                                                          \
+    if ((a)->icd.dtor) {                                                  \
+                                                                          \
+      unsigned _ut_i;                                                     \
+      for (_ut_i = 0; _ut_i < (len); _ut_i++) {                           \
+                                                                          \
+        (a)->icd.dtor(utarray_eltptr(a, (pos) + _ut_i));                  \
+                                                                          \
+      }                                                                   \
+                                                                          \
+    }                                                                     \
+    if ((a)->i > ((pos) + (len))) {                                       \
+                                                                          \
+      memmove(_utarray_eltptr(a, pos), _utarray_eltptr(a, (pos) + (len)), \
+              ((a)->i - ((pos) + (len))) * (a)->icd.sz);                  \
+                                                                          \
+    }                                                                     \
+    (a)->i -= (len);                                                      \
+                                                                          \
+  } while (0)
+
+#define utarray_renew(a, u) \
+  do {                      \
+                            \
+    if (a)                  \
+      utarray_clear(a);     \
+    else                    \
+      utarray_new(a, u);    \
+                            \
+  } while (0)
+
+#define utarray_clear(a)                            \
+  do {                                              \
+                                                    \
+    if ((a)->i > 0) {                               \
+                                                    \
+      if ((a)->icd.dtor) {                          \
+                                                    \
+        unsigned _ut_i;                             \
+        for (_ut_i = 0; _ut_i < (a)->i; _ut_i++) {  \
+                                                    \
+          (a)->icd.dtor(_utarray_eltptr(a, _ut_i)); \
+                                                    \
+        }                                           \
+                                                    \
+      }                                             \
+      (a)->i = 0;                                   \
+                                                    \
+    }                                               \
+                                                    \
+  } while (0)
+
+#define utarray_sort(a, cmp)                 \
+  do {                                       \
+                                             \
+    qsort((a)->d, (a)->i, (a)->icd.sz, cmp); \
+                                             \
+  } while (0)
+
+#define utarray_find(a, v, cmp) bsearch((v), (a)->d, (a)->i, (a)->icd.sz, cmp)
+
+#define utarray_front(a) (((a)->i) ? (_utarray_eltptr(a, 0)) : NULL)
+#define utarray_next(a, e)                                             \
+  (((e) == NULL) ? utarray_front(a)                                    \
+                 : (((a)->i != utarray_eltidx(a, e) + 1)               \
+                        ? _utarray_eltptr(a, utarray_eltidx(a, e) + 1) \
+                        : NULL))
+#define utarray_prev(a, e)                                             \
+  (((e) == NULL) ? utarray_back(a)                                     \
+                 : ((utarray_eltidx(a, e) != 0)                        \
+                        ? _utarray_eltptr(a, utarray_eltidx(a, e) - 1) \
+                        : NULL))
+#define utarray_back(a) (((a)->i) ? (_utarray_eltptr(a, (a)->i - 1)) : NULL)
+#define utarray_eltidx(a, e) (((char *)(e) - (a)->d) / (a)->icd.sz)
+
+/* last we pre-define a few icd for common utarrays of ints and strings */
+static void utarray_str_cpy(void *dst, const void *src) {
+
+  char **_src = (char **)src, **_dst = (char **)dst;
+  *_dst = (*_src == NULL) ? NULL : strdup(*_src);
+
+}
+
+static void utarray_str_dtor(void *elt) {
+
+  char **eltc = (char **)elt;
+  if (*eltc != NULL) free(*eltc);
+
+}
+
+static const UT_icd ut_str_icd UTARRAY_UNUSED = {
+
+    sizeof(char *), NULL, utarray_str_cpy, utarray_str_dtor};
+static const UT_icd ut_int_icd UTARRAY_UNUSED = {sizeof(int), NULL, NULL, NULL};
+static const UT_icd ut_ptr_icd UTARRAY_UNUSED = {sizeof(void *), NULL, NULL,
+                                                 NULL};
+
+#endif                                                         /* UTARRAY_H */
+

custom_mutators/grammar_mutator/GRAMMAR_VERSION

@@ -1 +1 @@
-b79d51a
+cbe5e32

custom_mutators/grammar_mutator/build_grammar_mutator.sh

@@ -14,7 +14,7 @@
 #                                <andreafioraldi@gmail.com>
 #
 # Copyright 2017 Battelle Memorial Institute. All rights reserved.
-# Copyright 2019-2020 AFLplusplus Project. All rights reserved.
+# Copyright 2019-2022 AFLplusplus Project. All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.

custom_mutators/honggfuzz/README.md

@@ -1,7 +1,7 @@
 # custum mutator: honggfuzz mangle
 
 this is the honggfuzz mutator in mangle.c as a custom mutator
-module for afl++. It is the original mangle.c, mangle.h and honggfuzz.h
+module for AFL++. It is the original mangle.c, mangle.h and honggfuzz.h
 with a lot of mocking around it :-)
 
 just type `make` to build

custom_mutators/honggfuzz/honggfuzz.c

@@ -65,9 +65,9 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) {
 /* When a new queue entry is added we check if there are new dictionary
    entries to add to honggfuzz structure */
 
-void afl_custom_queue_new_entry(my_mutator_t * data,
-                                const uint8_t *filename_new_queue,
-                                const uint8_t *filename_orig_queue) {
+uint8_t afl_custom_queue_new_entry(my_mutator_t * data,
+                                   const uint8_t *filename_new_queue,
+                                   const uint8_t *filename_orig_queue) {
 
   if (run.global->mutate.dictionaryCnt >= 1024) return;
 
@@ -97,6 +97,8 @@ void afl_custom_queue_new_entry(my_mutator_t * data,
 
   }
 
+  return 0;
+
 }
 
 /* we could set only_printable if is_ascii is set ... let's see

custom_mutators/libfuzzer/FuzzerLoop.cpp

@@ -1086,6 +1086,7 @@ ATTRIBUTE_INTERFACE size_t LLVMFuzzerMutate(uint8_t *Data, size_t Size,
                                             size_t MaxSize) {
 
   assert(fuzzer::F);
+  fuzzer::F->GetMD().StartMutationSequence();
   size_t r = fuzzer::F->GetMD().DefaultMutate(Data, Size, MaxSize);
 #ifdef  INTROSPECTION
   introspection_ptr = fuzzer::F->GetMD().WriteMutationSequence();

custom_mutators/libfuzzer/README.md

@@ -11,9 +11,11 @@ Note that this is currently a simple implementation and it is missing two featur
   * Dictionary support
 
 To update the source, all that is needed is that FuzzerDriver.cpp has to receive
+
 ```
 #include "libfuzzer.inc"
 ```
+
 before the closing namespace bracket.
 
 It is also libfuzzer.inc where the configuration of the libfuzzer mutations
@@ -21,4 +23,4 @@ are done.
 
 > Original repository: https://github.com/llvm/llvm-project
 > Path: compiler-rt/lib/fuzzer/*.{h|cpp}
-> Source commit: df3e903655e2499968fc7af64fb5fa52b2ee79bb
+> Source commit: df3e903655e2499968fc7af64fb5fa52b2ee79bb

custom_mutators/libfuzzer/libfuzzer.cpp

@@ -78,9 +78,9 @@ extern "C" my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) {
 /* When a new queue entry is added we check if there are new dictionary
    entries to add to honggfuzz structure */
 #if 0
-extern "C" void afl_custom_queue_new_entry(my_mutator_t * data,
-                                           const uint8_t *filename_new_queue,
-                                           const uint8_t *filename_orig_queue) {
+extern "C" uint8_t afl_custom_queue_new_entry(my_mutator_t * data,
+                                              const uint8_t *filename_new_queue,
+                                              const uint8_t *filename_orig_queue) {
 
   while (data->extras_cnt < afl_struct->extras_cnt) {
 
@@ -110,6 +110,8 @@ extern "C" void afl_custom_queue_new_entry(my_mutator_t * data,
 
   }
 
+  return 0;
+
 }
 
 #endif

custom_mutators/libfuzzer/libfuzzer.inc

@@ -2,7 +2,7 @@
 
 extern "C" ATTRIBUTE_INTERFACE void
 LLVMFuzzerMyInit(int (*Callback)(const uint8_t *Data, size_t Size), unsigned int Seed) {
-  Random Rand(Seed);
+  auto *Rand = new Random(Seed);
   FuzzingOptions Options;
   Options.Verbosity = 3;
   Options.MaxLen = 1024000;
@@ -30,7 +30,7 @@ LLVMFuzzerMyInit(int (*Callback)(const uint8_t *Data, size_t Size), unsigned int
   struct EntropicOptions Entropic;
   Entropic.Enabled = Options.Entropic;
   EF = new ExternalFunctions();
-  auto *MD = new MutationDispatcher(Rand, Options);
+  auto *MD = new MutationDispatcher(*Rand, Options);
   auto *Corpus = new InputCorpus(Options.OutputCorpus, Entropic);
   auto *F = new Fuzzer(Callback, *Corpus, *MD, Options);
 }

custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.cc

@@ -99,10 +99,12 @@ extern "C" size_t afl_custom_fuzz(MyMutator *mutator, // return value from afl_c
     std::string s = ProtoToData(*p);
     // Copy to a new buffer ( mutated_out )
     size_t mutated_size = s.size() <= max_size ? s.size() : max_size; // check if raw data's size is larger than max_size
-    uint8_t *mutated_out = new uint8_t[mutated_size+1];
-    memcpy(mutated_out, s.c_str(), mutated_size); // copy the mutated data
+
+    delete[] mutator->mutated_out;
+    mutator->mutated_out = new uint8_t[mutated_size];
+    memcpy(mutator->mutated_out, s.c_str(), mutated_size); // copy the mutated data
     // Assign the mutated data and return mutated_size
-    *out_buf = mutated_out;
+    *out_buf = mutator->mutated_out;
     return mutated_size;
 }
 

custom_mutators/libprotobuf-mutator-example/lpm_aflpp_custom_mutator_input.h

@@ -2,4 +2,9 @@
 #include "test.pb.h"
 
 class MyMutator : public protobuf_mutator::Mutator {
+public:
+    uint8_t *mutated_out = nullptr; 
+    ~MyMutator() {
+        delete[] mutated_out;
+    }
 };

custom_mutators/radamsa/libradamsa.c

@@ -4473,6 +4473,10 @@ static word prim_sys(word op, word a, word b, word c) {
         FD_CLOEXEC,
         F_DUPFD,
         F_DUPFD_CLOEXEC,
+#if defined(F_DUP2FD)
+        F_DUP2FD,
+        F_DUP2FD_CLOEXEC,
+#endif
         F_GETFD,
         F_SETFD,
         F_GETFL,

custom_mutators/rust/custom_mutator/src/lib.rs

@@ -53,7 +53,13 @@ pub trait RawCustomMutator {
         1
     }
 
-    fn queue_new_entry(&mut self, filename_new_queue: &Path, _filename_orig_queue: Option<&Path>) {}
+    fn queue_new_entry(
+        &mut self,
+        filename_new_queue: &Path,
+        _filename_orig_queue: Option<&Path>,
+    ) -> bool {
+        false
+    }
 
     fn queue_get(&mut self, filename: &Path) -> bool {
         true
@@ -84,7 +90,6 @@ pub mod wrappers {
 
     use std::{
         any::Any,
-        convert::TryInto,
         ffi::{c_void, CStr, OsStr},
         mem::ManuallyDrop,
         os::{raw::c_char, unix::ffi::OsStrExt},
@@ -176,6 +181,10 @@ pub mod wrappers {
     }
 
     /// Internal function used in the macro
+    /// # Safety
+    ///
+    /// May dereference all passed-in pointers.
+    /// Should not be called manually, but will be called by `afl-fuzz`
     pub unsafe fn afl_custom_fuzz_<M: RawCustomMutator>(
         data: *mut c_void,
         buf: *mut u8,
@@ -199,13 +208,10 @@ pub mod wrappers {
             } else {
                 Some(slice::from_raw_parts(add_buf, add_buf_size))
             };
-            match context
-                .mutator
-                .fuzz(buff_slice, add_buff_slice, max_size.try_into().unwrap())
-            {
+            match context.mutator.fuzz(buff_slice, add_buff_slice, max_size) {
                 Some(buffer) => {
                     *out_buf = buffer.as_ptr();
-                    buffer.len().try_into().unwrap()
+                    buffer.len()
                 }
                 None => {
                     // return the input buffer with 0-length to let AFL skip this mutation attempt
@@ -220,6 +226,10 @@ pub mod wrappers {
     }
 
     /// Internal function used in the macro
+    ///
+    /// # Safety
+    /// Dereferences the passed-in pointers up to `buf_size` bytes.
+    /// Should not be called directly.
     pub unsafe fn afl_custom_fuzz_count_<M: RawCustomMutator>(
         data: *mut c_void,
         buf: *const u8,
@@ -246,7 +256,7 @@ pub mod wrappers {
         data: *mut c_void,
         filename_new_queue: *const c_char,
         filename_orig_queue: *const c_char,
-    ) {
+    ) -> bool {
         match catch_unwind(|| {
             let mut context = FFIContext::<M>::from(data);
             if filename_new_queue.is_null() {
@@ -264,7 +274,7 @@ pub mod wrappers {
             };
             context
                 .mutator
-                .queue_new_entry(filename_new_queue, filename_orig_queue);
+                .queue_new_entry(filename_new_queue, filename_orig_queue)
         }) {
             Ok(ret) => ret,
             Err(err) => panic_handler("afl_custom_queue_new_entry", err),
@@ -272,6 +282,10 @@ pub mod wrappers {
     }
 
     /// Internal function used in the macro
+    ///
+    /// # Safety
+    /// May dereference the passed-in `data` pointer.
+    /// Should not be called directly.
     pub unsafe fn afl_custom_deinit_<M: RawCustomMutator>(data: *mut c_void) {
         match catch_unwind(|| {
             // drop the context
@@ -386,18 +400,16 @@ macro_rules! export_mutator {
         }
 
         #[no_mangle]
-        pub extern "C" fn afl_custom_fuzz_count(
+        pub unsafe extern "C" fn afl_custom_fuzz_count(
             data: *mut ::std::os::raw::c_void,
             buf: *const u8,
             buf_size: usize,
         ) -> u32 {
-            unsafe {
-                $crate::wrappers::afl_custom_fuzz_count_::<$mutator_type>(data, buf, buf_size)
-            }
+            $crate::wrappers::afl_custom_fuzz_count_::<$mutator_type>(data, buf, buf_size)
         }
 
         #[no_mangle]
-        pub extern "C" fn afl_custom_fuzz(
+        pub unsafe extern "C" fn afl_custom_fuzz(
             data: *mut ::std::os::raw::c_void,
             buf: *mut u8,
             buf_size: usize,
@@ -406,17 +418,15 @@ macro_rules! export_mutator {
             add_buf_size: usize,
             max_size: usize,
         ) -> usize {
-            unsafe {
-                $crate::wrappers::afl_custom_fuzz_::<$mutator_type>(
-                    data,
-                    buf,
-                    buf_size,
-                    out_buf,
-                    add_buf,
-                    add_buf_size,
-                    max_size,
-                )
-            }
+            $crate::wrappers::afl_custom_fuzz_::<$mutator_type>(
+                data,
+                buf,
+                buf_size,
+                out_buf,
+                add_buf,
+                add_buf_size,
+                max_size,
+            )
         }
 
         #[no_mangle]
@@ -424,7 +434,7 @@ macro_rules! export_mutator {
             data: *mut ::std::os::raw::c_void,
             filename_new_queue: *const ::std::os::raw::c_char,
             filename_orig_queue: *const ::std::os::raw::c_char,
-        ) {
+        ) -> bool {
             $crate::wrappers::afl_custom_queue_new_entry_::<$mutator_type>(
                 data,
                 filename_new_queue,
@@ -456,8 +466,8 @@ macro_rules! export_mutator {
         }
 
         #[no_mangle]
-        pub extern "C" fn afl_custom_deinit(data: *mut ::std::os::raw::c_void) {
-            unsafe { $crate::wrappers::afl_custom_deinit_::<$mutator_type>(data) }
+        pub unsafe extern "C" fn afl_custom_deinit(data: *mut ::std::os::raw::c_void) {
+            $crate::wrappers::afl_custom_deinit_::<$mutator_type>(data)
         }
     };
 }
@@ -542,8 +552,8 @@ pub trait CustomMutator {
         &mut self,
         filename_new_queue: &Path,
         filename_orig_queue: Option<&Path>,
-    ) -> Result<(), Self::Error> {
-        Ok(())
+    ) -> Result<bool, Self::Error> {
+        Ok(false)
     }
 
     fn queue_get(&mut self, filename: &Path) -> Result<bool, Self::Error> {
@@ -617,11 +627,16 @@ where
         }
     }
 
-    fn queue_new_entry(&mut self, filename_new_queue: &Path, filename_orig_queue: Option<&Path>) {
+    fn queue_new_entry(
+        &mut self,
+        filename_new_queue: &Path,
+        filename_orig_queue: Option<&Path>,
+    ) -> bool {
         match self.queue_new_entry(filename_new_queue, filename_orig_queue) {
             Ok(r) => r,
             Err(e) => {
                 Self::handle_error(e);
+                false
             }
         }
     }
@@ -696,16 +711,14 @@ mod default_mutator_describe {
 fn truncate_str_unicode_safe(s: &str, max_len: usize) -> &str {
     if s.len() <= max_len {
         s
+    } else if let Some((last_index, _)) = s
+        .char_indices()
+        .take_while(|(index, _)| *index <= max_len)
+        .last()
+    {
+        &s[..last_index]
     } else {
-        if let Some((last_index, _)) = s
-            .char_indices()
-            .take_while(|(index, _)| *index <= max_len)
-            .last()
-        {
-            &s[..last_index]
-        } else {
-            ""
-        }
+        ""
     }
 }
 

custom_mutators/symcc/symcc.c

@@ -101,9 +101,9 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) {
 
 /* When a new queue entry is added we run this input with the symcc
    instrumented binary */
-void afl_custom_queue_new_entry(my_mutator_t * data,
-                                const uint8_t *filename_new_queue,
-                                const uint8_t *filename_orig_queue) {
+uint8_t afl_custom_queue_new_entry(my_mutator_t * data,
+                                   const uint8_t *filename_new_queue,
+                                   const uint8_t *filename_orig_queue) {
 
   int         pipefd[2];
   struct stat st;
@@ -129,7 +129,7 @@ void afl_custom_queue_new_entry(my_mutator_t * data,
 
   int pid = fork();
 
-  if (pid == -1) return;
+  if (pid == -1) return 0;
 
   if (pid) {
 
@@ -147,7 +147,7 @@ void afl_custom_queue_new_entry(my_mutator_t * data,
         if (r <= 0) {
 
           close(pipefd[1]);
-          return;
+          return 0;
 
         }
 
@@ -232,6 +232,8 @@ void afl_custom_queue_new_entry(my_mutator_t * data,
 
   }
 
+  return 0;
+
 }
 
 uint32_t afl_custom_fuzz_count(my_mutator_t *data, const u8 *buf,

dictionaries/README.md

@@ -1,20 +1,18 @@
-# AFL dictionaries
+# AFL++ dictionaries
 
-(See [../README.md](../README.md) for the general instruction manual.)
+For the general instruction manual, see [docs/README.md](../docs/README.md).
 
-This subdirectory contains a set of dictionaries that can be used in
-conjunction with the -x option to allow the fuzzer to effortlessly explore the
-grammar of some of the more verbose data formats or languages. The basic
-principle behind the operation of fuzzer dictionaries is outlined in section 10
-of the "main" README.md for the project.
+This subdirectory contains a set of dictionaries that can be used in conjunction
+with the -x option to allow the fuzzer to effortlessly explore the grammar of
+some of the more verbose data formats or languages.
 
-These sets were done by Michal Zalewski, various contributors, and imported
-from oss-fuzz, go-fuzz and libfuzzer.
+These sets were done by Michal Zalewski, various contributors, and imported from
+oss-fuzz, go-fuzz and libfuzzer.
 
 Custom dictionaries can be added at will. They should consist of a
 reasonably-sized set of rudimentary syntax units that the fuzzer will then try
-to clobber together in various ways. Snippets between 2 and 16 bytes are
-usually the sweet spot.
+to clobber together in various ways. Snippets between 2 and 16 bytes are usually
+the sweet spot.
 
 Custom dictionaries can be created in two ways:
 
@@ -36,9 +34,9 @@ In the file mode, every name field can be optionally followed by @<num>, e.g.:
   `keyword_foo@1 = "foo"`
 
 Such entries will be loaded only if the requested dictionary level is equal or
-higher than this number. The default level is zero; a higher value can be set
-by appending @<num> to the dictionary file name, like so:
+higher than this number. The default level is zero; a higher value can be set by
+appending @<num> to the dictionary file name, like so:
 
   `-x path/to/dictionary.dct@2`
 
-Good examples of dictionaries can be found in xml.dict and png.dict.
+Good examples of dictionaries can be found in xml.dict and png.dict.

docs/Changelog.md

@@ -1,13 +1,86 @@
 # Changelog
 
-  This is the list of all noteworthy changes made in every public release of
-  the tool. See README.md for the general instruction manual.
+  This is the list of all noteworthy changes made in every public
+  release of the tool. See README.md for the general instruction manual.
 
 ## Staying informed
 
 Want to stay in the loop on major new features? Join our mailing list by
 sending a mail to <afl-users+subscribe@googlegroups.com>.
 
+### Version ++4.00c (release)
+  - complete documentation restructuring, made possible by Google Season
+    of Docs :) thank you Jana!
+  - we renamed several UI and fuzzer_stat entries to be more precise,
+    e.g. "unique crashes" -> "saved crashes", "total paths" ->
+    "corpus count", "current path" -> "current item".
+    This might need changing custom scripting!
+  - Nyx mode (full system emulation with snapshot capability) has been
+    added - thanks to @schumilo and @eqv!
+  - unicorn_mode:
+    - Moved to unicorn2! by Ziqiao Kong (@lazymio)
+    - Faster, more accurate emulation (newer QEMU base), risc-v support
+    - removed indirections in rust callbacks
+  - new binary-only fuzzing mode: coresight_mode for aarch64 CPUs :)
+    thanks to RICSecLab submitting!
+  - if instrumented libaries are dlopen()'ed after the forkserver you
+    will now see a crash. Before you would have colliding coverage.
+    We changed this to force fixing a broken setup rather then allowing
+    ineffective fuzzing.
+    See docs/best_practices.md how to fix such setups.
+  - afl-fuzz:
+    - cmplog binaries will need to be recompiled for this version
+      (it is better!)
+    - fix a regression introduced in 3.10 that resulted in less
+      coverage being detected. thanks to Collin May for reporting!
+    - ensure all spawned targets are killed on exit
+    - added AFL_IGNORE_PROBLEMS, plus checks to identify and abort on
+      incorrect LTO usage setups and enhanced the READMEs for better
+      information on how to deal with instrumenting libraries
+    - fix -n dumb mode (nobody should use this mode though)
+    - fix stability issue with LTO and cmplog
+    - better banner
+    - more effective cmplog mode
+    - more often update the UI when in input2stage mode
+  - qemu_mode/unicorn_mode: fixed OOB write when using libcompcov,
+      thanks to kotee4ko for reporting!
+  - frida_mode:
+    - better performance, bug fixes
+    - David Carlier added Android support :)
+  - afl-showmap, afl-tmin and afl-analyze:
+    - honor persistent mode for more speed. thanks to dloffre-snl
+      for reporting!
+    - fix bug where targets are not killed on timeouts
+    - moved hidden afl-showmap -A option to -H to be used for
+      coresight_mode
+  - Prevent accidentaly killing non-afl/fuzz services when aborting
+    afl-showmap and other tools.
+  - afl-cc:
+    - detect overflow reads on initial input buffer for asan
+    - new cmplog mode (incompatible with older afl++ versions)
+    - support llvm IR select instrumentation for default PCGUARD and LTO
+    - fix for shared linking on MacOS
+    - better selective instrumentation AFL_LLVM_{ALLOW|DENY}LIST
+      on filename matching (requires llvm 11 or newer)
+    - fixed a potential crash in targets for LAF string handling
+    - fixed a bad assert in LAF split switches
+    - added AFL_USE_TSAN thread sanitizer support
+    - llvm and LTO mode modified to work with new llvm 14-dev (again.)
+    - fix for AFL_REAL_LD
+    - more -z defs filtering
+    - make -v without options work
+  - added the very good grammar mutator "GramaTron" to the
+    custom_mutators
+  - added optimin, a faster and better corpus minimizer by
+    Adrian Herrera. Thank you!
+  - added afl-persistent-config script to set perform permanent system
+    configuration settings for fuzzing, for Linux and Macos.
+    thanks to jhertz!
+  - added xml, curl & exotic string functions to llvm dictionary feature
+  - fix AFL_PRELOAD issues on MacOS
+  - removed utils/afl_frida because frida_mode/ is now so much better
+  - added uninstall target to makefile (todo: update new readme!)
+
 ### Version ++3.14c (release)
   - afl-fuzz:
     - fix -F when a '/' was part of the parameter
@@ -26,7 +99,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
     - Fix to instrument global namespace functions in c++
     - Fix for llvm 13
     - support partial linking
-    - do honor AFL_LLVM_{ALLOW/DENY}LIST for LTO autodictionary and DICT2FILE
+    - do honor AFL_LLVM_{ALLOW/DENY}LIST for LTO autodictionary andDICT2FILE
     - We do support llvm versions from 3.8 to 5.0 again
   - frida_mode:
     - several fixes for cmplog
@@ -70,7 +143,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
     - on a crashing seed potentially the wrong input was disabled
     - added AFL_EXIT_ON_SEED_ISSUES env that will exit if a seed in
       -i dir crashes the target or results in a timeout. By default
-      afl++ ignores these and uses them for splicing instead.
+      AFL++ ignores these and uses them for splicing instead.
     - added AFL_EXIT_ON_TIME env that will make afl-fuzz exit fuzzing
       after no new paths have been found for n seconds
     - when AFL_FAST_CAL is set a variable path will now be calibrated
@@ -224,7 +297,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - Updated utils/afl_frida to be 5% faster, 7% on x86_x64
   - Added `AFL_KILL_SIGNAL` env variable (thanks @v-p-b)
   - @Edznux added a nice documentation on how to use rpc.statsd with
-    afl++ in docs/rpc_statsd.md, thanks!
+    AFL++ in docs/rpc_statsd.md, thanks!
 
 ### Version ++3.00c (release)
   - llvm_mode/ and gcc_plugin/ moved to instrumentation/
@@ -280,7 +353,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - custom mutators
     - added a new custom mutator: symcc -> https://github.com/eurecom-s3/symcc/
     - added a new custom mutator: libfuzzer that integrates libfuzzer mutations
-    - Our afl++ Grammar-Mutator is now better integrated into custom_mutators/
+    - Our AFL++ Grammar-Mutator is now better integrated into custom_mutators/
     - added INTROSPECTION support for custom modules
     - python fuzz function was not optional, fixed
     - some python mutator speed improvements
@@ -291,7 +364,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
 
 
 ### Version ++2.68c (release)
-  - added the GSoC excellent afl++ grammar mutator by Shengtuo to our
+  - added the GSoC excellent AFL++ grammar mutator by Shengtuo to our
     custom_mutators/ (see custom_mutators/README.md) - or get it here:
     https://github.com/AFLplusplus/Grammar-Mutator
   - a few QOL changes for Apple and its outdated gmake
@@ -314,12 +387,12 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - llvm_mode:
     - ported SanCov to LTO, and made it the default for LTO. better
       instrumentation locations
-    - Further llvm 12 support (fast moving target like afl++ :-) )
+    - Further llvm 12 support (fast moving target like AFL++ :-) )
     - deprecated LLVM SKIPSINGLEBLOCK env environment
 
 
 ### Version ++2.67c (release)
-  - Support for improved afl++ snapshot module:
+  - Support for improved AFL++ snapshot module:
     https://github.com/AFLplusplus/AFL-Snapshot-LKM
   - Due to the instrumentation needing more memory, the initial memory sizes
     for -m have been increased
@@ -421,7 +494,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
     files/stdin) - 10-100% performance increase
   - General support for 64 bit PowerPC, RiscV, Sparc etc.
   - fix afl-cmin.bash
-  - slightly better performance compilation options for afl++ and targets
+  - slightly better performance compilation options for AFL++ and targets
   - fixed afl-gcc/afl-as that could break on fast systems reusing pids in
     the same second
   - added lots of dictionaries from oss-fuzz, go-fuzz and Jakub Wilk
@@ -434,7 +507,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - afl-fuzz:
      - AFL_MAP_SIZE was not working correctly
      - better python detection
-     - an old, old bug in afl that would show negative stability in rare
+     - an old, old bug in AFL that would show negative stability in rare
        circumstances is now hopefully fixed
      - AFL_POST_LIBRARY was deprecated, use AFL_CUSTOM_MUTATOR_LIBRARY
        instead (see docs/custom_mutators.md)
@@ -493,8 +566,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - extended forkserver: map_size and more information is communicated to
     afl-fuzz (and afl-fuzz acts accordingly)
   - new environment variable: AFL_MAP_SIZE to specify the size of the shared map
-  - if AFL_CC/AFL_CXX is set but empty afl compilers did fail, fixed
-    (this bug is in vanilla afl too)
+  - if AFL_CC/AFL_CXX is set but empty AFL compilers did fail, fixed
+    (this bug is in vanilla AFL too)
   - added NO_PYTHON flag to disable python support when building afl-fuzz
   - more refactoring
 
@@ -508,7 +581,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - all:
     - big code changes to make afl-fuzz thread-safe so afl-fuzz can spawn
       multiple fuzzing threads in the future or even become a library
-    - afl basic tools now report on the environment variables picked up
+    - AFL basic tools now report on the environment variables picked up
     - more tools get environment variable usage info in the help output
     - force all output to stdout (some OK/SAY/WARN messages were sent to
       stdout, some to stderr)
@@ -657,7 +730,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - qemu and unicorn download scripts now try to download until the full
     download succeeded. f*ckin travis fails downloading 40% of the time!
   - more support for Android (please test!)
-  - added the few Android stuff we didnt have already from Google afl repository
+  - added the few Android stuff we didnt have already from Google AFL repository
   - removed unnecessary warnings
 
 
@@ -705,7 +778,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
 
   - big code refactoring:
     * all includes are now in include/
-    * all afl sources are now in src/ - see src/README.md
+    * all AFL sources are now in src/ - see src/README.md
     * afl-fuzz was split up in various individual files for including
       functionality in other programs (e.g. forkserver, memory map, etc.)
       for better readability.
@@ -721,7 +794,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - fix building on *BSD (thanks to tobias.kortkamp for the patch)
   - fix for a few features to support different map sized than 2^16
   - afl-showmap: new option -r now shows the real values in the buckets (stock
-    afl never did), plus shows tuple content summary information now
+    AFL never did), plus shows tuple content summary information now
   - small docu updates
   - NeverZero counters for QEMU
   - NeverZero counters for Unicorn
@@ -764,7 +837,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
     debugging
   - added -V time and -E execs option to better comparison runs, runs afl-fuzz
     for a specific time/executions.
-  - added a -s seed switch to allow afl run with a fixed initial
+  - added a -s seed switch to allow AFL run with a fixed initial
     seed that is not updated. This is good for performance and path discovery
     tests as the random numbers are deterministic then
   - llvm_mode LAF_... env variables can now be specified as AFL_LLVM_LAF_...
@@ -1512,7 +1585,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - Fixed a bug with installed copies of AFL trying to use QEMU mode. Spotted
     by G.M. Lime.
 
-  - Added last path / crash / hang times to fuzzer_stats, suggested by
+  - Added last find / crash / hang times to fuzzer_stats, suggested by
     Richard Hipp.
 
   - Fixed a typo, thanks to Jakub Wilk.
@@ -1585,7 +1658,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
 ### Version 1.63b:
 
   - Updated cgroups_asan/ with a new version from Sam, made a couple changes
-    to streamline it and keep parallel afl instances in separate groups.
+    to streamline it and keep parallel AFL instances in separate groups.
 
   - Fixed typos, thanks to Jakub Wilk.
 
@@ -2383,7 +2456,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
 
   - Added AFL_KEEP_ASSEMBLY for easier troubleshooting.
 
-  - Added an override for AFL_USE_ASAN if set at afl compile time. Requested by
+  - Added an override for AFL_USE_ASAN if set at AFL compile time. Requested by
     Hanno Boeck.
 
 ### Version 0.79b:
@@ -2724,7 +2797,7 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
   - Updated the documentation and added notes_for_asan.txt. Based on feedback
     from Hanno Boeck, Ben Laurie, and others.
 
-  - Moved the project to http://lcamtuf.coredump.cx/afl/.
+  - Moved the project to https://lcamtuf.coredump.cx/afl/.
 
 ### Version 0.46b:
 

docs/FAQ.md

@@ -1,243 +1,257 @@
-# Frequently asked questions about afl++
-
-## Contents
-
-  * [What is the difference between afl and afl++?](#what-is-the-difference-between-afl-and-afl)
-  * [I got a weird compile error from clang](#i-got-a-weird-compile-error-from-clang)
-  * [How to improve the fuzzing speed?](#how-to-improve-the-fuzzing-speed)
-  * [How do I fuzz a network service?](#how-do-i-fuzz-a-network-service)
-  * [How do I fuzz a GUI program?](#how-do-i-fuzz-a-gui-program)
-  * [What is an edge?](#what-is-an-edge)
-  * [Why is my stability below 100%?](#why-is-my-stability-below-100)
-  * [How can I improve the stability value?](#how-can-i-improve-the-stability-value)
+# Frequently asked questions (FAQ)
 
 If you find an interesting or important question missing, submit it via
-[https://github.com/AFLplusplus/AFLplusplus/issues](https://github.com/AFLplusplus/AFLplusplus/issues)
+[https://github.com/AFLplusplus/AFLplusplus/discussions](https://github.com/AFLplusplus/AFLplusplus/discussions).
 
-## What is the difference between afl and afl++?
+## General
 
-American Fuzzy Lop (AFL) was developed by Michał "lcamtuf" Zalewski starting in
-2013/2014, and when he left Google end of 2017 he stopped developing it.
+<details>
+  <summary id="what-is-the-difference-between-afl-and-aflplusplus">What is the difference between AFL and AFL++?</summary><p>
 
-At the end of 2019 the Google fuzzing team took over maintenance of AFL, however
-it is only accepting PRs from the community and is not developing enhancements
-anymore.
+  AFL++ is a superior fork to Google's AFL - more speed, more and better
+  mutations, more and better instrumentation, custom module support, etc.
 
-In the second quarter of 2019, 1 1/2 year later when no further development of
-AFL had happened and it became clear there would none be coming, afl++
-was born, where initially community patches were collected and applied
-for bug fixes and enhancements. Then from various AFL spin-offs - mostly academic
-research - features were integrated. This already resulted in a much advanced
-AFL.
+  American Fuzzy Lop (AFL) was developed by Michał "lcamtuf" Zalewski starting
+  in 2013/2014, and when he left Google end of 2017 he stopped developing it.
 
-Until the end of 2019 the afl++ team had grown to four active developers which
-then implemented their own research and features, making it now by far the most
-flexible and feature rich guided fuzzer available as open source.
-And in independent fuzzing benchmarks it is one of the best fuzzers available,
-e.g. [Fuzzbench Report](https://www.fuzzbench.com/reports/2020-08-03/index.html)
+  At the end of 2019, the Google fuzzing team took over maintenance of AFL,
+  however, it is only accepting PRs from the community and is not developing
+  enhancements anymore.
 
-## I got a weird compile error from clang
+  In the second quarter of 2019, 1 1/2 years later, when no further development
+  of AFL had happened and it became clear there would none be coming, AFL++ was
+  born, where initially community patches were collected and applied for bug
+  fixes and enhancements. Then from various AFL spin-offs - mostly academic
+  research - features were integrated. This already resulted in a much advanced
+  AFL.
 
-If you see this kind of error when trying to instrument a target with afl-cc/
-afl-clang-fast/afl-clang-lto:
-```
-/prg/tmp/llvm-project/build/bin/clang-13: symbol lookup error: /usr/local/bin/../lib/afl//cmplog-instructions-pass.so: undefined symbol: _ZNK4llvm8TypeSizecvmEv
-clang-13: error: unable to execute command: No such file or directory
-clang-13: error: clang frontend command failed due to signal (use -v to see invocation)
-clang version 13.0.0 (https://github.com/llvm/llvm-project 1d7cf550721c51030144f3cd295c5789d51c4aad)
-Target: x86_64-unknown-linux-gnu
-Thread model: posix
-InstalledDir: /prg/tmp/llvm-project/build/bin
-clang-13: note: diagnostic msg: 
-********************
-```
-Then this means that your OS updated the clang installation from an upgrade
-package and because of that the afl++ llvm plugins do not match anymore.
+  Until the end of 2019, the AFL++ team had grown to four active developers
+  which then implemented their own research and features, making it now by far
+  the most flexible and feature rich guided fuzzer available as open source. And
+  in independent fuzzing benchmarks it is one of the best fuzzers available,
+  e.g., [Fuzzbench
+  Report](https://www.fuzzbench.com/reports/2020-08-03/index.html).
+</p></details>
 
-Solution: `git pull ; make clean install` of afl++
+<details>
+  <summary id="is-afl-a-whitebox-graybox-or-blackbox-fuzzer">Is AFL++ a whitebox, graybox, or blackbox fuzzer?</summary><p>
 
-## How to improve the fuzzing speed?
+  The definition of the terms whitebox, graybox, and blackbox fuzzing varies
+  from one source to another. For example, "graybox fuzzing" could mean
+  binary-only or source code fuzzing, or something completely different.
+  Therefore, we try to avoid them.
 
-  1. Use [llvm_mode](docs/llvm_mode/README.md): afl-clang-lto (llvm >= 11) or afl-clang-fast (llvm >= 9 recommended)
-  2. Use [persistent mode](llvm_mode/README.persistent_mode.md) (x2-x20 speed increase)
-  3. Use the [afl++ snapshot module](https://github.com/AFLplusplus/AFL-Snapshot-LKM) (x2 speed increase)
-  4. If you do not use shmem persistent mode, use `AFL_TMPDIR` to put the input file directory on a tempfs location, see [docs/env_variables.md](docs/env_variables.md)
-  5. Improve Linux kernel performance: modify `/etc/default/grub`, set `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then `update-grub` and `reboot` (warning: makes the system less secure)
-  6. Running on an `ext2` filesystem with `noatime` mount option will be a bit faster than on any other journaling filesystem
-  7. Use your cores! [README.md:3.b) Using multiple cores/threads](../README.md#b-using-multiple-coresthreads)
+  [The Fuzzing Book](https://www.fuzzingbook.org/html/GreyboxFuzzer.html#AFL:-An-Effective-Greybox-Fuzzer)
+  describes the original AFL to be a graybox fuzzer. In that sense, AFL++ is
+  also a graybox fuzzer.
+</p></details>
 
-## How do I fuzz a network service?
+<details>
+  <summary id="where-can-i-find-tutorials">Where can I find tutorials?</summary><p>
 
-The short answer is - you cannot, at least not "out of the box".
+  We compiled a list of tutorials and exercises, see
+  [tutorials.md](tutorials.md).
+</p></details>
 
-Using a network channel is inadequate for several reasons:
-- it has a slow-down of x10-20 on the fuzzing speed
-- it does not scale to fuzzing multiple instances easily,
-- instead of one initial data packet often a back-and-forth interplay of packets is needed for stateful protocols (which is totally unsupported by most coverage aware fuzzers).
+<details>
+  <summary id="what-is-an-edge">What is an "edge"?</summary><p>
 
-The established method to fuzz network services is to modify the source code
-to read from a file or stdin (fd 0) (or even faster via shared memory, combine
-this with persistent mode [llvm_mode/README.persistent_mode.md](llvm_mode/README.persistent_mode.md)
-and you have a performance gain of x10 instead of a performance loss of over
-x10 - that is a x100 difference!).
+  A program contains `functions`, `functions` contain the compiled machine code.
+  The compiled machine code in a `function` can be in a single or many `basic
+  blocks`. A `basic block` is the **largest possible number of subsequent machine
+  code instructions** that has **exactly one entry point** (which can be be entered by
+  multiple other basic blocks) and runs linearly **without branching or jumping to
+  other addresses** (except at the end).
 
-If modifying the source is not an option (e.g. because you only have a binary
-and perform binary fuzzing) you can also use a shared library with AFL_PRELOAD
-to emulate the network. This is also much faster than the real network would be.
-See [utils/socket_fuzzing/](../utils/socket_fuzzing/).
+  ```
+  function() {
+    A:
+      some
+      code
+    B:
+      if (x) goto C; else goto D;
+    C:
+      some code
+      goto E
+    D:
+      some code
+      goto B
+    E:
+      return
+  }
+  ```
 
-There is an outdated afl++ branch that implements networking if you are
-desperate though: [https://github.com/AFLplusplus/AFLplusplus/tree/networking](https://github.com/AFLplusplus/AFLplusplus/tree/networking) - 
-however a better option is AFLnet ([https://github.com/aflnet/aflnet](https://github.com/aflnet/aflnet))
-which allows you to define network state with different type of data packets.
+  Every code block between two jump locations is a `basic block`.
 
-## How do I fuzz a GUI program?
+  An `edge` is then the unique relationship between two directly connected
+  `basic blocks` (from the code example above):
 
-If the GUI program can read the fuzz data from a file (via the command line,
-a fixed location or via an environment variable) without needing any user
-interaction then it would be suitable for fuzzing.
-
-Otherwise it is not possible without modifying the source code - which is a
-very good idea anyway as the GUI functionality is a huge CPU/time overhead
-for the fuzzing.
-
-So create a new `main()` that just reads the test case and calls the
-functionality for processing the input that the GUI program is using.
-
-## What is an "edge"?
-
-A program contains `functions`, `functions` contain the compiled machine code.
-The compiled machine code in a `function` can be in a single or many `basic blocks`.
-A `basic block` is the largest possible number of subsequent machine code
-instructions that has exactly one entrypoint (which can be be entered by multiple other basic blocks)
-and runs linearly without branching or jumping to other addresses (except at the end).
-```
-function() {
-  A:
-    some
-    code
-  B:
-    if (x) goto C; else goto D;
-  C:
-    some code
-    goto E
-  D:
-    some code
-    goto B
-  E:
-    return
-}
-```
-Every code block between two jump locations is a `basic block`.
-
-An `edge` is then the unique relationship between two directly connected `basic blocks` (from the
-code example above):
-```
-              Block A
-                |
+  ```
+                Block A
+                  |
+                  v
+                Block B  <------+
+              /        \       |
+              v          v      |
+          Block C    Block D --+
+              \
                 v
-              Block B  <------+
-             /        \       |
-            v          v      |
-         Block C    Block D --+
-             \
-              v
-              Block E
-```
-Every line between two blocks is an `edge`.
-Note that a few basic block loop to itself, this too would be an edge.
+                Block E
+  ```
 
-## Why is my stability below 100%?
+  Every line between two blocks is an `edge`. Note that a few basic block loop
+  to itself, this too would be an edge.
+</p></details>
 
-Stability is measured by how many percent of the edges in the target are
-"stable". Sending the same input again and again should take the exact same
-path through the target every time. If that is the case, the stability is 100%.
+## Targets
 
-If however randomness happens, e.g. a thread reading other external data,
-reaction to timing, etc. then in some of the re-executions with the same data
-the edge coverage result will be different accross runs.
-Those edges that change are then flagged "unstable".
+<details>
+  <summary id="how-can-i-fuzz-a-binary-only-target">How can I fuzz a binary-only target?</summary><p>
 
-The more "unstable" edges, the more difficult for afl++ to identify valid new
-paths.
+  AFL++ is a great fuzzer if you have the source code available.
 
-A value above 90% is usually fine and a value above 80% is also still ok, and
-even a value above 20% can still result in successful finds of bugs.
-However, it is recommended that for values below 90% or 80% you should take
-countermeasures to improve stability.
+  However, if there is only the binary program and no source code available,
+  then the standard non-instrumented mode is not effective.
 
-## How can I improve the stability value?
+  To learn how these binaries can be fuzzed, read
+  [fuzzing_binary-only_targets.md](fuzzing_binary-only_targets.md).
+</p></details>
 
-For fuzzing a 100% stable target that covers all edges is the best case.
-A 90% stable target that covers all edges is however better than a 100% stable
-target that ignores 10% of the edges.
+<details>
+  <summary id="how-can-i-fuzz-a-network-service">How can I fuzz a network service?</summary><p>
 
-With instability you basically have a partial coverage loss on an edge, with
-ignored functions you have a full loss on that edges.
+  The short answer is - you cannot, at least not "out of the box".
 
-There are functions that are unstable, but also provide value to coverage, eg
-init functions that use fuzz data as input for example.
-If however a function that has nothing to do with the input data is the
-source of instability, e.g. checking jitter, or is a hash map function etc.
-then it should not be instrumented.
+  For more information on fuzzing network services, see
+  [best_practices.md#fuzzing-a-network-service](best_practices.md#fuzzing-a-network-service).
+</p></details>
 
-To be able to exclude these functions (based on AFL++'s measured stability)
-the following process will allow to identify functions with variable edges.
+<details>
+  <summary id="how-can-i-fuzz-a-gui-program">How can I fuzz a GUI program?</summary><p>
 
-Four steps are required to do this and it also requires quite some knowledge
-of coding and/or disassembly and is effectively possible only with
-afl-clang-fast PCGUARD and afl-clang-lto LTO instrumentation.
+  Not all GUI programs are suitable for fuzzing. If the GUI program can read the
+  fuzz data from a file without needing any user interaction, then it would be
+  suitable for fuzzing.
 
-  1. First step: Instrument to be able to find the responsible function(s).
+  For more information on fuzzing GUI programs, see
+  [best_practices.md#fuzzing-a-gui-program](best_practices.md#fuzzing-a-gui-program).
+</p></details>
 
-     a) For LTO instrumented binaries this can be documented during compile
-        time, just set `export AFL_LLVM_DOCUMENT_IDS=/path/to/a/file`.
-        This file will have one assigned edge ID and the corresponding
-        function per line.
+## Performance
 
-     b) For PCGUARD instrumented binaries it is much more difficult. Here you
-        can either modify the __sanitizer_cov_trace_pc_guard function in
-        llvm_mode/afl-llvm-rt.o.c to write a backtrace to a file if the ID in
-        __afl_area_ptr[*guard] is one of the unstable edge IDs.
-        (Example code is already there).
-        Then recompile and reinstall llvm_mode and rebuild your target.
-        Run the recompiled target with afl-fuzz for a while and then check the
-        file that you wrote with the backtrace information.
-        Alternatively you can use `gdb` to hook __sanitizer_cov_trace_pc_guard_init
-        on start, check to which memory address the edge ID value is written
-        and set a write breakpoint to that address (`watch 0x.....`).
+<details>
+  <summary id="what-makes-a-good-performance">What makes a good performance?</summary><p>
 
-     c) in all other instrumentation types this is not possible. So just
-        recompile with the two mentioned above. This is just for
-        identifying the functions that have unstable edges.
+  Good performance generally means "making the fuzzing results better". This can
+  be influenced by various factors, for example, speed (finding lots of paths
+  quickly) or thoroughness (working with decreased speed, but finding better
+  mutations).
+</p></details>
 
-  2. Second step: Identify which edge ID numbers are unstable
+<details>
+  <summary id="how-can-i-improve-the-fuzzing-speed">How can I improve the fuzzing speed?</summary><p>
 
-     run the target with `export AFL_DEBUG=1` for a few minutes then terminate.
-     The out/fuzzer_stats file will then show the edge IDs that were identified
-     as unstable in the `var_bytes` entry. You can match these numbers
-     directly to the data you created in the first step.
-     Now you know which functions are responsible for the instability
+  There are a few things you can do to improve the fuzzing speed, see
+  [best_practices.md#improving-speed](best_practices.md#improving-speed).
+</p></details>
 
-  3. Third step: create a text file with the filenames/functions
+<details>
+  <summary id="why-is-my-stability-below-100percent">Why is my stability below 100%?</summary><p>
 
-     Identify which source code files contain the functions that you need to
-     remove from instrumentation, or just specify the functions you want to
-     skip for instrumentation. Note that optimization might inline functions!
+  Stability is measured by how many percent of the edges in the target are
+  "stable". Sending the same input again and again should take the exact same
+  path through the target every time. If that is the case, the stability is
+  100%.
 
-     Simply follow this document on how to do this: [llvm_mode/README.instrument_list.md](llvm_mode/README.instrument_list.md)
-     If PCGUARD is used, then you need to follow this guide (needs llvm 12+!):
-     [http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation](http://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation)
+  If, however, randomness happens, e.g., a thread reading other external data,
+  reaction to timing, etc., then in some of the re-executions with the same data
+  the edge coverage result will be different across runs. Those edges that
+  change are then flagged "unstable".
 
-     Only exclude those functions from instrumentation that provide no value
-     for coverage - that is if it does not process any fuzz data directly
-     or indirectly (e.g. hash maps, thread management etc.).
-     If however a function directly or indirectly handles fuzz data then you
-     should not put the function in a deny instrumentation list and rather
-     live with the instability it comes with.
+  The more "unstable" edges there are, the harder it is for AFL++ to identify
+  valid new paths.
 
-  4. Fourth step: recompile the target
+  A value above 90% is usually fine and a value above 80% is also still ok, and
+  even a value above 20% can still result in successful finds of bugs. However,
+  it is recommended that for values below 90% or 80% you should take
+  countermeasures to improve stability.
 
-     Recompile, fuzz it, be happy :)
+  For more information on stability and how to improve the stability value, see
+  [best_practices.md#improving-stability](best_practices.md#improving-stability).
+</p></details>
 
-     This link explains this process for [Fuzzbench](https://github.com/google/fuzzbench/issues/677)
+<details>
+  <summary id="what-are-power-schedules">What are power schedules?</summary><p>
+
+  Not every item in our queue/corpus is the same, some are more interesting,
+  others provide little value.
+  A power schedule measures how "interesting" a value is, and depending on
+  the calculated value spends more or less time mutating it.
+
+  AFL++ comes with several power schedules, initially ported from
+  [AFLFast](https://github.com/mboehme/aflfast), however, modified to be more
+  effective and several more modes added.
+
+  The most effective modes are `-p fast` (default) and `-p explore`.
+
+  If you fuzz with several parallel afl-fuzz instances, then it is beneficial
+  to assign a different schedule to each instance, however the majority should
+  be `fast` and `explore`.
+
+  It does not make sense to explain the details of the calculation and
+  reasoning behind all of the schedules. If you are interested, read the source
+  code and the AFLFast paper.
+</p></details>
+
+## Troubleshooting
+
+<details>
+  <summary id="fatal-forkserver-is-already-up-but-an-instrumented-dlopen-library-loaded-afterwards">FATAL: forkserver is already up but an instrumented dlopen library loaded afterwards</summary><p>
+
+  It can happen that you see this error on startup when fuzzing a target:
+
+  ```
+  [-] FATAL: forkserver is already up, but an instrumented dlopen() library
+             loaded afterwards. You must AFL_PRELOAD such libraries to be able
+             to fuzz them or LD_PRELOAD to run outside of afl-fuzz.
+             To ignore this set AFL_IGNORE_PROBLEMS=1.
+  ```
+
+  As the error describes, a dlopen() call is happening in the target that is
+  loading an instrumented library after the forkserver is already in place. This
+  is a problem for afl-fuzz because when the forkserver is started, we must know
+  the map size already and it can't be changed later.
+
+  The best solution is to simply set `AFL_PRELOAD=foo.so` to the libraries that
+  are dlopen'ed (e.g., use `strace` to see which), or to set a manual forkserver
+  after the final dlopen().
+
+  If this is not a viable option, you can set `AFL_IGNORE_PROBLEMS=1` but then
+  the existing map will be used also for the newly loaded libraries, which
+  allows it to work, however, the efficiency of the fuzzing will be partially
+  degraded.
+</p></details>
+
+<details>
+  <summary id="i-got-a-weird-compile-error-from-clang">I got a weird compile error from clang.</summary><p>
+
+  If you see this kind of error when trying to instrument a target with
+  afl-cc/afl-clang-fast/afl-clang-lto:
+
+  ```
+  /prg/tmp/llvm-project/build/bin/clang-13: symbol lookup error: /usr/local/bin/../lib/afl//cmplog-instructions-pass.so: undefined symbol: _ZNK4llvm8TypeSizecvmEv
+  clang-13: error: unable to execute command: No such file or directory
+  clang-13: error: clang frontend command failed due to signal (use -v to see invocation)
+  clang version 13.0.0 (https://github.com/llvm/llvm-project 1d7cf550721c51030144f3cd295c5789d51c4aad)
+  Target: x86_64-unknown-linux-gnu
+  Thread model: posix
+  InstalledDir: /prg/tmp/llvm-project/build/bin
+  clang-13: note: diagnostic msg:
+  ********************
+  ```
+
+  Then this means that your OS updated the clang installation from an upgrade
+  package and because of that the AFL++ llvm plugins do not match anymore.
+
+  Solution: `git pull ; make clean install` of AFL++.
+</p></details>

docs/INSTALL.md

@@ -1,85 +1,133 @@
-# Installation instructions
+# Building and installing AFL++
 
-  This document provides basic installation instructions and discusses known
-  issues for a variety of platforms. See README.md for the general instruction
-  manual.
+## Linux on x86
 
-## 1. Linux on x86
----------------
+An easy way to install AFL++ with everything compiled is available via docker:
+You can use the [Dockerfile](../Dockerfile) (which has gcc-10 and clang-11 -
+hence afl-clang-lto is available!) or just pull directly from the Docker Hub:
 
-This platform is expected to work well. Compile the program with:
-
-```bash
-make
+```shell
+docker pull aflplusplus/aflplusplus
+docker run -ti -v /location/of/your/target:/src aflplusplus/aflplusplus
 ```
 
-You can start using the fuzzer without installation, but it is also possible to
-install it with:
+This image is automatically generated when a push to the stable repo happens.
+You will find your target source code in `/src` in the container.
 
-```bash
+If you want to build AFL++ yourself, you have many options. The easiest choice
+is to build and install everything:
+
+```shell
+sudo apt-get update
+sudo apt-get install -y build-essential python3-dev automake git flex bison libglib2.0-dev libpixman-1-dev python3-setuptools
+# try to install llvm 11 and install the distro default if that fails
+sudo apt-get install -y lld-11 llvm-11 llvm-11-dev clang-11 || sudo apt-get install -y lld llvm llvm-dev clang
+sudo apt-get install -y gcc-$(gcc --version|head -n1|sed 's/.* //'|sed 's/\..*//')-plugin-dev libstdc++-$(gcc --version|head -n1|sed 's/.* //'|sed 's/\..*//')-dev
+sudo apt-get install -y ninja-build # for QEMU mode
+git clone https://github.com/AFLplusplus/AFLplusplus
+cd AFLplusplus
+make distrib
 sudo make install
 ```
 
-There are no special dependencies to speak of; you will need GNU make and a
-working compiler (gcc or clang). Some of the optional scripts bundled with the
-program may depend on bash, gdb, and similar basic tools.
+It is recommended to install the newest available gcc, clang and llvm-dev
+possible in your distribution!
 
-If you are using clang, please review README.llvm.md; the LLVM
-integration mode can offer substantial performance gains compared to the
-traditional approach.
+Note that `make distrib` also builds FRIDA mode, QEMU mode, unicorn_mode, and
+more. If you just want plain AFL++, then do `make all`. If you want some
+assisting tooling compiled but are not interested in binary-only targets, then
+instead choose:
 
-Likewise, if you are using GCC, please review instrumentation/README.gcc_plugin.md.
+```shell
+make source-only
+```
 
-You may have to change several settings to get optimal results (most notably,
-disable crash reporting utilities and switch to a different CPU governor), but
-afl-fuzz will guide you through that if necessary.
+These build targets exist:
 
-## 2. OpenBSD, FreeBSD, NetBSD on x86
+* all: the main afl++ binaries and llvm/gcc instrumentation
+* binary-only: everything for binary-only fuzzing: frida_mode, nyx_mode,
+  qemu_mode, frida_mode, unicorn_mode, coresight_mode, libdislocator,
+  libtokencap
+* source-only: everything for source code fuzzing: nyx_mode, libdislocator,
+  libtokencap
+* distrib: everything (for both binary-only and source code fuzzing)
+* man: creates simple man pages from the help option of the programs
+* install: installs everything you have compiled with the build options above
+* clean: cleans everything compiled, not downloads (unless not on a checkout)
+* deepclean: cleans everything including downloads
+* code-format: format the code, do this before you commit and send a PR please!
+* tests: runs test cases to ensure that all features are still working as they
+  should
+* unit: perform unit tests (based on cmocka)
+* help: shows these build options
 
-Similarly to Linux, these platforms are expected to work well and are
-regularly tested. Compile everything with GNU make:
+[Unless you are on Mac OS X](https://developer.apple.com/library/archive/qa/qa1118/_index.html),
+you can also build statically linked versions of the AFL++ binaries by passing
+the `STATIC=1` argument to make:
 
-```bash
+```shell
+make STATIC=1
+```
+
+These build options exist:
+
+* STATIC - compile AFL++ static
+* ASAN_BUILD - compiles with memory sanitizer for debug purposes
+* DEBUG - no optimization, -ggdb3, all warnings and -Werror
+* PROFILING - compile with profiling information (gprof)
+* INTROSPECTION - compile afl-fuzz with mutation introspection
+* NO_PYTHON - disable python support
+* NO_SPLICING - disables splicing mutation in afl-fuzz, not recommended for
+  normal fuzzing
+* AFL_NO_X86 - if compiling on non-intel/amd platforms
+* LLVM_CONFIG - if your distro doesn't use the standard name for llvm-config
+  (e.g., Debian)
+
+e.g.: `make ASAN_BUILD=1`
+
+## MacOS X on x86 and arm64 (M1)
+
+MacOS has some gotchas due to the idiosyncrasies of the platform.
+
+To build AFL, install llvm (and perhaps gcc) from brew and follow the general
+instructions for Linux. If possible, avoid Xcode at all cost.
+
+```shell
+brew install wget git make cmake llvm gdb coreutils
+```
+
+Be sure to setup `PATH` to point to the correct clang binaries and use the
+freshly installed clang, clang++, llvm-config, gmake and coreutils, e.g.:
+
+```shell
+# Depending on your MacOS system + brew version it is either
+export PATH="/opt/homebrew/opt/llvm/bin:$PATH"
+# or
+export PATH="/usr/local/opt/llvm/bin:$PATH"
+# you can check with "brew info llvm"
+
+export PATH="/usr/local/opt/coreutils/libexec/gnubin:/usr/local/bin:$PATH"
+export CC=clang
+export CXX=clang++
 gmake
-```
-
-Note that BSD make will *not* work; if you do not have gmake on your system,
-please install it first. As on Linux, you can use the fuzzer itself without
-installation, or install it with:
-
-```
+cd frida_mode
+gmake
+cd ..
 sudo gmake install
 ```
 
-Keep in mind that if you are using csh as your shell, the syntax of some of the
-shell commands given in the README.md and other docs will be different.
-
-The `llvm` requires a dynamically linked, fully-operational installation of
-clang. At least on FreeBSD, the clang binaries are static and do not include
-some of the essential tools, so if you want to make it work, you may need to
-follow the instructions in README.llvm.md.
-
-Beyond that, everything should work as advertised.
-
-The QEMU mode is currently supported only on Linux. I think it's just a QEMU
-problem, I couldn't get a vanilla copy of user-mode emulation support working
-correctly on BSD at all.
-
-## 3. MacOS X on x86 and arm64 (M1)
-
-MacOS X should work, but there are some gotchas due to the idiosyncrasies of
-the platform. On top of this, I have limited release testing capabilities
-and depend mostly on user feedback.
-
-To build AFL, install llvm (and perhaps gcc) from brew and follow the general
-instructions for Linux. If possible avoid Xcode at all cost.
-
-afl-gcc will fail unless you have GCC installed, but that is using outdated
-instrumentation anyway. You don't want that.
+`afl-gcc` will fail unless you have GCC installed, but that is using outdated
+instrumentation anyway. `afl-clang` might fail too depending on your PATH setup.
+But you don't want neither, you want `afl-clang-fast` anyway :) Note that
+`afl-clang-lto`, `afl-gcc-fast` and `qemu_mode` are not working on MacOS.
 
 The crash reporting daemon that comes by default with MacOS X will cause
-problems with fuzzing. You need to turn it off by following the instructions
-provided here: http://goo.gl/CCcd5u
+problems with fuzzing. You need to turn it off:
+
+```
+launchctl unload -w /System/Library/LaunchAgents/com.apple.ReportCrash.plist
+sudo launchctl unload -w /System/Library/LaunchDaemons/com.apple.ReportCrash.Root.plist
+```
 
 The `fork()` semantics on OS X are a bit unusual compared to other unix systems
 and definitely don't look POSIX-compliant. This means two things:
@@ -87,17 +135,17 @@ and definitely don't look POSIX-compliant. This means two things:
   - Fuzzing will be probably slower than on Linux. In fact, some folks report
     considerable performance gains by running the jobs inside a Linux VM on
     MacOS X.
-  - Some non-portable, platform-specific code may be incompatible with the
-    AFL forkserver. If you run into any problems, set `AFL_NO_FORKSRV=1` in the
+  - Some non-portable, platform-specific code may be incompatible with the AFL++
+    forkserver. If you run into any problems, set `AFL_NO_FORKSRV=1` in the
     environment before starting afl-fuzz.
 
 User emulation mode of QEMU does not appear to be supported on MacOS X, so
-black-box instrumentation mode (`-Q`) will not work.
-However Frida mode (`-O`) should work on x86 and arm64 MacOS boxes.
+black-box instrumentation mode (`-Q`) will not work. However, Frida mode (`-O`)
+works on both x86 and arm64 MacOS boxes.
 
 MacOS X supports SYSV shared memory used by AFL's instrumentation, but the
-default settings aren't usable with AFL++. The default settings on 10.14 seem
-to be:
+default settings aren't usable with AFL++. The default settings on 10.14 seem to
+be:
 
 ```bash
 $ ipcs -M
@@ -118,8 +166,8 @@ sysctl kern.sysv.shmmax=8388608
 sysctl kern.sysv.shmall=4096
 ```
 
-If you're running more than one instance of AFL you likely want to make `shmall`
-bigger and increase `shmseg` as well:
+If you're running more than one instance of AFL, you likely want to make
+`shmall` bigger and increase `shmseg` as well:
 
 ```bash
 sysctl kern.sysv.shmmax=8388608
@@ -127,91 +175,6 @@ sysctl kern.sysv.shmseg=48
 sysctl kern.sysv.shmall=98304
 ```
 
-See http://www.spy-hill.com/help/apple/SharedMemory.html for documentation for
-these settings and how to make them permanent.
-
-## 4. Linux or *BSD on non-x86 systems
-
-Standard build will fail on non-x86 systems, but you should be able to
-leverage two other options:
-
-  - The LLVM mode (see README.llvm.md), which does not rely on
-    x86-specific assembly shims. It's fast and robust, but requires a
-    complete installation of clang.
-  - The QEMU mode (see qemu_mode/README.md), which can be also used for
-    fuzzing cross-platform binaries. It's slower and more fragile, but
-    can be used even when you don't have the source for the tested app.
-
-If you're not sure what you need, you need the LLVM mode, which is built by
-default.
-
-...and compile your target program with afl-clang-fast or afl-clang-fast++
-instead of the traditional afl-gcc or afl-clang wrappers.
-
-## 5. Solaris on x86
-
-The fuzzer reportedly works on Solaris, but I have not tested this first-hand,
-and the user base is fairly small, so I don't have a lot of feedback.
-
-To get the ball rolling, you will need to use GNU make and GCC or clang. I'm
-being told that the stock version of GCC that comes with the platform does not
-work properly due to its reliance on a hardcoded location for 'as' (completely
-ignoring the `-B` parameter or `$PATH`).
-
-To fix this, you may want to build stock GCC from the source, like so:
-
-```sh
-./configure --prefix=$HOME/gcc --with-gnu-as --with-gnu-ld \
-  --with-gmp-include=/usr/include/gmp --with-mpfr-include=/usr/include/mpfr
-make
-sudo make install
-```
-
-Do *not* specify `--with-as=/usr/gnu/bin/as` - this will produce a GCC binary that
-ignores the `-B` flag and you will be back to square one.
-
-Note that Solaris reportedly comes with crash reporting enabled, which causes
-problems with crashes being misinterpreted as hangs, similarly to the gotchas
-for Linux and MacOS X. AFL does not auto-detect crash reporting on this
-particular platform, but you may need to run the following command:
-
-```sh
-coreadm -d global -d global-setid -d process -d proc-setid \
-  -d kzone -d log
-```
-
-User emulation mode of QEMU is not available on Solaris, so black-box
-instrumentation mode (`-Q`) will not work.
-
-## 6. Everything else
-
-You're on your own. On POSIX-compliant systems, you may be able to compile and
-run the fuzzer; and the LLVM and GCC plugin modes may offer a way to instrument
-non-x86 code.
-
-The fuzzer will run on Windows in WSL only. It will not work under Cygwin on in the normal Windows world. It
-could be ported to the latter platform fairly easily, but it's a pretty bad
-idea, because Cygwin is extremely slow. It makes much more sense to use
-VirtualBox or so to run a hardware-accelerated Linux VM; it will run around
-20x faster or so. If you have a *really* compelling use case for Cygwin, let
-me know.
-
-Although Android on x86 should theoretically work, the stock kernel may have
-SHM support compiled out, and if so, you may have to address that issue first.
-It's possible that all you need is this workaround:
-
-  https://github.com/pelya/android-shmem
-
-Joshua J. Drake notes that the Android linker adds a shim that automatically
-intercepts `SIGSEGV` and related signals. To fix this issue and be able to see
-crashes, you need to put this at the beginning of the fuzzed program:
-
-```sh
-  signal(SIGILL, SIG_DFL);
-  signal(SIGABRT, SIG_DFL);
-  signal(SIGBUS, SIG_DFL);
-  signal(SIGFPE, SIG_DFL);
-  signal(SIGSEGV, SIG_DFL);
-```
-
-You may need to `#include <signal.h>` first.
+See
+[http://www.spy-hill.com/help/apple/SharedMemory.html](http://www.spy-hill.com/help/apple/SharedMemory.html)
+for documentation for these settings and how to make them permanent.

docs/QuickStartGuide.md

@@ -1,50 +0,0 @@
-# AFL quick start guide
-
-You should read [README.md](../README.md) - it's pretty short. If you really can't, here's
-how to hit the ground running:
-
-1) Compile AFL with 'make'. If build fails, see [INSTALL.md](INSTALL.md) for tips.
-
-2) Find or write a reasonably fast and simple program that takes data from
-   a file or stdin, processes it in a test-worthy way, then exits cleanly.
-   If testing a network service, modify it to run in the foreground and read
-   from stdin. When fuzzing a format that uses checksums, comment out the
-   checksum verification code, too.
-
-   If this is not possible (e.g. in -Q(emu) mode) then use
-   AFL_CUSTOM_MUTATOR_LIBRARY to calculate the values with your own library.
-
-   The program must crash properly when a fault is encountered. Watch out for
-   custom SIGSEGV or SIGABRT handlers and background processes. For tips on
-   detecting non-crashing flaws, see section 11 in [README.md](README.md) .
-
-3) Compile the program / library to be fuzzed using afl-cc. A common way to
-   do this would be:
-
-   CC=/path/to/afl-cc CXX=/path/to/afl-c++ ./configure --disable-shared
-   make clean all
-
-4) Get a small but valid input file that makes sense to the program. When
-   fuzzing verbose syntax (SQL, HTTP, etc), create a dictionary as described in
-   dictionaries/README.md, too.
-
-5) If the program reads from stdin, run 'afl-fuzz' like so:
-
-   ./afl-fuzz -i testcase_dir -o findings_dir -- \
-     /path/to/tested/program [...program's cmdline...]
-
-   If the program takes input from a file, you can put @@ in the program's
-   command line; AFL will put an auto-generated file name in there for you.
-
-6) Investigate anything shown in red in the fuzzer UI by promptly consulting
-   [status_screen.md](status_screen.md).
-
-8) There is a basic docker build with 'docker build -t aflplusplus .'
-
-That's it. Sit back, relax, and - time permitting - try to skim through the
-following files:
-
-  - README.md                 - A general introduction to AFL,
-  - docs/perf_tips.md         - Simple tips on how to fuzz more quickly,
-  - docs/status_screen.md     - An explanation of the tidbits shown in the UI,
-  - docs/parallel_fuzzing.md  - Advice on running AFL on multiple cores.

docs/README.md

@@ -0,0 +1,65 @@
+# AFL++ documentation
+
+This is the overview of the AFL++ docs content.
+
+For general information on AFL++, see the
+[README.md of the repository](../README.md).
+
+Also take a look at our [FAQ.md](FAQ.md) and
+[best_practices.md](best_practices.md).
+
+## Fuzzing targets with the source code available
+
+You can find a quickstart for fuzzing targets with the source code available in
+the [README.md of the repository](../README.md#quick-start-fuzzing-with-afl).
+
+For in-depth information on the steps of the fuzzing process, see
+[fuzzing_in_depth.md](fuzzing_in_depth.md) or click on the following
+image and select a step.
+
+![Fuzzing process overview](https://raw.githubusercontent.com/AFLplusplus/AFLplusplus/dev/docs/resources/0_fuzzing_process_overview.drawio.svg "Fuzzing process overview")
+
+For further information on instrumentation, see the
+[READMEs in the instrumentation/ folder](../instrumentation/).
+
+### Instrumenting the target
+
+For more information, click on the following image and select a step.
+
+![Instrumenting the target](https://raw.githubusercontent.com/AFLplusplus/AFLplusplus/dev/docs/resources/1_instrument_target.drawio.svg "Instrumenting the target")
+
+### Preparing the fuzzing campaign
+
+For more information, click on the following image and select a step.
+
+![Preparing the fuzzing campaign](https://raw.githubusercontent.com/AFLplusplus/AFLplusplus/dev/docs/resources/2_prepare_campaign.drawio.svg "Preparing the fuzzing campaign")
+
+### Fuzzing the target
+
+For more information, click on the following image and select a step.
+
+![Fuzzing the target](https://raw.githubusercontent.com/AFLplusplus/AFLplusplus/dev/docs/resources/3_fuzz_target.drawio.svg "Fuzzing the target")
+
+### Managing the fuzzing campaign
+
+For more information, click on the following image and select a step.
+
+![Managing the fuzzing campaign](https://raw.githubusercontent.com/AFLplusplus/AFLplusplus/dev/docs/resources/4_manage_campaign.drawio.svg "Managing the fuzzing campaign")
+
+## Fuzzing other targets
+
+To learn about fuzzing other targets, see:
+
+* Binary-only: [fuzzing_binary-only_targets.md](fuzzing_binary-only_targets.md)
+* GUI programs:
+  [best_practices.md#fuzzing-a-gui-program](best_practices.md#fuzzing-a-gui-program)
+* Libraries: [frida_mode/README.md](../frida_mode/README.md)
+* Network services:
+  [best_practices.md#fuzzing-a-network-service](best_practices.md#fuzzing-a-network-service)
+* Non-linux: [unicorn_mode/README.md](../unicorn_mode/README.md)
+
+## Additional information
+
+* Tools that help fuzzing with AFL++:
+  [third_party_tools.md](third_party_tools.md)
+* Tutorials: [tutorials.md](tutorials.md)

docs/afl-fuzz_approach.md

@@ -0,0 +1,543 @@
+# The afl-fuzz approach
+
+AFL++ is a brute-force fuzzer coupled with an exceedingly simple but rock-solid
+instrumentation-guided genetic algorithm. It uses a modified form of edge
+coverage to effortlessly pick up subtle, local-scale changes to program control
+flow.
+
+Simplifying a bit, the overall algorithm can be summed up as:
+
+1) Load user-supplied initial test cases into the queue.
+
+2) Take the next input file from the queue.
+
+3) Attempt to trim the test case to the smallest size that doesn't alter the
+   measured behavior of the program.
+
+4) Repeatedly mutate the file using a balanced and well-researched variety of
+   traditional fuzzing strategies.
+
+5) If any of the generated mutations resulted in a new state transition recorded
+   by the instrumentation, add mutated output as a new entry in the queue.
+
+6) Go to 2.
+
+The discovered test cases are also periodically culled to eliminate ones that
+have been obsoleted by newer, higher-coverage finds; and undergo several other
+instrumentation-driven effort minimization steps.
+
+As a side result of the fuzzing process, the tool creates a small,
+self-contained corpus of interesting test cases. These are extremely useful for
+seeding other, labor- or resource-intensive testing regimes - for example, for
+stress-testing browsers, office applications, graphics suites, or closed-source
+tools.
+
+The fuzzer is thoroughly tested to deliver out-of-the-box performance far
+superior to blind fuzzing or coverage-only tools.
+
+## Understanding the status screen
+
+This section provides an overview of the status screen - plus tips for
+troubleshooting any warnings and red text shown in the UI.
+
+For the general instruction manual, see [README.md](README.md).
+
+### A note about colors
+
+The status screen and error messages use colors to keep things readable and
+attract your attention to the most important details. For example, red almost
+always means "consult this doc" :-)
+
+Unfortunately, the UI will only render correctly if your terminal is using
+traditional un*x palette (white text on black background) or something close to
+that.
+
+If you are using inverse video, you may want to change your settings, say:
+
+- For GNOME Terminal, go to `Edit > Profile` preferences, select the "colors"
+  tab, and from the list of built-in schemes, choose "white on black".
+- For the MacOS X Terminal app, open a new window using the "Pro" scheme via the
+  `Shell > New Window` menu (or make "Pro" your default).
+
+Alternatively, if you really like your current colors, you can edit config.h to
+comment out USE_COLORS, then do `make clean all`.
+
+We are not aware of any other simple way to make this work without causing other
+side effects - sorry about that.
+
+With that out of the way, let's talk about what's actually on the screen...
+
+### The status bar
+
+```
+american fuzzy lop ++3.01a (default) [fast] {0}
+```
+
+The top line shows you which mode afl-fuzz is running in (normal: "american
+fuzzy lop", crash exploration mode: "peruvian rabbit mode") and the version of
+AFL++. Next to the version is the banner, which, if not set with -T by hand,
+will either show the binary name being fuzzed, or the -M/-S main/secondary name
+for parallel fuzzing. Second to last is the power schedule mode being run
+(default: fast). Finally, the last item is the CPU id.
+
+### Process timing
+
+```
+  +----------------------------------------------------+
+  |        run time : 0 days, 8 hrs, 32 min, 43 sec    |
+  |   last new find : 0 days, 0 hrs, 6 min, 40 sec     |
+  | last uniq crash : none seen yet                    |
+  |  last uniq hang : 0 days, 1 hrs, 24 min, 32 sec    |
+  +----------------------------------------------------+
+```
+
+This section is fairly self-explanatory: it tells you how long the fuzzer has
+been running and how much time has elapsed since its most recent finds. This is
+broken down into "paths" (a shorthand for test cases that trigger new execution
+patterns), crashes, and hangs.
+
+When it comes to timing: there is no hard rule, but most fuzzing jobs should be
+expected to run for days or weeks; in fact, for a moderately complex project,
+the first pass will probably take a day or so. Every now and then, some jobs
+will be allowed to run for months.
+
+There's one important thing to watch out for: if the tool is not finding new
+paths within several minutes of starting, you're probably not invoking the
+target binary correctly and it never gets to parse the input files that are
+thrown at it; other possible explanations are that the default memory limit
+(`-m`) is too restrictive and the program exits after failing to allocate a
+buffer very early on; or that the input files are patently invalid and always
+fail a basic header check.
+
+If there are no new paths showing up for a while, you will eventually see a big
+red warning in this section, too :-)
+
+### Overall results
+
+```
+  +-----------------------+
+  |  cycles done : 0      |
+  |  total paths : 2095   |
+  | uniq crashes : 0      |
+  |   uniq hangs : 19     |
+  +-----------------------+
+```
+
+The first field in this section gives you the count of queue passes done so far
+- that is, the number of times the fuzzer went over all the interesting test
+  cases discovered so far, fuzzed them, and looped back to the very beginning.
+  Every fuzzing session should be allowed to complete at least one cycle; and
+  ideally, should run much longer than that.
+
+As noted earlier, the first pass can take a day or longer, so sit back and
+relax.
+
+To help make the call on when to hit `Ctrl-C`, the cycle counter is color-coded.
+It is shown in magenta during the first pass, progresses to yellow if new finds
+are still being made in subsequent rounds, then blue when that ends - and
+finally, turns green after the fuzzer hasn't been seeing any action for a longer
+while.
+
+The remaining fields in this part of the screen should be pretty obvious:
+there's the number of test cases ("paths") discovered so far, and the number of
+unique faults. The test cases, crashes, and hangs can be explored in real-time
+by browsing the output directory, see
+[#interpreting-output](#interpreting-output).
+
+### Cycle progress
+
+```
+  +-------------------------------------+
+  |  now processing : 1296 (61.86%)     |
+  | paths timed out : 0 (0.00%)         |
+  +-------------------------------------+
+```
+
+This box tells you how far along the fuzzer is with the current queue cycle: it
+shows the ID of the test case it is currently working on, plus the number of
+inputs it decided to ditch because they were persistently timing out.
+
+The "*" suffix sometimes shown in the first line means that the currently
+processed path is not "favored" (a property discussed later on).
+
+### Map coverage
+
+```
+  +--------------------------------------+
+  |    map density : 10.15% / 29.07%     |
+  | count coverage : 4.03 bits/tuple     |
+  +--------------------------------------+
+```
+
+The section provides some trivia about the coverage observed by the
+instrumentation embedded in the target binary.
+
+The first line in the box tells you how many branch tuples already were hit, in
+proportion to how much the bitmap can hold. The number on the left describes the
+current input; the one on the right is the value for the entire input corpus.
+
+Be wary of extremes:
+
+- Absolute numbers below 200 or so suggest one of three things: that the program
+  is extremely simple; that it is not instrumented properly (e.g., due to being
+  linked against a non-instrumented copy of the target library); or that it is
+  bailing out prematurely on your input test cases. The fuzzer will try to mark
+  this in pink, just to make you aware.
+- Percentages over 70% may very rarely happen with very complex programs that
+  make heavy use of template-generated code. Because high bitmap density makes
+  it harder for the fuzzer to reliably discern new program states, we recommend
+  recompiling the binary with `AFL_INST_RATIO=10` or so and trying again (see
+  [env_variables.md](env_variables.md)). The fuzzer will flag high percentages
+  in red. Chances are, you will never see that unless you're fuzzing extremely
+  hairy software (say, v8, perl, ffmpeg).
+
+The other line deals with the variability in tuple hit counts seen in the
+binary. In essence, if every taken branch is always taken a fixed number of
+times for all the inputs that were tried, this will read `1.00`. As we manage to
+trigger other hit counts for every branch, the needle will start to move toward
+`8.00` (every bit in the 8-bit map hit), but will probably never reach that
+extreme.
+
+Together, the values can be useful for comparing the coverage of several
+different fuzzing jobs that rely on the same instrumented binary.
+
+### Stage progress
+
+```
+  +-------------------------------------+
+  |  now trying : interest 32/8         |
+  | stage execs : 3996/34.4k (11.62%)   |
+  | total execs : 27.4M                 |
+  |  exec speed : 891.7/sec             |
+  +-------------------------------------+
+```
+
+This part gives you an in-depth peek at what the fuzzer is actually doing right
+now. It tells you about the current stage, which can be any of:
+
+- calibration - a pre-fuzzing stage where the execution path is examined to
+  detect anomalies, establish baseline execution speed, and so on. Executed very
+  briefly whenever a new find is being made.
+- trim L/S - another pre-fuzzing stage where the test case is trimmed to the
+  shortest form that still produces the same execution path. The length (L) and
+  stepover (S) are chosen in general relationship to file size.
+- bitflip L/S - deterministic bit flips. There are L bits toggled at any given
+  time, walking the input file with S-bit increments. The current L/S variants
+  are: `1/1`, `2/1`, `4/1`, `8/8`, `16/8`, `32/8`.
+- arith L/8 - deterministic arithmetics. The fuzzer tries to subtract or add
+  small integers to 8-, 16-, and 32-bit values. The stepover is always 8 bits.
+- interest L/8 - deterministic value overwrite. The fuzzer has a list of known
+  "interesting" 8-, 16-, and 32-bit values to try. The stepover is 8 bits.
+- extras - deterministic injection of dictionary terms. This can be shown as
+  "user" or "auto", depending on whether the fuzzer is using a user-supplied
+  dictionary (`-x`) or an auto-created one. You will also see "over" or
+  "insert", depending on whether the dictionary words overwrite existing data or
+  are inserted by offsetting the remaining data to accommodate their length.
+- havoc - a sort-of-fixed-length cycle with stacked random tweaks. The
+  operations attempted during this stage include bit flips, overwrites with
+  random and "interesting" integers, block deletion, block duplication, plus
+  assorted dictionary-related operations (if a dictionary is supplied in the
+  first place).
+- splice - a last-resort strategy that kicks in after the first full queue cycle
+  with no new paths. It is equivalent to 'havoc', except that it first splices
+  together two random inputs from the queue at some arbitrarily selected
+  midpoint.
+- sync - a stage used only when `-M` or `-S` is set (see
+  [fuzzing_in_depth.md:3c) Using multiple cores](fuzzing_in_depth.md#c-using-multiple-cores)).
+  No real fuzzing is involved, but the tool scans the output from other fuzzers
+  and imports test cases as necessary. The first time this is done, it may take
+  several minutes or so.
+
+The remaining fields should be fairly self-evident: there's the exec count
+progress indicator for the current stage, a global exec counter, and a benchmark
+for the current program execution speed. This may fluctuate from one test case
+to another, but the benchmark should be ideally over 500 execs/sec most of the
+time - and if it stays below 100, the job will probably take very long.
+
+The fuzzer will explicitly warn you about slow targets, too. If this happens,
+see the [best_practices.md#improving-speed](best_practices.md#improving-speed)
+for ideas on how to speed things up.
+
+### Findings in depth
+
+```
+  +--------------------------------------+
+  | favored paths : 879 (41.96%)         |
+  |  new edges on : 423 (20.19%)         |
+  | total crashes : 0 (0 unique)         |
+  |  total tmouts : 24 (19 unique)       |
+  +--------------------------------------+
+```
+
+This gives you several metrics that are of interest mostly to complete nerds.
+The section includes the number of paths that the fuzzer likes the most based on
+a minimization algorithm baked into the code (these will get considerably more
+air time), and the number of test cases that actually resulted in better edge
+coverage (versus just pushing the branch hit counters up). There are also
+additional, more detailed counters for crashes and timeouts.
+
+Note that the timeout counter is somewhat different from the hang counter; this
+one includes all test cases that exceeded the timeout, even if they did not
+exceed it by a margin sufficient to be classified as hangs.
+
+### Fuzzing strategy yields
+
+```
+  +-----------------------------------------------------+
+  |   bit flips : 57/289k, 18/289k, 18/288k             |
+  |  byte flips : 0/36.2k, 4/35.7k, 7/34.6k             |
+  | arithmetics : 53/2.54M, 0/537k, 0/55.2k             |
+  |  known ints : 8/322k, 12/1.32M, 10/1.70M            |
+  |  dictionary : 9/52k, 1/53k, 1/24k                   |
+  |havoc/splice : 1903/20.0M, 0/0                       |
+  |py/custom/rq : unused, 53/2.54M, unused              |
+  |    trim/eff : 20.31%/9201, 17.05%                   |
+  +-----------------------------------------------------+
+```
+
+This is just another nerd-targeted section keeping track of how many paths were
+netted, in proportion to the number of execs attempted, for each of the fuzzing
+strategies discussed earlier on. This serves to convincingly validate
+assumptions about the usefulness of the various approaches taken by afl-fuzz.
+
+The trim strategy stats in this section are a bit different than the rest. The
+first number in this line shows the ratio of bytes removed from the input files;
+the second one corresponds to the number of execs needed to achieve this goal.
+Finally, the third number shows the proportion of bytes that, although not
+possible to remove, were deemed to have no effect and were excluded from some of
+the more expensive deterministic fuzzing steps.
+
+Note that when deterministic mutation mode is off (which is the default because
+it is not very efficient) the first five lines display "disabled (default,
+enable with -D)".
+
+Only what is activated will have counter shown.
+
+### Path geometry
+
+```
+  +---------------------+
+  |    levels : 5       |
+  |   pending : 1570    |
+  |  pend fav : 583     |
+  | own finds : 0       |
+  |  imported : 0       |
+  | stability : 100.00% |
+  +---------------------+
+```
+
+The first field in this section tracks the path depth reached through the guided
+fuzzing process. In essence: the initial test cases supplied by the user are
+considered "level 1". The test cases that can be derived from that through
+traditional fuzzing are considered "level 2"; the ones derived by using these as
+inputs to subsequent fuzzing rounds are "level 3"; and so forth. The maximum
+depth is therefore a rough proxy for how much value you're getting out of the
+instrumentation-guided approach taken by afl-fuzz.
+
+The next field shows you the number of inputs that have not gone through any
+fuzzing yet. The same stat is also given for "favored" entries that the fuzzer
+really wants to get to in this queue cycle (the non-favored entries may have to
+wait a couple of cycles to get their chance).
+
+Next is the number of new paths found during this fuzzing section and imported
+from other fuzzer instances when doing parallelized fuzzing; and the extent to
+which identical inputs appear to sometimes produce variable behavior in the
+tested binary.
+
+That last bit is actually fairly interesting: it measures the consistency of
+observed traces. If a program always behaves the same for the same input data,
+it will earn a score of 100%. When the value is lower but still shown in purple,
+the fuzzing process is unlikely to be negatively affected. If it goes into red,
+you may be in trouble, since AFL++ will have difficulty discerning between
+meaningful and "phantom" effects of tweaking the input file.
+
+Now, most targets will just get a 100% score, but when you see lower figures,
+there are several things to look at:
+
+- The use of uninitialized memory in conjunction with some intrinsic sources of
+  entropy in the tested binary. Harmless to AFL, but could be indicative of a
+  security bug.
+- Attempts to manipulate persistent resources, such as left over temporary files
+  or shared memory objects. This is usually harmless, but you may want to
+  double-check to make sure the program isn't bailing out prematurely. Running
+  out of disk space, SHM handles, or other global resources can trigger this,
+  too.
+- Hitting some functionality that is actually designed to behave randomly.
+  Generally harmless. For example, when fuzzing sqlite, an input like `select
+  random();` will trigger a variable execution path.
+- Multiple threads executing at once in semi-random order. This is harmless when
+  the 'stability' metric stays over 90% or so, but can become an issue if not.
+  Here's what to try:
+  * Use afl-clang-fast from [instrumentation](../instrumentation/) - it uses a
+    thread-local tracking model that is less prone to concurrency issues,
+  * See if the target can be compiled or run without threads. Common
+    `./configure` options include `--without-threads`, `--disable-pthreads`, or
+    `--disable-openmp`.
+  * Replace pthreads with GNU Pth (https://www.gnu.org/software/pth/), which
+    allows you to use a deterministic scheduler.
+- In persistent mode, minor drops in the "stability" metric can be normal,
+  because not all the code behaves identically when re-entered; but major dips
+  may signify that the code within `__AFL_LOOP()` is not behaving correctly on
+  subsequent iterations (e.g., due to incomplete clean-up or reinitialization of
+  the state) and that most of the fuzzing effort goes to waste.
+
+The paths where variable behavior is detected are marked with a matching entry
+in the `<out_dir>/queue/.state/variable_behavior/` directory, so you can look
+them up easily.
+
+### CPU load
+
+```
+  [cpu: 25%]
+```
+
+This tiny widget shows the apparent CPU utilization on the local system. It is
+calculated by taking the number of processes in the "runnable" state, and then
+comparing it to the number of logical cores on the system.
+
+If the value is shown in green, you are using fewer CPU cores than available on
+your system and can probably parallelize to improve performance; for tips on how
+to do that, see
+[fuzzing_in_depth.md:3c) Using multiple cores](fuzzing_in_depth.md#c-using-multiple-cores).
+
+If the value is shown in red, your CPU is *possibly* oversubscribed, and running
+additional fuzzers may not give you any benefits.
+
+Of course, this benchmark is very simplistic; it tells you how many processes
+are ready to run, but not how resource-hungry they may be. It also doesn't
+distinguish between physical cores, logical cores, and virtualized CPUs; the
+performance characteristics of each of these will differ quite a bit.
+
+If you want a more accurate measurement, you can run the `afl-gotcpu` utility
+from the command line.
+
+## Interpreting output
+
+See [#understanding-the-status-screen](#understanding-the-status-screen) for
+information on how to interpret the displayed stats and monitor the health of
+the process. Be sure to consult this file especially if any UI elements are
+highlighted in red.
+
+The fuzzing process will continue until you press Ctrl-C. At a minimum, you want
+to allow the fuzzer to complete one queue cycle, which may take anywhere from a
+couple of hours to a week or so.
+
+There are three subdirectories created within the output directory and updated
+in real-time:
+
+- queue/   - test cases for every distinctive execution path, plus all the
+             starting files given by the user. This is the synthesized corpus.
+
+             Before using this corpus for any other purposes, you can shrink
+             it to a smaller size using the afl-cmin tool. The tool will find
+             a smaller subset of files offering equivalent edge coverage.
+
+- crashes/ - unique test cases that cause the tested program to receive a fatal
+             signal (e.g., SIGSEGV, SIGILL, SIGABRT). The entries are grouped by
+             the received signal.
+
+- hangs/   - unique test cases that cause the tested program to time out. The
+             default time limit before something is classified as a hang is the
+             larger of 1 second and the value of the -t parameter. The value can
+             be fine-tuned by setting AFL_HANG_TMOUT, but this is rarely
+             necessary.
+
+Crashes and hangs are considered "unique" if the associated execution paths
+involve any state transitions not seen in previously-recorded faults. If a
+single bug can be reached in multiple ways, there will be some count inflation
+early in the process, but this should quickly taper off.
+
+The file names for crashes and hangs are correlated with the parent,
+non-faulting queue entries. This should help with debugging.
+
+## Visualizing
+
+If you have gnuplot installed, you can also generate some pretty graphs for any
+active fuzzing task using afl-plot. For an example of how this looks like, see
+[https://lcamtuf.coredump.cx/afl/plot/](https://lcamtuf.coredump.cx/afl/plot/).
+
+You can also manually build and install afl-plot-ui, which is a helper utility
+for showing the graphs generated by afl-plot in a graphical window using GTK.
+You can build and install it as follows:
+
+```shell
+sudo apt install libgtk-3-0 libgtk-3-dev pkg-config
+cd utils/plot_ui
+make
+cd ../../
+sudo make install
+```
+
+To learn more about remote monitoring and metrics visualization with StatsD, see
+[rpc_statsd.md](rpc_statsd.md).
+
+### Addendum: status and plot files
+
+For unattended operation, some of the key status screen information can be also
+found in a machine-readable format in the fuzzer_stats file in the output
+directory. This includes:
+
+- `start_time`        - unix time indicating the start time of afl-fuzz
+- `last_update`       - unix time corresponding to the last update of this file
+- `run_time`          - run time in seconds to the last update of this file
+- `fuzzer_pid`        - PID of the fuzzer process
+- `cycles_done`       - queue cycles completed so far
+- `cycles_wo_finds`   - number of cycles without any new paths found
+- `execs_done`        - number of execve() calls attempted
+- `execs_per_sec`     - overall number of execs per second
+- `corpus_count`      - total number of entries in the queue
+- `corpus_favored`    - number of queue entries that are favored
+- `corpus_found`      - number of entries discovered through local fuzzing
+- `corpus_imported`   - number of entries imported from other instances
+- `max_depth`         - number of levels in the generated data set
+- `cur_item`          - currently processed entry number
+- `pending_favs`      - number of favored entries still waiting to be fuzzed
+- `pending_total`     - number of all entries waiting to be fuzzed
+- `corpus_variable`   - number of test cases showing variable behavior
+- `stability`         - percentage of bitmap bytes that behave consistently
+- `bitmap_cvg`        - percentage of edge coverage found in the map so far
+- `saved_crashes`     - number of unique crashes recorded
+- `saved_hangs`       - number of unique hangs encountered
+- `last_find`         - seconds since the last find was found
+- `last_crash`        - seconds since the last crash was found
+- `last_hang`         - seconds since the last hang was found
+- `execs_since_crash` - execs since the last crash was found
+- `exec_timeout`      - the -t command line value
+- `slowest_exec_ms`   - real time of the slowest execution in ms
+- `peak_rss_mb`       - max rss usage reached during fuzzing in MB
+- `edges_found`       - how many edges have been found
+- `var_byte_count`    - how many edges are non-deterministic
+- `afl_banner`        - banner text (e.g., the target name)
+- `afl_version`       - the version of AFL++ used
+- `target_mode`       - default, persistent, qemu, unicorn, non-instrumented
+- `command_line`      - full command line used for the fuzzing session
+
+Most of these map directly to the UI elements discussed earlier on.
+
+On top of that, you can also find an entry called `plot_data`, containing a
+plottable history for most of these fields. If you have gnuplot installed, you
+can turn this into a nice progress report with the included `afl-plot` tool.
+
+### Addendum: automatically sending metrics with StatsD
+
+In a CI environment or when running multiple fuzzers, it can be tedious to log
+into each of them or deploy scripts to read the fuzzer statistics. Using
+`AFL_STATSD` (and the other related environment variables `AFL_STATSD_HOST`,
+`AFL_STATSD_PORT`, `AFL_STATSD_TAGS_FLAVOR`) you can automatically send metrics
+to your favorite StatsD server. Depending on your StatsD server, you will be
+able to monitor, trigger alerts, or perform actions based on these metrics
+(e.g.: alert on slow exec/s for a new build, threshold of crashes, time since
+last crash > X, etc.).
+
+The selected metrics are a subset of all the metrics found in the status and in
+the plot file. The list is the following: `cycle_done`, `cycles_wo_finds`,
+`execs_done`,`execs_per_sec`, `corpus_count`, `corpus_favored`, `corpus_found`,
+`corpus_imported`, `max_depth`, `cur_item`, `pending_favs`, `pending_total`,
+`corpus_variable`, `saved_crashes`, `saved_hangs`, `total_crashes`,
+`slowest_exec_ms`, `edges_found`, `var_byte_count`, `havoc_expansion`. Their
+definitions can be found in the addendum above.
+
+When using multiple fuzzer instances with StatsD, it is *strongly* recommended
+to setup the flavor (`AFL_STATSD_TAGS_FLAVOR`) to match your StatsD server. This
+will allow you to see individual fuzzer performance, detect bad ones, see the
+progress of each strategy...

docs/best_practices.md

@@ -0,0 +1,192 @@
+# Best practices
+
+## Contents
+
+### Targets
+
+* [Fuzzing a target with source code available](#fuzzing-a-target-with-source-code-available)
+* [Fuzzing a target with dlopen() instrumented libraries](#fuzzing-a-target-with-dlopen-instrumented-libraries)
+* [Fuzzing a binary-only target](#fuzzing-a-binary-only-target)
+* [Fuzzing a GUI program](#fuzzing-a-gui-program)
+* [Fuzzing a network service](#fuzzing-a-network-service)
+
+### Improvements
+
+* [Improving speed](#improving-speed)
+* [Improving stability](#improving-stability)
+
+## Targets
+
+### Fuzzing a target with source code available
+
+To learn how to fuzz a target if source code is available, see
+[fuzzing_in_depth.md](fuzzing_in_depth.md).
+
+### Fuzzing a target with dlopen instrumented libraries
+
+If a source code based fuzzing target loads instrumented libraries with
+dlopen() after the forkserver has been activated and non-colliding coverage
+instrumentation is used (PCGUARD (which is the default), or LTO), then this
+an issue, because this would enlarge the coverage map, but afl-fuzz doesn't
+know about it.
+
+The solution is to use `AFL_PRELOAD` for all dlopen()'ed libraries to
+ensure that all coverage targets are present on startup in the target,
+even if accessed only later with dlopen().
+
+For PCGUARD instrumentation `abort()` is called if this is detected, for LTO
+there will either be no coverage for the instrumented dlopen()'ed libraries or
+you will see lots of crashes in the UI.
+
+Note that this is not an issue if you use the inferiour `afl-gcc-fast`,
+`afl-gcc` or`AFL_LLVM_INSTRUMENT=CLASSIC/NGRAM/CTX afl-clang-fast`
+instrumentation.
+
+### Fuzzing a binary-only target
+
+For a comprehensive guide, see
+[fuzzing_binary-only_targets.md](fuzzing_binary-only_targets.md).
+
+### Fuzzing a GUI program
+
+If the GUI program can read the fuzz data from a file (via the command line, a
+fixed location or via an environment variable) without needing any user
+interaction, then it would be suitable for fuzzing.
+
+Otherwise, it is not possible without modifying the source code - which is a
+very good idea anyway as the GUI functionality is a huge CPU/time overhead for
+the fuzzing.
+
+So create a new `main()` that just reads the test case and calls the
+functionality for processing the input that the GUI program is using.
+
+### Fuzzing a network service
+
+Fuzzing a network service does not work "out of the box".
+
+Using a network channel is inadequate for several reasons:
+- it has a slow-down of x10-20 on the fuzzing speed
+- it does not scale to fuzzing multiple instances easily,
+- instead of one initial data packet often a back-and-forth interplay of packets
+  is needed for stateful protocols (which is totally unsupported by most
+  coverage aware fuzzers).
+
+The established method to fuzz network services is to modify the source code to
+read from a file or stdin (fd 0) (or even faster via shared memory, combine this
+with persistent mode
+[instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md)
+and you have a performance gain of x10 instead of a performance loss of over x10
+- that is a x100 difference!).
+
+If modifying the source is not an option (e.g., because you only have a binary
+and perform binary fuzzing) you can also use a shared library with AFL_PRELOAD
+to emulate the network. This is also much faster than the real network would be.
+See [utils/socket_fuzzing/](../utils/socket_fuzzing/).
+
+There is an outdated AFL++ branch that implements networking if you are
+desperate though:
+[https://github.com/AFLplusplus/AFLplusplus/tree/networking](https://github.com/AFLplusplus/AFLplusplus/tree/networking)
+- however, a better option is AFLnet
+([https://github.com/aflnet/aflnet](https://github.com/aflnet/aflnet)) which
+allows you to define network state with different type of data packets.
+
+## Improvements
+
+### Improving speed
+
+1. Use [llvm_mode](../instrumentation/README.llvm.md): afl-clang-lto (llvm >=
+   11) or afl-clang-fast (llvm >= 9 recommended).
+2. Use [persistent mode](../instrumentation/README.persistent_mode.md) (x2-x20
+   speed increase).
+3. Instrument just what you are interested in, see
+   [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md).
+4. If you do not use shmem persistent mode, use `AFL_TMPDIR` to put the input
+   file directory on a tempfs location, see
+   [env_variables.md](env_variables.md).
+5. Improve Linux kernel performance: modify `/etc/default/grub`, set
+   `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off
+   mitigations=off no_stf_barrier noibpb noibrs nopcid nopti
+   nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off
+   spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then
+   `update-grub` and `reboot` (warning: makes the system less secure).
+6. Running on an `ext2` filesystem with `noatime` mount option will be a bit
+   faster than on any other journaling filesystem.
+7. Use your cores
+   ([fuzzing_in_depth.md:3c) Using multiple cores](fuzzing_in_depth.md#c-using-multiple-cores))!
+
+### Improving stability
+
+For fuzzing, a 100% stable target that covers all edges is the best case. A 90%
+stable target that covers all edges is, however, better than a 100% stable
+target that ignores 10% of the edges.
+
+With instability, you basically have a partial coverage loss on an edge, with
+ignored functions you have a full loss on that edges.
+
+There are functions that are unstable, but also provide value to coverage, e.g.,
+init functions that use fuzz data as input. If, however, a function that has
+nothing to do with the input data is the source of instability, e.g., checking
+jitter, or is a hash map function etc., then it should not be instrumented.
+
+To be able to exclude these functions (based on AFL++'s measured stability), the
+following process will allow to identify functions with variable edges.
+
+Four steps are required to do this and it also requires quite some knowledge of
+coding and/or disassembly and is effectively possible only with `afl-clang-fast`
+`PCGUARD` and `afl-clang-lto` `LTO` instrumentation.
+
+  1. Instrument to be able to find the responsible function(s):
+
+     a) For LTO instrumented binaries, this can be documented during compile
+        time, just set `export AFL_LLVM_DOCUMENT_IDS=/path/to/a/file`. This file
+        will have one assigned edge ID and the corresponding function per line.
+
+     b) For PCGUARD instrumented binaries, it is much more difficult. Here you
+        can either modify the `__sanitizer_cov_trace_pc_guard` function in
+        `instrumentation/afl-llvm-rt.o.c` to write a backtrace to a file if the
+        ID in `__afl_area_ptr[*guard]` is one of the unstable edge IDs. (Example
+        code is already there). Then recompile and reinstall `llvm_mode` and
+        rebuild your target. Run the recompiled target with `afl-fuzz` for a
+        while and then check the file that you wrote with the backtrace
+        information. Alternatively, you can use `gdb` to hook
+        `__sanitizer_cov_trace_pc_guard_init` on start, check to which memory
+        address the edge ID value is written, and set a write breakpoint to that
+        address (`watch 0x.....`).
+
+     c) In other instrumentation types, this is not possible. So just recompile
+        with the two mentioned above. This is just for identifying the functions
+        that have unstable edges.
+
+  2. Identify which edge ID numbers are unstable.
+
+     Run the target with `export AFL_DEBUG=1` for a few minutes then terminate.
+     The out/fuzzer_stats file will then show the edge IDs that were identified
+     as unstable in the `var_bytes` entry. You can match these numbers directly
+     to the data you created in the first step. Now you know which functions are
+     responsible for the instability
+
+  3. Create a text file with the filenames/functions
+
+     Identify which source code files contain the functions that you need to
+     remove from instrumentation, or just specify the functions you want to skip
+     for instrumentation. Note that optimization might inline functions!
+
+     Follow this document on how to do this:
+     [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md).
+
+     If `PCGUARD` is used, then you need to follow this guide (needs llvm 12+!):
+     [https://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation](https://clang.llvm.org/docs/SanitizerCoverage.html#partially-disabling-instrumentation)
+
+     Only exclude those functions from instrumentation that provide no value for
+     coverage - that is if it does not process any fuzz data directly or
+     indirectly (e.g., hash maps, thread management etc.). If, however, a
+     function directly or indirectly handles fuzz data, then you should not put
+     the function in a deny instrumentation list and rather live with the
+     instability it comes with.
+
+  4. Recompile the target
+
+     Recompile, fuzz it, be happy :)
+
+     This link explains this process for
+     [Fuzzbench](https://github.com/google/fuzzbench/issues/677).

docs/binaryonly_fuzzing.md

@@ -1,223 +0,0 @@
-# Fuzzing binary-only programs with afl++
-
-  afl++, libfuzzer and others are great if you have the source code, and
-  it allows for very fast and coverage guided fuzzing.
-
-  However, if there is only the binary program and no source code available,
-  then standard `afl-fuzz -n` (non-instrumented mode) is not effective.
-
-  The following is a description of how these binaries can be fuzzed with afl++.
-
-
-## TL;DR:
-
-  qemu_mode in persistent mode is the fastest - if the stability is
-  high enough. Otherwise try retrowrite, afl-dyninst and if these
-  fail too then try standard qemu_mode with AFL_ENTRYPOINT to where you need it.
-
-  If your target is a library use utils/afl_frida/.
-
-  If your target is non-linux then use unicorn_mode/.
-
-
-## QEMU
-
-  Qemu is the "native" solution to the program.
-  It is available in the ./qemu_mode/ directory and once compiled it can
-  be accessed by the afl-fuzz -Q command line option.
-  It is the easiest to use alternative and even works for cross-platform binaries.
-
-  The speed decrease is at about 50%.
-  However various options exist to increase the speed:
-   - using AFL_ENTRYPOINT to move the forkserver entry to a later basic block in
-     the binary (+5-10% speed)
-   - using persistent mode [qemu_mode/README.persistent.md](../qemu_mode/README.persistent.md)
-     this will result in 150-300% overall speed increase - so 3-8x the original
-     qemu_mode speed!
-   - using AFL_CODE_START/AFL_CODE_END to only instrument specific parts
-
-  Note that there is also honggfuzz: [https://github.com/google/honggfuzz](https://github.com/google/honggfuzz)
-  which now has a qemu_mode, but its performance is just 1.5% ...
-
-  As it is included in afl++ this needs no URL.
-
-  If you like to code a customized fuzzer without much work, we highly
-  recommend to check out our sister project libafl which will support QEMU
-  too:
-  [https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL)
-
-
-## AFL FRIDA
-
-  In frida_mode you can fuzz binary-only targets easily like with QEMU,
-  with the advantage that frida_mode also works on MacOS (both intel and M1).
-
-  If you want to fuzz a binary-only library then you can fuzz it with
-  frida-gum via utils/afl_frida/, you will have to write a harness to
-  call the target function in the library, use afl-frida.c as a template.
-
-  Both come with afl++ so this needs no URL.
-
-  You can also perform remote fuzzing with frida, e.g. if you want to fuzz
-  on iPhone or Android devices, for this you can use
-  [https://github.com/ttdennis/fpicker/](https://github.com/ttdennis/fpicker/)
-  as an intermediate that uses afl++ for fuzzing.
-
-  If you like to code a customized fuzzer without much work, we highly
-  recommend to check out our sister project libafl which supports Frida too:
-  [https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL)
-  Working examples already exist :-)
-
-
-## WINE+QEMU
-
-  Wine mode can run Win32 PE binaries with the QEMU instrumentation.
-  It needs Wine, python3 and the pefile python package installed.
-
-  As it is included in afl++ this needs no URL.
-
-
-## UNICORN
-
-  Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar.
-  In contrast to QEMU, Unicorn does not offer a full system or even userland
-  emulation. Runtime environment and/or loaders have to be written from scratch,
-  if needed. On top, block chaining has been removed. This means the speed boost
-  introduced in  the patched QEMU Mode of afl++ cannot simply be ported over to
-  Unicorn. For further information, check out [unicorn_mode/README.md](../unicorn_mode/README.md).
-
-  As it is included in afl++ this needs no URL.
-
-
-## AFL UNTRACER
-
-   If you want to fuzz a binary-only shared library then you can fuzz it with
-   utils/afl_untracer/, use afl-untracer.c as a template.
-   It is slower than AFL FRIDA (see above).
-
-
-## DYNINST
-
-  Dyninst is a binary instrumentation framework similar to Pintool and
-  Dynamorio (see far below). However whereas Pintool and Dynamorio work at
-  runtime, dyninst instruments the target at load time, and then let it run -
-  or save the  binary with the changes.
-  This is great for some things, e.g. fuzzing, and not so effective for others,
-  e.g. malware analysis.
-
-  So what we can do with dyninst is taking every basic block, and put afl's
-  instrumention code in there - and then save the binary.
-  Afterwards we can just fuzz the newly saved target binary with afl-fuzz.
-  Sounds great? It is. The issue though - it is a non-trivial problem to
-  insert instructions, which change addresses in the process space, so that
-  everything is still working afterwards. Hence more often than not binaries
-  crash when they are run.
-
-  The speed decrease is about 15-35%, depending on the optimization options
-  used with afl-dyninst.
-
-  So if Dyninst works, it is the best option available. Otherwise it just
-  doesn't work well.
-
-  [https://github.com/vanhauser-thc/afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst)
-
-
-## RETROWRITE, ZAFL, ... other binary rewriter
-
-  If you have an x86/x86_64 binary that still has its symbols, is compiled
-  with position independant code (PIC/PIE) and does not use most of the C++
-  features then the retrowrite solution might be for you.
-  It decompiles to ASM files which can then be instrumented with afl-gcc.
-
-  It is at about 80-85% performance.
-
-  [https://git.zephyr-software.com/opensrc/zafl](https://git.zephyr-software.com/opensrc/zafl)
-  [https://github.com/HexHive/retrowrite](https://github.com/HexHive/retrowrite)
-
-
-## MCSEMA
-
-  Theoretically you can also decompile to llvm IR with mcsema, and then
-  use llvm_mode to instrument the binary.
-  Good luck with that.
-
-  [https://github.com/lifting-bits/mcsema](https://github.com/lifting-bits/mcsema)
-
-
-## INTEL-PT
-
-  If you have a newer Intel CPU, you can make use of Intels processor trace.
-  The big issue with Intel's PT is the small buffer size and the complex
-  encoding of the debug information collected through PT.
-  This makes the decoding very CPU intensive and hence slow.
-  As a result, the overall speed decrease is about 70-90% (depending on
-  the implementation and other factors).
-
-  There are two afl intel-pt implementations:
-
-  1. [https://github.com/junxzm1990/afl-pt](https://github.com/junxzm1990/afl-pt)
-     => this needs Ubuntu 14.04.05 without any updates and the 4.4 kernel.
-
-  2. [https://github.com/hunter-ht-2018/ptfuzzer](https://github.com/hunter-ht-2018/ptfuzzer)
-     => this needs a 4.14 or 4.15 kernel. the "nopti" kernel boot option must
-        be used. This one is faster than the other.
-
-  Note that there is also honggfuzz: https://github.com/google/honggfuzz
-  But its IPT performance is just 6%!
-
-
-## CORESIGHT
-
-  Coresight is ARM's answer to Intel's PT.
-  There is no implementation so far which handles coresight and getting
-  it working on an ARM Linux is very difficult due to custom kernel building
-  on embedded systems is difficult. And finding one that has coresight in
-  the ARM chip is difficult too.
-  My guess is that it is slower than Qemu, but faster than Intel PT.
-
-  If anyone finds any coresight implementation for afl please ping me: vh@thc.org
-
-
-## PIN & DYNAMORIO
-
-  Pintool and Dynamorio are dynamic instrumentation engines, and they can be
-  used for getting basic block information at runtime.
-  Pintool is only available for Intel x32/x64 on Linux, Mac OS and Windows,
-  whereas Dynamorio is additionally available for ARM and AARCH64.
-  Dynamorio is also 10x faster than Pintool.
-
-  The big issue with Dynamorio (and therefore Pintool too) is speed.
-  Dynamorio has a speed decrease of 98-99%
-  Pintool has a speed decrease of 99.5%
-
-  Hence Dynamorio is the option to go for if everything else fails, and Pintool
-  only if Dynamorio fails too.
-
-  Dynamorio solutions:
-  * [https://github.com/vanhauser-thc/afl-dynamorio](https://github.com/vanhauser-thc/afl-dynamorio)
-  * [https://github.com/mxmssh/drAFL](https://github.com/mxmssh/drAFL)
-  * [https://github.com/googleprojectzero/winafl/](https://github.com/googleprojectzero/winafl/) <= very good but windows only
-
-  Pintool solutions:
-  * [https://github.com/vanhauser-thc/afl-pin](https://github.com/vanhauser-thc/afl-pin)
-  * [https://github.com/mothran/aflpin](https://github.com/mothran/aflpin)
-  * [https://github.com/spinpx/afl_pin_mode](https://github.com/spinpx/afl_pin_mode) <= only old Pintool version supported
-
-
-## Non-AFL solutions
-
-  There are many binary-only fuzzing frameworks.
-  Some are great for CTFs but don't work with large binaries, others are very
-  slow but have good path discovery, some are very hard to set-up ...
-
-  * QSYM: [https://github.com/sslab-gatech/qsym](https://github.com/sslab-gatech/qsym)
-  * Manticore: [https://github.com/trailofbits/manticore](https://github.com/trailofbits/manticore)
-  * S2E: [https://github.com/S2E](https://github.com/S2E)
-  * Tinyinst: [https://github.com/googleprojectzero/TinyInst](https://github.com/googleprojectzero/TinyInst) (Mac/Windows only)
-  * Jackalope: [https://github.com/googleprojectzero/Jackalope](https://github.com/googleprojectzero/Jackalope)
-  *  ... please send me any missing that are good
-
-
-## Closing words
-
-  That's it! News, corrections, updates? Send an email to vh@thc.org

docs/custom_mutators.md

@@ -1,16 +1,16 @@
 # Custom Mutators in AFL++
 
 This file describes how you can implement custom mutations to be used in AFL.
-For now, we support C/C++ library and Python module, collectivelly named as the
+For now, we support C/C++ library and Python module, collectively named as the
 custom mutator.
 
-There is also experimental support for Rust in `custom_mutators/rust`.
-Please refer to that directory for documentation.
-Run ```cargo doc -p custom_mutator --open``` in that directory to view the
-documentation in your web browser.
+There is also experimental support for Rust in `custom_mutators/rust`. For
+documentation, refer to that directory. Run `cargo doc -p custom_mutator --open`
+in that directory to view the documentation in your web browser.
 
 Implemented by
-- C/C++ library (`*.so`): Khaled Yakdan from Code Intelligence (<yakdan@code-intelligence.de>)
+- C/C++ library (`*.so`): Khaled Yakdan from Code Intelligence
+  (<yakdan@code-intelligence.de>)
 - Python module: Christian Holler from Mozilla (<choller@mozilla.com>)
 
 ## 1) Introduction
@@ -21,13 +21,17 @@ fuzzing by using libraries that perform mutations according to a given grammar.
 
 The custom mutator is passed to `afl-fuzz` via the `AFL_CUSTOM_MUTATOR_LIBRARY`
 or `AFL_PYTHON_MODULE` environment variable, and must export a fuzz function.
-Now afl also supports multiple custom mutators which can be specified in the same `AFL_CUSTOM_MUTATOR_LIBRARY` environment variable like this.
+Now AFL++ also supports multiple custom mutators which can be specified in the
+same `AFL_CUSTOM_MUTATOR_LIBRARY` environment variable like this.
+
 ```bash
 export AFL_CUSTOM_MUTATOR_LIBRARY="full/path/to/mutator_first.so;full/path/to/mutator_second.so"
 ```
-Please see [APIs](#2-apis) and [Usage](#3-usage) for detail.
 
-The custom mutation stage is set to be the first non-deterministic stage (right before the havoc stage).
+For details, see [APIs](#2-apis) and [Usage](#3-usage).
+
+The custom mutation stage is set to be the first non-deterministic stage (right
+before the havoc stage).
 
 Note: If `AFL_CUSTOM_MUTATOR_ONLY` is set, all mutations will solely be
 performed with the custom mutator.
@@ -35,6 +39,7 @@ performed with the custom mutator.
 ## 2) APIs
 
 C/C++:
+
 ```c
 void *afl_custom_init(afl_state_t *afl, unsigned int seed);
 unsigned int afl_custom_fuzz_count(void *data, const unsigned char *buf, size_t buf_size);
@@ -47,12 +52,13 @@ int afl_custom_post_trim(void *data, unsigned char success);
 size_t afl_custom_havoc_mutation(void *data, unsigned char *buf, size_t buf_size, unsigned char **out_buf, size_t max_size);
 unsigned char afl_custom_havoc_mutation_probability(void *data);
 unsigned char afl_custom_queue_get(void *data, const unsigned char *filename);
-void afl_custom_queue_new_entry(void *data, const unsigned char *filename_new_queue, const unsigned int *filename_orig_queue);
+u8 afl_custom_queue_new_entry(void *data, const unsigned char *filename_new_queue, const unsigned int *filename_orig_queue);
 const char* afl_custom_introspection(my_mutator_t *data);
 void afl_custom_deinit(void *data);
 ```
 
 Python:
+
 ```python
 def init(seed):
     pass
@@ -88,7 +94,7 @@ def queue_get(filename):
     return True
 
 def queue_new_entry(filename_new_queue, filename_orig_queue):
-    pass
+    return False
 
 def introspection():
     return string
@@ -101,7 +107,8 @@ def deinit():  # optional for Python
 
 - `init`:
 
-    This method is called when AFL++ starts up and is used to seed RNG and set up buffers and state.
+    This method is called when AFL++ starts up and is used to seed RNG and set
+    up buffers and state.
 
 - `queue_get` (optional):
 
@@ -110,27 +117,26 @@ def deinit():  # optional for Python
 
 - `fuzz_count` (optional):
 
-    When a queue entry is selected to be fuzzed, afl-fuzz selects the number
-    of fuzzing attempts with this input based on a few factors.
-    If however the custom mutator wants to set this number instead on how often
-    it is called for a specific queue entry, use this function.
-    This function is most useful if `AFL_CUSTOM_MUTATOR_ONLY` is **not** used.
+    When a queue entry is selected to be fuzzed, afl-fuzz selects the number of
+    fuzzing attempts with this input based on a few factors. If, however, the
+    custom mutator wants to set this number instead on how often it is called
+    for a specific queue entry, use this function. This function is most useful
+    if `AFL_CUSTOM_MUTATOR_ONLY` is **not** used.
 
 - `fuzz` (optional):
 
     This method performs custom mutations on a given input. It also accepts an
-    additional test case.
-    Note that this function is optional - but it makes sense to use it.
-    You would only skip this if `post_process` is used to fix checksums etc.
-    so if you are using it e.g. as a post processing library.
+    additional test case. Note that this function is optional - but it makes
+    sense to use it. You would only skip this if `post_process` is used to fix
+    checksums etc. so if you are using it, e.g., as a post processing library.
     Note that a length > 0 *must* be returned!
 
 - `describe` (optional):
 
-    When this function is called, it shall describe the current testcase,
-    generated by the last mutation. This will be called, for example,
-    to name the written testcase file after a crash occurred.
-    Using it can help to reproduce crashing mutations.
+    When this function is called, it shall describe the current test case,
+    generated by the last mutation. This will be called, for example, to name
+    the written test case file after a crash occurred. Using it can help to
+    reproduce crashing mutations.
 
 - `havoc_mutation` and `havoc_mutation_probability` (optional):
 
@@ -142,20 +148,21 @@ def deinit():  # optional for Python
 - `post_process` (optional):
 
     For some cases, the format of the mutated data returned from the custom
-    mutator is not suitable to directly execute the target with this input.
-    For example, when using libprotobuf-mutator, the data returned is in a
-    protobuf format which corresponds to a given grammar. In order to execute
-    the target, the protobuf data must be converted to the plain-text format
-    expected by the target. In such scenarios, the user can define the
-    `post_process` function. This function is then transforming the data into the
-    format expected by the API before executing the target.
+    mutator is not suitable to directly execute the target with this input. For
+    example, when using libprotobuf-mutator, the data returned is in a protobuf
+    format which corresponds to a given grammar. In order to execute the target,
+    the protobuf data must be converted to the plain-text format expected by the
+    target. In such scenarios, the user can define the `post_process` function.
+    This function is then transforming the data into the format expected by the
+    API before executing the target.
 
     This can return any python object that implements the buffer protocol and
     supports PyBUF_SIMPLE. These include bytes, bytearray, etc.
 
 - `queue_new_entry` (optional):
 
-    This methods is called after adding a new test case to the queue.
+    This methods is called after adding a new test case to the queue. If the
+    contents of the file was changed, return True, False otherwise.
 
 - `introspection` (optional):
 
@@ -167,8 +174,8 @@ def deinit():  # optional for Python
 
     The last method to be called, deinitializing the state.
 
-Note that there are also three functions for trimming as described in the
-next section.
+Note that there are also three functions for trimming as described in the next
+section.
 
 ### Trimming Support
 
@@ -176,8 +183,8 @@ The generic trimming routines implemented in AFL++ can easily destroy the
 structure of complex formats, possibly leading to a point where you have a lot
 of test cases in the queue that your Python module cannot process anymore but
 your target application still accepts. This is especially the case when your
-target can process a part of the input (causing coverage) and then errors out
-on the remaining input.
+target can process a part of the input (causing coverage) and then errors out on
+the remaining input.
 
 In such cases, it makes sense to implement a custom trimming routine. The API
 consists of multiple methods because after each trimming step, we have to go
@@ -188,8 +195,9 @@ trimmed input. Here's a quick API description:
 
     This method is called at the start of each trimming operation and receives
     the initial buffer. It should return the amount of iteration steps possible
-    on this input (e.g. if your input has n elements and you want to remove them
-    one by one, return n, if you do a binary search, return log(n), and so on).
+    on this input (e.g., if your input has n elements and you want to remove
+    them one by one, return n, if you do a binary search, return log(n), and so
+    on).
 
     If your trimming algorithm doesn't allow to determine the amount of
     (remaining) steps easily (esp. while running), then you can alternatively
@@ -201,21 +209,21 @@ trimmed input. Here's a quick API description:
 - `trim` (optional)
 
     This method is called for each trimming operation. It doesn't have any
-    arguments because we already have the initial buffer from `init_trim` and we
-    can memorize the current state in the data variables. This can also save
+    arguments because there is already the initial buffer from `init_trim` and
+    we can memorize the current state in the data variables. This can also save
     reparsing steps for each iteration. It should return the trimmed input
     buffer.
 
 - `post_trim` (optional)
 
     This method is called after each trim operation to inform you if your
-    trimming step was successful or not (in terms of coverage). If you receive
-    a failure here, you should reset your input to the last known good state.
-    In any case, this method must return the next trim iteration index (from 0
-    to the maximum amount of steps you returned in `init_trim`).
+    trimming step was successful or not (in terms of coverage). If you receive a
+    failure here, you should reset your input to the last known good state. In
+    any case, this method must return the next trim iteration index (from 0 to
+    the maximum amount of steps you returned in `init_trim`).
 
 Omitting any of three trimming methods will cause the trimming to be disabled
-and trigger a fallback to the builtin default trimming routine.
+and trigger a fallback to the built-in default trimming routine.
 
 ### Environment Variables
 
@@ -223,11 +231,10 @@ Optionally, the following environment variables are supported:
 
 - `AFL_CUSTOM_MUTATOR_ONLY`
 
-    Disable all other mutation stages. This can prevent broken testcases
-    (those that your Python module can't work with anymore) to fill up your
-    queue. Best combined with a custom trimming routine (see below) because
-    trimming can cause the same test breakage like havoc and splice.
-
+    Disable all other mutation stages. This can prevent broken test cases (those
+    that your Python module can't work with anymore) to fill up your queue. Best
+    combined with a custom trimming routine (see below) because trimming can
+    cause the same test breakage like havoc and splice.
 
 - `AFL_PYTHON_ONLY`
 
@@ -263,22 +270,27 @@ In case your setup is different, set the necessary variables like this:
 ### Custom Mutator Preparation
 
 For C/C++ mutators, the source code must be compiled as a shared object:
+
 ```bash
 gcc -shared -Wall -O3 example.c -o example.so
 ```
-Note that if you specify multiple custom mutators, the corresponding functions will
-be called in the order in which they are specified. e.g first `post_process` function of
-`example_first.so` will be called and then that of `example_second.so`.
+
+Note that if you specify multiple custom mutators, the corresponding functions
+will be called in the order in which they are specified. E.g., the first
+`post_process` function of `example_first.so` will be called and then that of
+`example_second.so`.
 
 ### Run
 
 C/C++
+
 ```bash
 export AFL_CUSTOM_MUTATOR_LIBRARY="/full/path/to/example_first.so;/full/path/to/example_second.so"
 afl-fuzz /path/to/program
 ```
 
 Python
+
 ```bash
 export PYTHONPATH=`dirname /full/path/to/example.py`
 export AFL_PYTHON_MODULE=example
@@ -287,8 +299,8 @@ afl-fuzz /path/to/program
 
 ## 4) Example
 
-Please see [example.c](../custom_mutators/examples/example.c) and
-[example.py](../custom_mutators/examples/example.py)
+See [example.c](../custom_mutators/examples/example.c) and
+[example.py](../custom_mutators/examples/example.py).
 
 ## 5) Other Resources
 
@@ -296,4 +308,4 @@ Please see [example.c](../custom_mutators/examples/example.c) and
     - [bruce30262/libprotobuf-mutator_fuzzing_learning](https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/tree/master/4_libprotobuf_aflpp_custom_mutator)
     - [thebabush/afl-libprotobuf-mutator](https://github.com/thebabush/afl-libprotobuf-mutator)
 - [XML Fuzzing@NullCon 2017](https://www.agarri.fr/docs/XML_Fuzzing-NullCon2017-PUBLIC.pdf)
-    - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663)
+    - [A bug detected by AFL + XML-aware mutators](https://bugs.chromium.org/p/chromium/issues/detail?id=930663)

docs/docs.md

@@ -1,122 +0,0 @@
-# Restructure afl++'s documentation
-
-## About us
-
-We are dedicated to everything around fuzzing, our main and most well known
-contribution is the fuzzer `afl++` which is part of all major Unix
-distributions (e.g. Debian, Arch, FreeBSD, etc.) and is deployed on Google's
-oss-fuzz and clusterfuzz. It is rated the top fuzzer on Google's fuzzbench.
-
-We are four individuals from Europe supported by a large community.
-
-All our tools are open source.
-
-## About the afl++ fuzzer project
-
-afl++ inherited it's documentation from the original Google afl project.
-Since then it has been massively improved - feature and performance wise -
-and although the documenation has likewise been continued it has grown out
-of proportion.
-The documentation is done by non-natives to the English language, plus
-none of us has a writer background.
-
-We see questions on afl++ usage on mailing lists (e.g. afl-users), discord
-channels, web forums and as issues in our repository.
-
-This only increases as afl++ has been on the top of Google's fuzzbench
-statistics (which measures the performance of fuzzers) and is now being
-integrated in Google's oss-fuzz and clusterfuzz - and is in many Unix
-packaging repositories, e.g. Debian, FreeBSD, etc.
-
-afl++ now has 44 (!) documentation files with 13k total lines of content.
-This is way too much.
-
-Hence afl++ needs a complete overhaul of it's documentation, both on a 
-organisation/structural level as well as the content.
-
-Overall the following actions have to be performed:
-  * Create a better structure of documentation so it is easier to find the
-    information that is being looked for, combining and/or splitting up the
-    existing documents as needed.
-  * Rewrite some documentation to remove duplication. Several information is
-    present several times in the documentation. These should be removed to
-    where needed so that we have as little bloat as possible.
-  * The documents have been written and modified by a lot of different people,
-    most of them non-native English speaker. Hence an overall review where
-    parts should be rewritten has to be performed and then the rewrite done.
-  * Create a cheat-sheet for a very short best-setup build and run of afl++
-  * Pictures explain more than 1000 words. We need at least 4 images that
-    explain the workflow with afl++:
-      - the build workflow
-      - the fuzzing workflow
-      - the fuzzing campaign management workflow
-      - the overall workflow that is an overview of the above
-      - maybe more? where the technical writes seems it necessary for
-        understanding.
-
-Requirements:
-  * Documentation has to be in Markdown format
-  * Images have to be either in SVG or PNG format.
-  * All documentation should be (moved) in(to) docs/
-
-The project does not require writing new documentation or tutorials beside the
-cheat sheet. The technical information for the cheat sheet will be provided by
-us.
-
-## Metrics
-
-afl++ is a the highest performant fuzzer publicly available - but is also the
-most feature rich and complex. With the publicity of afl++' success and
-deployment in Google projects internally and externally and availability as
-a package on most Linux distributions we see more and more issues being
-created and help requests on our Discord channel that would not be
-necessary if people would have read through all our documentation - which
-is unrealistic.
-
-We expect the the new documenation after this project to be cleaner, easier
-accessible and lighter to digest by our users, resulting in much less
-help requests. On the other hand the amount of users using afl++ should
-increase as well as it will be more accessible which would also increase
-questions again - but overall resulting in a reduction of help requests.
-
-In numbers: we currently have per week on average 5 issues on Github,
-10 questions on discord and 1 on mailing lists that would not be necessary
-with perfect documentation and perfect people.
-
-We would consider this project a success if afterwards we only have
-2 issues on Github and 3 questions on discord anymore that would be answered
-by reading the documentation. The mailing list is usually used by the most
-novice users and we don't expect any less questions there.
-
-## Project Budget
-
-We have zero experience with technical writers, so this is very hard for us
-to calculate. We expect it to be a lot of work though because of the amount
-of documentation we have that needs to be restructured and partially rewritten
-(44 documents with 13k total lines of content).
-
-We assume the daily rate of a very good and experienced technical writer in
-times of a pandemic to be ~500$ (according to web research), and calculate
-the overall amout of work to be around 20 days for everything incl. the
-graphics (but again - this is basically just guessing).
-
-Technical Writer                                              10000$
-Volunteer stipends                                                0$ (waved)
-T-Shirts for the top 10 contributors and helpers to this documentation project:
-	10 afl++ logo t-shirts 		20$ each		200$
-	10 shipping cost of t-shirts    10$ each		100$
-
-Total: 10.300$
-(in the submission form 10.280$ was entered)
-
-## Additional Information
-
-We have participated in Google Summer of Code in 2020 and hope to be selected
-again in 2021.
-
-We have no experience with a technical writer, but we will support that person
-with video calls, chats, emails and messaging, provide all necessary information
-and write technical contents that is required for the success of this project.
-It is clear to us that a technical writer knows how to write, but cannot know
-the technical details in a complex tooling like in afl++. This guidance, input,
-etc. has to come from us.

docs/features.md

@@ -0,0 +1,118 @@
+# Important features of AFL++
+
+AFL++ supports llvm from 3.8 up to version 12, very fast binary fuzzing with
+QEMU 5.1 with laf-intel and Redqueen, FRIDA mode, unicorn mode, gcc plugin, full
+*BSD, Mac OS, Solaris and Android support and much, much, much more.
+
+## Features and instrumentation
+
+| Feature/Instrumentation       | afl-gcc  | llvm      | gcc_plugin | FRIDA mode(9)  | QEMU mode(10)    | unicorn_mode(10) | nyx_mode(12) | coresight_mode(11) |
+| ------------------------------|:--------:|:---------:|:----------:|:--------------:|:----------------:|:----------------:|:------------:|:------------------:|
+| Threadsafe counters [A]       |          |    x(3)   |            |                |                  |                  |       x      |                    |
+| NeverZero           [B]       | x86[_64] |    x(1)   |      x     |        x       |         x        |         x        |              |                    |
+| Persistent Mode     [C]       |          |     x     |      x     | x86[_64]/arm64 | x86[_64]/arm[64] |         x        |              |                    |
+| LAF-Intel / CompCov [D]       |          |     x     |            |                | x86[_64]/arm[64] | x86[_64]/arm[64] |   x86[_64]   |                    |
+| CmpLog              [E]       |          |     x     |            | x86[_64]/arm64 | x86[_64]/arm[64] |                  |              |                    |
+| Selective Instrumentation [F] |          |     x     |      x     |        x       |         x        |                  |              |                    |
+| Non-Colliding Coverage    [G] |          |    x(4)   |            |                |       (x)(5)     |                  |              |                    |
+| Ngram prev_loc Coverage   [H] |          |    x(6)   |            |                |                  |                  |              |                    |
+| Context Coverage    [I]       |          |    x(6)   |            |                |                  |                  |              |                    |
+| Auto Dictionary     [J]       |          |    x(7)   |            |                |                  |                  |              |                    |
+| Snapshot Support    [K]       |          |   (x)(8)  |   (x)(8)   |                |       (x)(5)     |                  |       x      |                    |
+| Shared Memory Test cases  [L] |          |     x     |      x     | x86[_64]/arm64 |         x        |         x        |       x      |                    |
+
+## More information about features
+
+A. Default is not thread-safe coverage counter updates for better performance,
+   see [instrumentation/README.llvm.md](../instrumentation/README.llvm.md)
+
+B. On wrapping coverage counters (255 + 1), skip the 0 value and jump to 1
+   instead. This has shown to give better coverage data and is the default; see
+   [instrumentation/README.llvm.md](../instrumentation/README.llvm.md).
+
+C. Instead of forking, reiterate the fuzz target function in a loop (like
+   `LLVMFuzzerTestOneInput`. Great speed increase but only works with target
+   functions that do not keep state, leak memory, or exit; see
+   [instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md)
+
+D. Split any non-8-bit comparison to 8-bit comparison; see
+   [instrumentation/README.laf-intel.md](../instrumentation/README.laf-intel.md)
+
+E. CmpLog is our enhanced
+   [Redqueen](https://www.ndss-symposium.org/ndss-paper/redqueen-fuzzing-with-input-to-state-correspondence/)
+   implementation, see
+   [instrumentation/README.cmplog.md](../instrumentation/README.cmplog.md)
+
+F. Similar and compatible to clang 13+ sancov sanitize-coverage-allow/deny but
+   for all llvm versions and all our compile modes, only instrument what should
+   be instrumented, for more speed, directed fuzzing and less instability; see
+   [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md)
+
+G. Vanilla AFL uses coverage where edges could collide to the same coverage
+   bytes the larger the target is. Our default instrumentation in LTO and
+   afl-clang-fast (PCGUARD) uses non-colliding coverage that also makes it
+   faster. Vanilla AFL style is available with `AFL_LLVM_INSTRUMENT=AFL`; see
+   [instrumentation/README.llvm.md](../instrumentation/README.llvm.md).
+
+H.+I. Alternative coverage based on previous edges (NGRAM) or depending on the
+   caller (CTX), based on
+   [https://www.usenix.org/system/files/raid2019-wang-jinghan.pdf](https://www.usenix.org/system/files/raid2019-wang-jinghan.pdf);
+   see [instrumentation/README.llvm.md](../instrumentation/README.llvm.md).
+
+J. An LTO feature that creates a fuzzing dictionary based on comparisons found
+   during compilation/instrumentation. Automatic feature :) See
+   [instrumentation/README.lto.md](../instrumentation/README.lto.md)
+
+K. The snapshot feature requires a kernel module that was a lot of work to get
+   right and maintained so it is no longer supported. We have
+   [nyx_mode](../nyx_mode/README.md) instead.
+
+L. Faster fuzzing and less kernel syscall overhead by in-memory fuzz testcase
+   delivery, see
+   [instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md)
+
+## More information about instrumentation
+
+1. Default for LLVM >= 9.0, environment variable for older version due an
+   efficiency bug in previous llvm versions
+2. GCC creates non-performant code, hence it is disabled in gcc_plugin
+3. With `AFL_LLVM_THREADSAFE_INST`, disables NeverZero
+4. With pcguard mode and LTO mode for LLVM 11 and newer
+5. Upcoming, development in the branch
+6. Not compatible with LTO instrumentation and needs at least LLVM v4.1
+7. Automatic in LTO mode with LLVM 11 and newer, an extra pass for all LLVM
+   versions that write to a file to use with afl-fuzz' `-x`
+8. The snapshot LKM is currently unmaintained due to too many kernel changes
+   coming too fast :-(
+9. FRIDA mode is supported on Linux and MacOS for Intel and ARM
+10. QEMU/Unicorn is only supported on Linux
+11. Coresight mode is only available on AARCH64 Linux with a CPU with Coresight
+    extension
+12. Nyx mode is only supported on Linux and currently restricted to x86_x64
+
+## Integrated features and patches
+
+Among others, the following features and patches have been integrated:
+
+* NeverZero patch for afl-gcc, instrumentation, QEMU mode and unicorn_mode which
+  prevents a wrapping map value to zero, increases coverage
+* Persistent mode, deferred forkserver and in-memory fuzzing for QEMU mode
+* Unicorn mode which allows fuzzing of binaries from completely different
+  platforms (integration provided by domenukk)
+* The new CmpLog instrumentation for LLVM and QEMU inspired by
+  [Redqueen](https://github.com/RUB-SysSec/redqueen)
+* Win32 PE binary-only fuzzing with QEMU and Wine
+* AFLfast's power schedules by Marcel Böhme:
+  [https://github.com/mboehme/aflfast](https://github.com/mboehme/aflfast)
+* The MOpt mutator:
+  [https://github.com/puppet-meteor/MOpt-AFL](https://github.com/puppet-meteor/MOpt-AFL)
+* LLVM mode Ngram coverage by Adrian Herrera
+  [https://github.com/adrianherrera/afl-ngram-pass](https://github.com/adrianherrera/afl-ngram-pass)
+* LAF-Intel/CompCov support for instrumentation, QEMU mode and unicorn_mode
+  (with enhanced capabilities)
+* Radamsa and honggfuzz mutators (as custom mutators).
+* QBDI mode to fuzz android native libraries via Quarkslab's
+  [QBDI](https://github.com/QBDI/QBDI) framework
+* Frida and ptrace mode to fuzz binary-only libraries, etc.
+
+So all in all this is the best-of AFL that is out there :-)

docs/fuzzing_binary-only_targets.md

@@ -0,0 +1,309 @@
+# Fuzzing binary-only targets
+
+AFL++, libfuzzer, and other fuzzers are great if you have the source code of the
+target. This allows for very fast and coverage guided fuzzing.
+
+However, if there is only the binary program and no source code available, then
+standard `afl-fuzz -n` (non-instrumented mode) is not effective.
+
+For fast, on-the-fly instrumentation of black-box binaries, AFL++ still offers
+various support. The following is a description of how these binaries can be
+fuzzed with AFL++.
+
+## TL;DR:
+
+FRIDA mode and QEMU mode in persistent mode are the fastest - if persistent mode
+is possible and the stability is high enough.
+
+Otherwise, try Zafl, RetroWrite, Dyninst, and if these fail, too, then try
+standard FRIDA/QEMU mode with `AFL_ENTRYPOINT` to where you need it.
+
+If your target is non-linux, then use unicorn_mode.
+
+## Fuzzing binary-only targets with AFL++
+
+### QEMU mode
+
+QEMU mode is the "native" solution to the program. It is available in the
+./qemu_mode/ directory and, once compiled, it can be accessed by the afl-fuzz -Q
+command line option. It is the easiest to use alternative and even works for
+cross-platform binaries.
+
+For linux programs and its libraries, this is accomplished with a version of
+QEMU running in the lesser-known "user space emulation" mode. QEMU is a project
+separate from AFL++, but you can conveniently build the feature by doing:
+
+```shell
+cd qemu_mode
+./build_qemu_support.sh
+```
+
+The following setup to use QEMU mode is recommended:
+
+* run 1 afl-fuzz -Q instance with CMPLOG (`-c 0` + `AFL_COMPCOV_LEVEL=2`)
+* run 1 afl-fuzz -Q instance with QASAN (`AFL_USE_QASAN=1`)
+* run 1 afl-fuzz -Q instance with LAF (`AFL_PRELOAD=libcmpcov.so` +
+  `AFL_COMPCOV_LEVEL=2`), alternatively you can use FRIDA mode, just switch `-Q`
+  with `-O` and remove the LAF instance
+
+Then run as many instances as you have cores left with either -Q mode or - even
+better - use a binary rewriter like Dyninst, RetroWrite, ZAFL, etc.
+
+If [afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst) works for your
+binary, then you can use afl-fuzz normally and it will have twice the speed
+compared to QEMU mode (but slower than QEMU persistent mode). Note that several
+other binary rewriters exist, all with their advantages and caveats.
+
+The speed decrease of QEMU mode is at about 50%. However, various options exist
+to increase the speed:
+- using AFL_ENTRYPOINT to move the forkserver entry to a later basic block in
+  the binary (+5-10% speed)
+- using persistent mode
+  [qemu_mode/README.persistent.md](../qemu_mode/README.persistent.md) this will
+  result in a 150-300% overall speed increase - so 3-8x the original QEMU mode
+  speed!
+- using AFL_CODE_START/AFL_CODE_END to only instrument specific parts
+
+For additional instructions and caveats, see
+[qemu_mode/README.md](../qemu_mode/README.md). If possible, you should use the
+persistent mode, see
+[qemu_mode/README.persistent.md](../qemu_mode/README.persistent.md). The mode is
+approximately 2-5x slower than compile-time instrumentation, and is less
+conducive to parallelization.
+
+Note that there is also honggfuzz:
+[https://github.com/google/honggfuzz](https://github.com/google/honggfuzz) which
+now has a QEMU mode, but its performance is just 1.5% ...
+
+If you like to code a customized fuzzer without much work, we highly recommend
+to check out our sister project libafl which supports QEMU, too:
+[https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL)
+
+### WINE+QEMU
+
+Wine mode can run Win32 PE binaries with the QEMU instrumentation. It needs
+Wine, python3, and the pefile python package installed.
+
+It is included in AFL++.
+
+For more information, see
+[qemu_mode/README.wine.md](../qemu_mode/README.wine.md).
+
+### FRIDA mode
+
+In FRIDA mode, you can fuzz binary-only targets as easily as with QEMU mode.
+FRIDA mode is most of the times slightly faster than QEMU mode. It is also
+newer, lacks COMPCOV, and has the advantage that it works on MacOS (both intel
+and M1).
+
+To build FRIDA mode:
+
+```shell
+cd frida_mode
+gmake
+```
+
+For additional instructions and caveats, see
+[frida_mode/README.md](../frida_mode/README.md).
+
+If possible, you should use the persistent mode, see
+[instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md).
+The mode is approximately 2-5x slower than compile-time instrumentation, and is
+less conducive to parallelization. But for binary-only fuzzing, it gives a huge
+speed improvement if it is possible to use.
+
+If you want to fuzz a binary-only library, then you can fuzz it with frida-gum
+via frida_mode/. You will have to write a harness to call the target function in
+the library, use afl-frida.c as a template.
+
+You can also perform remote fuzzing with frida, e.g., if you want to fuzz on
+iPhone or Android devices, for this you can use
+[https://github.com/ttdennis/fpicker/](https://github.com/ttdennis/fpicker/) as
+an intermediate that uses AFL++ for fuzzing.
+
+If you like to code a customized fuzzer without much work, we highly recommend
+to check out our sister project libafl which supports Frida, too:
+[https://github.com/AFLplusplus/LibAFL](https://github.com/AFLplusplus/LibAFL).
+Working examples already exist :-)
+
+### Nyx mode
+
+Nyx is a full system emulation fuzzing environment with snapshot support that is
+built upon KVM and QEMU. It is only available on Linux and currently restricted
+to x86_x64.
+
+For binary-only fuzzing a special 5.10 kernel is required.
+
+See [nyx_mode/README.md](../nyx_mode/README.md).
+
+### Unicorn
+
+Unicorn is a fork of QEMU. The instrumentation is, therefore, very similar. In
+contrast to QEMU, Unicorn does not offer a full system or even userland
+emulation. Runtime environment and/or loaders have to be written from scratch,
+if needed. On top, block chaining has been removed. This means the speed boost
+introduced in the patched QEMU Mode of AFL++ cannot be ported over to Unicorn.
+
+For non-Linux binaries, you can use AFL++'s unicorn_mode which can emulate
+anything you want - for the price of speed and user written scripts.
+
+To build unicorn_mode:
+
+```shell
+cd unicorn_mode
+./build_unicorn_support.sh
+```
+
+For further information, check out
+[unicorn_mode/README.md](../unicorn_mode/README.md).
+
+### Shared libraries
+
+If the goal is to fuzz a dynamic library, then there are two options available.
+For both, you need to write a small harness that loads and calls the library.
+Then you fuzz this with either FRIDA mode or QEMU mode and either use
+`AFL_INST_LIBS=1` or `AFL_QEMU/FRIDA_INST_RANGES`.
+
+Another, less precise and slower option is to fuzz it with utils/afl_untracer/
+and use afl-untracer.c as a template. It is slower than FRIDA mode.
+
+For more information, see
+[utils/afl_untracer/README.md](../utils/afl_untracer/README.md).
+
+### Coresight
+
+Coresight is ARM's answer to Intel's PT. With AFL++ v3.15, there is a coresight
+tracer implementation available in `coresight_mode/` which is faster than QEMU,
+however, cannot run in parallel. Currently, only one process can be traced, it
+is WIP.
+
+Fore more information, see
+[coresight_mode/README.md](../coresight_mode/README.md).
+
+## Binary rewriters
+
+An alternative solution are binary rewriters. They are faster than the solutions
+native to AFL++ but don't always work.
+
+### ZAFL
+
+ZAFL is a static rewriting platform supporting x86-64 C/C++,
+stripped/unstripped, and PIE/non-PIE binaries. Beyond conventional
+instrumentation, ZAFL's API enables transformation passes (e.g., laf-Intel,
+context sensitivity, InsTrim, etc.).
+
+Its baseline instrumentation speed typically averages 90-95% of
+afl-clang-fast's.
+
+[https://git.zephyr-software.com/opensrc/zafl](https://git.zephyr-software.com/opensrc/zafl)
+
+### RetroWrite
+
+RetroWrite is a static binary rewriter that can be combined with AFL++. If you
+have an x86_64 binary that still has its symbols (i.e., not stripped binary), is
+compiled with position independent code (PIC/PIE), and does not contain C++
+exceptions, then the RetroWrite solution might be for you. It decompiles to ASM
+files which can then be instrumented with afl-gcc.
+
+Binaries that are statically instrumented for fuzzing using RetroWrite are close
+in performance to compiler-instrumented binaries and outperform the QEMU-based
+instrumentation.
+
+[https://github.com/HexHive/retrowrite](https://github.com/HexHive/retrowrite)
+
+### Dyninst
+
+Dyninst is a binary instrumentation framework similar to Pintool and DynamoRIO.
+However, whereas Pintool and DynamoRIO work at runtime, Dyninst instruments the
+target at load time and then let it run - or save the binary with the changes.
+This is great for some things, e.g., fuzzing, and not so effective for others,
+e.g., malware analysis.
+
+So, what you can do with Dyninst is taking every basic block and putting AFL++'s
+instrumentation code in there - and then save the binary. Afterwards, just fuzz
+the newly saved target binary with afl-fuzz. Sounds great? It is. The issue
+though - it is a non-trivial problem to insert instructions, which change
+addresses in the process space, so that everything is still working afterwards.
+Hence, more often than not binaries crash when they are run.
+
+The speed decrease is about 15-35%, depending on the optimization options used
+with afl-dyninst.
+
+[https://github.com/vanhauser-thc/afl-dyninst](https://github.com/vanhauser-thc/afl-dyninst)
+
+### Mcsema
+
+Theoretically, you can also decompile to llvm IR with mcsema, and then use
+llvm_mode to instrument the binary. Good luck with that.
+
+[https://github.com/lifting-bits/mcsema](https://github.com/lifting-bits/mcsema)
+
+## Binary tracers
+
+### Pintool & DynamoRIO
+
+Pintool and DynamoRIO are dynamic instrumentation engines. They can be used for
+getting basic block information at runtime. Pintool is only available for Intel
+x32/x64 on Linux, Mac OS, and Windows, whereas DynamoRIO is additionally
+available for ARM and AARCH64. DynamoRIO is also 10x faster than Pintool.
+
+The big issue with DynamoRIO (and therefore Pintool, too) is speed. DynamoRIO
+has a speed decrease of 98-99%, Pintool has a speed decrease of 99.5%.
+
+Hence, DynamoRIO is the option to go for if everything else fails and Pintool
+only if DynamoRIO fails, too.
+
+DynamoRIO solutions:
+* [https://github.com/vanhauser-thc/afl-dynamorio](https://github.com/vanhauser-thc/afl-dynamorio)
+* [https://github.com/mxmssh/drAFL](https://github.com/mxmssh/drAFL)
+* [https://github.com/googleprojectzero/winafl/](https://github.com/googleprojectzero/winafl/)
+  <= very good but windows only
+
+Pintool solutions:
+* [https://github.com/vanhauser-thc/afl-pin](https://github.com/vanhauser-thc/afl-pin)
+* [https://github.com/mothran/aflpin](https://github.com/mothran/aflpin)
+* [https://github.com/spinpx/afl_pin_mode](https://github.com/spinpx/afl_pin_mode)
+  <= only old Pintool version supported
+
+### Intel PT
+
+If you have a newer Intel CPU, you can make use of Intel's processor trace. The
+big issue with Intel's PT is the small buffer size and the complex encoding of
+the debug information collected through PT. This makes the decoding very CPU
+intensive and hence slow. As a result, the overall speed decrease is about
+70-90% (depending on the implementation and other factors).
+
+There are two AFL intel-pt implementations:
+
+1. [https://github.com/junxzm1990/afl-pt](https://github.com/junxzm1990/afl-pt)
+    => This needs Ubuntu 14.04.05 without any updates and the 4.4 kernel.
+
+2. [https://github.com/hunter-ht-2018/ptfuzzer](https://github.com/hunter-ht-2018/ptfuzzer)
+    => This needs a 4.14 or 4.15 kernel. The "nopti" kernel boot option must be
+    used. This one is faster than the other.
+
+Note that there is also honggfuzz:
+[https://github.com/google/honggfuzz](https://github.com/google/honggfuzz). But
+its IPT performance is just 6%!
+
+## Non-AFL++ solutions
+
+There are many binary-only fuzzing frameworks. Some are great for CTFs but don't
+work with large binaries, others are very slow but have good path discovery,
+some are very hard to set-up...
+
+* Jackalope:
+  [https://github.com/googleprojectzero/Jackalope](https://github.com/googleprojectzero/Jackalope)
+* Manticore:
+  [https://github.com/trailofbits/manticore](https://github.com/trailofbits/manticore)
+* QSYM:
+  [https://github.com/sslab-gatech/qsym](https://github.com/sslab-gatech/qsym)
+* S2E: [https://github.com/S2E](https://github.com/S2E)
+* TinyInst:
+  [https://github.com/googleprojectzero/TinyInst](https://github.com/googleprojectzero/TinyInst)
+  (Mac/Windows only)
+*  ... please send me any missing that are good
+
+## Closing words
+
+That's it! News, corrections, updates? Send an email to vh@thc.org.

docs/fuzzing_in_depth.md

@@ -0,0 +1,927 @@
+# Fuzzing with AFL++
+
+The following describes how to fuzz with a target if source code is available.
+If you have a binary-only target, go to
+[fuzzing_binary-only_targets.md](fuzzing_binary-only_targets.md).
+
+Fuzzing source code is a three-step process:
+
+1. Compile the target with a special compiler that prepares the target to be
+   fuzzed efficiently. This step is called "instrumenting a target".
+2. Prepare the fuzzing by selecting and optimizing the input corpus for the
+   target.
+3. Perform the fuzzing of the target by randomly mutating input and assessing if
+   that input was processed on a new path in the target binary.
+
+## 0. Common sense risks
+
+Please keep in mind that, similarly to many other computationally-intensive
+tasks, fuzzing may put a strain on your hardware and on the OS. In particular:
+
+- Your CPU will run hot and will need adequate cooling. In most cases, if
+  cooling is insufficient or stops working properly, CPU speeds will be
+  automatically throttled. That said, especially when fuzzing on less suitable
+  hardware (laptops, smartphones, etc.), it's not entirely impossible for
+  something to blow up.
+
+- Targeted programs may end up erratically grabbing gigabytes of memory or
+  filling up disk space with junk files. AFL++ tries to enforce basic memory
+  limits, but can't prevent each and every possible mishap. The bottom line is
+  that you shouldn't be fuzzing on systems where the prospect of data loss is
+  not an acceptable risk.
+
+- Fuzzing involves billions of reads and writes to the filesystem. On modern
+  systems, this will be usually heavily cached, resulting in fairly modest
+  "physical" I/O - but there are many factors that may alter this equation. It
+  is your responsibility to monitor for potential trouble; with very heavy I/O,
+  the lifespan of many HDDs and SSDs may be reduced.
+
+  A good way to monitor disk I/O on Linux is the `iostat` command:
+
+  ```shell
+  $ iostat -d 3 -x -k [...optional disk ID...]
+  ```
+
+  Using the `AFL_TMPDIR` environment variable and a RAM-disk, you can have the
+  heavy writing done in RAM to prevent the aforementioned wear and tear. For
+  example, the following line will run a Docker container with all this preset:
+
+  ```shell
+  # docker run -ti --mount type=tmpfs,destination=/ramdisk -e AFL_TMPDIR=/ramdisk aflplusplus/aflplusplus
+  ```
+
+## 1. Instrumenting the target
+
+### a) Selecting the best AFL++ compiler for instrumenting the target
+
+AFL++ comes with a central compiler `afl-cc` that incorporates various different
+kinds of compiler targets and instrumentation options. The following
+evaluation flow will help you to select the best possible.
+
+It is highly recommended to have the newest llvm version possible installed,
+anything below 9 is not recommended.
+
+```
++--------------------------------+
+| clang/clang++ 11+ is available | --> use LTO mode (afl-clang-lto/afl-clang-lto++)
++--------------------------------+     see [instrumentation/README.lto.md](instrumentation/README.lto.md)
+    |
+    | if not, or if the target fails with LTO afl-clang-lto/++
+    |
+    v
++---------------------------------+
+| clang/clang++ 3.8+ is available | --> use LLVM mode (afl-clang-fast/afl-clang-fast++)
++---------------------------------+     see [instrumentation/README.llvm.md](instrumentation/README.llvm.md)
+    |
+    | if not, or if the target fails with LLVM afl-clang-fast/++
+    |
+    v
+ +--------------------------------+
+ | gcc 5+ is available            | -> use GCC_PLUGIN mode (afl-gcc-fast/afl-g++-fast)
+ +--------------------------------+    see [instrumentation/README.gcc_plugin.md](instrumentation/README.gcc_plugin.md) and
+                                       [instrumentation/README.instrument_list.md](instrumentation/README.instrument_list.md)
+    |
+    | if not, or if you do not have a gcc with plugin support
+    |
+    v
+   use GCC mode (afl-gcc/afl-g++) (or afl-clang/afl-clang++ for clang)
+```
+
+Clickable README links for the chosen compiler:
+
+* [LTO mode - afl-clang-lto](../instrumentation/README.lto.md)
+* [LLVM mode - afl-clang-fast](../instrumentation/README.llvm.md)
+* [GCC_PLUGIN mode - afl-gcc-fast](../instrumentation/README.gcc_plugin.md)
+* GCC/CLANG modes (afl-gcc/afl-clang) have no README as they have no own
+  features
+
+You can select the mode for the afl-cc compiler by one of the following methods:
+
+* Using a symlink to afl-cc: afl-gcc, afl-g++, afl-clang, afl-clang++,
+   afl-clang-fast, afl-clang-fast++, afl-clang-lto, afl-clang-lto++,
+   afl-gcc-fast, afl-g++-fast (recommended!).
+* Using the environment variable `AFL_CC_COMPILER` with `MODE`.
+* Passing --afl-`MODE` command line options to the compiler via
+   `CFLAGS`/`CXXFLAGS`/`CPPFLAGS`.
+
+`MODE` can be one of the following:
+
+* LTO (afl-clang-lto*)
+* LLVM (afl-clang-fast*)
+* GCC_PLUGIN (afl-g*-fast) or GCC (afl-gcc/afl-g++)
+* CLANG(afl-clang/afl-clang++)
+
+Because no AFL++ specific command-line options are accepted (beside the
+--afl-MODE command), the compile-time tools make fairly broad use of environment
+variables, which can be listed with `afl-cc -hh` or looked up in
+[env_variables.md](env_variables.md).
+
+### b) Selecting instrumentation options
+
+If you instrument with LTO mode (afl-clang-fast/afl-clang-lto), the following
+options are available:
+
+* Splitting integer, string, float, and switch comparisons so AFL++ can easier
+  solve these. This is an important option if you do not have a very good and
+  large input corpus. This technique is called laf-intel or COMPCOV. To use
+  this, set the following environment variable before compiling the target:
+  `export AFL_LLVM_LAF_ALL=1`. You can read more about this in
+  [instrumentation/README.laf-intel.md](../instrumentation/README.laf-intel.md).
+* A different technique (and usually a better one than laf-intel) is to
+  instrument the target so that any compare values in the target are sent to
+  AFL++ which then tries to put these values into the fuzzing data at different
+  locations. This technique is very fast and good - if the target does not
+  transform input data before comparison. Therefore, this technique is called
+  `input to state` or `redqueen`. If you want to use this technique, then you
+  have to compile the target twice, once specifically with/for this mode by
+  setting `AFL_LLVM_CMPLOG=1`, and pass this binary to afl-fuzz via the `-c`
+  parameter. Note that you can compile also just a cmplog binary and use that
+  for both, however, there will be a performance penalty. You can read more
+  about this in
+  [instrumentation/README.cmplog.md](../instrumentation/README.cmplog.md).
+
+If you use LTO, LLVM, or GCC_PLUGIN mode
+(afl-clang-fast/afl-clang-lto/afl-gcc-fast), you have the option to selectively
+instrument _parts_ of the target that you are interested in. For afl-clang-fast,
+you have to use an llvm version newer than 10.0.0 or a mode other than
+DEFAULT/PCGUARD.
+
+This step can be done either by explicitly including parts to be instrumented or
+by explicitly excluding parts from instrumentation.
+
+* To instrument _only specified parts_, create a file (e.g., `allowlist.txt`)
+  with all the filenames and/or functions of the source code that should be
+  instrumented and then:
+
+  1. Just put one filename or function (prefixing with `fun: `) per line (no
+     directory information necessary for filenames) in the file `allowlist.txt`.
+
+     Example:
+
+     ```
+     foo.cpp        # will match foo/foo.cpp, bar/foo.cpp, barfoo.cpp etc.
+     fun: foo_func  # will match the function foo_func
+     ```
+
+  2. Set `export AFL_LLVM_ALLOWLIST=allowlist.txt` to enable selective positive
+     instrumentation.
+
+* Similarly to _exclude_ specified parts from instrumentation, create a file
+  (e.g., `denylist.txt`) with all the filenames of the source code that should
+  be skipped during instrumentation and then:
+
+  1. Same as above. Just put one filename or function per line in the file
+     `denylist.txt`.
+
+  2. Set `export AFL_LLVM_DENYLIST=denylist.txt` to enable selective negative
+     instrumentation.
+
+**NOTE:** During optimization functions might be
+inlined and then would not match the list! See
+[instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md).
+
+There are many more options and modes available, however, these are most of the
+time less effective. See:
+
+* [instrumentation/README.llvm.md#6) AFL++ Context Sensitive Branch Coverage](../instrumentation/README.llvm.md#6-afl-context-sensitive-branch-coverage)
+* [instrumentation/README.llvm.md#7) AFL++ N-Gram Branch Coverage](../instrumentation/README.llvm.md#7-afl-n-gram-branch-coverage)
+
+AFL++ performs "never zero" counting in its bitmap. You can read more about this
+here:
+* [instrumentation/README.llvm.md#8-neverzero-counters](../instrumentation/README.llvm.md#8-neverzero-counters)
+
+### c) Selecting sanitizers
+
+It is possible to use sanitizers when instrumenting targets for fuzzing, which
+allows you to find bugs that would not necessarily result in a crash.
+
+Note that sanitizers have a huge impact on CPU (= less executions per second)
+and RAM usage. Also, you should only run one afl-fuzz instance per sanitizer
+type. This is enough because e.g. a use-after-free bug will be picked up by ASAN
+(address sanitizer) anyway after syncing test cases from other fuzzing
+instances, so running more than one address sanitized target would be a waste.
+
+The following sanitizers have built-in support in AFL++:
+
+* ASAN = Address SANitizer, finds memory corruption vulnerabilities like
+  use-after-free, NULL pointer dereference, buffer overruns, etc. Enabled with
+  `export AFL_USE_ASAN=1` before compiling.
+* MSAN = Memory SANitizer, finds read accesses to uninitialized memory, e.g., a
+  local variable that is defined and read before it is even set. Enabled with
+  `export AFL_USE_MSAN=1` before compiling.
+* UBSAN = Undefined Behavior SANitizer, finds instances where - by the C and C++
+  standards - undefined behavior happens, e.g., adding two signed integers where
+  the result is larger than what a signed integer can hold. Enabled with `export
+  AFL_USE_UBSAN=1` before compiling.
+* CFISAN = Control Flow Integrity SANitizer, finds instances where the control
+  flow is found to be illegal. Originally this was rather to prevent return
+  oriented programming (ROP) exploit chains from functioning. In fuzzing, this
+  is mostly reduced to detecting type confusion vulnerabilities - which is,
+  however, one of the most important and dangerous C++ memory corruption
+  classes! Enabled with `export AFL_USE_CFISAN=1` before compiling.
+* TSAN = Thread SANitizer, finds thread race conditions. Enabled with `export
+  AFL_USE_TSAN=1` before compiling.
+* LSAN = Leak SANitizer, finds memory leaks in a program. This is not really a
+  security issue, but for developers this can be very valuable. Note that unlike
+  the other sanitizers above this needs `__AFL_LEAK_CHECK();` added to all areas
+  of the target source code where you find a leak check necessary! Enabled with
+  `export AFL_USE_LSAN=1` before compiling. To ignore the memory-leaking check
+  for certain allocations, `__AFL_LSAN_OFF();` can be used before memory is
+  allocated, and `__AFL_LSAN_ON();` afterwards. Memory allocated between these
+  two macros will not be checked for memory leaks.
+
+It is possible to further modify the behavior of the sanitizers at run-time by
+setting `ASAN_OPTIONS=...`, `LSAN_OPTIONS` etc. - the available parameters can
+be looked up in the sanitizer documentation of llvm/clang. afl-fuzz, however,
+requires some specific parameters important for fuzzing to be set. If you want
+to set your own, it might bail and report what it is missing.
+
+Note that some sanitizers cannot be used together, e.g., ASAN and MSAN, and
+others often cannot work together because of target weirdness, e.g., ASAN and
+CFISAN. You might need to experiment which sanitizers you can combine in a
+target (which means more instances can be run without a sanitized target, which
+is more effective).
+
+### d) Modifying the target
+
+If the target has features that make fuzzing more difficult, e.g., checksums,
+HMAC, etc., then modify the source code so that checks for these values are
+removed. This can even be done safely for source code used in operational
+products by eliminating these checks within these AFL++ specific blocks:
+
+```
+#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION
+  // say that the checksum or HMAC was fine - or whatever is required
+  // to eliminate the need for the fuzzer to guess the right checksum
+  return 0;
+#endif
+```
+
+All AFL++ compilers will set this preprocessor definition automatically.
+
+### e) Instrumenting the target
+
+In this step, the target source code is compiled so that it can be fuzzed.
+
+Basically, you have to tell the target build system that the selected AFL++
+compiler is used. Also - if possible - you should always configure the build
+system in such way that the target is compiled statically and not dynamically.
+How to do this is described below.
+
+The #1 rule when instrumenting a target is: avoid instrumenting shared libraries
+at all cost. You would need to set `LD_LIBRARY_PATH` to point to these, you
+could accidentally type "make install" and install them system wide - so don't.
+Really don't. **Always compile libraries you want to have instrumented as static
+and link these to the target program!**
+
+Then build the target. (Usually with `make`.)
+
+**NOTES**
+
+1. Sometimes configure and build systems are fickle and do not like stderr
+   output (and think this means a test failure) - which is something AFL++ likes
+   to do to show statistics. It is recommended to disable AFL++ instrumentation
+   reporting via `export AFL_QUIET=1`.
+
+2. Sometimes configure and build systems error on warnings - these should be
+   disabled (e.g., `--disable-werror` for some configure scripts).
+
+3. In case the configure/build system complains about AFL++'s compiler and
+   aborts, then set `export AFL_NOOPT=1` which will then just behave like the
+   real compiler and run the configure step separately. For building the target
+   afterwards this option has to be unset again!
+
+#### configure
+
+For `configure` build systems, this is usually done by:
+
+```
+CC=afl-clang-fast CXX=afl-clang-fast++ ./configure --disable-shared
+```
+
+Note that if you are using the (better) afl-clang-lto compiler, you also have to
+set `AR` to llvm-ar[-VERSION] and `RANLIB` to llvm-ranlib[-VERSION] - as is
+described in [instrumentation/README.lto.md](../instrumentation/README.lto.md).
+
+#### CMake
+
+For CMake build systems, this is usually done by:
+
+```
+mkdir build; cd build; cmake -DCMAKE_C_COMPILER=afl-cc -DCMAKE_CXX_COMPILER=afl-c++ ..
+```
+
+Note that if you are using the (better) afl-clang-lto compiler you also have to
+set AR to llvm-ar[-VERSION] and RANLIB to llvm-ranlib[-VERSION] - as is
+described in [instrumentation/README.lto.md](../instrumentation/README.lto.md).
+
+#### Meson Build System
+
+For the Meson Build System, you have to set the AFL++ compiler with the very
+first command!
+
+```
+CC=afl-cc CXX=afl-c++ meson
+```
+
+#### Other build systems or if configure/cmake didn't work
+
+Sometimes `cmake` and `configure` do not pick up the AFL++ compiler or the
+`RANLIB`/`AR` that is needed - because this was just not foreseen by the
+developer of the target. Or they have non-standard options. Figure out if there
+is a non-standard way to set this, otherwise set up the build normally and edit
+the generated build environment afterwards manually to point it to the right
+compiler (and/or `RANLIB` and `AR`).
+
+### f) Better instrumentation
+
+If you just fuzz a target program as-is, you are wasting a great opportunity for
+much more fuzzing speed.
+
+This variant requires the usage of afl-clang-lto, afl-clang-fast or
+afl-gcc-fast.
+
+It is the so-called `persistent mode`, which is much, much faster but requires
+that you code a source file that is specifically calling the target functions
+that you want to fuzz, plus a few specific AFL++ functions around it. See
+[instrumentation/README.persistent_mode.md](../instrumentation/README.persistent_mode.md)
+for details.
+
+Basically, if you do not fuzz a target in persistent mode, then you are just
+doing it for a hobby and not professionally :-).
+
+### g) libfuzzer fuzzer harnesses with LLVMFuzzerTestOneInput()
+
+libfuzzer `LLVMFuzzerTestOneInput()` harnesses are the defacto standard for
+fuzzing, and they can be used with AFL++ (and honggfuzz) as well!
+
+Compiling them is as simple as:
+
+```
+afl-clang-fast++ -fsanitize=fuzzer -o harness harness.cpp targetlib.a
+```
+
+You can even use advanced libfuzzer features like `FuzzedDataProvider`,
+`LLVMFuzzerInitialize()` etc. and they will work!
+
+The generated binary is fuzzed with afl-fuzz like any other fuzz target.
+
+Bonus: the target is already optimized for fuzzing due to persistent mode and
+shared-memory test cases and hence gives you the fastest speed possible.
+
+For more information, see
+[utils/aflpp_driver/README.md](../utils/aflpp_driver/README.md).
+
+## 2. Preparing the fuzzing campaign
+
+As you fuzz the target with mutated input, having as diverse inputs for the
+target as possible improves the efficiency a lot.
+
+### a) Collecting inputs
+
+To operate correctly, the fuzzer requires one or more starting files that
+contain a good example of the input data normally expected by the targeted
+application.
+
+Try to gather valid inputs for the target from wherever you can. E.g., if it is
+the PNG picture format, try to find as many PNG files as possible, e.g., from
+reported bugs, test suites, random downloads from the internet, unit test case
+data - from all kind of PNG software.
+
+If the input format is not known, you can also modify a target program to write
+normal data it receives and processes to a file and use these.
+
+You can find many good examples of starting files in the
+[testcases/](../testcases) subdirectory that comes with this tool.
+
+### b) Making the input corpus unique
+
+Use the AFL++ tool `afl-cmin` to remove inputs from the corpus that do not
+produce a new path/coverage in the target:
+
+1. Put all files from [step a](#a-collecting-inputs) into one directory, e.g.,
+   `INPUTS`.
+2. Run afl-cmin:
+   * If the target program is to be called by fuzzing as `bin/target INPUTFILE`,
+     replace the INPUTFILE argument that the target program would read from with
+     `@@`:
+
+     ```
+     afl-cmin -i INPUTS -o INPUTS_UNIQUE -- bin/target -someopt @@
+     ```
+
+   * If the target reads from stdin (standard input) instead, just omit the `@@`
+     as this is the default:
+
+     ```
+     afl-cmin -i INPUTS -o INPUTS_UNIQUE -- bin/target -someopt
+     ```
+
+This step is highly recommended, because afterwards the testcase corpus is not
+bloated with duplicates anymore, which would slow down the fuzzing progress!
+
+### c) Minimizing all corpus files
+
+The shorter the input files that still traverse the same path within the target,
+the better the fuzzing will be. This minimization is done with `afl-tmin`,
+however, it is a long process as this has to be done for every file:
+
+```
+mkdir input
+cd INPUTS_UNIQUE
+for i in *; do
+  afl-tmin -i "$i" -o "../input/$i" -- bin/target -someopt @@
+done
+```
+
+This step can also be parallelized, e.g., with `parallel`.
+
+Note that this step is rather optional though.
+
+### Done!
+
+The INPUTS_UNIQUE/ directory from [step b](#b-making-the-input-corpus-unique) -
+or even better the directory input/ if you minimized the corpus in
+[step c](#c-minimizing-all-corpus-files) - is the resulting input corpus
+directory to be used in fuzzing! :-)
+
+## 3. Fuzzing the target
+
+In this final step, fuzz the target. There are not that many important options
+to run the target - unless you want to use many CPU cores/threads for the
+fuzzing, which will make the fuzzing much more useful.
+
+If you just use one instance for fuzzing, then you are fuzzing just for fun and
+not seriously :-)
+
+### a) Running afl-fuzz
+
+Before you do even a test run of afl-fuzz, execute `sudo afl-system-config` (on
+the host if you execute afl-fuzz in a Docker container). This reconfigures the
+system for optimal speed - which afl-fuzz checks and bails otherwise. Set
+`export AFL_SKIP_CPUFREQ=1` for afl-fuzz to skip this check if you cannot run
+afl-system-config with root privileges on the host for whatever reason.
+
+Note:
+
+* There is also `sudo afl-persistent-config` which sets additional permanent
+  boot options for a much better fuzzing performance.
+* Both scripts improve your fuzzing performance but also decrease your system
+  protection against attacks! So set strong firewall rules and only expose SSH
+  as a network service if you use these (which is highly recommended).
+
+If you have an input corpus from [step 2](#2-preparing-the-fuzzing-campaign),
+then specify this directory with the `-i` option. Otherwise, create a new
+directory and create a file with any content as test data in there.
+
+If you do not want anything special, the defaults are already usually best,
+hence all you need is to specify the seed input directory with the result of
+step [2a) Collecting inputs](#a-collecting-inputs):
+
+```
+afl-fuzz -i input -o output -- bin/target -someopt @@
+```
+
+Note that the directory specified with `-o` will be created if it does not
+exist.
+
+It can be valuable to run afl-fuzz in a `screen` or `tmux` shell so you can log
+off, or afl-fuzz is not aborted if you are running it in a remote ssh session
+where the connection fails in between. Only do that though once you have
+verified that your fuzzing setup works! Run it like `screen -dmS afl-main --
+afl-fuzz -M main-$HOSTNAME -i ...` and it will start away in a screen session.
+To enter this session, type `screen -r afl-main`. You see - it makes sense to
+name the screen session same as the afl-fuzz `-M`/`-S` naming :-) For more
+information on screen or tmux, check their documentation.
+
+If you need to stop and re-start the fuzzing, use the same command line options
+(or even change them by selecting a different power schedule or another mutation
+mode!) and switch the input directory with a dash (`-`):
+
+```
+afl-fuzz -i - -o output -- bin/target -someopt @@
+```
+
+Adding a dictionary is helpful. You have to following options:
+
+* See the directory
+[dictionaries/](../dictionaries/), if something is already included for your
+data format, and tell afl-fuzz to load that dictionary by adding `-x
+dictionaries/FORMAT.dict`.
+* With `afl-clang-lto`, you have an autodictionary generation for which you need
+  to do nothing except to use afl-clang-lto as the compiler.
+* With `afl-clang-fast`, you can set
+  `AFL_LLVM_DICT2FILE=/full/path/to/new/file.dic` to automatically generate a
+  dictionary during target compilation.
+* You also have the option to generate a dictionary yourself during an
+  independent run of the target, see
+  [utils/libtokencap/README.md](../utils/libtokencap/README.md).
+* Finally, you can also write a dictionary file manually, of course.
+
+afl-fuzz has a variety of options that help to workaround target quirks like
+very specific locations for the input file (`-f`), performing deterministic
+fuzzing (`-D`) and many more. Check out `afl-fuzz -h`.
+
+We highly recommend that you set a memory limit for running the target with `-m`
+which defines the maximum memory in MB. This prevents a potential out-of-memory
+problem for your system plus helps you detect missing `malloc()` failure
+handling in the target. Play around with various `-m` values until you find one
+that safely works for all your input seeds (if you have good ones and then
+double or quadruple that).
+
+By default, afl-fuzz never stops fuzzing. To terminate AFL++, press Control-C or
+send a signal SIGINT. You can limit the number of executions or approximate
+runtime in seconds with options also.
+
+When you start afl-fuzz, you will see a user interface that shows what the
+status is:
+
+![resources/screenshot.png](resources/screenshot.png)
+
+All labels are explained in
+[afl-fuzz_approach.md#understanding-the-status-screen](afl-fuzz_approach.md#understanding-the-status-screen).
+
+### b) Keeping memory use and timeouts in check
+
+Memory limits are not enforced by afl-fuzz by default and the system may run out
+of memory. You can decrease the memory with the `-m` option, the value is in MB.
+If this is too small for the target, you can usually see this by afl-fuzz
+bailing with the message that it could not connect to the forkserver.
+
+Consider setting low values for `-m` and `-t`.
+
+For programs that are nominally very fast, but get sluggish for some inputs, you
+can also try setting `-t` values that are more punishing than what `afl-fuzz`
+dares to use on its own. On fast and idle machines, going down to `-t 5` may be
+a viable plan.
+
+The `-m` parameter is worth looking at, too. Some programs can end up spending a
+fair amount of time allocating and initializing megabytes of memory when
+presented with pathological inputs. Low `-m` values can make them give up sooner
+and not waste CPU time.
+
+### c) Using multiple cores
+
+If you want to seriously fuzz, then use as many cores/threads as possible to
+fuzz your target.
+
+On the same machine - due to the design of how AFL++ works - there is a maximum
+number of CPU cores/threads that are useful, use more and the overall
+performance degrades instead. This value depends on the target, and the limit is
+between 32 and 64 cores per machine.
+
+If you have the RAM, it is highly recommended run the instances with a caching
+of the test cases. Depending on the average test case size (and those found
+during fuzzing) and their number, a value between 50-500MB is recommended. You
+can set the cache size (in MB) by setting the environment variable
+`AFL_TESTCACHE_SIZE`.
+
+There should be one main fuzzer (`-M main-$HOSTNAME` option) and as many
+secondary fuzzers (e.g., `-S variant1`) as you have cores that you use. Every
+`-M`/`-S` entry needs a unique name (that can be whatever), however, the same
+`-o` output directory location has to be used for all instances.
+
+For every secondary fuzzer there should be a variation, e.g.:
+* one should fuzz the target that was compiled differently: with sanitizers
+  activated (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export
+  AFL_USE_CFISAN=1`)
+* one or two should fuzz the target with CMPLOG/redqueen (see above), at least
+  one cmplog instance should follow transformations (`-l AT`)
+* one to three fuzzers should fuzz a target compiled with laf-intel/COMPCOV (see
+  above). Important note: If you run more than one laf-intel/COMPCOV fuzzer and
+  you want them to share their intermediate results, the main fuzzer (`-M`) must
+  be one of them! (Although this is not really recommended.)
+
+All other secondaries should be used like this:
+* a quarter to a third with the MOpt mutator enabled: `-L 0`
+* run with a different power schedule, recommended are: `fast` (default),
+  `explore`, `coe`, `lin`, `quad`, `exploit`, and `rare` which you can set with
+  the `-p` option, e.g., `-p explore`. See the
+  [FAQ](FAQ.md#what-are-power-schedules) for details.
+* a few instances should use the old queue cycling with `-Z`
+
+Also, it is recommended to set `export AFL_IMPORT_FIRST=1` to load test cases
+from other fuzzers in the campaign first.
+
+If you have a large corpus, a corpus from a previous run or are fuzzing in a CI,
+then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`.
+
+You can also use different fuzzers. If you are using AFL spinoffs or AFL
+conforming fuzzers, then just use the same -o directory and give it a unique
+`-S` name. Examples are:
+* [Fuzzolic](https://github.com/season-lab/fuzzolic)
+* [symcc](https://github.com/eurecom-s3/symcc/)
+* [Eclipser](https://github.com/SoftSec-KAIST/Eclipser/)
+* [AFLsmart](https://github.com/aflsmart/aflsmart)
+* [FairFuzz](https://github.com/carolemieux/afl-rb)
+* [Neuzz](https://github.com/Dongdongshe/neuzz)
+* [Angora](https://github.com/AngoraFuzzer/Angora)
+
+A long list can be found at
+[https://github.com/Microsvuln/Awesome-AFL](https://github.com/Microsvuln/Awesome-AFL).
+
+However, you can also sync AFL++ with honggfuzz, libfuzzer with `-entropic=1`,
+etc. Just show the main fuzzer (`-M`) with the `-F` option where the queue/work
+directory of a different fuzzer is, e.g., `-F /src/target/honggfuzz`. Using
+honggfuzz (with `-n 1` or `-n 2`) and libfuzzer in parallel is highly
+recommended!
+
+### d) Using multiple machines for fuzzing
+
+Maybe you have more than one machine you want to fuzz the same target on. Start
+the `afl-fuzz` (and perhaps libfuzzer, honggfuzz, ...) orchestra as you like,
+just ensure that your have one and only one `-M` instance per server, and that
+its name is unique, hence the recommendation for `-M main-$HOSTNAME`.
+
+Now there are three strategies on how you can sync between the servers:
+* never: sounds weird, but this makes every server an island and has the chance
+  that each follow different paths into the target. You can make this even more
+  interesting by even giving different seeds to each server.
+* regularly (~4h): this ensures that all fuzzing campaigns on the servers "see"
+  the same thing. It is like fuzzing on a huge server.
+* in intervals of 1/10th of the overall expected runtime of the fuzzing you
+  sync. This tries a bit to combine both. Have some individuality of the paths
+  each campaign on a server explores, on the other hand if one gets stuck where
+  another found progress this is handed over making it unstuck.
+
+The syncing process itself is very simple. As the `-M main-$HOSTNAME` instance
+syncs to all `-S` secondaries as well as to other fuzzers, you have to copy only
+this directory to the other machines.
+
+Lets say all servers have the `-o out` directory in /target/foo/out, and you
+created a file `servers.txt` which contains the hostnames of all participating
+servers, plus you have an ssh key deployed to all of them, then run:
+
+```bash
+for FROM in `cat servers.txt`; do
+  for TO in `cat servers.txt`; do
+    rsync -rlpogtz --rsh=ssh $FROM:/target/foo/out/main-$FROM $TO:target/foo/out/
+  done
+done
+```
+
+You can run this manually, per cron job - as you need it. There is a more
+complex and configurable script in
+[utils/distributed_fuzzing](../utils/distributed_fuzzing).
+
+### e) The status of the fuzz campaign
+
+AFL++ comes with the `afl-whatsup` script to show the status of the fuzzing
+campaign.
+
+Just supply the directory that afl-fuzz is given with the `-o` option and you
+will see a detailed status of every fuzzer in that campaign plus a summary.
+
+To have only the summary, use the `-s` switch, e.g., `afl-whatsup -s out/`.
+
+If you have multiple servers, then use the command after a sync or you have to
+execute this script per server.
+
+Another tool to inspect the current state and history of a specific instance is
+afl-plot, which generates an index.html file and graphs that show how the
+fuzzing instance is performing. The syntax is `afl-plot instance_dir web_dir`,
+e.g., `afl-plot out/default /srv/www/htdocs/plot`.
+
+### f) Stopping fuzzing, restarting fuzzing, adding new seeds
+
+To stop an afl-fuzz run, press Control-C.
+
+To restart an afl-fuzz run, just reuse the same command line but replace the `-i
+directory` with `-i -` or set `AFL_AUTORESUME=1`.
+
+If you want to add new seeds to a fuzzing campaign, you can run a temporary
+fuzzing instance, e.g., when your main fuzzer is using `-o out` and the new
+seeds are in `newseeds/` directory:
+
+```
+AFL_BENCH_JUST_ONE=1 AFL_FAST_CAL=1 afl-fuzz -i newseeds -o out -S newseeds -- ./target
+```
+
+### g) Checking the coverage of the fuzzing
+
+The `corpus count` value is a bad indicator for checking how good the coverage
+is.
+
+A better indicator - if you use default llvm instrumentation with at least
+version 9 - is to use `afl-showmap` with the collect coverage option `-C` on the
+output directory:
+
+```
+$ afl-showmap -C -i out -o /dev/null -- ./target -params @@
+...
+[*] Using SHARED MEMORY FUZZING feature.
+[*] Target map size: 9960
+[+] Processed 7849 input files.
+[+] Captured 4331 tuples (highest value 255, total values 67130596) in '/dev/nul
+l'.
+[+] A coverage of 4331 edges were achieved out of 9960 existing (43.48%) with 7849 input files.
+```
+
+It is even better to check out the exact lines of code that have been reached -
+and which have not been found so far.
+
+An "easy" helper script for this is
+[https://github.com/vanhauser-thc/afl-cov](https://github.com/vanhauser-thc/afl-cov),
+just follow the README of that separate project.
+
+If you see that an important area or a feature has not been covered so far, then
+try to find an input that is able to reach that and start a new secondary in
+that fuzzing campaign with that seed as input, let it run for a few minutes,
+then terminate it. The main node will pick it up and make it available to the
+other secondary nodes over time. Set `export AFL_NO_AFFINITY=1` or `export
+AFL_TRY_AFFINITY=1` if you have no free core.
+
+Note that in nearly all cases you can never reach full coverage. A lot of
+functionality is usually dependent on exclusive options that would need
+individual fuzzing campaigns each with one of these options set. E.g., if you
+fuzz a library to convert image formats and your target is the png to tiff API,
+then you will not touch any of the other library APIs and features.
+
+### h) How long to fuzz a target?
+
+This is a difficult question. Basically, if no new path is found for a long time
+(e.g., for a day or a week), then you can expect that your fuzzing won't be
+fruitful anymore. However, often this just means that you should switch out
+secondaries for others, e.g., custom mutator modules, sync to very different
+fuzzers, etc.
+
+Keep the queue/ directory (for future fuzzings of the same or similar targets)
+and use them to seed other good fuzzers like libfuzzer with the -entropic switch
+or honggfuzz.
+
+### i) Improve the speed!
+
+* Use [persistent mode](../instrumentation/README.persistent_mode.md) (x2-x20
+  speed increase).
+* If you do not use shmem persistent mode, use `AFL_TMPDIR` to point the input
+  file on a tempfs location, see [env_variables.md](env_variables.md).
+* Linux: Improve kernel performance: modify `/etc/default/grub`, set
+  `GRUB_CMDLINE_LINUX_DEFAULT="ibpb=off ibrs=off kpti=off l1tf=off mds=off
+  mitigations=off no_stf_barrier noibpb noibrs nopcid nopti
+  nospec_store_bypass_disable nospectre_v1 nospectre_v2 pcid=off pti=off
+  spec_store_bypass_disable=off spectre_v2=off stf_barrier=off"`; then
+  `update-grub` and `reboot` (warning: makes the system more insecure) - you can
+  also just run `sudo afl-persistent-config`.
+* Linux: Running on an `ext2` filesystem with `noatime` mount option will be a
+  bit faster than on any other journaling filesystem.
+* Use your cores! See [3c) Using multiple cores](#c-using-multiple-cores).
+* Run `sudo afl-system-config` before starting the first afl-fuzz instance after
+  a reboot.
+
+### j) Going beyond crashes
+
+Fuzzing is a wonderful and underutilized technique for discovering non-crashing
+design and implementation errors, too. Quite a few interesting bugs have been
+found by modifying the target programs to call `abort()` when say:
+
+- Two bignum libraries produce different outputs when given the same
+  fuzzer-generated input.
+
+- An image library produces different outputs when asked to decode the same
+  input image several times in a row.
+
+- A serialization/deserialization library fails to produce stable outputs when
+  iteratively serializing and deserializing fuzzer-supplied data.
+
+- A compression library produces an output inconsistent with the input file when
+  asked to compress and then decompress a particular blob.
+
+Implementing these or similar sanity checks usually takes very little time; if
+you are the maintainer of a particular package, you can make this code
+conditional with `#ifdef FUZZING_BUILD_MODE_UNSAFE_FOR_PRODUCTION` (a flag also
+shared with libfuzzer and honggfuzz) or `#ifdef __AFL_COMPILER` (this one is
+just for AFL++).
+
+### k) Known limitations & areas for improvement
+
+Here are some of the most important caveats for AFL++:
+
+- AFL++ detects faults by checking for the first spawned process dying due to a
+  signal (SIGSEGV, SIGABRT, etc.). Programs that install custom handlers for
+  these signals may need to have the relevant code commented out. In the same
+  vein, faults in child processes spawned by the fuzzed target may evade
+  detection unless you manually add some code to catch that.
+
+- As with any other brute-force tool, the fuzzer offers limited coverage if
+  encryption, checksums, cryptographic signatures, or compression are used to
+  wholly wrap the actual data format to be tested.
+
+  To work around this, you can comment out the relevant checks (see
+  utils/libpng_no_checksum/ for inspiration); if this is not possible, you can
+  also write a postprocessor, one of the hooks of custom mutators. See
+  [custom_mutators.md](custom_mutators.md) on how to use
+  `AFL_CUSTOM_MUTATOR_LIBRARY`.
+
+- There are some unfortunate trade-offs with ASAN and 64-bit binaries. This
+  isn't due to any specific fault of afl-fuzz.
+
+- There is no direct support for fuzzing network services, background daemons,
+  or interactive apps that require UI interaction to work. You may need to make
+  simple code changes to make them behave in a more traditional way. Preeny may
+  offer a relatively simple option, too - see:
+  [https://github.com/zardus/preeny](https://github.com/zardus/preeny)
+
+  Some useful tips for modifying network-based services can be also found at:
+  [https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop](https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop)
+
+- Occasionally, sentient machines rise against their creators. If this happens
+  to you, please consult
+  [https://lcamtuf.coredump.cx/prep/](https://lcamtuf.coredump.cx/prep/).
+
+Beyond this, see [INSTALL.md](INSTALL.md) for platform-specific tips.
+
+## 4. Triaging crashes
+
+The coverage-based grouping of crashes usually produces a small data set that
+can be quickly triaged manually or with a very simple GDB or Valgrind script.
+Every crash is also traceable to its parent non-crashing test case in the queue,
+making it easier to diagnose faults.
+
+Having said that, it's important to acknowledge that some fuzzing crashes can be
+difficult to quickly evaluate for exploitability without a lot of debugging and
+code analysis work. To assist with this task, afl-fuzz supports a very unique
+"crash exploration" mode enabled with the `-C` flag.
+
+In this mode, the fuzzer takes one or more crashing test cases as the input and
+uses its feedback-driven fuzzing strategies to very quickly enumerate all code
+paths that can be reached in the program while keeping it in the crashing state.
+
+Mutations that do not result in a crash are rejected; so are any changes that do
+not affect the execution path.
+
+The output is a small corpus of files that can be very rapidly examined to see
+what degree of control the attacker has over the faulting address, or whether it
+is possible to get past an initial out-of-bounds read - and see what lies
+beneath.
+
+Oh, one more thing: for test case minimization, give afl-tmin a try. The tool
+can be operated in a very simple way:
+
+```shell
+./afl-tmin -i test_case -o minimized_result -- /path/to/program [...]
+```
+
+The tool works with crashing and non-crashing test cases alike. In the crash
+mode, it will happily accept instrumented and non-instrumented binaries. In the
+non-crashing mode, the minimizer relies on standard AFL++ instrumentation to
+make the file simpler without altering the execution path.
+
+The minimizer accepts the `-m`, `-t`, `-f`, and `@@` syntax in a manner
+compatible with afl-fuzz.
+
+Another tool in AFL++ is the afl-analyze tool. It takes an input file, attempts
+to sequentially flip bytes and observes the behavior of the tested program. It
+then color-codes the input based on which sections appear to be critical and
+which are not; while not bulletproof, it can often offer quick insights into
+complex file formats.
+
+## 5. CI fuzzing
+
+Some notes on continuous integration (CI) fuzzing - this fuzzing is different to
+normal fuzzing campaigns as these are much shorter runnings.
+
+1. Always:
+    * LTO has a much longer compile time which is diametrical to short fuzzing -
+      hence use afl-clang-fast instead.
+    * If you compile with CMPLOG, then you can save compilation time and reuse
+      that compiled target with the `-c` option and as the main fuzz target.
+      This will impact the speed by ~15% though.
+    * `AFL_FAST_CAL` - enables fast calibration, this halves the time the
+      saturated corpus needs to be loaded.
+    * `AFL_CMPLOG_ONLY_NEW` - only perform cmplog on new finds, not the initial
+      corpus as this very likely has been done for them already.
+    * Keep the generated corpus, use afl-cmin and reuse it every time!
+
+2. Additionally randomize the AFL++ compilation options, e.g.:
+    * 40% for `AFL_LLVM_CMPLOG`
+    * 10% for `AFL_LLVM_LAF_ALL`
+
+3. Also randomize the afl-fuzz runtime options, e.g.:
+    * 65% for `AFL_DISABLE_TRIM`
+    * 50% use a dictionary generated by `AFL_LLVM_DICT2FILE`
+    * 40% use MOpt (`-L 0`)
+    * 40% for `AFL_EXPAND_HAVOC_NOW`
+    * 20% for old queue processing (`-Z`)
+    * for CMPLOG targets, 60% for `-l 2`, 40% for `-l 3`
+
+4. Do *not* run any `-M` modes, just running `-S` modes is better for CI
+   fuzzing. `-M` enables old queue handling etc. which is good for a fuzzing
+   campaign but not good for short CI runs.
+
+How this can look like can, e.g., be seen at AFL++'s setup in Google's
+[oss-fuzz](https://github.com/google/oss-fuzz/blob/master/infra/base-images/base-builder/compile_afl)
+and
+[clusterfuzz](https://github.com/google/clusterfuzz/blob/master/src/clusterfuzz/_internal/bot/fuzzers/afl/launcher.py).
+
+## The End
+
+Check out the [FAQ](FAQ.md). Maybe it answers your question (that you might not
+even have known you had ;-) ).
+
+This is basically all you need to know to professionally run fuzzing campaigns.
+If you want to know more, the tons of texts in [docs/](./) will have you
+covered.
+
+Note that there are also a lot of tools out there that help fuzzing with AFL++
+(some might be deprecated or unsupported), see
+[third_party_tools.md](third_party_tools.md).

docs/ideas.md

@@ -1,58 +1,57 @@
-# Ideas for afl++
+# Ideas for AFL++
 
-In the following, we describe a variety of ideas that could be implemented
-for future AFL++ versions.
+In the following, we describe a variety of ideas that could be implemented for
+future AFL++ versions.
 
 ## Analysis software
 
-Currently analysis is done by using afl-plot, which is rather outdated.
-A GTK or browser tool to create run-time analysis based on fuzzer_stats,
-queue/id* information and plot_data that allows for zooming in and out,
-changing min/max display values etc. and doing that for a single run,
-different runs and campaigns vs campaigns.
-Interesting values are execs, and execs/s, edges discovered (total, when
-each edge was discovered and which other fuzzer share finding that edge),
-test cases executed.
-It should be clickable which value is X and Y axis, zoom factor, log scaling
-on-off, etc.
+Currently analysis is done by using afl-plot, which is rather outdated. A GTK or
+browser tool to create run-time analysis based on fuzzer_stats, queue/id*
+information and plot_data that allows for zooming in and out, changing min/max
+display values etc. and doing that for a single run, different runs and
+campaigns vs. campaigns. Interesting values are execs, and execs/s, edges
+discovered (total, when each edge was discovered and which other fuzzer share
+finding that edge), test cases executed. It should be clickable which value is X
+and Y axis, zoom factor, log scaling on-off, etc.
 
 Mentor: vanhauser-thc
 
 ## WASM Instrumentation
 
 Currently, AFL++ can be used for source code fuzzing and traditional binaries.
-With the rise of WASM as compile target, however, a novel way of
-instrumentation needs to be implemented for binaries compiled to Webassembly.
-This can either be done by inserting instrumentation directly into the
-WASM AST, or by patching feedback into a WASM VMs of choice, similar to
-the current Unicorn instrumentation.
+With the rise of WASM as a compile target, however, a novel way of instrumentation
+needs to be implemented for binaries compiled to Webassembly. This can either be
+done by inserting instrumentation directly into the WASM AST, or by patching
+feedback into a WASM VM of choice, similar to the current Unicorn
+instrumentation.
 
 Mentor: any
 
 ## Support other programming languages
 
-Other programming languages also use llvm hence they could (easily?) supported
-for fuzzing, e.g. mono, swift, go, kotlin native, fortran, ...
+Other programming languages also use llvm hence they could be (easily?) supported
+for fuzzing, e.g., mono, swift, go, kotlin native, fortran, ...
 
-GCC also supports: Objective-C, Fortran, Ada, Go, and D
-(according to [Gcc homepage](https://gcc.gnu.org/))
+GCC also supports: Objective-C, Fortran, Ada, Go, and D (according to
+[Gcc homepage](https://gcc.gnu.org/))
 
-LLVM is also used by: Rust, LLGo (Go), kaleidoscope (Haskell), flang (Fortran), emscripten (JavaScript, WASM), ilwasm (CIL (C#))
-(according to [LLVM frontends](https://gist.github.com/axic/62d66fb9d8bccca6cc48fa9841db9241))
+LLVM is also used by: Rust, LLGo (Go), kaleidoscope (Haskell), flang (Fortran),
+emscripten (JavaScript, WASM), ilwasm (CIL (C#)) (according to
+[LLVM frontends](https://gist.github.com/axic/62d66fb9d8bccca6cc48fa9841db9241))
 
 Mentor: vanhauser-thc
 
 ## Machine Learning
 
-Something with machine learning, better than [NEUZZ](https://github.com/dongdongshe/neuzz) :-)
-Either improve a single mutator thorugh learning of many different bugs
-(a bug class) or gather deep insights about a single target beforehand
-(CFG, DFG, VFG, ...?) and improve performance for a single target.
+Something with machine learning, better than
+[NEUZZ](https://github.com/dongdongshe/neuzz) :-) Either improve a single
+mutator through learning of many different bugs (a bug class) or gather deep
+insights about a single target beforehand (CFG, DFG, VFG, ...?) and improve
+performance for a single target.
 
 Mentor: domenukk
 
 ## Your idea!
 
-Finally, we are open to proposals!
-Create an issue at https://github.com/AFLplusplus/AFLplusplus/issues and let's discuss :-)
-
+Finally, we are open to proposals! Create an issue at
+https://github.com/AFLplusplus/AFLplusplus/issues and let's discuss :-)

docs/important_changes.md

@@ -0,0 +1,60 @@
+# Important changes in AFL++
+
+This document lists important changes in AFL++, for example, major behavior
+changes.
+
+## From version 3.00 onwards
+
+With AFL++ 4.00, we introduced the following changes from previous behaviors:
+  * the complete documentation was overhauled and restructured thanks to @llzmb!
+  * a new CMPLOG target format requires recompiling CMPLOG targets for use with
+    AFL++ 4.0 onwards
+  * better naming for several fields in the UI
+
+With AFL++ 3.15, we introduced the following changes from previous behaviors:
+  * afl-cmin and afl-showmap `-Ci` now descend into subdirectories like afl-fuzz
+    `-i` does (but note that afl-cmin.bash does not)
+
+With AFL++ 3.14, we introduced the following changes from previous behaviors:
+  * afl-fuzz: deterministic fuzzing is not a default for `-M main` anymore
+  * afl-cmin/afl-showmap -i now descends into subdirectories (afl-cmin.bash,
+    however, does not)
+
+With AFL++ 3.10, we introduced the following changes from previous behaviors:
+  * The '+' feature of the `-t` option now means to auto-calculate the timeout
+    with the value given being the maximum timeout. The original meaning of
+    "skipping timeouts instead of abort" is now inherent to the `-t` option.
+
+With AFL++ 3.00, we introduced changes that break some previous AFL and AFL++
+behaviors and defaults:
+  * There are no llvm_mode and gcc_plugin subdirectories anymore and there is
+    only one compiler: afl-cc. All previous compilers now symlink to this one.
+    All instrumentation source code is now in the `instrumentation/` folder.
+  * The gcc_plugin was replaced with a new version submitted by AdaCore that
+    supports more features. Thank you!
+  * QEMU mode got upgraded to QEMU 5.1, but to be able to build this a current
+    ninja build tool version and python3 setuptools are required. QEMU mode also
+    got new options like snapshotting, instrumenting specific shared libraries,
+    etc. Additionally QEMU 5.1 supports more CPU targets so this is really worth
+    it.
+  * When instrumenting targets, afl-cc will not supersede optimizations anymore
+    if any were given. This allows to fuzz targets build regularly like those
+    for debug or release versions.
+  * afl-fuzz:
+    * if neither `-M` or `-S` is specified, `-S default` is assumed, so more
+      fuzzers can easily be added later
+    * `-i` input directory option now descends into subdirectories. It also does
+      not fail on crashes and too large files, instead it skips them and uses
+      them for splicing mutations
+    * `-m` none is now the default, set memory limits (in MB) with, e.g., `-m
+      250`
+    * deterministic fuzzing is now disabled by default (unless using `-M`) and
+      can be enabled with `-D`
+    * a caching of test cases can now be performed and can be modified by
+      editing config.h for `TESTCASE_CACHE` or by specifying the environment
+      variable `AFL_TESTCACHE_SIZE` (in MB). Good values are between 50-500
+      (default: 50).
+    * `-M` mains do not perform trimming
+  * `examples/` got renamed to `utils/`
+  * `libtokencap/`, `libdislocator/`, and `qdbi_mode/` were moved to `utils/`
+  * afl-cmin/afl-cmin.bash now search first in `PATH` and last in `AFL_PATH`

docs/life_pro_tips.md

@@ -1,87 +0,0 @@
-# AFL "Life Pro Tips"
-
-Bite-sized advice for those who understand the basics, but can't be bothered
-to read or memorize every other piece of documentation for AFL.
-
-## Get more bang for your buck by using fuzzing dictionaries.
-
-See [dictionaries/README.md](../dictionaries/README.md) to learn how.
-
-## You can get the most out of your hardware by parallelizing AFL jobs.
-
-See [parallel_fuzzing.md](parallel_fuzzing.md) for step-by-step tips.
-
-## Improve the odds of spotting memory corruption bugs with libdislocator.so!
-
-It's easy. Consult [utils/libdislocator/README.md](../utils/libdislocator/README.md) for usage tips.
-
-## Want to understand how your target parses a particular input file?
-
-Try the bundled `afl-analyze` tool; it's got colors and all!
-
-## You can visually monitor the progress of your fuzzing jobs.
-
-Run the bundled `afl-plot` utility to generate browser-friendly graphs.
-
-## Need to monitor AFL jobs programmatically? 
-Check out the `fuzzer_stats` file in the AFL output dir or try `afl-whatsup`.
-
-## Puzzled by something showing up in red or purple in the AFL UI?
-It could be important - consult docs/status_screen.md right away!
-
-## Know your target? Convert it to persistent mode for a huge performance gain!
-Consult section #5 in README.llvm.md for tips.
-
-## Using clang? 
-Check out instrumentation/ for a faster alternative to afl-gcc!
-
-## Did you know that AFL can fuzz closed-source or cross-platform binaries?
-Check out qemu_mode/README.md and unicorn_mode/README.md for more.
-
-## Did you know that afl-fuzz can minimize any test case for you?
-Try the bundled `afl-tmin` tool - and get small repro files fast!
-
-## Not sure if a crash is exploitable? AFL can help you figure it out. Specify
-`-C` to enable the peruvian were-rabbit mode.
-
-## Trouble dealing with a machine uprising? Relax, we've all been there.
-
-Find essential survival tips at http://lcamtuf.coredump.cx/prep/.
-
-## Want to automatically spot non-crashing memory handling bugs?
-
-Try running an AFL-generated corpus through ASAN, MSAN, or Valgrind.
-
-## Good selection of input files is critical to a successful fuzzing job.
-
-See docs/perf_tips.md for pro tips.
-
-## You can improve the odds of automatically spotting stack corruption issues.
-
-Specify `AFL_HARDEN=1` in the environment to enable hardening flags.
-
-## Bumping into problems with non-reproducible crashes? 
-It happens, but usually
-isn't hard to diagnose. See section #7 in README.md for tips.
-
-## Fuzzing is not just about memory corruption issues in the codebase. 
-Add some
-sanity-checking `assert()` / `abort()` statements to effortlessly catch logic bugs.
-
-## Hey kid... pssst... want to figure out how AFL really works?
-
-Check out docs/technical_details.md for all the gory details in one place!
-
-## There's a ton of third-party helper tools designed to work with AFL!
-
-Be sure to check out docs/sister_projects.md before writing your own.
-
-## Need to fuzz the command-line arguments of a particular program?
-
-You can find a simple solution in utils/argv_fuzzing.
-
-## Attacking a format that uses checksums? 
-
-Remove the checksum-checking code or use a postprocessor!
-See `afl_custom_post_process` in custom_mutators/examples/example.c for more.
-

docs/parallel_fuzzing.md

@@ -1,259 +0,0 @@
-# Tips for parallel fuzzing
-
-This document talks about synchronizing afl-fuzz jobs on a single machine
-or across a fleet of systems. See README.md for the general instruction manual.
-
-Note that this document is rather outdated. please refer to the main document
-section on multiple core usage [../README.md#Using multiple cores](../README.md#b-using-multiple-coresthreads)
-for up to date strategies!
-
-## 1) Introduction
-
-Every copy of afl-fuzz will take up one CPU core. This means that on an
-n-core system, you can almost always run around n concurrent fuzzing jobs with
-virtually no performance hit (you can use the afl-gotcpu tool to make sure).
-
-In fact, if you rely on just a single job on a multi-core system, you will
-be underutilizing the hardware. So, parallelization is always the right way to
-go.
-
-When targeting multiple unrelated binaries or using the tool in
-"non-instrumented" (-n) mode, it is perfectly fine to just start up several
-fully separate instances of afl-fuzz. The picture gets more complicated when
-you want to have multiple fuzzers hammering a common target: if a hard-to-hit
-but interesting test case is synthesized by one fuzzer, the remaining instances
-will not be able to use that input to guide their work.
-
-To help with this problem, afl-fuzz offers a simple way to synchronize test
-cases on the fly.
-
-Note that afl++ has AFLfast's power schedules implemented.
-It is therefore a good idea to use different power schedules if you run
-several instances in parallel. See [power_schedules.md](power_schedules.md)
-
-Alternatively running other AFL spinoffs in parallel can be of value,
-e.g. Angora (https://github.com/AngoraFuzzer/Angora/)
-
-## 2) Single-system parallelization
-
-If you wish to parallelize a single job across multiple cores on a local
-system, simply create a new, empty output directory ("sync dir") that will be
-shared by all the instances of afl-fuzz; and then come up with a naming scheme
-for every instance - say, "fuzzer01", "fuzzer02", etc. 
-
-Run the first one ("main node", -M) like this:
-
-```
-./afl-fuzz -i testcase_dir -o sync_dir -M fuzzer01 [...other stuff...]
-```
-
-...and then, start up secondary (-S) instances like this:
-
-```
-./afl-fuzz -i testcase_dir -o sync_dir -S fuzzer02 [...other stuff...]
-./afl-fuzz -i testcase_dir -o sync_dir -S fuzzer03 [...other stuff...]
-```
-
-Each fuzzer will keep its state in a separate subdirectory, like so:
-
-  /path/to/sync_dir/fuzzer01/
-
-Each instance will also periodically rescan the top-level sync directory
-for any test cases found by other fuzzers - and will incorporate them into
-its own fuzzing when they are deemed interesting enough.
-For performance reasons only -M main node syncs the queue with everyone, the
--S secondary nodes will only sync from the main node.
-
-The difference between the -M and -S modes is that the main instance will
-still perform deterministic checks; while the secondary instances will
-proceed straight to random tweaks.
-
-Note that you must always have one -M main instance!
-Running multiple -M instances is wasteful!
-
-You can also monitor the progress of your jobs from the command line with the
-provided afl-whatsup tool. When the instances are no longer finding new paths,
-it's probably time to stop.
-
-WARNING: Exercise caution when explicitly specifying the -f option. Each fuzzer
-must use a separate temporary file; otherwise, things will go south. One safe
-example may be:
-
-```
-./afl-fuzz [...] -S fuzzer10 -f file10.txt ./fuzzed/binary @@
-./afl-fuzz [...] -S fuzzer11 -f file11.txt ./fuzzed/binary @@
-./afl-fuzz [...] -S fuzzer12 -f file12.txt ./fuzzed/binary @@
-```
-
-This is not a concern if you use @@ without -f and let afl-fuzz come up with the
-file name.
-
-## 3) Multiple -M mains
-
-
-There is support for parallelizing the deterministic checks.
-This is only needed where
- 
- 1. many new paths are found fast over a long time and it looks unlikely that
-    main node will ever catch up, and
- 2. deterministic fuzzing is actively helping path discovery (you can see this
-    in the main node for the first for lines in the "fuzzing strategy yields"
-    section. If the ration `found/attemps` is high, then it is effective. It
-    most commonly isn't.)
-
-Only if both are true it is beneficial to have more than one main.
-You can leverage this by creating -M instances like so:
-
-```
-./afl-fuzz -i testcase_dir -o sync_dir -M mainA:1/3 [...]
-./afl-fuzz -i testcase_dir -o sync_dir -M mainB:2/3 [...]
-./afl-fuzz -i testcase_dir -o sync_dir -M mainC:3/3 [...]
-```
-
-... where the first value after ':' is the sequential ID of a particular main
-instance (starting at 1), and the second value is the total number of fuzzers to
-distribute the deterministic fuzzing across. Note that if you boot up fewer
-fuzzers than indicated by the second number passed to -M, you may end up with
-poor coverage.
-
-## 4) Syncing with non-afl fuzzers or independant instances
-
-A -M main node can be told with the `-F other_fuzzer_queue_directory` option
-to sync results from other fuzzers, e.g. libfuzzer or honggfuzz.
-
-Only the specified directory will by synced into afl, not subdirectories.
-The specified directory does not need to exist yet at the start of afl.
-
-The `-F` option can be passed to the main node several times.
-
-## 5) Multi-system parallelization
-
-The basic operating principle for multi-system parallelization is similar to
-the mechanism explained in section 2. The key difference is that you need to
-write a simple script that performs two actions:
-
-  - Uses SSH with authorized_keys to connect to every machine and retrieve
-    a tar archive of the /path/to/sync_dir/<main_node(s)> directory local to
-    the machine.
-    It is best to use a naming scheme that includes host name and it's being
-    a main node (e.g. main1, main2) in the fuzzer ID, so that you can do
-    something like:
-
-    ```sh
-    for host in `cat HOSTLIST`; do
-      ssh user@$host "tar -czf - sync/$host_main*/" > $host.tgz
-    done
-    ```
-
-  - Distributes and unpacks these files on all the remaining machines, e.g.:
-
-    ```sh
-    for srchost in `cat HOSTLIST`; do
-      for dsthost in `cat HOSTLIST`; do
-        test "$srchost" = "$dsthost" && continue
-        ssh user@$srchost 'tar -kxzf -' < $dsthost.tgz
-      done
-    done
-    ```
-
-There is an example of such a script in utils/distributed_fuzzing/.
-
-There are other (older) more featured, experimental tools:
-  * https://github.com/richo/roving
-  * https://github.com/MartijnB/disfuzz-afl
-
-However these do not support syncing just main nodes (yet).
-
-When developing custom test case sync code, there are several optimizations
-to keep in mind:
-
-  - The synchronization does not have to happen very often; running the
-    task every 60 minutes or even less often at later fuzzing stages is
-    fine
-
-  - There is no need to synchronize crashes/ or hangs/; you only need to
-    copy over queue/* (and ideally, also fuzzer_stats).
-
-  - It is not necessary (and not advisable!) to overwrite existing files;
-    the -k option in tar is a good way to avoid that.
-
-  - There is no need to fetch directories for fuzzers that are not running
-    locally on a particular machine, and were simply copied over onto that
-    system during earlier runs.
-
-  - For large fleets, you will want to consolidate tarballs for each host,
-    as this will let you use n SSH connections for sync, rather than n*(n-1).
-
-    You may also want to implement staged synchronization. For example, you
-    could have 10 groups of systems, with group 1 pushing test cases only
-    to group 2; group 2 pushing them only to group 3; and so on, with group
-    eventually 10 feeding back to group 1.
-
-    This arrangement would allow test interesting cases to propagate across
-    the fleet without having to copy every fuzzer queue to every single host.
-
-  - You do not want a "main" instance of afl-fuzz on every system; you should
-    run them all with -S, and just designate a single process somewhere within
-    the fleet to run with -M.
-    
-  - Syncing is only necessary for the main nodes on a system. It is possible
-    to run main-less with only secondaries. However then you need to find out
-    which secondary took over the temporary role to be the main node. Look for
-    the `is_main_node` file in the fuzzer directories, eg. `sync-dir/hostname-*/is_main_node`
-
-It is *not* advisable to skip the synchronization script and run the fuzzers
-directly on a network filesystem; unexpected latency and unkillable processes
-in I/O wait state can mess things up.
-
-## 6) Remote monitoring and data collection
-
-You can use screen, nohup, tmux, or something equivalent to run remote
-instances of afl-fuzz. If you redirect the program's output to a file, it will
-automatically switch from a fancy UI to more limited status reports. There is
-also basic machine-readable information which is always written to the
-fuzzer_stats file in the output directory. Locally, that information can be
-interpreted with afl-whatsup.
-
-In principle, you can use the status screen of the main (-M) instance to
-monitor the overall fuzzing progress and decide when to stop. In this
-mode, the most important signal is just that no new paths are being found
-for a longer while. If you do not have a main instance, just pick any
-single secondary instance to watch and go by that.
-
-You can also rely on that instance's output directory to collect the
-synthesized corpus that covers all the noteworthy paths discovered anywhere
-within the fleet. Secondary (-S) instances do not require any special
-monitoring, other than just making sure that they are up.
-
-Keep in mind that crashing inputs are *not* automatically propagated to the
-main instance, so you may still want to monitor for crashes fleet-wide
-from within your synchronization or health checking scripts (see afl-whatsup).
-
-## 7) Asymmetric setups
-
-It is perhaps worth noting that all of the following is permitted:
-
-  - Running afl-fuzz with conjunction with other guided tools that can extend
-    coverage (e.g., via concolic execution). Third-party tools simply need to
-    follow the protocol described above for pulling new test cases from
-    out_dir/<fuzzer_id>/queue/* and writing their own finds to sequentially
-    numbered id:nnnnnn files in out_dir/<ext_tool_id>/queue/*.
-
-  - Running some of the synchronized fuzzers with different (but related)
-    target binaries. For example, simultaneously stress-testing several
-    different JPEG parsers (say, IJG jpeg and libjpeg-turbo) while sharing
-    the discovered test cases can have synergistic effects and improve the
-    overall coverage.
-
-    (In this case, running one -M instance per target is necessary.)
-
-  - Having some of the fuzzers invoke the binary in different ways.
-    For example, 'djpeg' supports several DCT modes, configurable with
-    a command-line flag, while 'dwebp' supports incremental and one-shot
-    decoding. In some scenarios, going after multiple distinct modes and then
-    pooling test cases will improve coverage.
-
-  - Much less convincingly, running the synchronized fuzzers with different
-    starting test cases (e.g., progressive and standard JPEG) or dictionaries.
-    The synchronization mechanism ensures that the test sets will get fairly
-    homogeneous over time, but it introduces some initial variability.

docs/perf_tips.md

@@ -1,208 +0,0 @@
-## Tips for performance optimization
-
-  This file provides tips for troubleshooting slow or wasteful fuzzing jobs.
-  See README.md for the general instruction manual.
-
-## 1. Keep your test cases small
-
-This is probably the single most important step to take! Large test cases do
-not merely take more time and memory to be parsed by the tested binary, but
-also make the fuzzing process dramatically less efficient in several other
-ways.
-
-To illustrate, let's say that you're randomly flipping bits in a file, one bit
-at a time. Let's assume that if you flip bit #47, you will hit a security bug;
-flipping any other bit just results in an invalid document.
-
-Now, if your starting test case is 100 bytes long, you will have a 71% chance of
-triggering the bug within the first 1,000 execs - not bad! But if the test case
-is 1 kB long, the probability that we will randomly hit the right pattern in
-the same timeframe goes down to 11%. And if it has 10 kB of non-essential
-cruft, the odds plunge to 1%.
-
-On top of that, with larger inputs, the binary may be now running 5-10x times
-slower than before - so the overall drop in fuzzing efficiency may be easily
-as high as 500x or so.
-
-In practice, this means that you shouldn't fuzz image parsers with your
-vacation photos. Generate a tiny 16x16 picture instead, and run it through
-`jpegtran` or `pngcrunch` for good measure. The same goes for most other types
-of documents.
-
-There's plenty of small starting test cases in ../testcases/ - try them out
-or submit new ones!
-
-If you want to start with a larger, third-party corpus, run `afl-cmin` with an
-aggressive timeout on that data set first.
-
-## 2. Use a simpler target
-
-Consider using a simpler target binary in your fuzzing work. For example, for
-image formats, bundled utilities such as `djpeg`, `readpng`, or `gifhisto` are
-considerably (10-20x) faster than the convert tool from ImageMagick - all while exercising roughly the same library-level image parsing code.
-
-Even if you don't have a lightweight harness for a particular target, remember
-that you can always use another, related library to generate a corpus that will
-be then manually fed to a more resource-hungry program later on.
-
-Also note that reading the fuzzing input via stdin is faster than reading from
-a file.
-
-## 3. Use LLVM persistent instrumentation
-
-The LLVM mode offers a "persistent", in-process fuzzing mode that can
-work well for certain types of self-contained libraries, and for fast targets,
-can offer performance gains up to 5-10x; and a "deferred fork server" mode
-that can offer huge benefits for programs with high startup overhead. Both
-modes require you to edit the source code of the fuzzed program, but the
-changes often amount to just strategically placing a single line or two.
-
-If there are important data comparisons performed (e.g. `strcmp(ptr, MAGIC_HDR)`)
-then using laf-intel (see instrumentation/README.laf-intel.md) will help `afl-fuzz` a lot
-to get to the important parts in the code.
-
-If you are only interested in specific parts of the code being fuzzed, you can
-instrument_files the files that are actually relevant. This improves the speed and
-accuracy of afl. See instrumentation/README.instrument_list.md
-
-## 4. Profile and optimize the binary
-
-Check for any parameters or settings that obviously improve performance. For
-example, the djpeg utility that comes with IJG jpeg and libjpeg-turbo can be
-called with:
-
-```bash
-  -dct fast -nosmooth -onepass -dither none -scale 1/4
-```
-
-...and that will speed things up. There is a corresponding drop in the quality
-of decoded images, but it's probably not something you care about.
-
-In some programs, it is possible to disable output altogether, or at least use
-an output format that is computationally inexpensive. For example, with image
-transcoding tools, converting to a BMP file will be a lot faster than to PNG.
-
-With some laid-back parsers, enabling "strict" mode (i.e., bailing out after
-first error) may result in smaller files and improved run time without
-sacrificing coverage; for example, for sqlite, you may want to specify -bail.
-
-If the program is still too slow, you can use `strace -tt` or an equivalent
-profiling tool to see if the targeted binary is doing anything silly.
-Sometimes, you can speed things up simply by specifying `/dev/null` as the
-config file, or disabling some compile-time features that aren't really needed
-for the job (try `./configure --help`). One of the notoriously resource-consuming
-things would be calling other utilities via `exec*()`, `popen()`, `system()`, or
-equivalent calls; for example, tar can invoke external decompression tools
-when it decides that the input file is a compressed archive.
-
-Some programs may also intentionally call `sleep()`, `usleep()`, or `nanosleep()`;
-vim is a good example of that. Other programs may attempt `fsync()` and so on.
-There are third-party libraries that make it easy to get rid of such code,
-e.g.:
-
-  https://launchpad.net/libeatmydata
-
-In programs that are slow due to unavoidable initialization overhead, you may
-want to try the LLVM deferred forkserver mode (see README.llvm.md),
-which can give you speed gains up to 10x, as mentioned above.
-
-Last but not least, if you are using ASAN and the performance is unacceptable,
-consider turning it off for now, and manually examining the generated corpus
-with an ASAN-enabled binary later on.
-
-## 5. Instrument just what you need
-
-Instrument just the libraries you actually want to stress-test right now, one
-at a time. Let the program use system-wide, non-instrumented libraries for
-any functionality you don't actually want to fuzz. For example, in most
-cases, it doesn't make to instrument `libgmp` just because you're testing a
-crypto app that relies on it for bignum math.
-
-Beware of programs that come with oddball third-party libraries bundled with
-their source code (Spidermonkey is a good example of this). Check `./configure`
-options to use non-instrumented system-wide copies instead.
-
-## 6. Parallelize your fuzzers
-
-The fuzzer is designed to need ~1 core per job. This means that on a, say,
-4-core system, you can easily run four parallel fuzzing jobs with relatively
-little performance hit. For tips on how to do that, see parallel_fuzzing.md.
-
-The `afl-gotcpu` utility can help you understand if you still have idle CPU
-capacity on your system. (It won't tell you about memory bandwidth, cache
-misses, or similar factors, but they are less likely to be a concern.)
-
-## 7. Keep memory use and timeouts in check
-
-Consider setting low values for `-m` and `-t`.
-
-For programs that are nominally very fast, but get sluggish for some inputs,
-you can also try setting `-t` values that are more punishing than what `afl-fuzz`
-dares to use on its own. On fast and idle machines, going down to `-t 5` may be
-a viable plan.
-
-The `-m` parameter is worth looking at, too. Some programs can end up spending
-a fair amount of time allocating and initializing megabytes of memory when
-presented with pathological inputs. Low `-m` values can make them give up sooner
-and not waste CPU time.
-
-## 8. Check OS configuration
-
-There are several OS-level factors that may affect fuzzing speed:
-
-  - If you have no risk of power loss then run your fuzzing on a tmpfs
-    partition. This increases the performance noticably.
-    Alternatively you can use `AFL_TMPDIR` to point to a tmpfs location to
-    just write the input file to a tmpfs.
-  - High system load. Use idle machines where possible. Kill any non-essential
-    CPU hogs (idle browser windows, media players, complex screensavers, etc).
-  - Network filesystems, either used for fuzzer input / output, or accessed by
-    the fuzzed binary to read configuration files (pay special attention to the
-    home directory - many programs search it for dot-files).
-  - Disable all the spectre, meltdown etc. security countermeasures in the
-    kernel if your machine is properly separated:
-
-```
-ibpb=off ibrs=off kpti=off l1tf=off mds=off mitigations=off
-no_stf_barrier noibpb noibrs nopcid nopti nospec_store_bypass_disable
-nospectre_v1 nospectre_v2 pcid=off pti=off spec_store_bypass_disable=off
-spectre_v2=off stf_barrier=off
-```
-    In most Linux distributions you can put this into a `/etc/default/grub`
-    variable.
-
-The following list of changes are made when executing `afl-system-config`:
- 
-  - On-demand CPU scaling. The Linux `ondemand` governor performs its analysis
-    on a particular schedule and is known to underestimate the needs of
-    short-lived processes spawned by `afl-fuzz` (or any other fuzzer). On Linux,
-    this can be fixed with:
-
-``` bash
-    cd /sys/devices/system/cpu
-    echo performance | tee cpu*/cpufreq/scaling_governor
-```
-
-    On other systems, the impact of CPU scaling will be different; when fuzzing,
-    use OS-specific tools to find out if all cores are running at full speed.
-  - Transparent huge pages. Some allocators, such as `jemalloc`, can incur a
-    heavy fuzzing penalty when transparent huge pages (THP) are enabled in the
-    kernel. You can disable this via:
-
-```bash
-    echo never > /sys/kernel/mm/transparent_hugepage/enabled
-```
-
-  - Suboptimal scheduling strategies. The significance of this will vary from
-    one target to another, but on Linux, you may want to make sure that the
-    following options are set:
-
-```bash
-    echo 1 >/proc/sys/kernel/sched_child_runs_first
-    echo 1 >/proc/sys/kernel/sched_autogroup_enabled
-```
-
-    Setting a different scheduling policy for the fuzzer process - say
-    `SCHED_RR` - can usually speed things up, too, but needs to be done with
-    care.
-

docs/resources/grafana-afl++.json

@@ -424,7 +424,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "fuzzing{type=\"unique_crashes\"}",
+          "expr": "fuzzing{type=\"saved_crashes\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -519,7 +519,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "fuzzing{type=\"unique_hangs\"}",
+          "expr": "fuzzing{type=\"saved_hangs\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -926,7 +926,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "fuzzing{type=\"cur_path\"}",
+          "expr": "fuzzing{type=\"cur_item\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -936,7 +936,7 @@
       "timeFrom": null,
       "timeRegions": [],
       "timeShift": null,
-      "title": "Curent path",
+      "title": "Current fuzz item",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1116,7 +1116,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "fuzzing{type=\"paths_favored\"}",
+          "expr": "fuzzing{type=\"corpus_favored\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1135,7 +1135,7 @@
         }
       ],
       "timeShift": null,
-      "title": "Path Favored",
+      "title": "Corpus Favored",
       "tooltip": {
         "shared": true,
         "sort": 0,
@@ -1428,7 +1428,7 @@
       "steppedLine": false,
       "targets": [
         {
-          "expr": "fuzzing{type=\"paths_imported\"}",
+          "expr": "fuzzing{type=\"corpus_imported\"}",
           "interval": "",
           "legendFormat": "",
           "refId": "A"
@@ -1447,7 +1447,7 @@
         }
       ],
       "timeShift": null,
-      "title": "Path Imported",
+      "title": "Corpus Imported",
       "tooltip": {
         "shared": true,
         "sort": 0,

docs/rpc_statsd.md

@@ -1,143 +1,190 @@
-# Remote monitoring with StatsD
+# Remote monitoring and metrics visualization
 
-StatsD allows you to receive and aggregate metrics from a wide range of applications and retransmit them to the backend of your choice.
-This enables you to create nice and readable dashboards containing all the information you need on your fuzzer instances.
-No need to write your own statistics parsing system, deploy and maintain it to all your instances, sync with your graph rendering system...
+AFL++ can send out metrics as StatsD messages. For remote monitoring and
+visualization of the metrics, you can set up a tool chain. For example, with
+Prometheus and Grafana. All tools are free and open source.
 
-The available metrics are :
+This enables you to create nice and readable dashboards containing all the
+information you need on your fuzzer instances. There is no need to write your
+own statistics parsing system, deploy and maintain it to all your instances, and
+sync with your graph rendering system.
+
+Compared to the default integrated UI of AFL++, this can help you to visualize
+trends and the fuzzing state over time. You might be able to see when the
+fuzzing process has reached a state of no progress and visualize what are the
+"best strategies" for your targets (according to your own criteria). You can do
+so without logging into each instance individually.
+
+![example visualization with Grafana](resources/statsd-grafana.png)
+
+This is an example visualization with Grafana. The dashboard can be imported
+with [this JSON template](resources/grafana-afl++.json).
+
+## AFL++ metrics and StatsD
+
+StatsD allows you to receive and aggregate metrics from a wide range of
+applications and retransmit them to a backend of your choice.
+
+From AFL++, StatsD can receive the following metrics:
+- cur_item
 - cycle_done
 - cycles_wo_finds
+- edges_found
 - execs_done
 - execs_per_sec
-- paths_total
-- paths_favored
-- paths_found
-- paths_imported
+- havoc_expansion
 - max_depth
-- cur_path
+- corpus_favored
+- corpus_found
+- corpus_imported
+- corpus_count
 - pending_favs
 - pending_total
-- variable_paths
-- unique_crashes
-- unique_hangs
-- total_crashes
 - slowest_exec_ms
-- edges_found
+- total_crashes
+- saved_crashes
+- saved_hangs
 - var_byte_count
-- havoc_expansion
+- corpus_variable
 
-Compared to the default integrated UI, these metrics give you the opportunity to visualize trends and fuzzing state over time.
-By doing so, you might be able to see when the fuzzing process has reached a state of no progress, visualize what are the "best strategies"
-(according to your own criteria) for your targets, etc. And doing so without requiring to log into each instance manually.
+Depending on your StatsD server, you will be able to monitor, trigger alerts, or
+perform actions based on these metrics (for example: alert on slow exec/s for a
+new build, threshold of crashes, time since last crash > X, and so on).
 
-An example visualisation may look like the following:
-![StatsD Grafana](visualization/statsd-grafana.png)
+## Setting environment variables in AFL++
 
-*Notes: The exact same dashboard can be imported with [this JSON template](statsd/grafana-afl++.json).*
+1. To enable the StatsD metrics collection on your fuzzer instances, set the
+   environment variable `AFL_STATSD=1`. By default, AFL++ will send the metrics
+   over UDP to 127.0.0.1:8125.
 
-## How to use
+2. To enable tags for each metric based on their format (banner and
+   afl_version), set the environment variable `AFL_STATSD_TAGS_FLAVOR`. By
+   default, no tags will be added to the metrics.
 
-To enable the StatsD reporting on your fuzzer instances, you need to set the environment variable `AFL_STATSD=1`.
+    The available values are the following:
+    -  `dogstatsd`
+    -  `influxdb`
+    -  `librato`
+    -  `signalfx`
 
-Setting `AFL_STATSD_TAGS_FLAVOR` to the provider of your choice will assign tags / labels to each metric based on their format.
-The possible values are  `dogstatsd`, `librato`, `signalfx` or `influxdb`.
-For more information on these env vars, check out `docs/env_variables.md`.
+    For more information on environment variables, see
+    [env_variables.md](env_variables.md).
 
-The simplest way of using this feature is to use any metric provider and change the host/port of your StatsD daemon,
-with `AFL_STATSD_HOST` and `AFL_STATSD_PORT`, if required (defaults are `localhost` and port `8125`).
-To get started, here are some instructions with free and open source tools.
-The following setup is based on Prometheus, statsd_exporter and Grafana.
-Grafana here is not mandatory, but gives you some nice graphs and features.
+    Note: When using multiple fuzzer instances with StatsD it is *strongly*
+    recommended to set up `AFL_STATSD_TAGS_FLAVOR` to match your StatsD server.
+    This will allow you to see individual fuzzer performance, detect bad ones,
+    and see the progress of each strategy.
 
-Depending on your setup and infrastructure, you may want to run these applications not on your fuzzer instances.
-Only one instance of these 3 application is required for all your fuzzers.
+3. Optional: To set the host and port of your StatsD daemon, set
+   `AFL_STATSD_HOST` and `AFL_STATSD_PORT`. The default values are `localhost`
+   and `8125`.
 
-To simplify everything, we will use Docker and docker-compose.
-Make sure you have them both installed. On most common Linux distributions, it's as simple as:
+## Installing and setting up StatsD, Prometheus, and Grafana
 
-```sh
-curl -fsSL https://get.docker.com -o get-docker.sh
-sh get-docker.sh
-```
+The easiest way to install and set up the infrastructure is with Docker and
+Docker Compose.
 
-Once that's done, we can create the infrastructure.
-Create and move into the directory of your choice. This will store all the configurations files required.
+Depending on your fuzzing setup and infrastructure, you may not want to run
+these applications on your fuzzer instances. This setup may be modified before
+use in a production environment; for example, adding passwords, creating volumes
+for storage, tweaking the metrics gathering to get host metrics (CPU, RAM, and
+so on).
 
-First, create a `docker-compose.yml` containing the following:
-```yml
-version: '3'
+For all your fuzzing instances, only one instance of Prometheus and Grafana is
+required. The
+[statsd exporter](https://registry.hub.docker.com/r/prom/statsd-exporter)
+converts the StatsD metrics to Prometheus. If you are using a provider that
+supports StatsD directly, you can skip this part of the setup."
 
-networks:
-  statsd-net:
-    driver: bridge
+You can create and move the infrastructure files into a directory of your
+choice. The directory will store all the required configuration files.
+
+To install and set up Prometheus and Grafana:
+
+1. Install Docker and Docker Compose:
+
+    ```sh
+    curl -fsSL https://get.docker.com -o get-docker.sh
+    sh get-docker.sh
+    ```
+
+2. Create a `docker-compose.yml` containing the following:
+
+    ```yml
+    version: '3'
 
-services:
-  prometheus:
-    image: prom/prometheus
-    container_name: prometheus
-    volumes:
-      - ./prometheus.yml:/prometheus.yml
-    command:
-      - '--config.file=/prometheus.yml'
-    restart: unless-stopped
-    ports:
-      - "9090:9090"
     networks:
-      - statsd-net
+      statsd-net:
+        driver: bridge
 
-  statsd_exporter:
-    image: prom/statsd-exporter
-    container_name: statsd_exporter
-    volumes:
-      - ./statsd_mapping.yml:/statsd_mapping.yml
-    command:
-      - "--statsd.mapping-config=/statsd_mapping.yml"
-    ports:
-      - "9102:9102/tcp"
-      - "8125:9125/udp"
-    networks:
-      - statsd-net
-  
-  grafana:
-    image: grafana/grafana
-    container_name: grafana
-    restart: unless-stopped
-    ports:
-        - "3000:3000"
-    networks:
-      - statsd-net
-```
+    services:
+      prometheus:
+        image: prom/prometheus
+        container_name: prometheus
+        volumes:
+          - ./prometheus.yml:/prometheus.yml
+        command:
+          - '--config.file=/prometheus.yml'
+        restart: unless-stopped
+        ports:
+          - "9090:9090"
+        networks:
+          - statsd-net
 
-Then `prometheus.yml`
-```yml
-global:
-  scrape_interval:      15s
-  evaluation_interval:  15s
+      statsd_exporter:
+        image: prom/statsd-exporter
+        container_name: statsd_exporter
+        volumes:
+          - ./statsd_mapping.yml:/statsd_mapping.yml
+        command:
+          - "--statsd.mapping-config=/statsd_mapping.yml"
+        ports:
+          - "9102:9102/tcp"
+          - "8125:9125/udp"
+        networks:
+          - statsd-net
 
-scrape_configs:
-  - job_name: 'fuzzing_metrics'
-    static_configs:
-      - targets: ['statsd_exporter:9102']
-```
+      grafana:
+        image: grafana/grafana
+        container_name: grafana
+        restart: unless-stopped
+        ports:
+            - "3000:3000"
+        networks:
+          - statsd-net
+    ```
 
-And finally `statsd_mapping.yml`
-```yml 
-mappings:
-- match: "fuzzing.*"
-  name: "fuzzing"
-  labels:
-      type: "$1"
-```
+3. Create a `prometheus.yml` containing the following:
 
-Run `docker-compose up -d`.
+    ```yml
+    global:
+      scrape_interval:      15s
+      evaluation_interval:  15s
 
-Everything should now be setup, you are now able to run your fuzzers with
+    scrape_configs:
+      - job_name: 'fuzzing_metrics'
+        static_configs:
+          - targets: ['statsd_exporter:9102']
+    ```
+
+4. Create a `statsd_mapping.yml` containing the following:
+
+    ```yml
+    mappings:
+    - match: "fuzzing.*"
+      name: "fuzzing"
+      labels:
+          type: "$1"
+    ```
+
+5. Run `docker-compose up -d`.
+
+## Running AFL++ with StatsD
+
+To run your fuzzing instances:
 
 ```
-AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -M test-fuzzer-1 -i i -o o ./bin/my-application @@
-AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -S test-fuzzer-2 -i i -o o ./bin/my-application @@
+AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -M test-fuzzer-1 -i i -o o [./bin/my-application] @@
+AFL_STATSD_TAGS_FLAVOR=dogstatsd AFL_STATSD=1 afl-fuzz -S test-fuzzer-2 -i i -o o [./bin/my-application] @@
 ...
-```
-
-This setup may be modified before use in a production environment. Depending on your needs: adding passwords, creating volumes for storage,
-tweaking the metrics gathering to get host metrics (CPU, RAM ...).
+```

docs/sister_projects.md

@@ -1,319 +0,0 @@
-# Sister projects
-
-This doc lists some of the projects that are inspired by, derived from,
-designed for, or meant to integrate with AFL. See README.md for the general
-instruction manual.
-
-!!!
-!!! This list is outdated and needs an update, missing: e.g. Angora, FairFuzz
-!!!
-
-## Support for other languages / environments:
-
-### Python AFL (Jakub Wilk)
-
-Allows fuzz-testing of Python programs. Uses custom instrumentation and its
-own forkserver.
-
-http://jwilk.net/software/python-afl
-
-### Go-fuzz (Dmitry Vyukov)
-
-AFL-inspired guided fuzzing approach for Go targets:
-
-https://github.com/dvyukov/go-fuzz
-
-### afl.rs (Keegan McAllister)
-
-Allows Rust features to be easily fuzzed with AFL (using the LLVM mode).
-
-https://github.com/kmcallister/afl.rs
-
-### OCaml support (KC Sivaramakrishnan)
-
-Adds AFL-compatible instrumentation to OCaml programs.
-
-https://github.com/ocamllabs/opam-repo-dev/pull/23
-http://canopy.mirage.io/Posts/Fuzzing
-
-### AFL for GCJ Java and other GCC frontends (-)
-
-GCC Java programs are actually supported out of the box - simply rename
-afl-gcc to afl-gcj. Unfortunately, by default, unhandled exceptions in GCJ do
-not result in abort() being called, so you will need to manually add a
-top-level exception handler that exits with SIGABRT or something equivalent.
-
-Other GCC-supported languages should be fairly easy to get working, but may
-face similar problems. See https://gcc.gnu.org/frontends.html for a list of
-options.
-
-## AFL-style in-process fuzzer for LLVM (Kostya Serebryany)
-
-Provides an evolutionary instrumentation-guided fuzzing harness that allows
-some programs to be fuzzed without the fork / execve overhead. (Similar
-functionality is now available as the "persistent" feature described in
-[the llvm_mode readme](../instrumentation/README.llvm.md))
-
-http://llvm.org/docs/LibFuzzer.html
-
-## TriforceAFL (Tim Newsham and Jesse Hertz)
-
-Leverages QEMU full system emulation mode to allow AFL to target operating
-systems and other alien worlds:
-
-https://www.nccgroup.trust/us/about-us/newsroom-and-events/blog/2016/june/project-triforce-run-afl-on-everything/
-
-## WinAFL (Ivan Fratric)
-
-As the name implies, allows you to fuzz Windows binaries (using DynamoRio).
-
-https://github.com/ivanfratric/winafl
-
-Another Windows alternative may be:
-
-https://github.com/carlosgprado/BrundleFuzz/
-
-## Network fuzzing
-
-### Preeny (Yan Shoshitaishvili)
-
-Provides a fairly simple way to convince dynamically linked network-centric
-programs to read from a file or not fork. Not AFL-specific, but described as
-useful by many users. Some assembly required.
-
-https://github.com/zardus/preeny
-
-## Distributed fuzzing and related automation
-
-### roving (Richo Healey)
-
-A client-server architecture for effortlessly orchestrating AFL runs across
-a fleet of machines. You don't want to use this on systems that face the
-Internet or live in other untrusted environments.
-
-https://github.com/richo/roving
-
-### Distfuzz-AFL (Martijn Bogaard)
-
-Simplifies the management of afl-fuzz instances on remote machines. The
-author notes that the current implementation isn't secure and should not
-be exposed on the Internet.
-
-https://github.com/MartijnB/disfuzz-afl
-
-### AFLDFF (quantumvm)
-
-A nice GUI for managing AFL jobs.
-
-https://github.com/quantumvm/AFLDFF
-
-### afl-launch (Ben Nagy)
-
-Batch AFL launcher utility with a simple CLI.
-
-https://github.com/bnagy/afl-launch
-
-### AFL Utils (rc0r)
-
-Simplifies the triage of discovered crashes, start parallel instances, etc.
-
-https://github.com/rc0r/afl-utils
-
-### AFL crash analyzer (floyd)
-
-Another crash triage tool:
-
-https://github.com/floyd-fuh/afl-crash-analyzer
-
-###  afl-extras (fekir)
-
-Collect data, parallel afl-tmin, startup scripts.
-
-https://github.com/fekir/afl-extras
-
-### afl-fuzzing-scripts (Tobias Ospelt)
-
-Simplifies starting up multiple parallel AFL jobs.
-
-https://github.com/floyd-fuh/afl-fuzzing-scripts/
-
-### afl-sid (Jacek Wielemborek)
-
-Allows users to more conveniently build and deploy AFL via Docker.
-
-https://github.com/d33tah/afl-sid
-
-Another Docker-related project:
-
-https://github.com/ozzyjohnson/docker-afl
-
-### afl-monitor (Paul S. Ziegler)
-
-Provides more detailed and versatile statistics about your running AFL jobs.
-
-https://github.com/reflare/afl-monitor
-
-### FEXM (Security in Telecommunications)
-
-Fully automated fuzzing framework, based on AFL
-
-https://github.com/fgsect/fexm
-
-## Crash triage, coverage analysis, and other companion tools:
-
-### afl-crash-analyzer (Tobias Ospelt)
-
-Makes it easier to navigate and annotate crashing test cases.
-
-https://github.com/floyd-fuh/afl-crash-analyzer/
-
-### Crashwalk (Ben Nagy)
-
-AFL-aware tool to annotate and sort through crashing test cases.
-
-https://github.com/bnagy/crashwalk
-
-### afl-cov (Michael Rash)
-
-Produces human-readable coverage data based on the output queue of afl-fuzz.
-
-https://github.com/mrash/afl-cov
-
-### afl-sancov (Bhargava Shastry)
-
-Similar to afl-cov, but uses clang sanitizer instrumentation.
-
-https://github.com/bshastry/afl-sancov
-
-### RecidiVM (Jakub Wilk)
-
-Makes it easy to estimate memory usage limits when fuzzing with ASAN or MSAN.
-
-http://jwilk.net/software/recidivm
-
-### aflize (Jacek Wielemborek)
-
-Automatically build AFL-enabled versions of Debian packages.
-
-https://github.com/d33tah/aflize
-
-### afl-ddmin-mod (Markus Teufelberger)
-
-A variant of afl-tmin that uses a more sophisticated (but slower)
-minimization algorithm.
-
-https://github.com/MarkusTeufelberger/afl-ddmin-mod
-
-### afl-kit (Kuang-che Wu)
-
-Replacements for afl-cmin and afl-tmin with additional features, such
-as the ability to filter crashes based on stderr patterns.
-
-https://github.com/kcwu/afl-kit
-
-## Narrow-purpose or experimental:
-
-### Cygwin support (Ali Rizvi-Santiago)
-
-Pretty self-explanatory. As per the author, this "mostly" ports AFL to
-Windows. Field reports welcome!
-
-https://github.com/arizvisa/afl-cygwin
-
-### Pause and resume scripts (Ben Nagy)
-
-Simple automation to suspend and resume groups of fuzzing jobs.
-
-https://github.com/bnagy/afl-trivia
-
-### Static binary-only instrumentation (Aleksandar Nikolich)
-
-Allows black-box binaries to be instrumented statically (i.e., by modifying
-the binary ahead of the time, rather than translating it on the run). Author
-reports better performance compared to QEMU, but occasional translation
-errors with stripped binaries.
-
-https://github.com/vanhauser-thc/afl-dyninst
-
-### AFL PIN (Parker Thompson)
-
-Early-stage Intel PIN instrumentation support (from before we settled on
-faster-running QEMU).
-
-https://github.com/mothran/aflpin
-
-### AFL-style instrumentation in llvm (Kostya Serebryany)
-
-Allows AFL-equivalent instrumentation to be injected at compiler level.
-This is currently not supported by AFL as-is, but may be useful in other
-projects.
-
-https://code.google.com/p/address-sanitizer/wiki/AsanCoverage#Coverage_counters
-
-### AFL JS (Han Choongwoo)
-
-One-off optimizations to speed up the fuzzing of JavaScriptCore (now likely
-superseded by LLVM deferred forkserver init - see README.llvm.md).
-
-https://github.com/tunz/afl-fuzz-js
-
-### AFL harness for fwknop (Michael Rash)
-
-An example of a fairly involved integration with AFL.
-
-https://github.com/mrash/fwknop/tree/master/test/afl
-
-### Building harnesses for DNS servers (Jonathan Foote, Ron Bowes)
-
-Two articles outlining the general principles and showing some example code.
-
-https://www.fastly.com/blog/how-to-fuzz-server-american-fuzzy-lop
-https://goo.gl/j9EgFf
-
-### Fuzzer shell for SQLite (Richard Hipp)
-
-A simple SQL shell designed specifically for fuzzing the underlying library.
-
-http://www.sqlite.org/src/artifact/9e7e273da2030371
-
-### Support for Python mutation modules (Christian Holler)
-
-now integrated in AFL++, originally from here
-https://github.com/choller/afl/blob/master/docs/mozilla/python_modules.txt
-
-### Support for selective instrumentation (Christian Holler)
-
-now integrated in AFL++, originally from here
-https://github.com/choller/afl/blob/master/docs/mozilla/partial_instrumentation.txt
-
-### Syzkaller (Dmitry Vyukov)
-
-A similar guided approach as applied to fuzzing syscalls:
-
-https://github.com/google/syzkaller/wiki/Found-Bugs
-https://github.com/dvyukov/linux/commit/33787098ffaaa83b8a7ccf519913ac5fd6125931
-http://events.linuxfoundation.org/sites/events/files/slides/AFL%20filesystem%20fuzzing%2C%20Vault%202016_0.pdf
-
-
-### Kernel Snapshot Fuzzing using Unicornafl (Security in Telecommunications)
-
-https://github.com/fgsect/unicorefuzz
-
-### Android support (ele7enxxh)
-
-Based on a somewhat dated version of AFL:
-
-https://github.com/ele7enxxh/android-afl
-
-### CGI wrapper (floyd)
-
-Facilitates the testing of CGI scripts.
-
-https://github.com/floyd-fuh/afl-cgi-wrapper
-
-### Fuzzing difficulty estimation (Marcel Boehme)
-
-A fork of AFL that tries to quantify the likelihood of finding additional
-paths or crashes at any point in a fuzzing job.
-
-https://github.com/mboehme/pythia