diff --git a/afl-cmin.py b/afl-cmin.py index aad7bc16..5ceace8f 100755 --- a/afl-cmin.py +++ b/afl-cmin.py @@ -28,6 +28,38 @@ import shutil import subprocess import sys +# https://more-itertools.readthedocs.io/en/stable/_modules/more_itertools/recipes.html#batched +from sys import hexversion + +def _batched(iterable, n, *, strict=False): + """Batch data into tuples of length *n*. If the number of items in + *iterable* is not divisible by *n*: + * The last batch will be shorter if *strict* is ``False``. + * :exc:`ValueError` will be raised if *strict* is ``True``. + + >>> list(batched('ABCDEFG', 3)) + [('A', 'B', 'C'), ('D', 'E', 'F'), ('G',)] + + On Python 3.13 and above, this is an alias for :func:`itertools.batched`. + """ + if n < 1: + raise ValueError('n must be at least one') + iterator = iter(iterable) + while batch := tuple(itertools.islice(iterator, n)): + if strict and len(batch) != n: + raise ValueError('batched(): incomplete batch') + yield batch + + +if hexversion >= 0x30D00A2: # pragma: no cover + from itertools import batched as itertools_batched + def batched(iterable, n, *, strict=False): + return itertools_batched(iterable, n, strict=strict) +else: + batched = _batched + + batched.__doc__ = _batched.__doc__ + try: from tqdm import tqdm except ImportError: @@ -574,7 +606,7 @@ def main(): workers.append(p) chunk = max(1, min(128, len(files) // args.workers)) - jobs = list(itertools.batched(enumerate(files), chunk)) + jobs = list(batched(enumerate(files), chunk)) jobs += [None] * args.workers # sentinel dispatcher = JobDispatcher(job_queue, jobs) diff --git a/docs/SAND.md b/docs/SAND.md index b9d6647f..6012c956 100644 --- a/docs/SAND.md +++ b/docs/SAND.md @@ -21,7 +21,7 @@ For a normal fuzzing workflow, we have: For SAND fuzzing workflow, this is slightly different: 1. Build target project _without_ any sanitizers to get `target_native`, which we will define as a "native binary". It is usually done by using `afl-clang-fast/lto(++)` to compile your project _without_ `AFL_USE_ASAN/UBSAN/MSAN`. -2. Build target project with AFL_USE_ASAN=1 AFL_SAN_NO_INST=1 to get `target_asan`. Do note this step can be repeated for multiple sanitizers, like MSAN, UBSAN etc. It is also possible to have ASAN and UBSAN to build together. +2. Build target project with AFL_USE_ASAN=1 AFL_LLVM_ONLY_FSRV=1 to get `target_asan`. Do note this step can be repeated for multiple sanitizers, like MSAN, UBSAN etc. It is also possible to have ASAN and UBSAN to build together. 3. Fuzz the target with `afl-fuzz -i seeds -o out -w ./target_asan -- ./target_native`. Note `-w` can be specified multiple times. Then you get: @@ -44,11 +44,11 @@ Just like the normal building process, except using `afl-clang-fast` 2. Build the sanitizers-enabled binaries. ```bash -AFL_SAN_NO_INST=1 AFL_USE_UBSAN=1 AFL_USE_ASAN=1 afl-clang-fast test-instr.c -o ./asanubsan -AFL_SAN_NO_INST=1 AFL_USE_MSAN=1 afl-clang-fast test-instr.c -o ./msan +AFL_LLVM_ONLY_FSRV=1 AFL_USE_UBSAN=1 AFL_USE_ASAN=1 afl-clang-fast test-instr.c -o ./asanubsan +AFL_LLVM_ONLY_FSRV=1 AFL_USE_MSAN=1 afl-clang-fast test-instr.c -o ./msan ``` -Do note `AFL_SAN_NO_INST=1` is crucial, this enables forkservers but disables pc instrumentation. Do not reuse sanitizers-enabled binaries built _without_ `AFL_SAN_NO_INST=1`. This will mess up SAND execution pattern. +Do note `AFL_LLVM_ONLY_FSRV=1` is crucial, this enables forkservers but disables pc instrumentation. You are allowed to reuse sanitizers-enabled binaries, i.e. binaries built _without_ `AFL_LLVM_ONLY_FSRV=1`, at a cost of reduced speed. 3. Start fuzzing diff --git a/docs/env_variables.md b/docs/env_variables.md index 3052663a..a492a1bc 100644 --- a/docs/env_variables.md +++ b/docs/env_variables.md @@ -111,6 +111,10 @@ fairly broad use of environment variables instead: - Note: both `AFL_CFISAN_VERBOSE=1` and `AFL_UBSAN_VERBOSE=1` are disabled by default as verbose output can significantly slow down fuzzing performance. Use these options only during debugging or when additional crash diagnostics are required + - `AFL_LLVM_ONLY_FSRV`/`AFL_GCC_ONLY_FSRV` will inject forkserver but not pc instrumentation. Please note this is different compared to `AFL_LLVM_DISABLE_INSTRUMENTATION`, which will totally disable forkserver implementation. This env is pretty useful in two cases: + - [SAND](./SAND.md). In this case, the binaries built in this way will serve as extra oracles. Check the corresponding documents for details. + - Compatible with LibAFL ForkserverExecutor implementation and thus faster to repeatedly run, compared to simple CommandExecutor. + - `TMPDIR` is used by afl-as for temporary files; if this variable is not set, the tool defaults to /tmp. diff --git a/include/envs.h b/include/envs.h index 433b51a5..eda9ceaf 100644 --- a/include/envs.h +++ b/include/envs.h @@ -10,6 +10,7 @@ static char *afl_environment_deprecated[] = { "AFL_DEFER_FORKSRV", "AFL_POST_LIBRARY", "AFL_PERSISTENT", + "AFL_SAN_NO_INST", NULL }; @@ -118,7 +119,7 @@ static char *afl_environment_variables[] = { "AFL_CFISAN_VERBOSE", "AFL_USE_LSAN", "AFL_WINE_PATH", "AFL_NO_SNAPSHOT", "AFL_EXPAND_HAVOC_NOW", "AFL_USE_FASAN", "AFL_USE_QASAN", "AFL_PRINT_FILENAMES", "AFL_PIZZA_MODE", "AFL_NO_FASTRESUME", - "AFL_SAN_ABSTRACTION", "AFL_SAN_NO_INST", "AFL_SAN_RECOVER", + "AFL_SAN_ABSTRACTION", "AFL_LLVM_ONLY_FSRV", "AFL_GCC_ONLY_FRSV", "AFL_SAN_RECOVER", "AFL_PRELOAD_DISCRIMINATE_FORKSERVER_PARENT", NULL}; extern char *afl_environment_variables[]; diff --git a/instrumentation/SanitizerCoverageLTO.so.cc b/instrumentation/SanitizerCoverageLTO.so.cc index c27ff36a..7d2e45b3 100644 --- a/instrumentation/SanitizerCoverageLTO.so.cc +++ b/instrumentation/SanitizerCoverageLTO.so.cc @@ -327,7 +327,7 @@ class ModuleSanitizerCoverageLTOLegacyPass : public ModulePass { }; - if (!getenv("AFL_SAN_NO_INST")) { + if (!getenv("AFL_LLVM_ONLY_FSRV")) { return ModuleSancov.instrumentModule(M, DTCallback, PDTCallback); @@ -389,7 +389,7 @@ PreservedAnalyses ModuleSanitizerCoverageLTO::run(Module &M, }; - if (!getenv("AFL_SAN_NO_INST")) { + if (!getenv("AFL_LLVM_ONLY_FSRV")) { if (ModuleSancov.instrumentModule(M, DTCallback, PDTCallback)) return PreservedAnalyses::none(); diff --git a/instrumentation/SanitizerCoveragePCGUARD.so.cc b/instrumentation/SanitizerCoveragePCGUARD.so.cc index 670f06c1..7c70a42e 100644 --- a/instrumentation/SanitizerCoveragePCGUARD.so.cc +++ b/instrumentation/SanitizerCoveragePCGUARD.so.cc @@ -272,7 +272,7 @@ PreservedAnalyses ModuleSanitizerCoverageAFL::run(Module &M, // TODO: Support LTO or llvm classic? // Note we still need afl-compiler-rt so we just disable the instrumentation // here. - if (!getenv("AFL_SAN_NO_INST")) { + if (!getenv("AFL_LLVM_ONLY_FSRV")) { if (ModuleSancov.instrumentModule(M, DTCallback, PDTCallback)) return PreservedAnalyses::none(); diff --git a/instrumentation/afl-compiler-rt.o.c b/instrumentation/afl-compiler-rt.o.c index 24d2ed9e..10a271cd 100644 --- a/instrumentation/afl-compiler-rt.o.c +++ b/instrumentation/afl-compiler-rt.o.c @@ -1241,6 +1241,11 @@ void __afl_manual_init(void) { } + if (getenv("AFL_LLVM_ONLY_FSRV") || getenv("AFL_GCC_ONLY_FRSV")) { + fprintf(stderr, "DEBUG: Overwrite area_ptr to dummy due to AFL_LLVM_ONLY_FSRV/AFL_GCC_ONLY_FRSV\n"); + __afl_area_ptr = __afl_area_ptr_dummy; + } + if (!init_done) { __afl_start_forkserver(); diff --git a/instrumentation/afl-gcc-pass.so.cc b/instrumentation/afl-gcc-pass.so.cc index 26f7bd19..8509d60f 100644 --- a/instrumentation/afl-gcc-pass.so.cc +++ b/instrumentation/afl-gcc-pass.so.cc @@ -462,6 +462,7 @@ static struct plugin_info afl_plugin = { .help = G_("AFL gcc plugin\n\ \n\ Set AFL_QUIET in the environment to silence it.\n\ +Set AFL_GCC_ONLY_FRSV in the environment to disable instrumentation.\n\ \n\ Set AFL_INST_RATIO in the environment to a number from 0 to 100\n\ to control how likely a block will be chosen for instrumentation.\n\ @@ -502,9 +503,12 @@ int plugin_init(struct plugin_name_args *info, case it was specified in the command line's -frandom-seed for reproducible instrumentation. */ srandom(get_random_seed(false)); + bool fsrv_only = !!getenv("AFL_GCC_ONLY_FRSV"); const char *name = info->base_name; - register_callback(name, PLUGIN_INFO, NULL, &afl_plugin); + if (!fsrv_only) { + register_callback(name, PLUGIN_INFO, NULL, &afl_plugin); + } afl_pass *aflp = new afl_pass(quiet, inst_ratio); struct register_pass_info pass_info = { @@ -516,14 +520,18 @@ int plugin_init(struct plugin_name_args *info, }; - register_callback(name, PLUGIN_PASS_MANAGER_SETUP, NULL, &pass_info); - register_callback(name, PLUGIN_FINISH, afl_pass::plugin_finalize, - pass_info.pass); + if (!fsrv_only) { + register_callback(name, PLUGIN_PASS_MANAGER_SETUP, NULL, &pass_info); + register_callback(name, PLUGIN_FINISH, afl_pass::plugin_finalize, + pass_info.pass); + } if (!quiet) ACTF(G_("%s instrumentation at ratio of %u%% in %s mode."), aflp->out_of_line ? G_("Call-based") : G_("Inline"), inst_ratio, getenv("AFL_HARDEN") ? G_("hardened") : G_("non-hardened")); + else if (fsrv_only) + ACTF("Instrumentation disabled due to AFL_GCC_ONLY_FRSV"); return 0; diff --git a/instrumentation/afl-llvm-pass.so.cc b/instrumentation/afl-llvm-pass.so.cc index 3fb644af..e7839612 100644 --- a/instrumentation/afl-llvm-pass.so.cc +++ b/instrumentation/afl-llvm-pass.so.cc @@ -225,7 +225,7 @@ bool AFLCoverage::runOnModule(Module &M) { if (getenv("AFL_DEBUG")) debug = 1; #if LLVM_VERSION_MAJOR >= 11 /* use new pass manager */ - if (getenv("AFL_SAN_NO_INST")) { + if (getenv("AFL_LLVM_ONLY_FSRV")) { if (debug) { fprintf(stderr, "Instrument disabled\n"); } return PreservedAnalyses::all(); @@ -233,7 +233,7 @@ bool AFLCoverage::runOnModule(Module &M) { } #else - if (getenv("AFL_SAN_NO_INST")) { + if (getenv("AFL_LLVM_ONLY_FSRV")) { if (debug) { fprintf(stderr, "Instrument disabled\n"); } return true; diff --git a/src/afl-cc.c b/src/afl-cc.c index c735ef84..2892db14 100644 --- a/src/afl-cc.c +++ b/src/afl-cc.c @@ -244,7 +244,7 @@ static inline void insert_object(aflcc_state_t *aflcc, u8 *obj, u8 *fmt, /* Insert params into the new argv, make clang load the pass. */ static inline void load_llvm_pass(aflcc_state_t *aflcc, u8 *pass) { - if (getenv("AFL_SAN_NO_INST")) { + if (getenv("AFL_LLVM_ONLY_FSRV")) { if (!be_quiet) { DEBUGF("SAND: Coverage instrumentation disabled\n"); } return; @@ -2097,7 +2097,7 @@ void add_native_pcguard(aflcc_state_t *aflcc) { * anyway. */ if (aflcc->have_rust_asanrt) { return; } - if (getenv("AFL_SAN_NO_INST")) { + if (getenv("AFL_LLVM_ONLY_FSRV")) { if (!be_quiet) { DEBUGF("SAND: Coverage instrumentation disabled\n"); } return; @@ -2138,7 +2138,7 @@ void add_native_pcguard(aflcc_state_t *aflcc) { */ void add_optimized_pcguard(aflcc_state_t *aflcc) { - if (getenv("AFL_SAN_NO_INST")) { + if (getenv("AFL_LLVM_ONLY_FSRV")) { if (!be_quiet) { DEBUGF("SAND: Coverage instrumentation disabled\n"); } return; @@ -2600,7 +2600,7 @@ void add_assembler(aflcc_state_t *aflcc) { /* Add params to launch the gcc plugins for instrumentation. */ void add_gcc_plugin(aflcc_state_t *aflcc) { - if (getenv("AFL_SAN_NO_INST")) { + if (getenv("AFL_GCC_ONLY_FSRV")) { if (!be_quiet) { DEBUGF("SAND: Coverage instrumentation disabled\n"); } return; diff --git a/src/afl-common.c b/src/afl-common.c index f4eb54b2..e61b8388 100644 --- a/src/afl-common.c +++ b/src/afl-common.c @@ -819,8 +819,15 @@ void check_environment_vars(char **envp) { WARNF("AFL environment variable %s is deprecated!", afl_environment_deprecated[i]); - issue_detected = 1; - + + if (strncmp(afl_environment_deprecated[i], + "AFL_SAN_NO_INST", strlen(afl_environment_deprecated[i])) == 0) { + WARNF("AFL_LLVM_ONLY_FSRV/AFL_GCC_ONLY_FSRV is induced and set instead."); + setenv("AFL_GCC_ONLY_FSRV", "1", 0); + setenv("AFL_LLVM_ONLY_FSRV", "1", 0); + } else { + issue_detected = 1; + } } else { i++; diff --git a/src/afl-fuzz-sanfuzz.c b/src/afl-fuzz-sanfuzz.c index 3b25c57b..81c48449 100644 --- a/src/afl-fuzz-sanfuzz.c +++ b/src/afl-fuzz-sanfuzz.c @@ -36,9 +36,11 @@ void sanfuzz_exec_child(afl_forkserver_t *fsrv, char **argv) { argv[0] != fsrv->asanfuzz_binary) { argv[0] = fsrv->asanfuzz_binary; - } + // In case users provide the normally instrumented binaries, this servers as the last + // resort to avoid collecting incorrect coverage. + setenv("AFL_LLVM_ONLY_FSRV", "1", 0); execv(fsrv->target_path, argv); }