Merge pull request #2226 from AFLplusplus/dev

push to stable

GNUmakefile.llvm

@@ -163,7 +163,7 @@ endif
 
 # sanity check.
 # Are versions of clang --version and llvm-config --version equal?
-CLANGVER = $(shell $(CC) --version | sed -E -ne '/^.*version\ (1?[0-9]\.[0-9]\.[0-9]).*/s//\1/p')
+CLANGVER = $(shell $(CC) --version | sed -E -ne '/^.*version\ ([12]?[0-9]\.[0-9]\.[0-9]).*/s//\1/p')
 
 # I disable this because it does not make sense with what we did before (marc)
 # We did exactly set these 26 lines above with these values, and it would break

dictionaries/jsonschema.dict

@@ -0,0 +1,120 @@
+#
+# AFL dictionary for JSON Schema
+# https://json-schema.org/
+# -----------------------
+#
+
+"\"$schema\""
+"\"$id\""
+"\"$ref\""
+"\"$defs\""
+"\"definitions\""
+"\"enum\""
+"\"const\""
+"\"type\""
+
+# Annotations
+
+"\"title\""
+"\"description\""
+"\"default\""
+"\"examples\""
+"\"$comment\""
+"\"readOnly\""
+"\"writeOnly\""
+"\"deprecated\""
+
+# Types
+
+"\"string\""
+"\"integer\""
+"\"number\""
+"\"object\""
+"\"array\""
+"\"null\""
+"\"boolean\""
+
+# String
+
+"\"minLength\""
+"\"maxLength\""
+"\"pattern\""
+"\"format\""
+"\"contentMediaType\""
+"\"contentEncoding\""
+"\"contentSchema\""
+
+# Formats
+
+"\"date-time\""
+"\"time\""
+"\"date\""
+"\"duration\""
+"\"email\""
+"\"idn-email\""
+"\"hostname\""
+"\"idn-hostname\""
+"\"ipv4\""
+"\"ipv6\""
+"\"uuid\""
+"\"uri\""
+"\"uri-reference\""
+"\"iri\""
+"\"iri-reference\""
+"\"uri-template\""
+"\"json-pointer\""
+"\"relative-json-pointer\""
+"\"regex\""
+
+# Numeric
+
+"\"multipleOf\""
+"\"minimum\""
+"\"exclusiveMinimum\""
+"\"maximum\""
+"\"exclusiveMaximum\""
+
+# Object
+
+"\"properties\""
+"\"patternProperties\""
+"\"additionalProperties\""
+"\"unevaluatedProperties\""
+"\"required\""
+"\"propertyNames\""
+"\"minProperties\""
+"\"maxProperties\""
+"\"dependencies\""
+
+# Array
+
+"\"items\""
+"\"prefixItems\""
+"\"additionalItems\""
+"\"unevaluatedItems\""
+"\"contains\""
+"\"minContains\""
+"\"maxContains\""
+"\"minItems\""
+"\"maxItems\""
+"\"uniqueItems\""
+
+# Booleans
+
+"true"
+"false"
+
+# Composition
+
+"\"allOf\""
+"\"anyOf\""
+"\"oneOf\""
+"\"not\""
+
+# Conditions
+
+"\"dependentRequired\""
+"\"dependentSchemas\""
+"\"if\""
+"\"then\""
+"\"else\""

docs/Changelog.md

@@ -16,6 +16,8 @@
     - because of bad math and undefined behaviour fixes we have to change
       the CMPLOG map. **YOU NEED TO RECOMPILE CMPLOG TARGETS**
     - fixed custom_post_process for calibration
+    - fixes for AFL_EXIT_ON_TIME and AFL_EXIT_WHEN_DONE, changed behaviour of
+      AFL_EXIT_WHEN_DONE to finish when really done :-)
   - frida_mode:
     - AFL_FRIDA_PERSISTENT_ADDR can now be be any reachable address not just
       a function entry
@@ -35,6 +37,9 @@
     - new runtime (!) variable: `AFL_OLD_FORKSERVER` to use the old vanilla
       AFL type forkserver. Useful for symcc/symqemu/nautilus/etc. with
       AFL_LLVM_INSTRUMENT=CLASSIC
+    - new compile time variable: `AFL_OPT_LEVEL` to set a specific optimization
+      level, default is `3`
+    - correctly explain how to get the correct map size for large targets
   - code formatting updated to llvm 18
   - improved custom_mutators/aflpp/standalone/aflpp-standalone
   - added custom_mutators/autotokens/standalone/autotokens-standalone

docs/env_variables.md

@@ -64,6 +64,9 @@ fairly broad use of environment variables instead:
     optimizations, set `AFL_DONT_OPTIMIZE`. However, if `-O...` and/or
     `-fno-unroll-loops` are set, these are not overridden.
 
+  - The optimization level can also be set with `AFL_OPT_LEVEL`, e.g.
+    `AFL_OPT_LEVEL=z` for `-Oz`, default is `3`
+
   - Setting `AFL_HARDEN` automatically adds code hardening options when invoking
     the downstream compiler. This currently includes `-D_FORTIFY_SOURCE=2` and
     `-fstack-protector-all`. The setting is useful for catching non-crashing
@@ -323,6 +326,11 @@ mode.
     [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md)
     for more information.
 
+    Setting `AFL_GCC_DISABLE_VERSION_CHECK=1` will disable the GCC plugin 
+    version check if the target GCC plugin differs from the system-installed
+    version, resolving issues caused by version mismatches between GCC and 
+    the plugin. 
+
     Setting `AFL_GCC_OUT_OF_LINE=1` will instruct afl-gcc-fast to instrument the
     code with calls to an injected subroutine instead of the much more efficient
     inline instrumentation.
@@ -424,9 +432,8 @@ checks or alter some of the more exotic semantics of the tool:
     types of automated jobs.
 
   - `AFL_EXIT_WHEN_DONE` causes afl-fuzz to terminate when all existing paths
-    have been fuzzed and there were no new finds for a while. This would be
+    have been fuzzed and there were no new finds for a while. This is basically
-    normally indicated by the cycle counter in the UI turning green. May be
+    when the fuzzing state says `state: finished`
-    convenient for some types of automated jobs.
 
   - Setting `AFL_EXPAND_HAVOC_NOW` will start in the extended havoc mode that
     includes costly mutations. afl-fuzz automatically enables this mode when

include/envs.h

@@ -49,12 +49,12 @@ static char *afl_environment_variables[] = {
     "AFL_FRIDA_PERSISTENT_RET", "AFL_FRIDA_STALKER_ADJACENT_BLOCKS",
     "AFL_FRIDA_STALKER_IC_ENTRIES", "AFL_FRIDA_STALKER_NO_BACKPATCH",
     "AFL_FRIDA_STATS_FILE", "AFL_FRIDA_STATS_INTERVAL", "AFL_FRIDA_TRACEABLE",
-    "AFL_FRIDA_VERBOSE", "AFL_OLD_FORKSERVER",
+    "AFL_FRIDA_VERBOSE", "AFL_OLD_FORKSERVER", "AFL_OPT_LEVEL",
     "AFL_FUZZER_ARGS",  // oss-fuzz
     "AFL_FUZZER_STATS_UPDATE_INTERVAL", "AFL_GDB", "AFL_GCC_ALLOWLIST",
-    "AFL_GCC_DENYLIST", "AFL_GCC_BLOCKLIST", "AFL_GCC_INSTRUMENT_FILE",
+    "AFL_GCC_DENYLIST", "AFL_GCC_BLOCKLIST", "AFL_GCC_DISABLE_VERSION_CHECK",
-    "AFL_GCC_OUT_OF_LINE", "AFL_GCC_SKIP_NEVERZERO", "AFL_GCJ",
+    "AFL_GCC_INSTRUMENT_FILE", "AFL_GCC_OUT_OF_LINE", "AFL_GCC_SKIP_NEVERZERO",
-    "AFL_HANG_TMOUT", "AFL_FORKSRV_INIT_TMOUT", "AFL_HARDEN",
+    "AFL_GCJ", "AFL_HANG_TMOUT", "AFL_FORKSRV_INIT_TMOUT", "AFL_HARDEN",
     "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IGNORE_PROBLEMS",
     "AFL_IGNORE_PROBLEMS_COVERAGE", "AFL_IGNORE_SEED_PROBLEMS",
     "AFL_IGNORE_TIMEOUTS", "AFL_IGNORE_UNKNOWN_ENVS", "AFL_IMPORT_FIRST",

instrumentation/README.gcc_plugin.md

@@ -51,7 +51,10 @@ The idea and much of the implementation comes from Laszlo Szekeres.
 In order to leverage this mechanism, you need to have modern enough GCC (>=
 version 4.5.0) and the plugin development headers installed on your system. That
 should be all you need. On Debian machines, these headers can be acquired by
-installing the `gcc-VERSION-plugin-dev` packages.
+installing the `gcc-VERSION-plugin-dev` packages. If you're compiling a GCC 
+plugin that differs from the system-installed version and encounter issues 
+with version checks, you can use the `AFL_GCC_DISABLE_VERSION_CHECK` environment 
+variable.
 
 To build the instrumentation itself, type `make`. This will generate binaries
 called `afl-gcc-fast` and `afl-g++-fast` in the parent directory.

instrumentation/afl-compiler-rt.o.c

@@ -367,6 +367,12 @@ static void __afl_map_shm(void) {
 
     }
 
+    if (__afl_debug) {
+
+      fprintf(stderr, "DEBUG: AFL_MAP_SIZE=%u\n", __afl_map_size);
+
+    }
+
     if (__afl_final_loc > MAP_SIZE) {
 
       char *ptr;
@@ -413,7 +419,7 @@ static void __afl_map_shm(void) {
 
     if ((ptr = getenv("AFL_MAP_SIZE")) != NULL) { val = atoi(ptr); }
 
-    if (val > MAP_INITIAL_SIZE) {
+    if (val > MAP_INITIAL_SIZE && val > __afl_final_loc) {
 
       __afl_map_size = val;
 
@@ -630,6 +636,8 @@ static void __afl_map_shm(void) {
 
       __afl_area_ptr_dummy = (u8 *)malloc(__afl_map_size);
 
+    }
+
     if (__afl_area_ptr_dummy) {
 
       if (__afl_selective_coverage_start_off) {
@@ -648,8 +656,6 @@ static void __afl_map_shm(void) {
 
   }
 
-  }
-
   id_str = getenv(CMPLOG_SHM_ENV_VAR);
 
   if (__afl_debug) {

instrumentation/afl-gcc-cmplog-pass.so.cc

@@ -370,7 +370,8 @@ Set AFL_QUIET in the environment to silence it.\n\
 int plugin_init(struct plugin_name_args   *info,
                 struct plugin_gcc_version *version) {
 
-  if (!plugin_default_version_check(version, &gcc_version))
+  if (!plugin_default_version_check(version, &gcc_version) &&
+      !getenv("AFL_GCC_DISABLE_VERSION_CHECK"))
     FATAL(G_("GCC and plugin have incompatible versions, expected GCC %s, "
              "is %s"),
           gcc_version.basever, version->basever);

instrumentation/afl-gcc-cmptrs-pass.so.cc

@@ -338,7 +338,8 @@ Set AFL_QUIET in the environment to silence it.\n\
 int plugin_init(struct plugin_name_args   *info,
                 struct plugin_gcc_version *version) {
 
-  if (!plugin_default_version_check(version, &gcc_version))
+  if (!plugin_default_version_check(version, &gcc_version) &&
+      !getenv("AFL_GCC_DISABLE_VERSION_CHECK"))
     FATAL(G_("GCC and plugin have incompatible versions, expected GCC %s, "
              "is %s"),
           gcc_version.basever, version->basever);

instrumentation/afl-gcc-pass.so.cc

@@ -478,7 +478,8 @@ Specify -frandom-seed for reproducible instrumentation.\n\
 int plugin_init(struct plugin_name_args   *info,
                 struct plugin_gcc_version *version) {
 
-  if (!plugin_default_version_check(version, &gcc_version))
+  if (!plugin_default_version_check(version, &gcc_version) &&
+      !getenv("AFL_GCC_DISABLE_VERSION_CHECK"))
     FATAL(G_("GCC and plugin have incompatible versions, expected GCC %s, "
              "is %s"),
           gcc_version.basever, version->basever);

src/afl-cc.c

@@ -99,6 +99,7 @@ typedef enum {
 } compiler_mode_id;
 
 static u8   cwd[4096];
+static char opt_level = '3';
 
 char instrument_mode_string[18][18] = {
 
@@ -881,9 +882,17 @@ static void instrument_mode_old_environ(aflcc_state_t *aflcc) {
 */
 static void instrument_mode_new_environ(aflcc_state_t *aflcc) {
 
+  u8 *ptr2;
+
+  if ((ptr2 = getenv("AFL_OPT_LEVEL"))) {
+
+    opt_level = ptr2[0];  // ignore invalid data
+
+  }
+
   if (!getenv("AFL_LLVM_INSTRUMENT")) { return; }
 
-  u8 *ptr2 = strtok(getenv("AFL_LLVM_INSTRUMENT"), ":,;");
+  ptr2 = strtok(getenv("AFL_LLVM_INSTRUMENT"), ":,;");
 
   while (ptr2) {
 
@@ -2561,6 +2570,33 @@ void add_gcc_plugin(aflcc_state_t *aflcc) {
 
 }
 
+char *get_opt_level() {
+
+  static char levels[8][8] = {"-O0", "-O1", "-O2",    "-O3",
+                              "-Oz", "-Os", "-Ofast", "-Og"};
+  switch (opt_level) {
+
+    case '0':
+      return levels[0];
+    case '1':
+      return levels[1];
+    case '2':
+      return levels[2];
+    case 'z':
+      return levels[4];
+    case 's':
+      return levels[5];
+    case 'f':
+      return levels[6];
+    case 'g':
+      return levels[7];
+    default:
+      return levels[3];
+
+  }
+
+}
+
 /* Add some miscellaneous params required by our instrumentation. */
 void add_misc_params(aflcc_state_t *aflcc) {
 
@@ -2592,7 +2628,7 @@ void add_misc_params(aflcc_state_t *aflcc) {
   if (!getenv("AFL_DONT_OPTIMIZE")) {
 
     insert_param(aflcc, "-g");
-    if (!aflcc->have_o) insert_param(aflcc, "-O3");
+    if (!aflcc->have_o) insert_param(aflcc, get_opt_level());
     if (!aflcc->have_unroll) insert_param(aflcc, "-funroll-loops");
     // if (strlen(aflcc->march_opt) > 1 && aflcc->march_opt[0] == '-')
     //     insert_param(aflcc, aflcc->march_opt);
@@ -2922,6 +2958,8 @@ static void maybe_usage(aflcc_state_t *aflcc, int argc, char **argv) {
         SAYF(
             "\nGCC Plugin-specific environment variables:\n"
             "  AFL_GCC_CMPLOG: log operands of comparisons (RedQueen mutator)\n"
+            "  AFL_GCC_DISABLE_VERSION_CHECK: disable GCC plugin version "
+            "control\n"
             "  AFL_GCC_OUT_OF_LINE: disable inlined instrumentation\n"
             "  AFL_GCC_SKIP_NEVERZERO: do not skip zero on trace counters\n"
             "  AFL_GCC_INSTRUMENT_FILE: enable selective instrumentation by "

src/afl-forkserver.c

@@ -495,9 +495,9 @@ static void report_error_and_exit(int error) {
       FATAL(
           "AFL_MAP_SIZE is not set and fuzzing target reports that the "
           "required size is very large. Solution: Run the fuzzing target "
-          "stand-alone with the environment variable AFL_DEBUG=1 set and set "
+          "stand-alone with the environment variable AFL_DUMP_MAP_SIZE=1 set "
-          "the value for __afl_final_loc in the AFL_MAP_SIZE environment "
+          "the displayed value in the AFL_MAP_SIZE environment variable for "
-          "variable for afl-fuzz.");
+          "afl-fuzz.");
       break;
     case FS_ERROR_MAP_ADDR:
       FATAL(

src/afl-fuzz-init.c

@@ -1714,13 +1714,15 @@ static u8 delete_files(u8 *path, u8 *prefix) {
 
   while ((d_ent = readdir(d))) {
 
-    if (d_ent->d_name[0] != '.' &&
+    if ((d_ent->d_name[0] != '.' &&
-        (!prefix || !strncmp(d_ent->d_name, prefix, strlen(prefix)))) {
+         (!prefix || !strncmp(d_ent->d_name, prefix, strlen(prefix))))
+        /* heiko: don't forget the SHA1 files */
+        || strspn(d_ent->d_name, "0123456789abcdef") == 2 * 20 /* TODO use 2 * HASH_LENGTH */
+       ) {
 
       u8 *fname = alloc_printf("%s/%s", path, d_ent->d_name);
       if (unlink(fname)) { PFATAL("Unable to delete '%s'", fname); }
       ck_free(fname);
-
     }
 
   }

src/afl-fuzz-stats.c

@@ -56,6 +56,8 @@ char *get_fuzzing_state(afl_state_t *afl) {
 
     if (unlikely(percent_cur >= 80 && percent_total >= 80)) {
 
+      if (unlikely(afl->afl_env.afl_exit_when_done)) { afl->stop_soon = 2; }
+
       return fuzzing_state[3];
 
     } else if (unlikely(percent_cur >= 55 && percent_total >= 55)) {
@@ -822,15 +824,6 @@ void show_stats_normal(afl_state_t *afl) {
 
   }
 
-  /* Honor AFL_EXIT_WHEN_DONE and AFL_BENCH_UNTIL_CRASH. */
-
-  if (unlikely(!afl->non_instrumented_mode && afl->cycles_wo_finds > 100 &&
-               !afl->pending_not_fuzzed && afl->afl_env.afl_exit_when_done)) {
-
-    afl->stop_soon = 2;
-
-  }
-
   /* AFL_EXIT_ON_TIME. */
 
   /* If no coverage was found yet, check whether run time is greater than
@@ -998,14 +991,14 @@ void show_stats_normal(afl_state_t *afl) {
     } else
 
       /* Subsequent cycles, but we're still making finds. */
-      if (afl->cycles_wo_finds < 25 || min_wo_finds < 30) {
+      if (afl->cycles_wo_finds < 2 || min_wo_finds <= 30) {
 
         strcpy(tmp, cYEL);
 
       } else
 
         /* No finds for a long time and no test cases to try. */
-        if (afl->cycles_wo_finds > 100 && !afl->pending_not_fuzzed &&
+        if (afl->cycles_wo_finds > 1 && !afl->pending_not_fuzzed &&
             min_wo_finds > 120) {
 
           strcpy(tmp, cLGN);
@@ -1656,15 +1649,6 @@ void show_stats_pizza(afl_state_t *afl) {
 
   }
 
-  /* Honor AFL_EXIT_WHEN_DONE and AFL_BENCH_UNTIL_CRASH. */
-
-  if (unlikely(!afl->non_instrumented_mode && afl->cycles_wo_finds > 100 &&
-               !afl->pending_not_fuzzed && afl->afl_env.afl_exit_when_done)) {
-
-    afl->stop_soon = 2;
-
-  }
-
   /* AFL_EXIT_ON_TIME. */
 
   /* If no coverage was found yet, check whether run time is greater than
@@ -1813,14 +1797,14 @@ void show_stats_pizza(afl_state_t *afl) {
     } else
 
       /* Subsequent cycles, but we're still making finds. */
-      if (afl->cycles_wo_finds < 25 || min_wo_finds < 30) {
+      if (afl->cycles_wo_finds < 2 || min_wo_finds <= 30) {
 
         strcpy(tmp, cYEL);
 
       } else
 
         /* No finds for a long time and no test cases to try. */
-        if (afl->cycles_wo_finds > 100 && !afl->pending_not_fuzzed &&
+        if (afl->cycles_wo_finds > 1 && !afl->pending_not_fuzzed &&
             min_wo_finds > 120) {
 
           strcpy(tmp, cLGN);

src/afl-fuzz.c

@@ -2937,10 +2937,10 @@ int main(int argc, char **argv_orig, char **envp) {
                       3600 */
                    )) {
 
-        if (afl->use_splicing) {
-
         ++afl->cycles_wo_finds;
 
+        if (afl->use_splicing) {
+
           if (unlikely(afl->shm.cmplog_mode &&
                        afl->cmplog_max_filesize < MAX_FILE)) {