Merge pull request #2226 from AFLplusplus/dev

push to stable
2025-06-08 08:11:34 +00:00 · 2024-10-30 13:20:48 +01:00 · 2024-10-30 13:20:48 +01:00 · d0587a3ac4
commit d0587a3ac4
parent 78b7e14c73 d1fd072b79
15 changed files with 224 additions and 56 deletions
--- a/GNUmakefile.llvm
+++ b/GNUmakefile.llvm
@ -163,7 +163,7 @@ endif

 # sanity check.
 # Are versions of clang --version and llvm-config --version equal?
-CLANGVER = $(shell $(CC) --version | sed -E -ne '/^.*version\ (1?[0-9]\.[0-9]\.[0-9]).*/s//\1/p')
+CLANGVER = $(shell $(CC) --version | sed -E -ne '/^.*version\ ([12]?[0-9]\.[0-9]\.[0-9]).*/s//\1/p')

 # I disable this because it does not make sense with what we did before (marc)
 # We did exactly set these 26 lines above with these values, and it would break
--- a/dictionaries/jsonschema.dict
+++ b/dictionaries/jsonschema.dict
@ -0,0 +1,120 @@
+#
+# AFL dictionary for JSON Schema
+# https://json-schema.org/
+# -----------------------
+#
+
+"\"$schema\""
+"\"$id\""
+"\"$ref\""
+"\"$defs\""
+"\"definitions\""
+"\"enum\""
+"\"const\""
+"\"type\""
+
+# Annotations
+
+"\"title\""
+"\"description\""
+"\"default\""
+"\"examples\""
+"\"$comment\""
+"\"readOnly\""
+"\"writeOnly\""
+"\"deprecated\""
+
+# Types
+
+"\"string\""
+"\"integer\""
+"\"number\""
+"\"object\""
+"\"array\""
+"\"null\""
+"\"boolean\""
+
+# String
+
+"\"minLength\""
+"\"maxLength\""
+"\"pattern\""
+"\"format\""
+"\"contentMediaType\""
+"\"contentEncoding\""
+"\"contentSchema\""
+
+# Formats
+
+"\"date-time\""
+"\"time\""
+"\"date\""
+"\"duration\""
+"\"email\""
+"\"idn-email\""
+"\"hostname\""
+"\"idn-hostname\""
+"\"ipv4\""
+"\"ipv6\""
+"\"uuid\""
+"\"uri\""
+"\"uri-reference\""
+"\"iri\""
+"\"iri-reference\""
+"\"uri-template\""
+"\"json-pointer\""
+"\"relative-json-pointer\""
+"\"regex\""
+
+# Numeric
+
+"\"multipleOf\""
+"\"minimum\""
+"\"exclusiveMinimum\""
+"\"maximum\""
+"\"exclusiveMaximum\""
+
+# Object
+
+"\"properties\""
+"\"patternProperties\""
+"\"additionalProperties\""
+"\"unevaluatedProperties\""
+"\"required\""
+"\"propertyNames\""
+"\"minProperties\""
+"\"maxProperties\""
+"\"dependencies\""
+
+# Array
+
+"\"items\""
+"\"prefixItems\""
+"\"additionalItems\""
+"\"unevaluatedItems\""
+"\"contains\""
+"\"minContains\""
+"\"maxContains\""
+"\"minItems\""
+"\"maxItems\""
+"\"uniqueItems\""
+
+# Booleans
+
+"true"
+"false"
+
+# Composition
+
+"\"allOf\""
+"\"anyOf\""
+"\"oneOf\""
+"\"not\""
+
+# Conditions
+
+"\"dependentRequired\""
+"\"dependentSchemas\""
+"\"if\""
+"\"then\""
+"\"else\""
--- a/docs/Changelog.md
+++ b/docs/Changelog.md
@ -16,6 +16,8 @@
    - because of bad math and undefined behaviour fixes we have to change
      the CMPLOG map. **YOU NEED TO RECOMPILE CMPLOG TARGETS**
    - fixed custom_post_process for calibration
+    - fixes for AFL_EXIT_ON_TIME and AFL_EXIT_WHEN_DONE, changed behaviour of
+      AFL_EXIT_WHEN_DONE to finish when really done :-)
  - frida_mode:
    - AFL_FRIDA_PERSISTENT_ADDR can now be be any reachable address not just
      a function entry
@ -35,6 +37,9 @@
    - new runtime (!) variable: `AFL_OLD_FORKSERVER` to use the old vanilla
      AFL type forkserver. Useful for symcc/symqemu/nautilus/etc. with
      AFL_LLVM_INSTRUMENT=CLASSIC
+    - new compile time variable: `AFL_OPT_LEVEL` to set a specific optimization
+      level, default is `3`
+    - correctly explain how to get the correct map size for large targets
  - code formatting updated to llvm 18
  - improved custom_mutators/aflpp/standalone/aflpp-standalone
  - added custom_mutators/autotokens/standalone/autotokens-standalone
--- a/docs/env_variables.md
+++ b/docs/env_variables.md
@ -64,6 +64,9 @@ fairly broad use of environment variables instead:
    optimizations, set `AFL_DONT_OPTIMIZE`. However, if `-O...` and/or
    `-fno-unroll-loops` are set, these are not overridden.

+  - The optimization level can also be set with `AFL_OPT_LEVEL`, e.g.
+    `AFL_OPT_LEVEL=z` for `-Oz`, default is `3`
+
  - Setting `AFL_HARDEN` automatically adds code hardening options when invoking
    the downstream compiler. This currently includes `-D_FORTIFY_SOURCE=2` and
    `-fstack-protector-all`. The setting is useful for catching non-crashing
@ -323,6 +326,11 @@ mode.
    [instrumentation/README.instrument_list.md](../instrumentation/README.instrument_list.md)
    for more information.

+    Setting `AFL_GCC_DISABLE_VERSION_CHECK=1` will disable the GCC plugin 
+    version check if the target GCC plugin differs from the system-installed
+    version, resolving issues caused by version mismatches between GCC and 
+    the plugin. 
+
    Setting `AFL_GCC_OUT_OF_LINE=1` will instruct afl-gcc-fast to instrument the
    code with calls to an injected subroutine instead of the much more efficient
    inline instrumentation.
@ -424,9 +432,8 @@ checks or alter some of the more exotic semantics of the tool:
    types of automated jobs.

  - `AFL_EXIT_WHEN_DONE` causes afl-fuzz to terminate when all existing paths
-    have been fuzzed and there were no new finds for a while. This would be
-    normally indicated by the cycle counter in the UI turning green. May be
-    convenient for some types of automated jobs.
+    have been fuzzed and there were no new finds for a while. This is basically
+    when the fuzzing state says `state: finished`

  - Setting `AFL_EXPAND_HAVOC_NOW` will start in the extended havoc mode that
    includes costly mutations. afl-fuzz automatically enables this mode when
--- a/include/envs.h
+++ b/include/envs.h
@ -49,12 +49,12 @@ static char *afl_environment_variables[] = {
    "AFL_FRIDA_PERSISTENT_RET", "AFL_FRIDA_STALKER_ADJACENT_BLOCKS",
    "AFL_FRIDA_STALKER_IC_ENTRIES", "AFL_FRIDA_STALKER_NO_BACKPATCH",
    "AFL_FRIDA_STATS_FILE", "AFL_FRIDA_STATS_INTERVAL", "AFL_FRIDA_TRACEABLE",
-    "AFL_FRIDA_VERBOSE", "AFL_OLD_FORKSERVER",
+    "AFL_FRIDA_VERBOSE", "AFL_OLD_FORKSERVER", "AFL_OPT_LEVEL",
    "AFL_FUZZER_ARGS",  // oss-fuzz
    "AFL_FUZZER_STATS_UPDATE_INTERVAL", "AFL_GDB", "AFL_GCC_ALLOWLIST",
-    "AFL_GCC_DENYLIST", "AFL_GCC_BLOCKLIST", "AFL_GCC_INSTRUMENT_FILE",
-    "AFL_GCC_OUT_OF_LINE", "AFL_GCC_SKIP_NEVERZERO", "AFL_GCJ",
-    "AFL_HANG_TMOUT", "AFL_FORKSRV_INIT_TMOUT", "AFL_HARDEN",
+    "AFL_GCC_DENYLIST", "AFL_GCC_BLOCKLIST", "AFL_GCC_DISABLE_VERSION_CHECK",
+    "AFL_GCC_INSTRUMENT_FILE", "AFL_GCC_OUT_OF_LINE", "AFL_GCC_SKIP_NEVERZERO",
+    "AFL_GCJ", "AFL_HANG_TMOUT", "AFL_FORKSRV_INIT_TMOUT", "AFL_HARDEN",
    "AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IGNORE_PROBLEMS",
    "AFL_IGNORE_PROBLEMS_COVERAGE", "AFL_IGNORE_SEED_PROBLEMS",
    "AFL_IGNORE_TIMEOUTS", "AFL_IGNORE_UNKNOWN_ENVS", "AFL_IMPORT_FIRST",
--- a/instrumentation/README.gcc_plugin.md
+++ b/instrumentation/README.gcc_plugin.md
@ -51,7 +51,10 @@ The idea and much of the implementation comes from Laszlo Szekeres.
 In order to leverage this mechanism, you need to have modern enough GCC (>=
 version 4.5.0) and the plugin development headers installed on your system. That
 should be all you need. On Debian machines, these headers can be acquired by
-installing the `gcc-VERSION-plugin-dev` packages.
+installing the `gcc-VERSION-plugin-dev` packages. If you're compiling a GCC 
+plugin that differs from the system-installed version and encounter issues 
+with version checks, you can use the `AFL_GCC_DISABLE_VERSION_CHECK` environment 
+variable.

 To build the instrumentation itself, type `make`. This will generate binaries
 called `afl-gcc-fast` and `afl-g++-fast` in the parent directory.
--- a/instrumentation/afl-compiler-rt.o.c
+++ b/instrumentation/afl-compiler-rt.o.c
@ -367,6 +367,12 @@ static void __afl_map_shm(void) {

    }

+    if (__afl_debug) {
+
+      fprintf(stderr, "DEBUG: AFL_MAP_SIZE=%u\n", __afl_map_size);
+
+    }
+
    if (__afl_final_loc > MAP_SIZE) {

      char *ptr;
@ -413,7 +419,7 @@ static void __afl_map_shm(void) {

    if ((ptr = getenv("AFL_MAP_SIZE")) != NULL) { val = atoi(ptr); }

-    if (val > MAP_INITIAL_SIZE) {
+    if (val > MAP_INITIAL_SIZE && val > __afl_final_loc) {

      __afl_map_size = val;

@ -630,6 +636,8 @@ static void __afl_map_shm(void) {

      __afl_area_ptr_dummy = (u8 *)malloc(__afl_map_size);

+    }
+
    if (__afl_area_ptr_dummy) {

      if (__afl_selective_coverage_start_off) {
@ -648,8 +656,6 @@ static void __afl_map_shm(void) {

  }

-  }
-
  id_str = getenv(CMPLOG_SHM_ENV_VAR);

  if (__afl_debug) {
--- a/instrumentation/afl-gcc-cmplog-pass.so.cc
+++ b/instrumentation/afl-gcc-cmplog-pass.so.cc
@ -370,7 +370,8 @@ Set AFL_QUIET in the environment to silence it.\n\
 int plugin_init(struct plugin_name_args   *info,
                struct plugin_gcc_version *version) {

-  if (!plugin_default_version_check(version, &gcc_version))
+  if (!plugin_default_version_check(version, &gcc_version) &&
+      !getenv("AFL_GCC_DISABLE_VERSION_CHECK"))
    FATAL(G_("GCC and plugin have incompatible versions, expected GCC %s, "
             "is %s"),
          gcc_version.basever, version->basever);
--- a/instrumentation/afl-gcc-cmptrs-pass.so.cc
+++ b/instrumentation/afl-gcc-cmptrs-pass.so.cc
@ -338,7 +338,8 @@ Set AFL_QUIET in the environment to silence it.\n\
 int plugin_init(struct plugin_name_args   *info,
                struct plugin_gcc_version *version) {

-  if (!plugin_default_version_check(version, &gcc_version))
+  if (!plugin_default_version_check(version, &gcc_version) &&
+      !getenv("AFL_GCC_DISABLE_VERSION_CHECK"))
    FATAL(G_("GCC and plugin have incompatible versions, expected GCC %s, "
             "is %s"),
          gcc_version.basever, version->basever);
--- a/instrumentation/afl-gcc-pass.so.cc
+++ b/instrumentation/afl-gcc-pass.so.cc
@ -478,7 +478,8 @@ Specify -frandom-seed for reproducible instrumentation.\n\
 int plugin_init(struct plugin_name_args   *info,
                struct plugin_gcc_version *version) {

-  if (!plugin_default_version_check(version, &gcc_version))
+  if (!plugin_default_version_check(version, &gcc_version) &&
+      !getenv("AFL_GCC_DISABLE_VERSION_CHECK"))
    FATAL(G_("GCC and plugin have incompatible versions, expected GCC %s, "
             "is %s"),
          gcc_version.basever, version->basever);
--- a/src/afl-cc.c
+++ b/src/afl-cc.c
@ -99,6 +99,7 @@ typedef enum {
 } compiler_mode_id;

 static u8   cwd[4096];
+static char opt_level = '3';

 char instrument_mode_string[18][18] = {

@ -881,9 +882,17 @@ static void instrument_mode_old_environ(aflcc_state_t *aflcc) {
 */
 static void instrument_mode_new_environ(aflcc_state_t *aflcc) {

+  u8 *ptr2;
+
+  if ((ptr2 = getenv("AFL_OPT_LEVEL"))) {
+
+    opt_level = ptr2[0];  // ignore invalid data
+
+  }
+
  if (!getenv("AFL_LLVM_INSTRUMENT")) { return; }

-  u8 *ptr2 = strtok(getenv("AFL_LLVM_INSTRUMENT"), ":,;");
+  ptr2 = strtok(getenv("AFL_LLVM_INSTRUMENT"), ":,;");

  while (ptr2) {

@ -2561,6 +2570,33 @@ void add_gcc_plugin(aflcc_state_t *aflcc) {

 }

+char *get_opt_level() {
+
+  static char levels[8][8] = {"-O0", "-O1", "-O2",    "-O3",
+                              "-Oz", "-Os", "-Ofast", "-Og"};
+  switch (opt_level) {
+
+    case '0':
+      return levels[0];
+    case '1':
+      return levels[1];
+    case '2':
+      return levels[2];
+    case 'z':
+      return levels[4];
+    case 's':
+      return levels[5];
+    case 'f':
+      return levels[6];
+    case 'g':
+      return levels[7];
+    default:
+      return levels[3];
+
+  }
+
+}
+
 /* Add some miscellaneous params required by our instrumentation. */
 void add_misc_params(aflcc_state_t *aflcc) {

@ -2592,7 +2628,7 @@ void add_misc_params(aflcc_state_t *aflcc) {
  if (!getenv("AFL_DONT_OPTIMIZE")) {

    insert_param(aflcc, "-g");
-    if (!aflcc->have_o) insert_param(aflcc, "-O3");
+    if (!aflcc->have_o) insert_param(aflcc, get_opt_level());
    if (!aflcc->have_unroll) insert_param(aflcc, "-funroll-loops");
    // if (strlen(aflcc->march_opt) > 1 && aflcc->march_opt[0] == '-')
    //     insert_param(aflcc, aflcc->march_opt);
@ -2922,6 +2958,8 @@ static void maybe_usage(aflcc_state_t *aflcc, int argc, char **argv) {
        SAYF(
            "\nGCC Plugin-specific environment variables:\n"
            "  AFL_GCC_CMPLOG: log operands of comparisons (RedQueen mutator)\n"
+            "  AFL_GCC_DISABLE_VERSION_CHECK: disable GCC plugin version "
+            "control\n"
            "  AFL_GCC_OUT_OF_LINE: disable inlined instrumentation\n"
            "  AFL_GCC_SKIP_NEVERZERO: do not skip zero on trace counters\n"
            "  AFL_GCC_INSTRUMENT_FILE: enable selective instrumentation by "
--- a/src/afl-forkserver.c
+++ b/src/afl-forkserver.c
@ -495,9 +495,9 @@ static void report_error_and_exit(int error) {
      FATAL(
          "AFL_MAP_SIZE is not set and fuzzing target reports that the "
          "required size is very large. Solution: Run the fuzzing target "
-          "stand-alone with the environment variable AFL_DEBUG=1 set and set "
-          "the value for __afl_final_loc in the AFL_MAP_SIZE environment "
-          "variable for afl-fuzz.");
+          "stand-alone with the environment variable AFL_DUMP_MAP_SIZE=1 set "
+          "the displayed value in the AFL_MAP_SIZE environment variable for "
+          "afl-fuzz.");
      break;
    case FS_ERROR_MAP_ADDR:
      FATAL(
--- a/src/afl-fuzz-init.c
+++ b/src/afl-fuzz-init.c
@ -1714,13 +1714,15 @@ static u8 delete_files(u8 *path, u8 *prefix) {

  while ((d_ent = readdir(d))) {

-    if (d_ent->d_name[0] != '.' &&
-        (!prefix || !strncmp(d_ent->d_name, prefix, strlen(prefix)))) {
+    if ((d_ent->d_name[0] != '.' &&
+         (!prefix || !strncmp(d_ent->d_name, prefix, strlen(prefix))))
+        /* heiko: don't forget the SHA1 files */
+        || strspn(d_ent->d_name, "0123456789abcdef") == 2 * 20 /* TODO use 2 * HASH_LENGTH */
+       ) {

      u8 *fname = alloc_printf("%s/%s", path, d_ent->d_name);
      if (unlink(fname)) { PFATAL("Unable to delete '%s'", fname); }
      ck_free(fname);
-
    }

  }
--- a/src/afl-fuzz-stats.c
+++ b/src/afl-fuzz-stats.c
@ -56,6 +56,8 @@ char *get_fuzzing_state(afl_state_t *afl) {

    if (unlikely(percent_cur >= 80 && percent_total >= 80)) {

+      if (unlikely(afl->afl_env.afl_exit_when_done)) { afl->stop_soon = 2; }
+
      return fuzzing_state[3];

    } else if (unlikely(percent_cur >= 55 && percent_total >= 55)) {
@ -822,15 +824,6 @@ void show_stats_normal(afl_state_t *afl) {

  }

-  /* Honor AFL_EXIT_WHEN_DONE and AFL_BENCH_UNTIL_CRASH. */
-
-  if (unlikely(!afl->non_instrumented_mode && afl->cycles_wo_finds > 100 &&
-               !afl->pending_not_fuzzed && afl->afl_env.afl_exit_when_done)) {
-
-    afl->stop_soon = 2;
-
-  }
-
  /* AFL_EXIT_ON_TIME. */

  /* If no coverage was found yet, check whether run time is greater than
@ -998,14 +991,14 @@ void show_stats_normal(afl_state_t *afl) {
    } else

      /* Subsequent cycles, but we're still making finds. */
-      if (afl->cycles_wo_finds < 25 || min_wo_finds < 30) {
+      if (afl->cycles_wo_finds < 2 || min_wo_finds <= 30) {

        strcpy(tmp, cYEL);

      } else

        /* No finds for a long time and no test cases to try. */
-        if (afl->cycles_wo_finds > 100 && !afl->pending_not_fuzzed &&
+        if (afl->cycles_wo_finds > 1 && !afl->pending_not_fuzzed &&
            min_wo_finds > 120) {

          strcpy(tmp, cLGN);
@ -1656,15 +1649,6 @@ void show_stats_pizza(afl_state_t *afl) {

  }

-  /* Honor AFL_EXIT_WHEN_DONE and AFL_BENCH_UNTIL_CRASH. */
-
-  if (unlikely(!afl->non_instrumented_mode && afl->cycles_wo_finds > 100 &&
-               !afl->pending_not_fuzzed && afl->afl_env.afl_exit_when_done)) {
-
-    afl->stop_soon = 2;
-
-  }
-
  /* AFL_EXIT_ON_TIME. */

  /* If no coverage was found yet, check whether run time is greater than
@ -1813,14 +1797,14 @@ void show_stats_pizza(afl_state_t *afl) {
    } else

      /* Subsequent cycles, but we're still making finds. */
-      if (afl->cycles_wo_finds < 25 || min_wo_finds < 30) {
+      if (afl->cycles_wo_finds < 2 || min_wo_finds <= 30) {

        strcpy(tmp, cYEL);

      } else

        /* No finds for a long time and no test cases to try. */
-        if (afl->cycles_wo_finds > 100 && !afl->pending_not_fuzzed &&
+        if (afl->cycles_wo_finds > 1 && !afl->pending_not_fuzzed &&
            min_wo_finds > 120) {

          strcpy(tmp, cLGN);
--- a/src/afl-fuzz.c
+++ b/src/afl-fuzz.c
@ -2937,10 +2937,10 @@ int main(int argc, char **argv_orig, char **envp) {
                      3600 */
                   )) {

-        if (afl->use_splicing) {
-
        ++afl->cycles_wo_finds;

+        if (afl->use_splicing) {
+
          if (unlikely(afl->shm.cmplog_mode &&
                       afl->cmplog_max_filesize < MAX_FILE)) {