Merge pull request #1860 from AFLplusplus/dev

push to stable

TODO.md

@@ -2,8 +2,9 @@
 
 ## Should
 
+ - afl-showmap -f support
+ - afl-fuzz multicore wrapper script
  - afl-crash-analysis
- - test cmplog for less than 16bit
  - support persistent and deferred fork server in afl-showmap?
  - better autodetection of shifting runtime timeout values
  - Update afl->pending_not_fuzzed for MOpt

docs/fuzzing_in_depth.md

@@ -599,32 +599,40 @@ during fuzzing) and their number, a value between 50-500MB is recommended. You
 can set the cache size (in MB) by setting the environment variable
 `AFL_TESTCACHE_SIZE`.
 
-There should be one main fuzzer (`-M main-$HOSTNAME` option) and as many
-secondary fuzzers (e.g., `-S variant1`) as you have cores that you use. Every
-`-M`/`-S` entry needs a unique name (that can be whatever), however, the same
-`-o` output directory location has to be used for all instances.
+There should be one main fuzzer (`-M main-$HOSTNAME` option - set also
+`AFL_FINAL_SYNC=1`) and as many secondary fuzzers (e.g., `-S variant1`) as you
+have cores that you use. Every `-M`/`-S` entry needs a unique name (that can be
+whatever), however, the same `-o` output directory location has to be used for
+all instances.
 
 For every secondary fuzzer there should be a variation, e.g.:
-* one should fuzz the target that was compiled differently: with sanitizers
-  activated (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export
-  AFL_USE_CFISAN=1`)
+* one should fuzz the target that was compiled with sanitizers activated
+  (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export AFL_USE_CFISAN=1`)
 * one or two should fuzz the target with CMPLOG/redqueen (see above), at least
-  one cmplog instance should follow transformations (`-l AT`)
+  one cmplog instance should follow transformations (`-l 2AT`)
 * one to three fuzzers should fuzz a target compiled with laf-intel/COMPCOV (see
   above). Important note: If you run more than one laf-intel/COMPCOV fuzzer and
   you want them to share their intermediate results, the main fuzzer (`-M`) must
-  be one of them! (Although this is not really recommended.)
+  be one of them (although this is not really recommended).
 
-All other secondaries should be used like this:
-* 10-20% with the MOpt mutator enabled: `-L 0`
-* run with a different power schedule, recommended are: `fast` (default),
+The other secondaries should be run like this:
+* 10% with the MOpt mutator enabled: `-L 0`
+* 10% should use the old queue cycling with `-Z`
+* 50-70% should run with `AFL_DISABLE_TRIM`
+* 40% should run with `-P explore` and 20% with `-P exploit`
+* If you use `-a` then set 30% of the instances to not use `-a`; if you did
+  not set `-a` (why??), then set 30% to `-a ascii` and 30% to `-a binary`.
+* run each with a different power schedule, recommended are: `fast` (default),
   `explore`, `coe`, `lin`, `quad`, `exploit`, and `rare` which you can set with
   the `-p` option, e.g., `-p explore`. See the
   [FAQ](FAQ.md#what-are-power-schedules) for details.
-* a few instances should use the old queue cycling with `-Z`
+
+It can be useful to set `AFL_IGNORE_SEED_PROBLEMS=1` to skip over seeds that
+crash or timeout during startup.
 
 Also, it is recommended to set `export AFL_IMPORT_FIRST=1` to load test cases
-from other fuzzers in the campaign first.
+from other fuzzers in the campaign first. But note that can slow down the start
+of the first fuzz by quite a lot of you have many fuzzers and/or many seeds.
 
 If you have a large corpus, a corpus from a previous run or are fuzzing in a CI,
 then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`.

include/afl-fuzz.h

@@ -610,6 +610,7 @@ typedef struct afl_state {
 
   u32 stage_cur, stage_max;             /* Stage progression                */
   s32 splicing_with;                    /* Splicing with which test case?   */
+  s64 smallest_favored;                 /* smallest queue id favored        */
 
   u32 main_node_id, main_node_max;      /*   Main instance job splitting    */
 

src/afl-fuzz-init.c

@@ -942,6 +942,7 @@ void perform_dry_run(afl_state_t *afl) {
           if (!q->was_fuzzed) {
 
             q->was_fuzzed = 1;
+            afl->reinit_table = 1;
             --afl->pending_not_fuzzed;
             --afl->active_items;
 
@@ -982,6 +983,7 @@ void perform_dry_run(afl_state_t *afl) {
           if (!q->was_fuzzed) {
 
             q->was_fuzzed = 1;
+            afl->reinit_table = 1;
             --afl->pending_not_fuzzed;
             --afl->active_items;
 
@@ -1113,6 +1115,7 @@ void perform_dry_run(afl_state_t *afl) {
         if (!q->was_fuzzed) {
 
           q->was_fuzzed = 1;
+          afl->reinit_table = 1;
           --afl->pending_not_fuzzed;
           --afl->active_items;
 
@@ -1291,6 +1294,7 @@ void perform_dry_run(afl_state_t *afl) {
           if (!p->was_fuzzed) {
 
             p->was_fuzzed = 1;
+            afl->reinit_table = 1;
             --afl->pending_not_fuzzed;
             --afl->active_items;
 
@@ -1311,6 +1315,7 @@ void perform_dry_run(afl_state_t *afl) {
           if (!q->was_fuzzed) {
 
             q->was_fuzzed = 1;
+            afl->reinit_table = 1;
             --afl->pending_not_fuzzed;
             --afl->active_items;
 

src/afl-fuzz-one.c

@@ -3442,7 +3442,12 @@ abandon_entry:
     --afl->pending_not_fuzzed;
     afl->queue_cur->was_fuzzed = 1;
     afl->reinit_table = 1;
-    if (afl->queue_cur->favored) { --afl->pending_favored; }
+    if (afl->queue_cur->favored) {
+
+      --afl->pending_favored;
+      afl->smallest_favored = -1;
+
+    }
 
   }
 
@@ -5905,7 +5910,13 @@ pacemaker_fuzzing:
 
             --afl->pending_not_fuzzed;
             afl->queue_cur->was_fuzzed = 1;
-            if (afl->queue_cur->favored) { --afl->pending_favored; }
+            afl->reinit_table = 1
+            if (afl->queue_cur->favored) {
+
+              --afl->pending_favored;
+              afl->smallest_favored = -1;
+
+            }
 
           }
 

src/afl-fuzz-queue.c

@@ -80,6 +80,7 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q,
   if (unlikely(weight < 0.1)) { weight = 0.1; }
   if (unlikely(q->favored)) { weight *= 5; }
   if (unlikely(!q->was_fuzzed)) { weight *= 2; }
+  if (unlikely(q->fs_redundant)) { weight *= 0.8; }
 
   return weight;
 
@@ -830,6 +831,8 @@ void cull_queue(afl_state_t *afl) {
   /* Let's see if anything in the bitmap isn't captured in temp_v.
      If yes, and if it has a afl->top_rated[] contender, let's use it. */
 
+  afl->smallest_favored = -1;
+
   for (i = 0; i < afl->fsrv.map_size; ++i) {
 
     if (afl->top_rated[i] && (temp_v[i >> 3] & (1 << (i & 7)))) {
@@ -853,7 +856,16 @@ void cull_queue(afl_state_t *afl) {
         afl->top_rated[i]->favored = 1;
         ++afl->queued_favored;
 
-        if (!afl->top_rated[i]->was_fuzzed) { ++afl->pending_favored; }
+        if (!afl->top_rated[i]->was_fuzzed) {
+
+          ++afl->pending_favored;
+          if (unlikely(afl->smallest_favored < 0)) {
+
+            afl->smallest_favored = (s64)afl->top_rated[i]->id;
+
+          }
+
+        }
 
       }
 
@@ -871,6 +883,8 @@ void cull_queue(afl_state_t *afl) {
 
   }
 
+  afl->reinit_table = 1;
+
 }
 
 /* Calculate case desirability score to adjust the length of havoc fuzzing.

src/afl-fuzz.c

@@ -2369,7 +2369,7 @@ int main(int argc, char **argv_orig, char **envp) {
 
   } else {
 
-    ACTF("skipping initial seed calibration due option override");
+    ACTF("skipping initial seed calibration due option override!");
     usleep(1000);
 
   }
@@ -2707,23 +2707,53 @@ int main(int argc, char **argv_orig, char **envp) {
 
       if (likely(!afl->old_seed_selection)) {
 
-        if (unlikely(prev_queued_items < afl->queued_items ||
-                     afl->reinit_table)) {
+        if (likely(afl->pending_favored && afl->smallest_favored >= 0)) {
 
-          // we have new queue entries since the last run, recreate alias table
-          prev_queued_items = afl->queued_items;
-          create_alias_table(afl);
+          afl->current_entry = afl->smallest_favored;
+
+          /*
+
+                    } else {
+
+                      for (s32 iter = afl->queued_items - 1; iter >= 0; --iter)
+             {
+
+                        if (unlikely(afl->queue_buf[iter]->favored &&
+                                     !afl->queue_buf[iter]->was_fuzzed)) {
+
+                          afl->current_entry = iter;
+                          break;
+
+                        }
+
+                      }
+
+          */
+
+          afl->queue_cur = afl->queue_buf[afl->current_entry];
+
+        } else {
+
+          if (unlikely(prev_queued_items < afl->queued_items ||
+                       afl->reinit_table)) {
+
+            // we have new queue entries since the last run, recreate alias
+            // table
+            prev_queued_items = afl->queued_items;
+            create_alias_table(afl);
+
+          }
+
+          do {
+
+            afl->current_entry = select_next_queue_entry(afl);
+
+          } while (unlikely(afl->current_entry >= afl->queued_items));
+
+          afl->queue_cur = afl->queue_buf[afl->current_entry];
 
         }
 
-        do {
-
-          afl->current_entry = select_next_queue_entry(afl);
-
-        } while (unlikely(afl->current_entry >= afl->queued_items));
-
-        afl->queue_cur = afl->queue_buf[afl->current_entry];
-
       }
 
       skipped_fuzz = fuzz_one(afl);