From 78848f863767cee6543166bd52d67e0051641360 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Thu, 31 Aug 2023 09:25:41 +0200 Subject: [PATCH 1/7] pendfav --- src/afl-fuzz.c | 46 +++++++++++++++++++++++++++++++++------------- 1 file changed, 33 insertions(+), 13 deletions(-) diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 90c255e3..c8cc7da6 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2707,23 +2707,43 @@ int main(int argc, char **argv_orig, char **envp) { if (likely(!afl->old_seed_selection)) { - if (unlikely(prev_queued_items < afl->queued_items || - afl->reinit_table)) { + if (likely(afl->pending_favored)) { - // we have new queue entries since the last run, recreate alias table - prev_queued_items = afl->queued_items; - create_alias_table(afl); + for (u32 iter = 0; iter < afl->queued_items; ++iter) { + + if (unlikely(afl->queue_buf[iter]->favored && + !afl->queue_buf[iter]->was_fuzzed)) { + + afl->current_entry = iter; + afl->queue_cur = afl->queue_buf[afl->current_entry]; + break; + + } + + } + + } else { + + if (unlikely(prev_queued_items < afl->queued_items || + afl->reinit_table)) { + + // we have new queue entries since the last run, recreate alias + // table + prev_queued_items = afl->queued_items; + create_alias_table(afl); + + } + + do { + + afl->current_entry = select_next_queue_entry(afl); + + } while (unlikely(afl->current_entry >= afl->queued_items)); + + afl->queue_cur = afl->queue_buf[afl->current_entry]; } - do { - - afl->current_entry = select_next_queue_entry(afl); - - } while (unlikely(afl->current_entry >= afl->queued_items)); - - afl->queue_cur = afl->queue_buf[afl->current_entry]; - } skipped_fuzz = fuzz_one(afl); From a809c3c50ce32fde390769b607b020dd68730474 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 3 Sep 2023 11:22:54 +0200 Subject: [PATCH 2/7] less impact --- frida_mode/src/instrument/instrument_arm64.c | 14 ++++------ include/afl-fuzz.h | 1 + src/afl-fuzz-one.c | 10 +++++-- src/afl-fuzz-queue.c | 13 ++++++++- src/afl-fuzz.c | 28 +++++++++++++------- 5 files changed, 45 insertions(+), 21 deletions(-) diff --git a/frida_mode/src/instrument/instrument_arm64.c b/frida_mode/src/instrument/instrument_arm64.c index a0c66697..1147275f 100644 --- a/frida_mode/src/instrument/instrument_arm64.c +++ b/frida_mode/src/instrument/instrument_arm64.c @@ -402,17 +402,13 @@ bool instrument_write_inline(GumArm64Writer *cw, GumAddress code_addr, } - /* - * The mov instruction supports up to a 16-bit offset. If our offset is out of - * range, then it can end up clobbering the op-code portion of the instruction - * rather than just the operands. So return false and fall back to the + /* + * The mov instruction supports up to a 16-bit offset. If our offset is out of + * range, then it can end up clobbering the op-code portion of the instruction + * rather than just the operands. So return false and fall back to the * alternative instrumentation. */ - if (area_offset > UINT16_MAX) { - - return false; - - } + if (area_offset > UINT16_MAX) { return false; } code.code.mov_x0_curr_loc |= area_offset << 5; diff --git a/include/afl-fuzz.h b/include/afl-fuzz.h index d02e852e..217a720a 100644 --- a/include/afl-fuzz.h +++ b/include/afl-fuzz.h @@ -610,6 +610,7 @@ typedef struct afl_state { u32 stage_cur, stage_max; /* Stage progression */ s32 splicing_with; /* Splicing with which test case? */ + s64 smallest_favored; /* smallest queue id favored */ u32 main_node_id, main_node_max; /* Main instance job splitting */ diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 2ad4697e..ae39abe8 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -3442,7 +3442,12 @@ abandon_entry: --afl->pending_not_fuzzed; afl->queue_cur->was_fuzzed = 1; afl->reinit_table = 1; - if (afl->queue_cur->favored) { --afl->pending_favored; } + if (afl->queue_cur->favored) { + + --afl->pending_favored; + afl->smallest_favored = -1; + + } } @@ -5905,7 +5910,8 @@ pacemaker_fuzzing: --afl->pending_not_fuzzed; afl->queue_cur->was_fuzzed = 1; - if (afl->queue_cur->favored) { --afl->pending_favored; } + if (afl->queue_cur->favored) { --afl->pending_favored; + afl->smallest_favored = -1; } } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 14ba1ace..5f915c9a 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -826,6 +826,8 @@ void cull_queue(afl_state_t *afl) { /* Let's see if anything in the bitmap isn't captured in temp_v. If yes, and if it has a afl->top_rated[] contender, let's use it. */ + afl->smallest_favored = -1; + for (i = 0; i < afl->fsrv.map_size; ++i) { if (afl->top_rated[i] && (temp_v[i >> 3] & (1 << (i & 7)))) { @@ -849,7 +851,16 @@ void cull_queue(afl_state_t *afl) { afl->top_rated[i]->favored = 1; ++afl->queued_favored; - if (!afl->top_rated[i]->was_fuzzed) { ++afl->pending_favored; } + if (!afl->top_rated[i]->was_fuzzed) { + + ++afl->pending_favored; + if (unlikely(afl->smallest_favored > (s64)afl->top_rated[i]->id)) { + + afl->smallest_favored = (s64)afl->top_rated[i]->id; + + } + + } } diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index c8cc7da6..d34b52db 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2707,20 +2707,30 @@ int main(int argc, char **argv_orig, char **envp) { if (likely(!afl->old_seed_selection)) { - if (likely(afl->pending_favored)) { + if (likely(afl->pending_favored && afl->smallest_favored >= 0)) { - for (u32 iter = 0; iter < afl->queued_items; ++iter) { + afl->current_entry = afl->smallest_favored; - if (unlikely(afl->queue_buf[iter]->favored && - !afl->queue_buf[iter]->was_fuzzed)) { + /* - afl->current_entry = iter; - afl->queue_cur = afl->queue_buf[afl->current_entry]; - break; + } else { - } + for (s32 iter = afl->queued_items - 1; iter >= 0; --iter) + { - } + if (unlikely(afl->queue_buf[iter]->favored && + !afl->queue_buf[iter]->was_fuzzed)) { + + afl->current_entry = iter; + break; + + } + + } + + */ + + afl->queue_cur = afl->queue_buf[afl->current_entry]; } else { From cd6b89eb74cc501a67b1c14a4433a496b2053eec Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Sun, 3 Sep 2023 13:54:54 +0200 Subject: [PATCH 3/7] nit --- src/afl-fuzz-queue.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 5f915c9a..2b102879 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -854,7 +854,7 @@ void cull_queue(afl_state_t *afl) { if (!afl->top_rated[i]->was_fuzzed) { ++afl->pending_favored; - if (unlikely(afl->smallest_favored > (s64)afl->top_rated[i]->id)) { + if (unlikely(afl->smallest_favored < 0)) { afl->smallest_favored = (s64)afl->top_rated[i]->id; From 87b33740ea426bac276a9eb4bc5f201bd396b6dc Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 4 Sep 2023 13:38:09 +0200 Subject: [PATCH 4/7] ensure table reinit, downgrade redundant --- src/afl-fuzz-init.c | 5 +++++ src/afl-fuzz-one.c | 1 + src/afl-fuzz-queue.c | 3 +++ src/afl-fuzz.c | 2 +- 4 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/afl-fuzz-init.c b/src/afl-fuzz-init.c index 9fc0cc57..35932913 100644 --- a/src/afl-fuzz-init.c +++ b/src/afl-fuzz-init.c @@ -942,6 +942,7 @@ void perform_dry_run(afl_state_t *afl) { if (!q->was_fuzzed) { q->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; @@ -982,6 +983,7 @@ void perform_dry_run(afl_state_t *afl) { if (!q->was_fuzzed) { q->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; @@ -1113,6 +1115,7 @@ void perform_dry_run(afl_state_t *afl) { if (!q->was_fuzzed) { q->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; @@ -1291,6 +1294,7 @@ void perform_dry_run(afl_state_t *afl) { if (!p->was_fuzzed) { p->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; @@ -1311,6 +1315,7 @@ void perform_dry_run(afl_state_t *afl) { if (!q->was_fuzzed) { q->was_fuzzed = 1; + afl->reinit_table = 1; --afl->pending_not_fuzzed; --afl->active_items; diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index 2ad4697e..c2b7e583 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -5905,6 +5905,7 @@ pacemaker_fuzzing: --afl->pending_not_fuzzed; afl->queue_cur->was_fuzzed = 1; + afl->reinit_table = 1 if (afl->queue_cur->favored) { --afl->pending_favored; } } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index 14ba1ace..78c1d654 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -80,6 +80,7 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, if (unlikely(weight < 0.1)) { weight = 0.1; } if (unlikely(q->favored)) { weight *= 5; } if (unlikely(!q->was_fuzzed)) { weight *= 2; } + if (unlikely(q->fs_redundant)) { weight *= 0.2; } return weight; @@ -867,6 +868,8 @@ void cull_queue(afl_state_t *afl) { } + afl->reinit_table = 1; + } /* Calculate case desirability score to adjust the length of havoc fuzzing. diff --git a/src/afl-fuzz.c b/src/afl-fuzz.c index 90c255e3..768a5bbd 100644 --- a/src/afl-fuzz.c +++ b/src/afl-fuzz.c @@ -2369,7 +2369,7 @@ int main(int argc, char **argv_orig, char **envp) { } else { - ACTF("skipping initial seed calibration due option override"); + ACTF("skipping initial seed calibration due option override!"); usleep(1000); } From 6c7c4b821adae1ed244973127426ef2e966f6350 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Mon, 11 Sep 2023 14:56:34 +0200 Subject: [PATCH 5/7] reweight redundant --- src/afl-fuzz-one.c | 3 +++ src/afl-fuzz-queue.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/src/afl-fuzz-one.c b/src/afl-fuzz-one.c index db88f239..2003be1f 100644 --- a/src/afl-fuzz-one.c +++ b/src/afl-fuzz-one.c @@ -5912,9 +5912,12 @@ pacemaker_fuzzing: afl->queue_cur->was_fuzzed = 1; afl->reinit_table = 1 if (afl->queue_cur->favored) { + --afl->pending_favored; afl->smallest_favored = -1; + } + } } diff --git a/src/afl-fuzz-queue.c b/src/afl-fuzz-queue.c index ce9718b0..4b9627f7 100644 --- a/src/afl-fuzz-queue.c +++ b/src/afl-fuzz-queue.c @@ -80,7 +80,7 @@ double compute_weight(afl_state_t *afl, struct queue_entry *q, if (unlikely(weight < 0.1)) { weight = 0.1; } if (unlikely(q->favored)) { weight *= 5; } if (unlikely(!q->was_fuzzed)) { weight *= 2; } - if (unlikely(q->fs_redundant)) { weight *= 0.2; } + if (unlikely(q->fs_redundant)) { weight *= 0.8; } return weight; From 19c387a824d8e5c34b2babb20bab1e47fa152649 Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 12 Sep 2023 09:54:05 +0200 Subject: [PATCH 6/7] update multicore recommendation --- docs/fuzzing_in_depth.md | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/docs/fuzzing_in_depth.md b/docs/fuzzing_in_depth.md index 5a5acbb2..6a217641 100644 --- a/docs/fuzzing_in_depth.md +++ b/docs/fuzzing_in_depth.md @@ -599,32 +599,40 @@ during fuzzing) and their number, a value between 50-500MB is recommended. You can set the cache size (in MB) by setting the environment variable `AFL_TESTCACHE_SIZE`. -There should be one main fuzzer (`-M main-$HOSTNAME` option) and as many -secondary fuzzers (e.g., `-S variant1`) as you have cores that you use. Every -`-M`/`-S` entry needs a unique name (that can be whatever), however, the same -`-o` output directory location has to be used for all instances. +There should be one main fuzzer (`-M main-$HOSTNAME` option - set also +`AFL_FINAL_SYNC=1`) and as many secondary fuzzers (e.g., `-S variant1`) as you +have cores that you use. Every `-M`/`-S` entry needs a unique name (that can be +whatever), however, the same `-o` output directory location has to be used for +all instances. For every secondary fuzzer there should be a variation, e.g.: -* one should fuzz the target that was compiled differently: with sanitizers - activated (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export - AFL_USE_CFISAN=1`) +* one should fuzz the target that was compiled with sanitizers activated + (`export AFL_USE_ASAN=1 ; export AFL_USE_UBSAN=1 ; export AFL_USE_CFISAN=1`) * one or two should fuzz the target with CMPLOG/redqueen (see above), at least - one cmplog instance should follow transformations (`-l AT`) + one cmplog instance should follow transformations (`-l 2AT`) * one to three fuzzers should fuzz a target compiled with laf-intel/COMPCOV (see above). Important note: If you run more than one laf-intel/COMPCOV fuzzer and you want them to share their intermediate results, the main fuzzer (`-M`) must - be one of them! (Although this is not really recommended.) + be one of them (although this is not really recommended). -All other secondaries should be used like this: -* 10-20% with the MOpt mutator enabled: `-L 0` -* run with a different power schedule, recommended are: `fast` (default), +The other secondaries should be run like this: +* 10% with the MOpt mutator enabled: `-L 0` +* 10% should use the old queue cycling with `-Z` +* 50-70% should run with `AFL_DISABLE_TRIM` +* 40% should run with `-P explore` and 20% with `-P exploit` +* If you use `-a` then set 30% of the instances to not use `-a`; if you did + not set `-a` (why??), then set 30% to `-a ascii` and 30% to `-a binary`. +* run each with a different power schedule, recommended are: `fast` (default), `explore`, `coe`, `lin`, `quad`, `exploit`, and `rare` which you can set with the `-p` option, e.g., `-p explore`. See the [FAQ](FAQ.md#what-are-power-schedules) for details. -* a few instances should use the old queue cycling with `-Z` + +It can be useful to set `AFL_IGNORE_SEED_PROBLEMS=1` to skip over seeds that +crash or timeout during startup. Also, it is recommended to set `export AFL_IMPORT_FIRST=1` to load test cases -from other fuzzers in the campaign first. +from other fuzzers in the campaign first. But note that can slow down the start +of the first fuzz by quite a lot of you have many fuzzers and/or many seeds. If you have a large corpus, a corpus from a previous run or are fuzzing in a CI, then also set `export AFL_CMPLOG_ONLY_NEW=1` and `export AFL_FAST_CAL=1`. From 4f4ce24690c682af500ee235c57055d87dfb9c9d Mon Sep 17 00:00:00 2001 From: vanhauser-thc Date: Tue, 12 Sep 2023 10:26:52 +0200 Subject: [PATCH 7/7] update todo --- TODO.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/TODO.md b/TODO.md index 7968452e..eb934e3f 100644 --- a/TODO.md +++ b/TODO.md @@ -2,8 +2,9 @@ ## Should + - afl-showmap -f support + - afl-fuzz multicore wrapper script - afl-crash-analysis - - test cmplog for less than 16bit - support persistent and deferred fork server in afl-showmap? - better autodetection of shifting runtime timeout values - Update afl->pending_not_fuzzed for MOpt