afl-cmin -T support

afl-cmin

@@ -103,9 +103,10 @@ function usage() {
 "  -o dir        - output directory for minimized files\n" \
 "\n" \
 "Execution control settings:\n" \
+"  -T tasks      - how many parallel tasks to run (default: 1, all=nproc)\n" \
 "  -f file       - location read by the fuzzed program (stdin)\n" \
 "  -m megs       - memory limit for child process ("mem_limit" MB)\n" \
-"  -t msec       - run time limit for child process (default: none)\n" \
+"  -t msec       - run time limit for child process (default: 5000)\n" \
 "  -O            - use binary-only instrumentation (FRIDA mode)\n" \
 "  -Q            - use binary-only instrumentation (QEMU mode)\n" \
 "  -U            - use unicorn-based instrumentation (unicorn mode)\n" \
@@ -119,7 +120,6 @@ function usage() {
 "For additional tips, please consult README.md\n" \
 "\n" \
 "Environment variables used:\n" \
-"AFL_ALLOW_TMP: allow unsafe use of input/output directories under {/var}/tmp\n" \
 "AFL_CRASH_EXITCODE: optional child exit code to be interpreted as crash\n" \
 "AFL_FORKSRV_INIT_TMOUT: time the fuzzer waits for the forkserver to come up\n" \
 "AFL_KEEP_TRACES: leave the temporary <out_dir>/.traces directory\n" \
@@ -159,13 +159,19 @@ BEGIN {
   # process options
   Opterr = 1    # default is to diagnose
   Optind = 1    # skip ARGV[0]
-  while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUXY?")) != -1) {
+  while ((_go_c = getopt(ARGC, ARGV, "hi:o:f:m:t:eACOQUXYT:?")) != -1) {
     if (_go_c == "i") {
       if (!Optarg) usage()
       if (in_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"}
       in_dir = Optarg
       continue
     } else 
+    if (_go_c == "T") {
+      if (!Optarg) usage()
+      if (threads) { print "Option "_go_c" is only allowed once" > "/dev/stderr"}
+      threads = Optarg
+      continue
+    } else 
     if (_go_c == "o") {
       if (!Optarg) usage()
       if (out_dir) { print "Option "_go_c" is only allowed once" > "/dev/stderr"}
@@ -253,21 +259,30 @@ BEGIN {
   # Do a sanity check to discourage the use of /tmp, since we can't really
   # handle this safely from an awk script.
 
-  if (!ENVIRON["AFL_ALLOW_TMP"]) {
+  #if (!ENVIRON["AFL_ALLOW_TMP"]) {
-    dirlist[0] = in_dir
+  #  dirlist[0] = in_dir
-    dirlist[1] = target_bin
+  #  dirlist[1] = target_bin
-    dirlist[2] = out_dir
+  #  dirlist[2] = out_dir
-    dirlist[3] = stdin_file
+  #  dirlist[3] = stdin_file
-    "pwd" | getline dirlist[4] # current directory
+  #  "pwd" | getline dirlist[4] # current directory
-    for (dirind in dirlist) {
+  #  for (dirind in dirlist) {
-      dir = dirlist[dirind]
+  #    dir = dirlist[dirind]
+  #
+  #      if (dir ~ /^(\/var)?\/tmp/) {
+  #        print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr"
+  #        exit 1
+  #      }
+  #    }
+  #  delete dirlist
+  #}
 
-      if (dir ~ /^(\/var)?\/tmp/) {
+  if (threads && stdin_file) {
-        print "[-] Error: do not use this script in /tmp or /var/tmp." > "/dev/stderr"
+    print "[-] Error: -T and -f cannot be used together." > "/dev/stderr"
-        exit 1
+    exit 1
-      }
+  }
-    }
+
-    delete dirlist
+  if (!threads && !stdin_file) {
+    print "[*] Are you aware of the '-T all' parallelize option that massively improves the speed for large corpuses?"
   }
 
   # If @@ is specified, but there's no -f, let's come up with a temporary input
@@ -350,6 +365,18 @@ BEGIN {
     exit 1
   }
 
+  if (threads) {
+    "nproc" | getline nproc
+    if (threads == "all") {
+      threads = nproc
+    } else {
+      if (!(threads > 1 && threads <= nproc)) {
+        print "[-] Error: -T option must be between 1 and "nproc" or \"all\"." > "/dev/stderr"
+        exit 1
+      }
+    }
+  }
+
   # Check for the more efficient way to copy files...
   if (0 != system("mkdir -p -m 0700 "trace_dir)) {
     print "[-] Error: Cannot create directory "trace_dir > "/dev/stderr"
@@ -459,27 +486,81 @@ BEGIN {
   # STEP 1: Collecting traces #
   #############################
 
+  if (threads) {
+
+    inputsperfile = in_count / threads
+    if (in_count % threads) {
+      inputsperfile++;
+    }
+
+    cnt = 0;
+    tmpfile=out_dir "/.filelist"
+    for (instance = 1; instance < threads; instance++) {
+      for (i = 0; i < inputsperfile; i++) {
+        print in_dir"/"infilesSmallToBigFull[cnt] >> tmpfile"."instance
+        cnt++
+      }
+    }
+    for (; cnt < in_count; cnt++) {
+      print in_dir"/"infilesSmallToBigFull[cnt] >> tmpfile"."threads
+    }
+
+    print "ls -l "tmpfile"*"
+
+  }
+
   print "[*] Obtaining traces for "in_count" input files in '"in_dir"'."
 
   cur = 0;
-  if (!stdin_file) {
-    print "    Processing "in_count" files (forkserver mode)..."
-#    print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string
-    retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string)
-  } else {
-    print "    Processing "in_count" files (forkserver mode)..."
-#    print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null"
-    retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
-  }
 
-  if (retval && (!AFL_CMIN_CRASHES_ONLY && !AFL_CMIN_ALLOW_ANY)) {
+  if (threads > 1) {
-    print "[!] Exit code "retval" != 0 received from afl-showmap (this means a crashing or timeout input is likely present), terminating..."
 
-    if (!ENVIRON["AFL_KEEP_TRACES"]) {
+    print "[*] Creating " threads " parallel tasks with about " inputsperfile " each."
-      system("rm -rf "trace_dir" 2>/dev/null")
+    for (i = 1; i <= threads; i++) {
-      system("rmdir "out_dir)
+
+      if (!stdin_file) {
+#        print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &"
+        retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -- \""target_bin"\" "prog_args_string"; > "tmpfile"."i".done ; } &")
+      } else {
+        stdin_file=tmpfile"."i".stdin"
+#        print " { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &"
+        retval = system(" { "AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -I \""tmpfile"."i"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null; > "tmpfile"."i".done ; } &")
+      }
     }
-    exit retval
+    print "[*] Waiting for parallel tasks to complete ..."
+    # wait for all processes to finish
+    ok=0
+    while (ok < threads) {
+      ok=0
+      for (i = 1; i <= threads; i++) {
+        if (system("test -f "tmpfile"."i".done") == 0) {
+          ok++
+        }
+      }
+    }
+    print "[*] Done!"
+    system("rm -f "tmpfile"*")
+  } else {
+    if (!stdin_file) {
+      print "    Processing "in_count" files (forkserver mode)..."
+#      print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string
+      retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -- \""target_bin"\" "prog_args_string)
+    } else {
+      print "    Processing "in_count" files (forkserver mode)..."
+#    print AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null"
+      retval = system(AFL_MAP_SIZE AFL_CMIN_ALLOW_ANY AFL_CMIN_CRASHES_ONLY"\""showmap"\" -m "mem_limit" -t "timeout" -o \""trace_dir"\" -Z "extra_par" -i \""in_dir"\" -H \""stdin_file"\" -- \""target_bin"\" "prog_args_string" </dev/null")
+    }
+
+    if (retval && (!AFL_CMIN_CRASHES_ONLY && !AFL_CMIN_ALLOW_ANY)) {
+      print "[!] Exit code "retval" != 0 received from afl-showmap (this means a crashing or timeout input is likely present), terminating..."
+
+      if (!ENVIRON["AFL_KEEP_TRACES"]) {
+        system("rm -rf "trace_dir" 2>/dev/null")
+        system("rmdir "out_dir)
+      }
+      exit retval
+    }
+
   }
 
   #######################################################

docs/Changelog.md

@@ -14,7 +14,7 @@
   - afl-showmap:
     - added custom mutator post_process and send support
     - add `-I filelist` option, an alternative to `-i in_dir`
-  - afl-cmin.bash:
+  - afl-cmin + afl-cmin.bash:
     - `-T threads` parallel task support, huge speedup!
   - a new grammar custom mutator atnwalk was submitted by @voidptr127 !
 

src/afl-showmap.c

@@ -894,9 +894,7 @@ u32 execute_testcases_filelist(u8 *fn) {
   while (fgets(buf, sizeof(buf), f) != NULL) {
 
     struct stat st;
-
+    u8         *fn2 = buf, *fn3;
-    u8 *fn2 = buf, *fn3;
-    ;
 
     while (*fn2 == ' ') {
 
@@ -904,14 +902,11 @@ u32 execute_testcases_filelist(u8 *fn) {
 
     }
 
-    if (*fn2) {
+    while (*fn2 &&
+           (fn2[strlen(fn2) - 1] == '\r' || fn2[strlen(fn2) - 1] == '\n' ||
+            fn2[strlen(fn2) - 1] == ' ')) {
 
-      while (fn2[strlen(fn2) - 1] == '\r' || fn2[strlen(fn2) - 1] == '\n' ||
+      fn2[strlen(fn2) - 1] = 0;
-             fn2[strlen(fn2) - 1] == ' ') {
-
-        fn2[strlen(fn2) - 1] = 0;
-
-      }
 
     }
 
@@ -926,6 +921,8 @@ u32 execute_testcases_filelist(u8 *fn) {
 
     }
 
+    ++done;
+
     if (!S_ISREG(st.st_mode) || !st.st_size) { continue; }
 
     if ((fn3 = strrchr(fn2, '/'))) {
@@ -946,9 +943,12 @@ u32 execute_testcases_filelist(u8 *fn) {
 
     }
 
-    if (!collect_coverage)
+    if (!collect_coverage) {
+
       snprintf(outfile, sizeof(outfile), "%s/%s", out_file, fn3);
 
+    }
+
     if (read_file(fn2)) {
 
       if (wait_for_gdb) {
@@ -961,7 +961,6 @@ u32 execute_testcases_filelist(u8 *fn) {
 
       showmap_run_target_forkserver(fsrv, in_data, in_len);
       ck_free(in_data);
-      ++done;
 
       if (child_crashed && debug) { WARNF("crashed: %s", fn2); }