Add a test case for the custom mutator

- Update the Makefile in examples/custom_mutators
- Add a test program for testing the custom mutator
- Update test.sh for testing the custom mutator
- [TODO] Update the result checking criterias of the custom mutator in
test.sh

examples/custom_mutators/Makefile

@@ -1,2 +1,7 @@
-all:
+all: libexamplemutator.so
+
+libexamplemutator.so:
 	$(CC) $(CFLAGS) -fPIC -shared -g -I ../../include example.c -o libexamplemutator.so
+
+clean:
+	rm -rf libexamplemutator.so

test/test-custom-mutator.c

@@ -0,0 +1,20 @@
+/**
+ * Reference: https://github.com/bruce30262/libprotobuf-mutator_fuzzing_learning/blob/master/4_libprotobuf_aflpp_custom_mutator/vuln.c
+ */
+
+#include <stdio.h>
+#include <string.h>
+#include <math.h>
+#include <stdlib.h>
+#include <unistd.h>
+
+int main(int argc, char *argv[])
+{
+    char str[100]={ };
+    read(0, str, 100);
+    int *ptr = NULL;
+    if( str[0] == 'P') {
+        *ptr = 123;
+    }
+    return 0;
+}

test/test.sh

@@ -60,8 +60,8 @@ unset AFL_QEMU_PERSISTENT_GPR
 unset AFL_QEMU_PERSISTENT_RET
 unset AFL_QEMU_PERSISTENT_HOOK
 unset AFL_QEMU_PERSISTENT_CNT
-unset AFL_POST_LIBRARY
+
-unset AFL_CUSTOM_MUTATOR_LIBRARY
+export unset AFL_CUSTOM_MUTATOR_LIBRARY=unset AFL_POST_LIBRARY
 unset AFL_PYTHON_MODULE
 unset AFL_PRELOAD
 unset LD_PRELOAD
@@ -73,7 +73,7 @@ export ASAN_OPTIONS=detect_leaks=0:allocator_may_return_null=1:abort_on_error=1:
 # on OpenBSD we need to work with llvm from /usr/local/bin
 test -e /usr/local/bin/opt && {
   export PATH=/usr/local/bin:${PATH}
-} 
+}
 # on MacOS X we prefer afl-clang over afl-gcc, because
 # afl-gcc does not work there
 test `uname -s` = 'Darwin' -o `uname -s` = 'FreeBSD' && {
@@ -142,11 +142,11 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc
       CODE=1
     }
     rm -f test-compcov.harden
-  } || { 
+  } || {
     $ECHO "$RED[!] ${AFL_GCC} hardened mode compilation failed"
     CODE=1
   }
-  # now we want to be sure that afl-fuzz is working  
+  # now we want to be sure that afl-fuzz is working
   # make sure core_pattern is set to core on linux
   (test "$(uname -s)" = "Linux" && test "$(sysctl kernel.core_pattern)" != "kernel.core_pattern = core" && {
     $ECHO "$YELLOW[-] we should not run afl-fuzz with enabled core dumps. Run 'sudo sh afl-system-config'.$RESET"
@@ -210,13 +210,13 @@ test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc
     unset AFL_QUIET
   }
   rm -f test-instr.plain
- } || { 
+ } || {
   $ECHO "$YELLOW[-] afl is not compiled, cannot test"
   INCOMPLETE=1
  }
-} || { 
+} || {
  $ECHO "$YELLOW[-] not an intel platform, cannot test afl-gcc"
-} 
+}
 
 $ECHO "$BLUE[*] Testing: llvm_mode, afl-showmap, afl-fuzz, afl-cmin and afl-tmin"
 test -e ../afl-clang-fast -a -e ../split-switches-pass.so && {
@@ -248,7 +248,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && {
           CODE=1
         }
       }
-    } || { 
+    } || {
       $ECHO "$RED[!] llvm_mode instrumentation failed"
       CODE=1
     }
@@ -265,11 +265,11 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && {
       CODE=1
     }
     rm -f test-compcov.harden
-  } || { 
+  } || {
     $ECHO "$RED[!] llvm_mode hardened mode compilation failed"
     CODE=1
   }
-  # now we want to be sure that afl-fuzz is working  
+  # now we want to be sure that afl-fuzz is working
   (test "$(uname -s)" = "Linux" && test "$(sysctl kernel.core_pattern)" != "kernel.core_pattern = core" && {
     $ECHO "$YELLOW[-] we should not run afl-fuzz with enabled core dumps. Run 'sudo sh afl-system-config'.$RESET"
     true
@@ -373,7 +373,7 @@ test -e ../afl-clang-fast -a -e ../split-switches-pass.so && {
       $ECHO "$RED[!] llvm_mode whitelist feature failed"
       CODE=1
     }
-  } || { 
+  } || {
     $ECHO "$RED[!] llvm_mode whitelist feature compilation failed"
     CODE=1
   }
@@ -426,7 +426,7 @@ test -e ../afl-clang-lto -a -e ../afl-llvm-lto-instrumentation.so && {
           CODE=1
         }
       }
-    } || { 
+    } || {
       $ECHO "$RED[!] llvm_mode LTO instrumentation failed"
       CODE=1
     }
@@ -447,7 +447,7 @@ test -e ../afl-clang-lto -a -e ../afl-llvm-lto-instrumentation.so && {
 #      $ECHO "$RED[!] llvm_mode LTO whitelist feature failed"
 #      CODE=1
 #    }
-#  } || { 
+#  } || {
 #    $ECHO "$RED[!] llvm_mode LTO whitelist feature compilation failed"
 #    CODE=1
 #  }
@@ -483,7 +483,7 @@ test -e ../afl-gcc-fast -a -e ../afl-gcc-rt.o && {
       diff test-instr.plain.0 test-instr.plain.1 > /dev/null 2>&1 && {
         $ECHO "$RED[!] gcc_plugin instrumentation should be different on different input but is not"
         CODE=1
-      } || { 
+      } || {
         $ECHO "$GREEN[+] gcc_plugin instrumentation present and working correctly"
         TUPLES=`echo 0|../afl-showmap -m ${MEM_LIMIT} -o /dev/null -- ./test-instr.plain.gccpi 2>&1 | grep Captur | awk '{print$3}'`
         test "$TUPLES" -gt 3 -a "$TUPLES" -lt 7 && {
@@ -516,7 +516,7 @@ test -e ../afl-gcc-fast -a -e ../afl-gcc-rt.o && {
     $ECHO "$RED[!] gcc_plugin hardened mode compilation failed"
     CODE=1
   }
-  # now we want to be sure that afl-fuzz is working  
+  # now we want to be sure that afl-fuzz is working
   (test "$(uname -s)" = "Linux" && test "$(sysctl kernel.core_pattern)" != "kernel.core_pattern = core" && {
     $ECHO "$YELLOW[-] we should not run afl-fuzz with enabled core dumps. Run 'sudo sh afl-system-config'.$RESET"
     true
@@ -552,11 +552,11 @@ test -e ../afl-gcc-fast -a -e ../afl-gcc-rt.o && {
   test -e test-compcov && {
     echo 1 | ../afl-showmap -m ${MEM_LIMIT} -o - -r -- ./test-compcov 2>&1 | grep -q "Captured 1 tuples" && {
       $ECHO "$GREEN[+] gcc_plugin whitelist feature works correctly"
-    } || { 
+    } || {
       $ECHO "$RED[!] gcc_plugin whitelist feature failed"
       CODE=1
     }
-  } || { 
+  } || {
     $ECHO "$RED[!] gcc_plugin whitelist feature compilation failed"
     CODE=1
   }
@@ -585,7 +585,7 @@ test -e ../libtokencap.so && {
   AFL_TOKEN_FILE=token.out LD_PRELOAD=../libtokencap.so DYLD_INSERT_LIBRARIES=../libtokencap.so DYLD_FORCE_FLAT_NAMESPACE=1 ./test-compcov foobar > /dev/null 2>&1
   grep -q BUGMENOT token.out > /dev/null 2>&1 && {
     $ECHO "$GREEN[+] libtokencap did successfully capture tokens"
-  } || { 
+  } || {
     $ECHO "$RED[!] libtokencap did not capture tokens"
     CODE=1
   }
@@ -604,7 +604,7 @@ test -e ../libdislocator.so && {
     $ECHO "$RED[!] libdislocator did not detect the memory corruption"
     CODE=1
   } || {
-    $ECHO "$GREEN[+] libdislocator did successfully detect the memory corruption" 
+    $ECHO "$GREEN[+] libdislocator did successfully detect the memory corruption"
   }
   rm -f test.out core test-compcov.core core.test-compcov
 } || {
@@ -700,7 +700,7 @@ test -e ../afl-qemu-trace && {
         test -e ../libcompcov.so && {
           $ECHO "$GREY[*] running afl-fuzz for qemu_mode compcov, this will take approx 10 seconds"
           {
-            export AFL_PRELOAD=../libcompcov.so 
+            export AFL_PRELOAD=../libcompcov.so
             export AFL_COMPCOV_LEVEL=2
             ../afl-fuzz -m ${MEM_LIMIT} -V10 -Q -i in -o out -- ./test-compcov >>errors 2>&1
             unset AFL_PRELOAD
@@ -720,10 +720,10 @@ test -e ../afl-qemu-trace && {
           INCOMPLETE=1
         }
         rm -f errors
-      } || { 
+      } || {
        $ECHO "$YELLOW[-] not an intel or arm platform, cannot test qemu_mode compcov"
       }
-      
+
       test "$SYS" = "i686" -o "$SYS" = "x86_64" -o "$SYS" = "amd64" -o "$SYS" = "i86pc" -o "$SYS" = "aarch64" -o ! "${SYS%%arm*}" && {
         $ECHO "$GREY[*] running afl-fuzz for persistent qemu_mode, this will take approx 10 seconds"
         {
@@ -757,9 +757,9 @@ test -e ../afl-qemu-trace && {
           exit 1
         }
         rm -rf in out errors
-      } || { 
+      } || {
        $ECHO "$YELLOW[-] not an intel or arm platform, cannot test persistent qemu_mode"
-      } 
+      }
 
       test -e ../qemu_mode/unsigaction/unsigaction32.so && {
         ${AFL_CC} -o test-unsigaction32 -m32 test-unsigaction.c >> errors 2>&1 && {
@@ -824,7 +824,7 @@ test -e ../afl-qemu-trace && {
     $ECHO "$RED[!] gcc compilation of test targets failed - what is going on??"
     CODE=1
   }
-  
+
   rm -f test-instr test-compcov
 } || {
   $ECHO "$YELLOW[-] qemu_mode is not compiled, cannot test"
@@ -887,12 +887,86 @@ test -d ../unicorn_mode/unicornafl && {
     $ECHO "$RED[!] missing sample binaries in unicorn_mode/samples/ - what is going on??"
     CODE=1
   }
-  
+
 } || {
   $ECHO "$YELLOW[-] unicorn_mode is not compiled, cannot test"
   INCOMPLETE=1
 }
 
+$ECHO "$BLUE[*] Testing: custom mutator"
+unset AFL_CC  # Line 474 sets AFL_CC to "gcc". We reset it to use the default compiler
+CUSTOM_MUTATOR_PATH=../examples/custom_mutators
+test -e test-custom-mutator.c -a -e ${CUSTOM_MUTATOR_PATH}/example.c -a -e ${CUSTOM_MUTATOR_PATH}/example.c && {
+  # Compile the vulnerable program
+  ../afl-clang-fast -o test-custom-mutator test-custom-mutator.c
+  # Compile the custom mutator
+  make -C ../examples/custom_mutators libexamplemutator.so
+  test -e test-custom-mutator -a -e ${CUSTOM_MUTATOR_PATH}/libexamplemutator.so && {
+    # Create input directory
+    mkdir -p in
+    echo 00000 > in/in
+
+    # Run afl-fuzz w/ the C mutator
+    $ECHO "$GREY[*] running afl-fuzz for the C mutator, this will take approx 10 seconds"
+    {
+      export AFL_CUSTOM_MUTATOR_LIBRARY=${CUSTOM_MUTATOR_PATH}/libexamplemutator.so
+      ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -- ./test-custom-mutator
+      unset AFL_CUSTOM_MUTATOR_LIBRARY
+    } >>errors 2>&1
+
+    # Check results
+    test -n "$( ls out/queue/id:000001* 2>/dev/null )" && {  # TODO: update here
+      $ECHO "$GREEN[+] afl-fuzz is working correctly with the C mutator"
+    } || {
+      echo CUT------------------------------------------------------------------CUT
+      cat errors
+      echo CUT------------------------------------------------------------------CUT
+      $ECHO "$RED[!] afl-fuzz is not working correctly with the C mutator"
+      CODE=1
+    }
+
+    # Clean
+    rm -rf out errors
+
+    # Run afl-fuzz w/ the Python mutator
+    $ECHO "$GREY[*] running afl-fuzz for the Python mutator, this will take approx 10 seconds"
+    {
+      export PYTHONPATH=${CUSTOM_MUTATOR_PATH}
+      export AFL_PYTHON_MODULE=example
+      ../afl-fuzz -V10 -m ${MEM_LIMIT} -i in -o out -- ./test-custom-mutator
+      unset PYTHONPATH
+      unset AFL_PYTHON_MODULE
+    } >>errors 2>&1
+
+    # Check results
+    test -n "$( ls out/queue/id:000001* 2>/dev/null )" && {  # TODO: update here
+      $ECHO "$GREEN[+] afl-fuzz is working correctly with the Python mutator"
+    } || {
+      echo CUT------------------------------------------------------------------CUT
+      cat errors
+      echo CUT------------------------------------------------------------------CUT
+      $ECHO "$RED[!] afl-fuzz is not working correctly with the Python mutator"
+      CODE=1
+    }
+
+    # Clean
+    rm -rf in out errors
+    rm -rf ${CUSTOM_MUTATOR_PATH}/__pycache__/
+  } || {
+    ls .
+    ls ${CUSTOM_MUTATOR_PATH}
+    $ECHO "$RED[!] cannot compile the test program or the custom mutator"
+    CODE=1
+  }
+
+  make -C ../examples/custom_mutators clean > /dev/null 2>&1
+  rm -f test-custom-mutator
+} || {
+  $ECHO "$YELLOW[-] no custom mutators in $CUSTOM_MUTATOR_PATH, cannot test"
+  INCOMPLETE=1
+}
+unset CUSTOM_MUTATOR_PATH
+
 $ECHO "$GREY[*] all test cases completed.$RESET"
 test "$INCOMPLETE" = "0" && $ECHO "$GREEN[+] all test cases executed"
 test "$INCOMPLETE" = "1" && $ECHO "$YELLOW[-] not all test cases were executed"