add autotokens-standalone

custom_mutators/autotokens/autotokens.cpp

@@ -39,6 +39,7 @@ extern "C" {
 #ifndef AFL_TXT_MAX_LEN
   #define AFL_TXT_MAX_LEN 65535
 #endif
+#define AUTOTOKENS_TXT_MIN_LEN 1
 
 #if AUTOTOKENS_SPLICE_MIN >= AUTOTOKENS_SIZE_MIN
   #error SPLICE_MIN must be lower than SIZE_MIN
@@ -57,8 +58,9 @@ typedef struct my_mutator {
   if (unlikely(debug)) fprintf
 #define IFDEBUG if (unlikely(debug))
 
+int module_disabled = 0;
+
 static afl_state *afl_ptr;
-static int        module_disabled = 0;
 static int        auto_disable = AUTOTOKENS_AUTO_DISABLE;
 static int        debug = AUTOTOKENS_DEBUG;
 static int        only_fav = AUTOTOKENS_ONLY_FAV;
@@ -104,9 +106,9 @@ static void first_run(void *data) {
   if (afl_ptr->custom_only || !auto_disable) { return; }
 
   if (unlikely(afl_ptr->active_items == 1 &&
-               afl_ptr->queue_cur->len < AFL_TXT_MIN_LEN)) {
+               afl_ptr->queue_cur->len < AUTOTOKENS_TXT_MIN_LEN)) {
 
-    if (afl_ptr->extras_cnt > 8) {
+    if (afl_ptr->extras_cnt) {
 
       u32 valid = 0;
 
@@ -237,7 +239,7 @@ extern "C" u32 afl_custom_fuzz_count(void *data, const u8 *buf,
 
 }
 
-extern "C" size_t afl_custom_fuzz(my_mutator_t *data, u8 *buf, size_t buf_size,
+extern "C" size_t afl_custom_fuzz(void *data, u8 *buf, size_t buf_size,
                                   u8 **out_buf, u8 *add_buf,
                                   size_t add_buf_size, size_t max_size) {
 
@@ -655,6 +657,7 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
     if (current_id > whitespace_ids + 6 && afl_ptr->active_items == 1 &&
         afl_ptr->queue_cur->len < AFL_TXT_MIN_LEN) {
 
+    retry_thin_air:
       DEBUGF(stderr, "Creating an entry from thin air...\n");
       structure = new vector<u32>();
       u32 item, prev, cnt = current_id >> 1;
@@ -684,8 +687,6 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
 
     }
 
-    create_from_thin_air = 0;
-
   }
 
   if (entry == file_mapping.end()) {
@@ -693,7 +694,7 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
     // this input file was not analyzed for tokens yet, so let's do it!
     size_t len = afl_ptr->queue_cur->len;
 
-    if (len < AFL_TXT_MIN_LEN) {
+    if (len < AUTOTOKENS_TXT_MIN_LEN) {
 
       file_mapping[fn] = structure;  // NULL ptr so we don't read the file again
       s = NULL;
@@ -895,6 +896,7 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
 
     if (tokens.size() < AUTOTOKENS_SIZE_MIN) {
 
+      if (create_from_thin_air) { goto retry_thin_air; }
       file_mapping[fn] = NULL;
       s = NULL;
       DEBUGF(stderr, "too few tokens\n");
@@ -955,7 +957,7 @@ extern "C" unsigned char afl_custom_queue_get(void                *data,
 
 }
 
-extern "C" my_mutator_t *afl_custom_init(afl_state *afl, unsigned int seed) {
+extern "C" void *afl_custom_init(afl_state_t *afl, unsigned int seed) {
 
   (void)(seed);
   my_mutator_t *data = (my_mutator_t *)calloc(1, sizeof(my_mutator_t));
@@ -1070,7 +1072,7 @@ extern "C" my_mutator_t *afl_custom_init(afl_state *afl, unsigned int seed) {
   id_to_token[current_id] = "'";
   ++current_id;
 
-  return data;
+  return (void *)data;
 
 }
 

custom_mutators/autotokens/standalone/Makefile

@@ -0,0 +1,19 @@
+
+CFLAGS = -g -O3 -funroll-loops -fPIC -D_STANDALONE_MODULE=1 -Wno-implicit-function-declaration
+CXXFLAGS= -g -O3 -funroll-loops -fPIC -D_STANDALONE_MODULE=1
+
+all: autotokens-standalone
+
+autotokens.o:	../autotokens.cpp
+	$(CXX) $(CXXFLAGS) -I../../../include -I. -I../.. -c ../autotokens.cpp
+
+autotokens-standalone:	autotokens-standalone.c	autotokens.o
+	$(CC) $(CFLAGS) -DBIN_PATH=\"foo\" -I../../../include -I. -c autotokens-standalone.c
+	$(CC) $(CFLAGS) -DBIN_PATH=\"foo\" -I../../../include -I. -c ../../../src/afl-performance.c
+	$(CC) $(CFLAGS) -DBIN_PATH=\"foo\" -I../../../include -I. -c ../../../src/afl-fuzz-extras.c
+	$(CC) $(CFLAGS) -DBIN_PATH=\"foo\" -I../../../include -I. -c ../../../src/afl-fuzz-queue.c
+	$(CC) $(CFLAGS) -DBIN_PATH=\"foo\" -I../../../include -I. -c ../../../src/afl-common.c
+	$(CXX) $(CFLAGS) -DBIN_PATH=\"foo\" -I../../../include -I. -o autotokens-standalone *.o
+
+clean:
+	rm -f *.o *~ autotokens-standalone core

custom_mutators/autotokens/standalone/README.md

@@ -0,0 +1,12 @@
+# Autotokens standalone mutator
+
+this is a standalone version of the AFL++ autotokens custom mutator.
+
+just type `make` to build.
+
+You *MUST* use a dictionary file to have an effective grammarless grammar fuzzer!
+
+```
+autotokens-standalone -h # to see all parameteres
+autotokens-standalone -x foo.dict inputfile outputfile # example
+```

custom_mutators/autotokens/standalone/autotokens-standalone.c

@@ -0,0 +1,187 @@
+#include "afl-fuzz.h"
+#include "afl-mutations.h"
+
+#include <unistd.h>
+#include <getopt.h>
+
+static int            max_havoc = 16, verbose;
+static unsigned char *dict;
+
+extern int module_disabled;
+
+void *afl_custom_init(afl_state_t *, unsigned int);
+
+int main(int argc, char *argv[]) {
+
+  if (argc > 1 && strncmp(argv[1], "-h", 2) == 0) {
+
+    printf(
+        "Syntax: %s [-v] [-m maxmutations] [-x dict] [inputfile [outputfile "
+        "[splicefile]]]\n\n",
+        argv[0]);
+    printf("Reads a testcase from a file (not stdin!),\n");
+    printf("writes to stdout when '-' or\n");
+    printf(
+        "no output filename is given. As an optional third parameter you can "
+        "give a file\n");
+    printf("for splicing. Maximum input and output length is 1MB.\n");
+    printf("Options:\n");
+    printf("  -v      verbose debug output to stderr.\n");
+    printf("  -m val  max mutations (1-val, val default is 16)\n");
+    printf("  -x file dictionary file (AFL++ format)\n");
+    return 0;
+
+  }
+
+  FILE          *in = stdin, *out = stdout, *splice = NULL;
+  unsigned char *inbuf = malloc(1024 * 1024), *outbuf = NULL, *splicebuf = NULL;
+  int            splicelen = 0, opt;
+
+  while ((opt = getopt(argc, argv, "vm:x:")) > 0) {
+
+    switch (opt) {
+
+      case 'm':
+        max_havoc = atoi(optarg);
+        break;
+      case 'v':
+        verbose = 1;
+        break;
+      case 'x':
+        dict = optarg;
+        break;
+      default:
+        fprintf(stderr, "Error: unknown parameter -%c\n", opt);
+        exit(-1);
+
+    }
+
+  }
+
+  if (max_havoc < 1) {
+
+    fprintf(stderr, "Error: illegal -m value\n");
+    exit(-1);
+
+  }
+
+  if (argc > optind && strcmp(argv[optind], "-") != 0) {
+
+    if ((in = fopen(argv[optind], "r")) == NULL) {
+
+      perror(argv[1]);
+      return -1;
+
+    }
+
+    if (verbose) fprintf(stderr, "Input: %s\n", argv[optind]);
+
+  }
+
+  size_t inlen = fread(inbuf, 1, 1024 * 1024, in);
+
+  if (!inlen) {
+
+    fprintf(stderr, "Error: empty file %s\n",
+            argv[optind] ? argv[optind] : "stdin");
+    return -1;
+
+  }
+
+  if (argc > optind + 1 && strcmp(argv[optind + 1], "-") != 0) {
+
+    if ((out = fopen(argv[optind + 1], "w")) == NULL) {
+
+      perror(argv[optind + 1]);
+      return -1;
+
+    }
+
+    if (verbose) fprintf(stderr, "Output: %s\n", argv[optind + 1]);
+
+  }
+
+  if (argc > optind + 2) {
+
+    if ((splice = fopen(argv[optind + 2], "r")) == NULL) {
+
+      perror(argv[optind + 2]);
+      return -1;
+
+    }
+
+    if (verbose) fprintf(stderr, "Splice: %s\n", argv[optind + 2]);
+    splicebuf = malloc(1024 * 1024);
+    size_t splicelen = fread(splicebuf, 1, 1024 * 1024, splice);
+    if (!splicelen) {
+
+      fprintf(stderr, "Error: empty file %s\n", argv[optind + 2]);
+      return -1;
+
+    }
+
+    if (verbose) fprintf(stderr, "Mutation splice length: %zu\n", splicelen);
+
+  }
+
+  /* configure autotokens */
+  setenv("AUTOTOKENS_LEARN_DICT", "1", 0);
+  setenv("AUTOTOKENS_CREATE_FROM_THIN_AIR", "1", 0);
+
+  /* fake AFL++ state */
+  afl_state_t *afl = (afl_state_t *)calloc(1, sizeof(afl_state_t));
+  afl->queue_cycle = afl->havoc_div = afl->active_items = afl->queued_items = 1;
+  afl->shm.cmplog_mode = 0;
+  afl->fsrv.dev_urandom_fd = open("/dev/urandom", O_RDONLY);
+  if (afl->fsrv.dev_urandom_fd < 0) { PFATAL("Unable to open /dev/urandom"); }
+
+  rand_set_seed(afl, getpid());
+
+  if (dict) {
+
+    load_extras(afl, dict);
+    if (verbose)
+      fprintf(stderr, "Loaded dictionary: %s (%u entries)\n", dict,
+              afl->extras_cnt);
+
+  }
+
+  // setup a fake queue entry
+  afl->queue_buf = malloc(64);
+  afl->queue_buf[0] = afl->queue_cur =
+      (struct queue_entry *)malloc(sizeof(struct queue_entry));
+  afl->queue_cur->testcase_buf = inbuf;
+  afl->queue_cur->fname = (u8 *)argv[optind];
+  afl->queue_cur->len = inlen;
+  afl->queue_cur->perf_score = 100;
+  afl->queue_cur->favored = afl->queue_cur->is_ascii = 1;
+  // afl->custom_only = 1;
+
+  void *data = (void *)afl_custom_init(afl, (u32)0);
+
+  u8 res = afl_custom_queue_get(inbuf, (u8 *)argv[optind]);
+
+  if (verbose) fprintf(stderr, "Mutation input length: %zu\n", inlen);
+  unsigned int outlen = afl_custom_fuzz(data, inbuf, inlen, &outbuf, splicebuf,
+                                        splicelen, 1024 * 1024);
+
+  if (outlen == 0 || !outbuf) {
+
+    fprintf(stderr, "Error: no mutation data returned.\n");
+    return -1;
+
+  }
+
+  if (verbose) fprintf(stderr, "Mutation output length: %u\n", outlen);
+
+  if (fwrite(outbuf, 1, outlen, out) != outlen) {
+
+    fprintf(stderr, "Warning: incomplete write.\n");
+    return -1;
+
+  }
+
+  return 0;
+
+}
+

docs/Changelog.md

@@ -32,7 +32,8 @@
       AFL type forkserver. Useful for symcc/symqemu/nautilus/etc. with
       AFL_LLVM_INSTRUMENT=CLASSIC
   - code formatting updated to llvm 18
-  - improve /custom_mutators/aflpp/standalone/aflpp-standalone
+  - improved custom_mutators/aflpp/standalone/aflpp-standalone
+  - added custom_mutators/autotokens/standalone/autotokens-standalone
 
 
 ### Version ++4.21c (release)