more fixes

custom_mutators/gramatron/README.md

@@ -23,8 +23,8 @@ You have to set the grammar file to use with `GRAMMATRON_AUTOMATION`:
 ```
 export AFL_DISABLE_TRIM=1
 export AFL_CUSTOM_MUTATOR_ONLY=1
-export AFL_CUSTOM_MUTATOR_LIBRARY=./grammatron.so
+export AFL_CUSTOM_MUTATOR_LIBRARY=./gramatron.so
-export GRAMMATRON_AUTOMATION=grammars/ruby/source_automata.json
+export GRAMATRON_AUTOMATION=grammars/ruby/source_automata.json
 afl-fuzz -i in -o out -- ./target
 ```
 

custom_mutators/gramatron/gramfuzz-util.c

@@ -4,6 +4,11 @@
 #include <assert.h>
 #include "afl-fuzz.h"
 #include "gramfuzz.h"
+#ifdef _GNU_SOURCE
+  #undef _GNU_SOURCE
+#endif
+#define _GNU_SOURCE
+#include <sys/mman.h>
 
 /* Dynamic Array for adding to the input repr
  * */
@@ -178,7 +183,7 @@ void write_input(Array *input, u8 *fn) {
   // If the input has already been flushed, then skip silently
   if (fp == NULL) {
 
-    printf("\n File could not be open, exiting");
+    fprintf(stderr, "\n File '%s' could not be open, exiting\n", fn);
     exit(1);
 
   }
@@ -196,22 +201,13 @@ void write_input(Array *input, u8 *fn) {
 
 }
 
-// Read the input representation into memory
+Array *parse_input(state *pda, FILE *fp) {
-Array *read_input(state *pda, u8 *fn) {
 
-  FILE *    fp;
   terminal *term;
   state *   state_ptr;
   trigger * trigger;
   int       trigger_idx;
   Array *   input = (Array *)calloc(1, sizeof(Array));
-  fp = fopen(fn, "rb");
-  if (fp == NULL) {
-
-    printf("\nFile:%s does not exist..exiting", fn);
-    exit(1);
-
-  }
 
   // Read the length parameters
   fread(&input->used, sizeof(size_t), 1, fp);
@@ -219,6 +215,12 @@ Array *read_input(state *pda, u8 *fn) {
   fread(&input->inputlen, sizeof(size_t), 1, fp);
 
   terminal *start_ptr = (terminal *)calloc(input->size, sizeof(terminal));
+  if (!start_ptr) {
+
+    fprintf(stderr, "alloc failed!\n");
+    return NULL;
+
+  }
 
   // Read the dynamic array to memory
   fread(start_ptr, input->size * sizeof(terminal), 1, fp);
@@ -242,9 +244,51 @@ Array *read_input(state *pda, u8 *fn) {
   // printf("\nUsed:%zu Size:%zu Inputlen:%zu", input->used, input->size,
   // input->inputlen);
 
-  fclose(fp);
-
   return input;
 
 }
 
+Array *open_input(state *pda, u8 *data, size_t len) {
+
+  int fd = memfd_create("foo", O_RDWR);
+  if (fd < 0) {
+
+    fprintf(stderr, "Error: memfd_create failed\n");
+    return NULL;
+
+  }
+
+  ck_write(fd, data, len, "memfd_create");
+  lseek(fd, 0, SEEK_SET);
+  FILE *f = fdopen(fd, "rb");
+  if (!f) {
+
+    fprintf(stderr, "Error: fdopen failed\n");
+    return NULL;
+
+  }
+
+  Array *res = parse_input(pda, f);
+  fclose(f);
+  return res;
+
+}
+
+// Read the input representation into memory
+Array *read_input(state *pda, u8 *fn) {
+
+  FILE *fp;
+  fp = fopen(fn, "rb");
+  if (fp == NULL) {
+
+    fprintf(stderr, "\n File '%s' does not exist, exiting\n", fn);
+    exit(1);
+
+  }
+
+  Array *res = parse_input(pda, fp);
+  fclose(fp);
+  return res;
+
+}
+

custom_mutators/gramatron/gramfuzz.c

@@ -159,7 +159,7 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) {
   // u32 recur_len = 0;       // The number of recursive features
   // data->mutator_buf = NULL;
 
-  char *automaton_file = getenv("GRAMMATRON_AUTOMATION");
+  char *automaton_file = getenv("GRAMATRON_AUTOMATION");
   if (automaton_file) {
 
     pda = create_pda(automaton_file);
@@ -168,7 +168,7 @@ my_mutator_t *afl_custom_init(afl_state_t *afl, unsigned int seed) {
 
     fprintf(stderr,
             "\nError: GrammaTron needs an automation json file set in "
-            "AFL_GRAMMATRON_AUTOMATON\n");
+            "AFL_GRAMATRON_AUTOMATON\n");
     exit(-1);
 
   }
@@ -208,18 +208,18 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size,
         doMult(data->orig_walk, data->recurIdx, data->recurlen);
     data->mut_alloced = 1;
 
-  } else if (data->mut_idx == 2) {  // Perform splice mutation
+    /*} else if (data->mut_idx == 2) {  // Perform splice mutation
 
       // Read the input representation for the splice candidate
-    u8 *   automaton_fn = alloc_printf("%s.aut", add_buf);
+      //u8 *   automaton_fn = alloc_printf("%s.aut", add_buf);
-    Array *spliceCandidate = read_input(pda, automaton_fn);
+      Array *spliceCandidate = open_input(pda, add_buf, add_buf_size);
 
       data->mutated_walk =
           performSpliceOne(data->orig_walk, data->statemap, spliceCandidate);
       data->mut_alloced = 1;
       free(spliceCandidate->start);
       free(spliceCandidate);
-    ck_free(automaton_fn);
+      //ck_free(automaton_fn);*/
 
   } else {  // Generate an input from scratch
 
@@ -262,6 +262,10 @@ u8 afl_custom_queue_new_entry(my_mutator_t * data,
 
   automaton_fn = alloc_printf("%s.aut", filename_new_queue);
   // Check if this method is being called during initialization
+
+  // fprintf(stderr, "new: %s, old: %s, auto: %s\n",
+  // filename_new_queue,filename_orig_queue,automaton_fn);
+
   if (filename_orig_queue) {
 
     write_input(data->mutated_walk, automaton_fn);

custom_mutators/gramatron/gramfuzz.h

@@ -1,25 +1,25 @@
 #ifndef _GRAMFUZZ_H
 
-  #define _GRAMFUZZ_H
+#define _GRAMFUZZ_H
 
-  #include <json-c/json.h>
+#include <json-c/json.h>
-  #include <unistd.h>
+#include <unistd.h>
-  #include "hashmap.h"
+#include "hashmap.h"
-  #include "uthash.h"
+#include "uthash.h"
-  #include "utarray.h"
+#include "utarray.h"
 
-  #define INIT_INPUTS 100  // No. of initial inputs to be generated
+#define INIT_INPUTS 100  // No. of initial inputs to be generated
 
 // Set this as `numstates` + 1 where `numstates` is retrieved from gen automata
 // json #define STATES 63
 
-  #define INIT_SIZE 100  // Initial size of the dynamic array holding the input
+#define INIT_SIZE 100  // Initial size of the dynamic array holding the input
 
-  #define SPLICE_CORPUS 10000
+#define SPLICE_CORPUS 10000
-  #define RECUR_THRESHOLD 6
+#define RECUR_THRESHOLD 6
-  #define SIZE_THRESHOLD 2048
+#define SIZE_THRESHOLD 2048
 
-  #define FLUSH_INTERVAL \
+#define FLUSH_INTERVAL \
   3600  // Inputs that gave new coverage will be dumped every FLUSH_INTERVAL
         // seconds
 
@@ -199,6 +199,7 @@ Array *performSpliceGF(state *, Array *, afl_state_t *);
 void   dump_input(u8 *, char *, int *);
 void   write_input(Array *, u8 *);
 Array *read_input(state *, u8 *);
+Array *open_input(state *, u8 *, size_t);
 state *pda;
 
 // // AFL-specific struct

custom_mutators/gramatron/uthash.h

@@ -59,6 +59,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
       *_da_dst = (char *)(src);           \
                                           \
     } while (0)
+
 #else
   #define DECLTYPE_ASSIGN(dst, src) \
     do {                            \
@@ -66,6 +67,7 @@ SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
       (dst) = DECLTYPE(dst)(src);   \
                                     \
     } while (0)
+
 #endif
 
 /* a number of the hash function use uint32_t which isn't defined on Pre VS2010
@@ -138,6 +140,7 @@ typedef unsigned char uint8_t;
       (oomed) = 1;               \
                                  \
     } while (0)
+\
   #define IF_HASH_NONFATAL_OOM(x) x
 
 #else
@@ -154,7 +157,8 @@ typedef unsigned char uint8_t;
 
 /* initial number of buckets */
 #define HASH_INITIAL_NUM_BUCKETS 32U    /* initial number of buckets        */
-#define HASH_INITIAL_NUM_BUCKETS_LOG2 5U /* lg2 of initial number of buckets \
+#define HASH_INITIAL_NUM_BUCKETS_LOG2                                    \
+  5U                                 /* lg2 of initial number of buckets \
                                       */
 #define HASH_BKT_CAPACITY_THRESH 10U    /* expand when bucket count reaches */
 
@@ -376,6 +380,8 @@ typedef unsigned char uint8_t;
                                                                          \
     } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \
                                                                          \
+                                                                         \
+                                                                         \
   } while (0)
 
 #ifdef NO_DECLTYPE
@@ -397,6 +403,8 @@ typedef unsigned char uint8_t;
                                                                            \
       } while ((_hs_iter = HH_FROM_ELMT((head)->hh.tbl, _hs_iter)->next)); \
                                                                            \
+                                                                           \
+                                                                           \
     } while (0)
 #endif
 
@@ -639,6 +647,7 @@ typedef unsigned char uint8_t;
     HASH_FIND(hh, head, findstr, _uthash_hfstr_keylen, out);          \
                                                                       \
   } while (0)
+\
 #define HASH_ADD_STR(head, strfield, add)                                     \
   do {                                                                        \
                                                                               \
@@ -646,6 +655,7 @@ typedef unsigned char uint8_t;
     HASH_ADD(hh, head, strfield[0], _uthash_hastr_keylen, add);               \
                                                                               \
   } while (0)
+\
 #define HASH_REPLACE_STR(head, strfield, add, replaced)                       \
   do {                                                                        \
                                                                               \
@@ -653,6 +663,7 @@ typedef unsigned char uint8_t;
     HASH_REPLACE(hh, head, strfield[0], _uthash_hrstr_keylen, add, replaced); \
                                                                               \
   } while (0)
+\
 #define HASH_FIND_INT(head, findint, out) \
   HASH_FIND(hh, head, findint, sizeof(int), out)
 #define HASH_ADD_INT(head, intfield, add) \
@@ -679,6 +690,7 @@ typedef unsigned char uint8_t;
       exit(-1);                     \
                                     \
     } while (0)
+\
   #define HASH_FSCK(hh, head, where)                                          \
     do {                                                                      \
                                                                               \
@@ -748,6 +760,7 @@ typedef unsigned char uint8_t;
       }                                                                       \
                                                                               \
     } while (0)
+
 #else
   #define HASH_FSCK(hh, head, where)
 #endif
@@ -764,6 +777,7 @@ typedef unsigned char uint8_t;
       write(HASH_EMIT_KEYS, keyptr, (unsigned long)fieldlen); \
                                                               \
     } while (0)
+
 #else
   #define HASH_EMIT_KEY(hh, head, keyptr, fieldlen)
 #endif
@@ -806,6 +820,7 @@ typedef unsigned char uint8_t;
     }                                                            \
                                                                  \
   } while (0)
+
 /* FNV-1a variation */
 #define HASH_FNV(key, keylen, hashv)                             \
   do {                                                           \
@@ -1098,6 +1113,7 @@ typedef unsigned char uint8_t;
       hashv = _mur_h1;                                               \
                                                                      \
     } while (0)
+
 #endif                                     /* HASH_USING_NO_STRICT_ALIASING */
 
 /* iterate over items in a known bucket to find desired item */
@@ -1335,6 +1351,7 @@ typedef unsigned char uint8_t;
               _hs_psize--;                                                     \
                                                                                \
             } else if ((cmpfcn(DECLTYPE(head)(                                 \
+                                                                               \
                                    ELMT_FROM_HH((head)->hh.tbl, _hs_p)),       \
                                DECLTYPE(head)(ELMT_FROM_HH((head)->hh.tbl,     \
                                                            _hs_q)))) <= 0) {   \

include/afl-fuzz.h

@@ -1005,6 +1005,8 @@ void setup_custom_mutators(afl_state_t *);
 void destroy_custom_mutators(afl_state_t *);
 u8   trim_case_custom(afl_state_t *, struct queue_entry *q, u8 *in_buf,
                       struct custom_mutator *mutator);
+void run_afl_custom_queue_new_entry(afl_state_t *, struct queue_entry *, u8 *,
+                                    u8 *);
 
 /* Python */
 #ifdef USE_PYTHON

src/afl-fuzz-init.c

@@ -881,11 +881,7 @@ void perform_dry_run(afl_state_t *afl) {
 
     u32 read_len = MIN(q->len, (u32)MAX_FILE);
     use_mem = afl_realloc(AFL_BUF_PARAM(in), read_len);
-    if (read(fd, use_mem, read_len) != (ssize_t)read_len) {
+    ck_read(fd, use_mem, read_len, q->fname);
-
-      FATAL("Short read from '%s'", q->fname);
-
-    }
 
     close(fd);
 
@@ -1350,6 +1346,12 @@ void pivot_inputs(afl_state_t *afl) {
 
     if (q->passed_det) { mark_as_det_done(afl, q); }
 
+    if (afl->custom_mutators_count) {
+
+      run_afl_custom_queue_new_entry(afl, q, q->fname, NULL);
+
+    }
+
     ++id;
 
   }

src/afl-fuzz-mutators.c

@@ -31,6 +31,45 @@ struct custom_mutator *load_custom_mutator(afl_state_t *, const char *);
 struct custom_mutator *load_custom_mutator_py(afl_state_t *, char *);
 #endif
 
+void run_afl_custom_queue_new_entry(afl_state_t *afl, struct queue_entry *q,
+                                    u8 *fname, u8 *mother_fname) {
+
+  if (afl->custom_mutators_count) {
+
+    u8 updated = 0;
+
+    LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, {
+
+      if (el->afl_custom_queue_new_entry) {
+
+        if (el->afl_custom_queue_new_entry(el->data, fname, mother_fname)) {
+
+          updated = 1;
+
+        }
+
+      }
+
+    });
+
+    if (updated) {
+
+      struct stat st;
+      if (stat(fname, &st)) { PFATAL("File %s is gone!", fname); }
+      if (!st.st_size) {
+
+        FATAL("File %s became empty in custom mutator!", fname);
+
+      }
+
+      q->len = st.st_size;
+
+    }
+
+  }
+
+}
+
 void setup_custom_mutators(afl_state_t *afl) {
 
   /* Try mutator library first */

src/afl-fuzz-queue.c

@@ -474,42 +474,10 @@ void add_to_queue(afl_state_t *afl, u8 *fname, u32 len, u8 passed_det) {
 
   if (afl->custom_mutators_count) {
 
-    u8 updated = 0;
-
-    LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, {
-
-      if (el->afl_custom_queue_new_entry) {
-
-        u8 *fname_orig = NULL;
-
     /* At the initialization stage, queue_cur is NULL */
     if (afl->queue_cur && !afl->syncing_party) {
 
-          fname_orig = afl->queue_cur->fname;
+      run_afl_custom_queue_new_entry(afl, q, fname, afl->queue_cur->fname);
-
-        }
-
-        if (el->afl_custom_queue_new_entry(el->data, fname, fname_orig)) {
-
-          updated = 1;
-
-        }
-
-      }
-
-    });
-
-    if (updated) {
-
-      struct stat st;
-      if (stat(fname, &st)) { PFATAL("File %s is gone!", fname); }
-      if (!st.st_size) {
-
-        FATAL("File %s became empty in custom mutator!", fname);
-
-      }
-
-      q->len = st.st_size;
 
     }