Merge pull request #361 from rish9101/pre_save_format

Add post library API as custom mutator and rename pre_save
This commit is contained in:
van Hauser 2020-05-13 16:39:23 +02:00 committed by GitHub
commit c4fe6f5277
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 73 additions and 99 deletions

View File

@ -33,7 +33,7 @@ C/C++:
```c ```c
void *afl_custom_init(afl_t *afl, unsigned int seed); void *afl_custom_init(afl_t *afl, unsigned int seed);
size_t afl_custom_fuzz(void *data, uint8_t *buf, size_t buf_size, u8 **out_buf, uint8_t *add_buf, size_t add_buf_size, size_t max_size); size_t afl_custom_fuzz(void *data, uint8_t *buf, size_t buf_size, u8 **out_buf, uint8_t *add_buf, size_t add_buf_size, size_t max_size);
size_t afl_custom_pre_save(void *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf); size_t afl_custom_post_process(void *data, uint8_t *buf, size_t buf_size, uint8_t **out_buf);
int32_t afl_custom_init_trim(void *data, uint8_t *buf, size_t buf_size); int32_t afl_custom_init_trim(void *data, uint8_t *buf, size_t buf_size);
size_t afl_custom_trim(void *data, uint8_t **out_buf); size_t afl_custom_trim(void *data, uint8_t **out_buf);
int32_t afl_custom_post_trim(void *data, int success) { int32_t afl_custom_post_trim(void *data, int success) {
@ -51,7 +51,7 @@ def init(seed):
def fuzz(buf, add_buf, max_size): def fuzz(buf, add_buf, max_size):
return mutated_out return mutated_out
def pre_save(buf): def post_process(buf):
return out_buf return out_buf
def init_trim(buf): def init_trim(buf):
@ -92,7 +92,7 @@ def queue_new_entry(filename_new_queue, filename_orig_queue):
This method performs custom mutations on a given input. It also accepts an This method performs custom mutations on a given input. It also accepts an
additional test case. additional test case.
Note that this function is optional - but it makes sense to use it. Note that this function is optional - but it makes sense to use it.
You would only skip this if `pre_send` is used to fix checksums etc. You would only skip this if `post_process` is used to fix checksums etc.
so you are using it e.g. as a post processing library. so you are using it e.g. as a post processing library.
- `havoc_mutation` and `havoc_mutation_probability` (optional): - `havoc_mutation` and `havoc_mutation_probability` (optional):
@ -102,7 +102,7 @@ def queue_new_entry(filename_new_queue, filename_orig_queue):
`havoc_mutation_probability`, returns the probability that `havoc_mutation` `havoc_mutation_probability`, returns the probability that `havoc_mutation`
is called in havoc. By default, it is 6%. is called in havoc. By default, it is 6%.
- `pre_save` (optional): - `post_process` (optional):
For some cases, the format of the mutated data returned from the custom For some cases, the format of the mutated data returned from the custom
mutator is not suitable to directly execute the target with this input. mutator is not suitable to directly execute the target with this input.
@ -110,7 +110,7 @@ def queue_new_entry(filename_new_queue, filename_orig_queue):
protobuf format which corresponds to a given grammar. In order to execute protobuf format which corresponds to a given grammar. In order to execute
the target, the protobuf data must be converted to the plain-text format the target, the protobuf data must be converted to the plain-text format
expected by the target. In such scenarios, the user can define the expected by the target. In such scenarios, the user can define the
`pre_save` function. This function is then transforming the data into the `post_process` function. This function is then transforming the data into the
format expected by the API before executing the target. format expected by the API before executing the target.
- `queue_new_entry` (optional): - `queue_new_entry` (optional):
@ -222,7 +222,7 @@ For C/C++ mutator, the source code must be compiled as a shared object:
gcc -shared -Wall -O3 example.c -o example.so gcc -shared -Wall -O3 example.c -o example.so
``` ```
Note that if you specify multiple custom mutators, the corresponding functions will Note that if you specify multiple custom mutators, the corresponding functions will
be called in the order in which they are specified. e.g first `pre_save` function of be called in the order in which they are specified. e.g first `post_process` function of
`example_first.so` will be called and then that of `example_second.so` `example_first.so` will be called and then that of `example_second.so`
### Run ### Run

View File

@ -38,7 +38,7 @@ typedef struct my_mutator {
BUF_VAR(u8, data); BUF_VAR(u8, data);
BUF_VAR(u8, havoc); BUF_VAR(u8, havoc);
BUF_VAR(u8, trim); BUF_VAR(u8, trim);
BUF_VAR(u8, pre_save); BUF_VAR(u8, post_process);
} my_mutator_t; } my_mutator_t;
@ -139,11 +139,11 @@ size_t afl_custom_fuzz(my_mutator_t *data, uint8_t *buf, size_t buf_size,
* @return Size of the output buffer after processing or the needed amount. * @return Size of the output buffer after processing or the needed amount.
* A return of 0 indicates an error. * A return of 0 indicates an error.
*/ */
size_t afl_custom_pre_save(my_mutator_t *data, uint8_t *buf, size_t buf_size, size_t afl_custom_post_process(my_mutator_t *data, uint8_t *buf, size_t buf_size,
uint8_t **out_buf) { uint8_t **out_buf) {
uint8_t *pre_save_buf = maybe_grow(BUF_PARAMS(data, pre_save), buf_size + 5); uint8_t *post_process_buf = maybe_grow(BUF_PARAMS(data, post_process), buf_size + 5);
if (!pre_save_buf) { if (!post_process_buf) {
perror("custom mutator realloc failed."); perror("custom mutator realloc failed.");
*out_buf = NULL; *out_buf = NULL;
@ -151,14 +151,14 @@ size_t afl_custom_pre_save(my_mutator_t *data, uint8_t *buf, size_t buf_size,
} }
memcpy(pre_save_buf + 5, buf, buf_size); memcpy(post_process_buf + 5, buf, buf_size);
pre_save_buf[0] = 'A'; post_process_buf[0] = 'A';
pre_save_buf[1] = 'F'; post_process_buf[1] = 'F';
pre_save_buf[2] = 'L'; post_process_buf[2] = 'L';
pre_save_buf[3] = '+'; post_process_buf[3] = '+';
pre_save_buf[4] = '+'; post_process_buf[4] = '+';
*out_buf = pre_save_buf; *out_buf = post_process_buf;
return buf_size + 5; return buf_size + 5;
@ -364,7 +364,7 @@ void afl_custom_queue_new_entry(my_mutator_t * data,
*/ */
void afl_custom_deinit(my_mutator_t *data) { void afl_custom_deinit(my_mutator_t *data) {
free(data->pre_save_buf); free(data->post_process_buf);
free(data->havoc_buf); free(data->havoc_buf);
free(data->data_buf); free(data->data_buf);
free(data->fuzz_buf); free(data->fuzz_buf);

View File

@ -120,7 +120,7 @@ def fuzz(buf, add_buf, max_size):
# #
# return next_index # return next_index
# #
# def pre_save(buf): # def post_process(buf):
# ''' # '''
# Called just before the execution to write the test case in the format # Called just before the execution to write the test case in the format
# expected by the target # expected by the target

View File

@ -83,7 +83,7 @@ typedef struct post_state {
} post_state_t; } post_state_t;
void *afl_postprocess_init(void *afl) { void *afl_postprocess_init(void *afl, unsigned int seed) {
post_state_t *state = malloc(sizeof(post_state_t)); post_state_t *state = malloc(sizeof(post_state_t));
if (!state) { if (!state) {

View File

@ -43,7 +43,7 @@ typedef struct post_state {
} post_state_t; } post_state_t;
void *afl_postprocess_init(void *afl) { void *afl_postprocess_init(void *afl, unsigned int seed) {
post_state_t *state = malloc(sizeof(post_state_t)); post_state_t *state = malloc(sizeof(post_state_t));
if (!state) { if (!state) {

View File

@ -260,7 +260,7 @@ enum {
/* 00 */ PY_FUNC_INIT, /* 00 */ PY_FUNC_INIT,
/* 01 */ PY_FUNC_FUZZ, /* 01 */ PY_FUNC_FUZZ,
/* 02 */ PY_FUNC_PRE_SAVE, /* 02 */ PY_FUNC_POST_PROCESS,
/* 03 */ PY_FUNC_INIT_TRIM, /* 03 */ PY_FUNC_INIT_TRIM,
/* 04 */ PY_FUNC_POST_TRIM, /* 04 */ PY_FUNC_POST_TRIM,
/* 05 */ PY_FUNC_TRIM, /* 05 */ PY_FUNC_TRIM,
@ -283,8 +283,8 @@ typedef struct py_mutator {
u8 * fuzz_buf; u8 * fuzz_buf;
size_t fuzz_size; size_t fuzz_size;
u8 * pre_save_buf; u8 * post_process_buf;
size_t pre_save_size; size_t post_process_size;
u8 * trim_buf; u8 * trim_buf;
size_t trim_size; size_t trim_size;
@ -545,11 +545,9 @@ typedef struct afl_state {
struct extra_data *a_extras; /* Automatically selected extras */ struct extra_data *a_extras; /* Automatically selected extras */
u32 a_extras_cnt; /* Total number of tokens available */ u32 a_extras_cnt; /* Total number of tokens available */
/* afl_postprocess API */ /* afl_postprocess API - Now supported via custom mutators */
void *(*post_init)(struct afl_state *afl);
size_t (*post_handler)(void *data, u8 *buf, u32 len, u8 **out_buf); struct custom_mutator * post_library_mutator;
void *(*post_deinit)(void *data);
void *post_data;
/* CmpLog */ /* CmpLog */
@ -623,8 +621,8 @@ struct custom_mutator {
const char *name; const char *name;
void * dh; void * dh;
u8 * pre_save_buf; u8 * post_process_buf;
size_t pre_save_size; size_t post_process_size;
u8 stacked_custom_prob, stacked_custom; u8 stacked_custom_prob, stacked_custom;
void *data; /* custom mutator data ptr */ void *data; /* custom mutator data ptr */
@ -675,7 +673,7 @@ struct custom_mutator {
* It can chose to alter buf in-place, if the space is large enough. * It can chose to alter buf in-place, if the space is large enough.
* @return Size of the output buffer. * @return Size of the output buffer.
*/ */
size_t (*afl_custom_pre_save)(void *data, u8 *buf, size_t buf_size, size_t (*afl_custom_post_process)(void *data, u8 *buf, size_t buf_size,
u8 **out_buf); u8 **out_buf);
/** /**
@ -825,7 +823,7 @@ u8 trim_case_custom(afl_state_t *, struct queue_entry *q, u8 *in_buf,
struct custom_mutator *load_custom_mutator_py(afl_state_t *, char *); struct custom_mutator *load_custom_mutator_py(afl_state_t *, char *);
void finalize_py_module(void *); void finalize_py_module(void *);
size_t pre_save_py(void *, u8 *, size_t, u8 **); size_t post_process_py(void *, u8 *, size_t, u8 **);
s32 init_trim_py(void *, u8 *, size_t); s32 init_trim_py(void *, u8 *, size_t);
s32 post_trim_py(void *, u8); s32 post_trim_py(void *, u8);
size_t trim_py(void *, u8 **); size_t trim_py(void *, u8 **);

View File

@ -51,17 +51,6 @@ u8 common_fuzz_cmplog_stuff(afl_state_t *afl, u8 *out_buf, u32 len) {
u8 fault; u8 fault;
if (afl->post_handler) {
u8 *post_buf = NULL;
size_t post_len =
afl->post_handler(afl->post_data, out_buf, len, &post_buf);
if (!post_buf || !post_len) { return 0; }
out_buf = post_buf;
len = post_len;
}
write_to_testcase(afl, out_buf, len); write_to_testcase(afl, out_buf, len);

View File

@ -297,12 +297,16 @@ void setup_post(afl_state_t *afl) {
dh = dlopen(fn, RTLD_NOW); dh = dlopen(fn, RTLD_NOW);
if (!dh) { FATAL("%s", dlerror()); } if (!dh) { FATAL("%s", dlerror()); }
afl->post_handler = dlsym(dh, "afl_postprocess"); struct custom_mutator * mutator;
if (!afl->post_handler) { FATAL("Symbol 'afl_postprocess' not found."); } mutator = ck_alloc(sizeof(struct custom_mutator));
afl->post_init = dlsym(dh, "afl_postprocess_init"); memset(mutator, 0, sizeof(struct custom_mutator));
if (!afl->post_init) { FATAL("Symbol 'afl_postprocess_init' not found."); }
afl->post_deinit = dlsym(dh, "afl_postprocess_deinit"); mutator->afl_custom_post_process = dlsym(dh, "afl_postprocess");
if (!afl->post_deinit) { if (!mutator->afl_custom_post_process) { FATAL("Symbol 'afl_postprocess' not found."); }
mutator->afl_custom_init = dlsym(dh, "afl_postprocess_init");
if (!mutator->afl_custom_init) { FATAL("Symbol 'afl_postprocess_init' not found."); }
mutator->afl_custom_deinit = dlsym(dh, "afl_postprocess_deinit");
if (!mutator->afl_custom_post_process) {
FATAL("Symbol 'afl_postprocess_deinit' not found."); FATAL("Symbol 'afl_postprocess_deinit' not found.");
@ -310,16 +314,10 @@ void setup_post(afl_state_t *afl) {
/* Do a quick test. It's better to segfault now than later =) */ /* Do a quick test. It's better to segfault now than later =) */
u8 *post_buf = NULL; mutator->data = mutator->afl_custom_init(afl, rand_below(afl, 0xFFFFFFFF));
afl->post_data = afl->post_init(afl); if (!mutator->data) { FATAL("Could not initialize post handler."); }
if (!afl->post_data) { FATAL("Could not initialize post handler."); }
size_t post_len = afl->post_handler(afl->post_data, tbuf, tlen, &post_buf); afl->post_library_mutator = mutator;
if (!post_len || !post_buf) {
SAYF("Empty return in test post handler for buf=\"hello\\0\".");
}
OKF("Postprocessor installed successfully."); OKF("Postprocessor installed successfully.");

View File

@ -108,6 +108,8 @@ void setup_custom_mutators(afl_state_t *afl) {
#endif #endif
if (afl->post_library_mutator) list_append(&afl->custom_mutator_list, afl->post_library_mutator);
} }
void destroy_custom_mutators(afl_state_t *afl) { void destroy_custom_mutators(afl_state_t *afl) {
@ -120,11 +122,11 @@ void destroy_custom_mutators(afl_state_t *afl) {
if (el->afl_custom_deinit) el->afl_custom_deinit(el->data); if (el->afl_custom_deinit) el->afl_custom_deinit(el->data);
if (el->dh) dlclose(el->dh); if (el->dh) dlclose(el->dh);
if (el->pre_save_buf) { if (el->post_process_buf) {
ck_free(el->pre_save_buf); ck_free(el->post_process_buf);
el->pre_save_buf = NULL; el->post_process_buf = NULL;
el->pre_save_size = 0; el->post_process_size = 0;
} }
@ -170,10 +172,10 @@ struct custom_mutator *load_custom_mutator(afl_state_t *afl, const char *fn) {
mutator->afl_custom_deinit = dlsym(dh, "afl_custom_deinit"); mutator->afl_custom_deinit = dlsym(dh, "afl_custom_deinit");
if (!mutator->afl_custom_deinit) FATAL("Symbol 'afl_custom_init' not found."); if (!mutator->afl_custom_deinit) FATAL("Symbol 'afl_custom_init' not found.");
/* "afl_custom_pre_save", optional */ /* "afl_custom_post_process", optional */
mutator->afl_custom_pre_save = dlsym(dh, "afl_custom_pre_save"); mutator->afl_custom_post_process = dlsym(dh, "afl_custom_post_process");
if (!mutator->afl_custom_pre_save) if (!mutator->afl_custom_post_process)
ACTF("optional symbol 'afl_custom_pre_save' not found."); ACTF("optional symbol 'afl_custom_post_process' not found.");
u8 notrim = 0; u8 notrim = 0;
/* "afl_custom_init_trim", optional */ /* "afl_custom_init_trim", optional */

View File

@ -142,8 +142,8 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) {
py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "mutate"); py_functions[PY_FUNC_FUZZ] = PyObject_GetAttrString(py_module, "mutate");
if (!py_functions[PY_FUNC_FUZZ]) if (!py_functions[PY_FUNC_FUZZ])
WARNF("fuzz function not found in python module"); WARNF("fuzz function not found in python module");
py_functions[PY_FUNC_PRE_SAVE] = py_functions[PY_FUNC_POST_PROCESS] =
PyObject_GetAttrString(py_module, "pre_save"); PyObject_GetAttrString(py_module, "post_process");
py_functions[PY_FUNC_INIT_TRIM] = py_functions[PY_FUNC_INIT_TRIM] =
PyObject_GetAttrString(py_module, "init_trim"); PyObject_GetAttrString(py_module, "init_trim");
py_functions[PY_FUNC_POST_TRIM] = py_functions[PY_FUNC_POST_TRIM] =
@ -165,9 +165,9 @@ static py_mutator_t *init_py_module(afl_state_t *afl, u8 *module_name) {
if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) { if (!py_functions[py_idx] || !PyCallable_Check(py_functions[py_idx])) {
if (py_idx == PY_FUNC_PRE_SAVE) { if (py_idx == PY_FUNC_POST_PROCESS) {
// Implenting the pre_save API is optional for now // Implenting the post_process API is optional for now
if (PyErr_Occurred()) { PyErr_Print(); } if (PyErr_Occurred()) { PyErr_Print(); }
} else if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) { } else if (py_idx >= PY_FUNC_INIT_TRIM && py_idx <= PY_FUNC_TRIM) {
@ -309,8 +309,8 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl,
struct custom_mutator *mutator; struct custom_mutator *mutator;
mutator = ck_alloc(sizeof(struct custom_mutator)); mutator = ck_alloc(sizeof(struct custom_mutator));
mutator->pre_save_buf = NULL; mutator->post_process_buf = NULL;
mutator->pre_save_size = 0; mutator->post_process_size = 0;
mutator->name = module_name; mutator->name = module_name;
ACTF("Loading Python mutator library from '%s'...", module_name); ACTF("Loading Python mutator library from '%s'...", module_name);
@ -330,9 +330,9 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl,
is quite different from the custom mutator. */ is quite different from the custom mutator. */
mutator->afl_custom_fuzz = fuzz_py; mutator->afl_custom_fuzz = fuzz_py;
if (py_functions[PY_FUNC_PRE_SAVE]) { if (py_functions[PY_FUNC_POST_PROCESS]) {
mutator->afl_custom_pre_save = pre_save_py; mutator->afl_custom_post_process = post_process_py;
} }
@ -384,7 +384,7 @@ struct custom_mutator *load_custom_mutator_py(afl_state_t *afl,
} }
size_t pre_save_py(void *py_mutator, u8 *buf, size_t buf_size, u8 **out_buf) { size_t post_process_py(void *py_mutator, u8 *buf, size_t buf_size, u8 **out_buf) {
size_t py_out_buf_size; size_t py_out_buf_size;
PyObject * py_args, *py_value; PyObject * py_args, *py_value;
@ -395,14 +395,14 @@ size_t pre_save_py(void *py_mutator, u8 *buf, size_t buf_size, u8 **out_buf) {
if (!py_value) { if (!py_value) {
Py_DECREF(py_args); Py_DECREF(py_args);
FATAL("Failed to convert arguments in custom pre_save"); FATAL("Failed to convert arguments in custom post_process");
} }
PyTuple_SetItem(py_args, 0, py_value); PyTuple_SetItem(py_args, 0, py_value);
py_value = PyObject_CallObject( py_value = PyObject_CallObject(
((py_mutator_t *)py_mutator)->py_functions[PY_FUNC_PRE_SAVE], py_args); ((py_mutator_t *)py_mutator)->py_functions[PY_FUNC_POST_PROCESS], py_args);
Py_DECREF(py_args); Py_DECREF(py_args);
@ -410,18 +410,18 @@ size_t pre_save_py(void *py_mutator, u8 *buf, size_t buf_size, u8 **out_buf) {
py_out_buf_size = PyByteArray_Size(py_value); py_out_buf_size = PyByteArray_Size(py_value);
ck_maybe_grow(BUF_PARAMS(pre_save), py_out_buf_size); ck_maybe_grow(BUF_PARAMS(post_process), py_out_buf_size);
memcpy(py->pre_save_buf, PyByteArray_AsString(py_value), py_out_buf_size); memcpy(py->post_process_buf, PyByteArray_AsString(py_value), py_out_buf_size);
Py_DECREF(py_value); Py_DECREF(py_value);
*out_buf = py->pre_save_buf; *out_buf = py->post_process_buf;
return py_out_buf_size; return py_out_buf_size;
} else { } else {
PyErr_Print(); PyErr_Print();
FATAL("Python custom mutator: pre_save call failed."); FATAL("Python custom mutator: post_process call failed.");
} }

View File

@ -97,10 +97,10 @@ void write_to_testcase(afl_state_t *afl, void *mem, u32 len) {
LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, { LIST_FOREACH(&afl->custom_mutator_list, struct custom_mutator, {
if (el->afl_custom_pre_save) { if (el->afl_custom_post_process) {
new_size = new_size =
el->afl_custom_pre_save(el->data, new_mem, new_size, &new_buf); el->afl_custom_post_process(el->data, new_mem, new_size, &new_buf);
} }
@ -110,7 +110,7 @@ void write_to_testcase(afl_state_t *afl, void *mem, u32 len) {
if (unlikely(!new_buf && (new_size <= 0))) { if (unlikely(!new_buf && (new_size <= 0))) {
FATAL("Custom_pre_save failed (ret: %lu)", (long unsigned)new_size); FATAL("Custom_post_process failed (ret: %lu)", (long unsigned)new_size);
} else if (likely(new_buf)) { } else if (likely(new_buf)) {
@ -119,7 +119,7 @@ void write_to_testcase(afl_state_t *afl, void *mem, u32 len) {
} else { } else {
/* custom mutators do not has a custom_pre_save function */ /* custom mutators do not has a custom_post_process function */
afl_fsrv_write_to_testcase(&afl->fsrv, mem, len); afl_fsrv_write_to_testcase(&afl->fsrv, mem, len);
} }
@ -690,18 +690,6 @@ u8 common_fuzz_stuff(afl_state_t *afl, u8 *out_buf, u32 len) {
u8 fault; u8 fault;
if (afl->post_handler) {
u8 *post_buf = NULL;
size_t post_len =
afl->post_handler(afl->post_data, out_buf, len, &post_buf);
if (!post_buf || !post_len) { return 0; }
out_buf = post_buf;
len = post_len;
}
write_to_testcase(afl, out_buf, len); write_to_testcase(afl, out_buf, len);
fault = fuzz_run_target(afl, &afl->fsrv, afl->fsrv.exec_tmout); fault = fuzz_run_target(afl, &afl->fsrv, afl->fsrv.exec_tmout);

View File

@ -371,7 +371,6 @@ void read_afl_environment(afl_state_t *afl, char **envp) {
void afl_state_deinit(afl_state_t *afl) { void afl_state_deinit(afl_state_t *afl) {
if (afl->post_deinit) { afl->post_deinit(afl->post_data); }
if (afl->in_place_resume) { ck_free(afl->in_dir); } if (afl->in_place_resume) { ck_free(afl->in_dir); }
if (afl->sync_id) { ck_free(afl->out_dir); } if (afl->sync_id) { ck_free(afl->out_dir); }
if (afl->pass_stats) { ck_free(afl->pass_stats); } if (afl->pass_stats) { ck_free(afl->pass_stats); }