ExternalVendorCode/AFLplusplus
1
0
Fork 0
mirror of https://github.com/AFLplusplus/AFLplusplus.git synced 2025-06-13 18:48:08 +00:00
Code Issues Packages Projects Releases Wiki Activity

Finish refactoring APIs for the custom mutator and Python module

Browse Source 
- Remove AFL_PYTHON_ONLY (env) and python_only (variable)
- Unify fuzz API of the custom mutator and Python module
- Merge the custom mutator into the old python_stage, which is now renamed to custom_mutator_stage
This commit is contained in:
h1994st
2020-03-03 19:48:13 -05:00
parent 90506479e7
commit df46521658
11 changed files with 115 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.gitignore vendored
View File

@ -18,9 +18,11 @@ afl-qemu-trace
afl-showmap
afl-tmin
afl-analyze.8
afl-as.8
afl-clang-fast++.8
afl-clang-fast.8
afl-cmin.8
afl-cmin.bash.8
afl-fuzz.8
afl-gcc.8
afl-gcc-fast.8

2
docs/custom_mutator.md
View File

@ -13,7 +13,7 @@ a given grammar.
The custom mutator library is passed to afl-fuzz via the
AFL_CUSTOM_MUTATOR_LIBRARY environment variable. The library must export
the afl_custom_mutator() function and must be compiled as a shared object.
the afl_custom_fuzz() function and must be compiled as a shared object.
For example:
```
$CC -shared -Wall -O3 <lib-name>.c -o <lib-name>.so

12
docs/env_variables.md
View File

@ -223,15 +223,15 @@ checks or alter some of the more exotic semantics of the tool:
for more.
- Setting AFL_CUSTOM_MUTATOR_LIBRARY to a shared library with
afl_custom_mutator() creates additional mutations through this library.
afl_custom_fuzz() creates additional mutations through this library.
If afl-fuzz is compiled with Python (which is autodetected during builing
afl-fuzz), setting AFL_PYTHON_MODULE to a Python module can also provide
additional mutations.
If AFL_CUSTOM_MUTATOR_ONLY is also set, all mutations will solely be
performed with/from the library. See [custom_mutator.md](custom_mutator.md)
- For AFL_PYTHON_MODULE and AFL_PYTHON_ONLY - they require afl-fuzz to
be compiled with Python (which is autodetected during builing afl-fuzz).
Please see [python_mutators.md](python_mutators.md).
performed with/from the library/Python module.
This feature allows to configure custom mutators which can be very helpful
in e.g. fuzzing XML or other highly flexible structured input.
Please see [custom_mutator.md](custom_mutator.md) or [python_mutators.md](python_mutators.md).
- AFL_FAST_CAL keeps the calibration stage about 2.5x faster (albeit less
precise), which can help when starting a session against a slow target.

50
include/afl-fuzz.h
View File

@ -276,8 +276,7 @@ extern u8 cal_cycles, /* Calibration cycles defaults */
no_unlink, /* do not unlink cur_input */
use_stdin, /* use stdin for sending data */
debug, /* Debug mode */
custom_only, /* Custom mutator only mode */
python_only; /* Python-only mode */
custom_only; /* Custom mutator only mode */
extern u32 stats_update_freq; /* Stats update frequency (execs) */
@ -471,6 +470,8 @@ struct custom_mutator {
* Initialize the custom mutator.
*
* (Optional)
*
* @param seed Seed used for the mutation.
*/
void (*afl_custom_init)(unsigned int seed);
@ -479,17 +480,18 @@ struct custom_mutator {
*
* (Optional for now. Required in the future)
*
* @param[in] data Input data to be mutated
* @param[in] size Size of input data
* @param[in] buf Input data to be mutated
* @param[in] buf_size Size of input data
* @param[in] add_buf Buffer containing the additional test case
* @param[in] add_buf_size Size of the additional test case
* @param[out] mutated_out Buffer to store the mutated input
* @param[in] max_size Maximum size of the mutated output. The mutation must not
* produce data larger than max_size.
* @param[in] seed Seed used for the mutation. The mutation should produce the
* same output given the same seed.
* @return Size of the mutated output.
*/
size_t (*afl_custom_fuzz)(u8* data, size_t size, u8* mutated_out,
size_t max_size, unsigned int seed);
size_t (*afl_custom_fuzz)(u8* buf, size_t buf_size,
u8* add_buf, size_t add_buf_size,
u8* mutated_out, size_t max_size);
/**
* A post-processing function to use right before AFL writes the test case to
@ -498,12 +500,14 @@ struct custom_mutator {
* (Optional) If this functionality is not needed, simply don't define this
* function.
*
* @param[in] data Buffer containing the test case to be executed
* @param[in] size Size of the test case
* @param[out] new_data Buffer to store the test case after processing
* @return Size of data after processing
* @param[in] buf Buffer containing the test case to be executed
* @param[in] buf_size Size of the test case
* @param[out] out_buf Pointer to the buffer of storing the test case after
* processing. External library should allocate memory for out_buf. AFL++
* will release the memory after saving the test case.
* @return Size of the output buffer after processing
*/
size_t (*afl_custom_pre_save)(u8* data, size_t size, u8** new_data);
size_t (*afl_custom_pre_save)(u8* buf, size_t buf_size, u8** out_buf);
/**
* This method is called at the start of each trimming operation and receives
@ -521,11 +525,11 @@ struct custom_mutator {
*
* (Optional)
*
* @param data Buffer containing the test case
* @param size Size of the test case
* @param buf Buffer containing the test case
* @param buf_size Size of the test case
* @return The amount of possible iteration steps to trim the input
*/
u32 (*afl_custom_init_trim)(u8* data, size_t size);
u32 (*afl_custom_init_trim)(u8* buf, size_t buf_size);
/**
* This method is called for each trimming operation. It doesn't have any
@ -538,10 +542,12 @@ struct custom_mutator {
*
* (Optional)
*
* @param[out] ret Buffer containing the trimmed test case
* @param[out] ret_len Size of the trimmed test case
* @param[out] out_buf Pointer to the buffer containing the trimmed test case.
* External library should allocate memory for out_buf. AFL++ will release
* the memory after saving the test case.
* @param[out] out_buf_size Pointer to the size of the trimmed test case
*/
void (*afl_custom_trim)(u8** ret, size_t* ret_len);
void (*afl_custom_trim)(u8** out_buf, size_t* out_buf_size);
/**
* This method is called after each trim operation to inform you if your
@ -627,9 +633,9 @@ int init_py_module(u8*);
void finalize_py_module();
void init_py(unsigned int seed);
/* TODO: unify fuzz interface for custom mutator and Python mutator */
size_t fuzz_py(u8*, size_t, u8*, size_t, unsigned int);
void fuzz_py_original(char*, size_t, char*, size_t, char**, size_t*);
size_t fuzz_py(u8* buf, size_t buf_size,
u8* add_buf, size_t add_buf_size,
u8* mutated_out, size_t max_size);
size_t pre_save_py(u8* data, size_t size, u8** new_data);
u32 init_trim_py(u8*, size_t);
u32 post_trim_py(u8);

2
include/envs.h
View File

@ -24,7 +24,7 @@ const char *afl_environment_variables[] = {
"AFL_NO_X86", // not really an env but we dont want to warn on it
"AFL_PATH", "AFL_PERFORMANCE_FILE",
//"AFL_PERSISTENT", // not implemented anymore, so warn additionally
"AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE", "AFL_PYTHON_ONLY",
"AFL_POST_LIBRARY", "AFL_PRELOAD", "AFL_PYTHON_MODULE",
"AFL_QEMU_COMPCOV", "AFL_QEMU_COMPCOV_DEBUG", "AFL_QEMU_DEBUG_MAPS",
"AFL_QEMU_DISABLE_CACHE", "AFL_QEMU_PERSISTENT_ADDR",
"AFL_QEMU_PERSISTENT_CNT", "AFL_QEMU_PERSISTENT_GPR",

3
src/afl-fuzz-globals.c
View File

@ -88,8 +88,7 @@ u8 cal_cycles = CAL_CYCLES, /* Calibration cycles defaults */
no_unlink, /* do not unlink cur_input */
use_stdin = 1, /* use stdin for sending data */
be_quiet, /* is AFL_QUIET set? */
custom_only, /* Custom mutator only mode */
python_only; /* Python-only mode */
custom_only; /* Custom mutator only mode */
u32 stats_update_freq = 1; /* Stats update frequency (execs) */

4
src/afl-fuzz-mutators.c
View File

@ -27,6 +27,7 @@
void setup_custom_mutator(void) {
/* Try mutator library first */
u8* fn = getenv("AFL_CUSTOM_MUTATOR_LIBRARY");
if (fn) {
@ -41,6 +42,7 @@ void setup_custom_mutator(void) {
return;
}
/* Try Python module */
#ifdef USE_PYTHON
u8* module_name = getenv("AFL_PYTHON_MODULE");
@ -286,7 +288,7 @@ void load_custom_mutator_py(const char* module_name) {
/* "afl_custom_fuzz" should not be NULL, but the interface of Python mutator
is quite different from the custom mutator. */
mutator->afl_custom_fuzz = NULL;
mutator->afl_custom_fuzz = fuzz_py;
if (py_functions[PY_FUNC_PRE_SAVE])
mutator->afl_custom_pre_save = pre_save_py;

121
src/afl-fuzz-one.c
View File

@ -482,56 +482,6 @@ u8 fuzz_one_original(char** argv) {
if (use_radamsa > 1) goto radamsa_stage;
// custom_stage: // not used - yet
if (mutator->afl_custom_fuzz) {
stage_short = "custom";
stage_name = "custom mutator";
stage_max = len << 3;
stage_val_type = STAGE_VAL_NONE;
const u32 max_seed_size = 4096 * 4096;
u8* mutated_buf = ck_alloc(max_seed_size);
orig_hit_cnt = queued_paths + unique_crashes;
for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) {
size_t orig_size = (size_t)len;
size_t mutated_size = mutator->afl_custom_fuzz(in_buf, orig_size,
mutated_buf, max_seed_size,
UR(UINT32_MAX));
if (mutated_size > 0) {
out_buf = ck_realloc(out_buf, mutated_size);
memcpy(out_buf, mutated_buf, mutated_size);
if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) {
goto abandon_entry;
}
}
}
ck_free(mutated_buf);
new_hit_cnt = queued_paths + unique_crashes;
stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt;
stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max;
if (custom_only) {
/* Skip other stages */
ret_val = 0;
goto abandon_entry;
}
}
if (cmplog_mode) {
if (input_to_state_stage(argv, in_buf, out_buf, len, queue_cur->exec_cksum))
@ -551,11 +501,7 @@ u8 fuzz_one_original(char** argv) {
: havoc_max_mult * 100)) ||
queue_cur->passed_det) {
#ifdef USE_PYTHON
goto python_stage;
#else
goto havoc_stage;
#endif
goto custom_mutator_stage;
}
@ -564,11 +510,7 @@ u8 fuzz_one_original(char** argv) {
if (master_max && (queue_cur->exec_cksum % master_max) != master_id - 1) {
#ifdef USE_PYTHON
goto python_stage;
#else
goto havoc_stage;
#endif
goto custom_mutator_stage;
}
@ -1583,24 +1525,25 @@ skip_extras:
if (!queue_cur->passed_det) mark_as_det_done(queue_cur);
#ifdef USE_PYTHON
python_stage:
/**********************************
* EXTERNAL MUTATORS (Python API) *
**********************************/
custom_mutator_stage:
/*******************
* CUSTOM MUTATORS *
*******************/
if (!py_module) goto havoc_stage;
if (!mutator) goto havoc_stage;
if (!mutator->afl_custom_fuzz) goto havoc_stage;
stage_name = "python";
stage_short = "python";
stage_name = "custom mutator";
stage_short = "custom";
stage_max = HAVOC_CYCLES * perf_score / havoc_div / 100;
stage_val_type = STAGE_VAL_NONE;
if (stage_max < HAVOC_MIN) stage_max = HAVOC_MIN;
orig_hit_cnt = queued_paths + unique_crashes;
const u32 max_seed_size = 4096 * 4096;
u8* mutated_buf = ck_alloc(max_seed_size);
char* retbuf = NULL;
size_t retlen = 0;
orig_hit_cnt = queued_paths + unique_crashes;
for (stage_cur = 0; stage_cur < stage_max; ++stage_cur) {
@ -1647,26 +1590,24 @@ python_stage:
ck_read(fd, new_buf, target->len, target->fname);
close(fd);
fuzz_py_original(out_buf, len, new_buf, target->len, &retbuf, &retlen);
size_t mutated_size = mutator->afl_custom_fuzz(out_buf, len,
new_buf, target->len,
mutated_buf, max_seed_size);
ck_free(new_buf);
if (retbuf) {
if (mutated_size > 0) {
if (!retlen) goto abandon_entry;
out_buf = ck_realloc(out_buf, mutated_size);
memcpy(out_buf, mutated_buf, mutated_size);
if (common_fuzz_stuff(argv, retbuf, retlen)) {
if (common_fuzz_stuff(argv, out_buf, (u32)mutated_size)) {
free(retbuf);
ck_free(mutated_buf);
goto abandon_entry;
}
/* Reset retbuf/retlen */
free(retbuf);
retbuf = NULL;
retlen = 0;
/* If we're finding new stuff, let's run for a bit longer, limits
permitting. */
@ -1687,12 +1628,13 @@ python_stage:
}
ck_free(mutated_buf);
new_hit_cnt = queued_paths + unique_crashes;
stage_finds[STAGE_PYTHON] += new_hit_cnt - orig_hit_cnt;
stage_cycles[STAGE_PYTHON] += stage_max;
stage_finds[STAGE_CUSTOM_MUTATOR] += new_hit_cnt - orig_hit_cnt;
stage_cycles[STAGE_CUSTOM_MUTATOR] += stage_max;
if (python_only) {
if (custom_only) {
/* Skip other stages */
ret_val = 0;
@ -1700,8 +1642,6 @@ python_stage:
}
#endif
/****************
* RANDOM HAVOC *
****************/
@ -2270,11 +2210,10 @@ retry_splicing:
out_buf = ck_alloc_nozero(len);
memcpy(out_buf, in_buf, len);
#ifdef USE_PYTHON
goto python_stage;
#else
goto havoc_stage;
#endif
goto custom_mutator_stage;
/* ???: While integrating Python module, the author decided to jump to
python stage, but the reason behind this is not clear.*/
// goto havoc_stage;
}

122
src/afl-fuzz-python.c
View File

@ -159,67 +159,16 @@ void init_py(unsigned int seed) {
}
}
void fuzz_py_original(char* buf, size_t buflen,
char* add_buf, size_t add_buflen,
char** ret, size_t* retlen) {
size_t fuzz_py(u8* buf, size_t buf_size,
u8* add_buf, size_t add_buf_size,
u8* mutated_out, size_t max_size) {
if (py_module != NULL) {
PyObject *py_args, *py_value;
py_args = PyTuple_New(2);
py_value = PyByteArray_FromStringAndSize(buf, buflen);
if (!py_value) {
Py_DECREF(py_args);
fprintf(stderr, "Cannot convert argument\n");
return;
}
PyTuple_SetItem(py_args, 0, py_value);
py_value = PyByteArray_FromStringAndSize(add_buf, add_buflen);
if (!py_value) {
Py_DECREF(py_args);
fprintf(stderr, "Cannot convert argument\n");
return;
}
PyTuple_SetItem(py_args, 1, py_value);
py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args);
Py_DECREF(py_args);
if (py_value != NULL) {
*retlen = PyByteArray_Size(py_value);
*ret = malloc(*retlen);
memcpy(*ret, PyByteArray_AsString(py_value), *retlen);
Py_DECREF(py_value);
} else {
PyErr_Print();
fprintf(stderr, "Call failed\n");
return;
}
}
}
size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
unsigned int seed) {
size_t out_size;
size_t mutated_size;
PyObject *py_args, *py_value;
py_args = PyTuple_New(3);
py_value = PyByteArray_FromStringAndSize(data, size);
/* buf */
py_value = PyByteArray_FromStringAndSize(buf, buf_size);
if (!py_value) {
Py_DECREF(py_args);
@ -229,6 +178,18 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
PyTuple_SetItem(py_args, 0, py_value);
/* add_buf */
py_value = PyByteArray_FromStringAndSize(add_buf, add_buf_size);
if (!py_value) {
Py_DECREF(py_args);
FATAL("Failed to convert arguments");
}
PyTuple_SetItem(py_args, 1, py_value);
/* max_size */
#if PY_MAJOR_VERSION >= 3
py_value = PyLong_FromLong(max_size);
#else
@ -241,20 +202,6 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
}
PyTuple_SetItem(py_args, 1, py_value);
#if PY_MAJOR_VERSION >= 3
py_value = PyLong_FromLong(seed);
#else
py_value = PyInt_FromLong(seed);
#endif
if (!py_value) {
Py_DECREF(py_args);
FATAL("Failed to convert arguments");
}
PyTuple_SetItem(py_args, 2, py_value);
py_value = PyObject_CallObject(py_functions[PY_FUNC_FUZZ], py_args);
@ -263,11 +210,10 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
if (py_value != NULL) {
out_size = PyByteArray_Size(py_value);
memcpy(mutated_out, PyByteArray_AsString(py_value), out_size);
mutated_size = PyByteArray_Size(py_value);
memcpy(mutated_out, PyByteArray_AsString(py_value), mutated_size);
Py_DECREF(py_value);
return out_size;
return mutated_size;
} else {
@ -278,12 +224,12 @@ size_t fuzz_py(u8* data, size_t size, u8* mutated_out, size_t max_size,
}
size_t pre_save_py(u8* data, size_t size, u8** new_data) {
size_t pre_save_py(u8* buf, size_t buf_size, u8** out_buf) {
size_t new_size;
size_t out_buf_size;
PyObject *py_args, *py_value;
py_args = PyTuple_New(2);
py_value = PyByteArray_FromStringAndSize(data, size);
py_value = PyByteArray_FromStringAndSize(buf, buf_size);
if (!py_value) {
Py_DECREF(py_args);
@ -299,11 +245,11 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) {
if (py_value != NULL) {
new_size = PyByteArray_Size(py_value);
*new_data = malloc(new_size);
memcpy(*new_data, PyByteArray_AsString(py_value), new_size);
out_buf_size = PyByteArray_Size(py_value);
*out_buf = malloc(out_buf_size);
memcpy(*out_buf, PyByteArray_AsString(py_value), out_buf_size);
Py_DECREF(py_value);
return new_size;
return out_buf_size;
} else {
@ -314,12 +260,12 @@ size_t pre_save_py(u8* data, size_t size, u8** new_data) {
}
u32 init_trim_py(u8* buf, size_t buflen) {
u32 init_trim_py(u8* buf, size_t buf_size) {
PyObject *py_args, *py_value;
py_args = PyTuple_New(1);
py_value = PyByteArray_FromStringAndSize(buf, buflen);
py_value = PyByteArray_FromStringAndSize(buf, buf_size);
if (!py_value) {
Py_DECREF(py_args);
@ -389,7 +335,7 @@ u32 post_trim_py(u8 success) {
}
void trim_py(u8** ret, size_t* retlen) {
void trim_py(u8** out_buf, size_t* out_buf_size) {
PyObject *py_args, *py_value;
@ -399,9 +345,9 @@ void trim_py(u8** ret, size_t* retlen) {
if (py_value != NULL) {
*retlen = PyByteArray_Size(py_value);
*ret = malloc(*retlen);
memcpy(*ret, PyByteArray_AsString(py_value), *retlen);
*out_buf_size = PyByteArray_Size(py_value);
*out_buf = malloc(*out_buf_size);
memcpy(*out_buf, PyByteArray_AsString(py_value), *out_buf_size);
Py_DECREF(py_value);
} else {

3
src/afl-fuzz-run.c
View File

@ -309,11 +309,12 @@ void write_to_testcase(void* mem, u32 len) {
lseek(fd, 0, SEEK_SET);
if (mutator->afl_custom_pre_save) {
if (mutator && mutator->afl_custom_pre_save) {
u8* new_data;
size_t new_size = mutator->afl_custom_pre_save(mem, len, &new_data);
ck_write(fd, new_data, new_size, out_file);
ck_free(new_data);
} else {

18
src/afl-fuzz.c
View File

@ -158,7 +158,6 @@ static void usage(u8* argv0, int more_help) {
"AFL_CUSTOM_MUTATOR_LIBRARY: lib with afl_custom_fuzz() to mutate inputs\n"
"AFL_CUSTOM_MUTATOR_ONLY: avoid AFL++'s internal mutators\n"
"AFL_PYTHON_MODULE: mutate and trim inputs with the specified Python module\n"
"AFL_PYTHON_ONLY: skip AFL++'s own mutators\n"
"AFL_DEBUG: extra debugging output for Python mode trimming\n"
"AFL_DISABLE_TRIM: disable the trimming of test cases\n"
"AFL_NO_UI: switch status screen off\n"
@ -658,11 +657,10 @@ int main(int argc, char** argv, char** envp) {
OKF("afl-tmin fork server patch from github.com/nccgroup/TriforceAFL");
OKF("MOpt Mutator from github.com/puppet-meteor/MOpt-AFL");
if (sync_id && force_deterministic &&
(getenv("AFL_CUSTOM_MUTATOR_ONLY") || getenv("AFL_PYTHON_ONLY")))
if (sync_id && force_deterministic && getenv("AFL_CUSTOM_MUTATOR_ONLY"))
WARNF(
"Using -M master with the AFL_..._ONLY mutator options will result in "
"no deterministic mutations being done!");
"Using -M master with the AFL_CUSTOM_MUTATOR_ONLY mutator options will "
"result in no deterministic mutations being done!");
check_environment_vars(envp);
@ -832,16 +830,6 @@ int main(int argc, char** argv, char** envp) {
if (get_afl_env("AFL_DEBUG")) debug = 1;
if (get_afl_env("AFL_PYTHON_ONLY")) {
/* This ensures we don't proceed to havoc/splice */
python_only = 1;
/* Ensure we also skip all deterministic steps */
skip_deterministic = 1;
}
if (get_afl_env("AFL_CUSTOM_MUTATOR_ONLY")) {
/* This ensures we don't proceed to havoc/splice */