ExternalVendorCode/AFLplusplus
1
0
Fork 0
mirror of https://github.com/AFLplusplus/AFLplusplus.git synced 2025-06-18 20:48:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

add afl-ld-lto for LTO

Browse Source
This commit is contained in:
van Hauser
2020-06-04 13:57:16 +02:00
parent f1ad9bdaea
commit 855ee06247
5 changed files with 440 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
docs/Changelog.md
View File

@ -22,6 +22,8 @@ sending a mail to <afl-users+subscribe@googlegroups.com>.
better coverage. The original afl instrumentation can be set via
AFL_LLVM_INSTRUMENT=AFL. This is automatically done when the WHITELIST
feature is used.
- some targets want a ld variant for LD that is not gcc/clang but ld, added
afl-ld-lto to solve this
- lowered minimum required llvm version to 3.4 (except LLVMInsTrim,
which needs 3.8.0)
- WHITELIST feature now supports wildcards (thanks to sirmc)

10
llvm_mode/GNUmakefile
View File

@ -246,7 +246,7 @@ ifeq "$(TEST_MMAP)" "1"
LDFLAGS += -Wno-deprecated-declarations
endif
PROGS = ../afl-clang-fast ../afl-llvm-pass.so ../afl-llvm-lto-whitelist.so ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so ../cmplog-routines-pass.so ../cmplog-instructions-pass.so
PROGS = ../afl-clang-fast ../afl-llvm-pass.so ../afl-ld-lto ../afl-llvm-lto-whitelist.so ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../libLLVMInsTrim.so ../afl-llvm-rt.o ../afl-llvm-rt-32.o ../afl-llvm-rt-64.o ../compare-transform-pass.so ../split-compares-pass.so ../split-switches-pass.so ../cmplog-routines-pass.so ../cmplog-instructions-pass.so
# If prerequisites are not given, warn, do not build anything, and exit with code 0
ifeq "$(LLVMVER)" ""
@ -304,7 +304,7 @@ afl-common.o: ../src/afl-common.c
$(CC) $(CFLAGS) -c $< -o $@ $(LDFLAGS)
../afl-clang-fast: afl-clang-fast.c afl-common.o | test_deps
$(CC) $(CFLAGS) $< afl-common.o -o $@ $(LDFLAGS) -DCFLAGS_OPT=\"$(CFLAGS_OPT)\" -Dxxx
$(CC) $(CFLAGS) $< afl-common.o -o $@ $(LDFLAGS) -DCFLAGS_OPT=\"$(CFLAGS_OPT)\"
ln -sf afl-clang-fast ../afl-clang-fast++
ifneq "$(AFL_CLANG_FLTO)" ""
ifeq "$(LLVM_LTO)" "1"
@ -330,6 +330,11 @@ ifeq "$(LLVM_LTO)" "1"
$(CXX) $(CLANG_CFL) -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) afl-llvm-common.o
endif
../afl-ld-lto: afl-ld-lto.c
ifeq "$(LLVM_LTO)" "1"
$(CC) $(CFLAGS) $< -o $@
endif
../afl-llvm-lto-instrumentation.so: afl-llvm-lto-instrumentation.so.cc afl-llvm-common.o
ifeq "$(LLVM_LTO)" "1"
$(CXX) $(CLANG_CFL) -Wno-writable-strings -fno-rtti -fPIC -std=$(LLVM_STDCXX) -shared $< -o $@ $(CLANG_LFL) afl-llvm-common.o
@ -392,6 +397,7 @@ install: all
install -d -m 755 $${DESTDIR}$(BIN_PATH) $${DESTDIR}$(HELPER_PATH) $${DESTDIR}$(DOC_PATH) $${DESTDIR}$(MISC_PATH)
if [ -f ../afl-clang-fast -a -f ../libLLVMInsTrim.so -a -f ../afl-llvm-rt.o ]; then set -e; install -m 755 ../afl-clang-fast $${DESTDIR}$(BIN_PATH); ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-fast++; install -m 755 ../libLLVMInsTrim.so ../afl-llvm-pass.so ../afl-llvm-rt.o $${DESTDIR}$(HELPER_PATH); fi
if [ -f ../afl-clang-lto ]; then set -e; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-lto; ln -sf afl-clang-fast $${DESTDIR}$(BIN_PATH)/afl-clang-lto++; install -m 755 ../afl-llvm-lto-instrumentation.so ../afl-llvm-lto-instrim.so ../afl-llvm-rt-lto*.o ../afl-llvm-lto-whitelist.so $${DESTDIR}$(HELPER_PATH); fi
if [ -f ../afl-ld-lto ]; then set -e; install -m 755 ../afl-ld-lto $${DESTDIR}$(BIN_PATH); fi
if [ -f ../afl-llvm-rt-32.o ]; then set -e; install -m 755 ../afl-llvm-rt-32.o $${DESTDIR}$(HELPER_PATH); fi
if [ -f ../afl-llvm-rt-64.o ]; then set -e; install -m 755 ../afl-llvm-rt-64.o $${DESTDIR}$(HELPER_PATH); fi
if [ -f ../compare-transform-pass.so ]; then set -e; install -m 755 ../compare-transform-pass.so $${DESTDIR}$(HELPER_PATH); fi

76
llvm_mode/README.lto.md
View File

@ -14,9 +14,11 @@ This version requires a current llvm 11 compiled from the github master.
4. AUTODICTIONARY feature! see below
5. If any problems arise be sure to set `AR=llvm-ar RANLIB=llvm-ranlib` also
note that if that target uses _init functions or early constructors then
also set `AFL_LLVM_MAP_DYNAMIC=1` as your target will crash otherwise
5. If any problems arise be sure to set `AR=llvm-ar RANLIB=llvm-ranlib`.
Some targets might need `LD=afl-clang-lto` and others `LD=afl-ld-lto`.
6. If a target uses _init functions or early constructors then additionally
set `AFL_LLVM_MAP_DYNAMIC=1` as your target will crash otherwise!
## Introduction and problem description
@ -61,7 +63,8 @@ AUTODICTIONARY: 11 strings found
## Getting llvm 11
### Installing llvm 11
### Installing llvm 11 from the llvm repository
Installing the llvm snapshot builds is easy and mostly painless:
In the follow line change `NAME` for your Debian or Ubuntu release name
@ -80,7 +83,7 @@ apt-get install -y clang-11 clang-tools-11 libc++1-11 libc++-11-dev \
libomp5-11 lld-11 lldb-11 llvm-11 llvm-11-dev llvm-11-runtime llvm-11-tools
```
### Building llvm 11
### Building llvm 11 yourself
Building llvm from github takes quite some long time and is not painless:
```
@ -117,6 +120,9 @@ export AFL_LLVM_INSTRUMENT=CFG
make
```
NOTE: some targets also need to set the linker, try both `afl-clang-lto` and
`afl-ld-lto` for this for `LD=` for `configure`.
## AUTODICTIONARY feature
Setting `AFL_LLVM_LTO_AUTODICTIONARY` will generate a dictionary in the
@ -135,6 +141,51 @@ to be dynamic - the original afl way, which is slower).
AFL_LLVM_MAP_DYNAMIC can be set so the shared memory address is dynamic (which
is safer but also slower).
## Solving difficult targets
Some targets are difficult because the configure script does unusual stuff that
is unexpected for afl. See the next chapter `Potential issues` how to solve
these.
An example of a hard to solve target is ffmpeg. Here is how to successfully
instrument it:
1. Get and extract the current ffmpeg and change to it's directory
2. Running configure with --cc=clang fails and various other items will fail
when compiling, so we have to trick configure:
```
./configure --enable-lto --disable-shared
```
3. Now the configuration is done - and we edit the settings in `./ffbuild/config.mak`
(-: the original line, +: what to change it into):
```
-CC=gcc
+CC=afl-clang-lto
-CXX=g++
+CXX=afl-clang-lto++
-AS=gcc
+AS=llvm-as
-LD=gcc
+LD=afl-clang-lto++
-DEPCC=gcc
+DEPCC=afl-clang-lto
-DEPAS=gcc
+DEPAS=afl-clang-lto++
-AR=ar
+AR=llvm-ar
-AR_CMD=ar
+AR_CMD=llvm-ar
-NM_CMD=nm -g
+NM_CMD=llvm-nm -g
-RANLIB=ranlib -D
+RANLIB=llvm-ranlib -D
```
4. Then type make, wait for a long time and you are done :)
## Potential issues
### compiling libraries fails
@ -154,6 +205,16 @@ and on some target you have to to AR=/RANLIB= even for make as the configure scr
Other targets ignore environment variables and need the parameters set via
`./configure --cc=... --cxx= --ranlib= ...` etc. (I am looking at you ffmpeg!).
If you see this message
```
assembler command failed ...
```
then try setting `llvm-as` for configure:
```
AS=llvm-as ...
```
### compiling programs still fail
afl-clang-lto is still work in progress.
@ -166,11 +227,12 @@ Hence if building a target with afl-clang-lto fails try to build it with llvm11
and LTO enabled (`CC=clang-11` `CXX=clang++-11` `CFLAGS=-flto=full` and
`CXXFLAGS=-flto=full`).
An example that does not build with llvm 11 and LTO is ffmpeg.
If this succeeeds then there is an issue with afl-clang-lto. Please report at
[https://github.com/AFLplusplus/AFLplusplus/issues/226](https://github.com/AFLplusplus/AFLplusplus/issues/226)
Even some targets where clang-11 fails can be build if the fail is just in
`./configure`, see `Solving difficult targets` above.
### Target crashes immediately
If the target is using early constructors (priority values smaller than 6)

358
llvm_mode/afl-ld-lto.c Normal file
View File

@ -0,0 +1,358 @@
/*
american fuzzy lop++ - wrapper for llvm 11+ lld
-----------------------------------------------
Written by Marc Heuse <mh@mh-sec.de> for afl++
Maintained by Marc Heuse <mh@mh-sec.de>,
Heiko Eißfeldt <heiko.eissfeldt@hexco.de>
Andrea Fioraldi <andreafioraldi@gmail.com>
Dominik Maier <domenukk@gmail.com>
Copyright 2019-2020 AFLplusplus Project. All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at:
http://www.apache.org/licenses/LICENSE-2.0
The sole purpose of this wrapper is to preprocess clang LTO files when
linking with lld and performing the instrumentation on the whole program.
*/
#define AFL_MAIN
#define _GNU_SOURCE
#include "config.h"
#include "types.h"
#include "debug.h"
#include "alloc-inl.h"
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include <ctype.h>
#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/wait.h>
#include <sys/time.h>
#include <dirent.h>
#define MAX_PARAM_COUNT 4096
static u8 **ld_params; /* Parameters passed to the real 'ld' */
static u8 *afl_path = AFL_PATH;
static u8 *real_ld = AFL_REAL_LD;
static u8 be_quiet, /* Quiet mode (no stderr output) */
debug, /* AFL_DEBUG */
passthrough, /* AFL_LD_PASSTHROUGH - no link+optimize*/
just_version; /* Just show version? */
static u32 ld_param_cnt = 1; /* Number of params to 'ld' */
/* Examine and modify parameters to pass to 'ld', 'llvm-link' and 'llmv-ar'.
Note that the file name is always the last parameter passed by GCC,
so we exploit this property to keep the code "simple". */
static void edit_params(int argc, char **argv) {
u32 i, instrim = 0, gold_pos = 0, gold_present = 0, rt_present = 0,
rt_lto_present = 0, inst_present = 0;
char *ptr;
ld_params = ck_alloc(4096 * sizeof(u8 *));
ld_params[0] = (u8 *)real_ld;
if (!passthrough) {
for (i = 1; i < argc; i++) {
if (strstr(argv[i], "/afl-llvm-rt-lto.o") != NULL) rt_lto_present = 1;
if (strstr(argv[i], "/afl-llvm-rt.o") != NULL) rt_present = 1;
if (strstr(argv[i], "/afl-llvm-lto-instr") != NULL) inst_present = 1;
}
for (i = 1; i < argc && !gold_pos; i++) {
if (strcmp(argv[i], "-plugin") == 0) {
if (strncmp(argv[i], "-plugin=", strlen("-plugin=")) == 0) {
if (strcasestr(argv[i], "LLVMgold.so") != NULL)
gold_present = gold_pos = i + 1;
} else if (i < argc && strcasestr(argv[i + 1], "LLVMgold.so") != NULL) {
gold_present = gold_pos = i + 2;
}
}
}
if (!gold_pos) {
for (i = 1; i + 1 < argc && !gold_pos; i++) {
if (argv[i][0] != '-') {
if (argv[i - 1][0] == '-') {
switch (argv[i - 1][1]) {
case 'b':
break;
case 'd':
break;
case 'e':
break;
case 'F':
break;
case 'f':
break;
case 'I':
break;
case 'l':
break;
case 'L':
break;
case 'm':
break;
case 'o':
break;
case 'O':
break;
case 'p':
if (index(argv[i - 1], '=') == NULL) gold_pos = i;
break;
case 'R':
break;
case 'T':
break;
case 'u':
break;
case 'y':
break;
case 'z':
break;
case '-': {
if (strcmp(argv[i - 1], "--oformat") == 0) break;
if (strcmp(argv[i - 1], "--output") == 0) break;
if (strncmp(argv[i - 1], "--opt-remarks-", 14) == 0) break;
gold_pos = i;
break;
}
default:
gold_pos = i;
}
} else
gold_pos = i;
}
}
}
if (!gold_pos) gold_pos = 1;
}
if (getenv("AFL_LLVM_INSTRIM"))
instrim = 1;
else if ((ptr = getenv("AFL_LLVM_INSTRUMENT")) &&
(strcasestr(ptr, "CFG") == 0 || strcasestr(ptr, "INSTRIM") == 0))
instrim = 1;
if (debug)
SAYF(cMGN "[D] " cRST
"passthrough=%s instrim=%d, gold_pos=%d, gold_present=%s "
"inst_present=%s rt_present=%s rt_lto_present=%s\n",
passthrough ? "true" : "false", instrim, gold_pos,
gold_present ? "true" : "false", inst_present ? "true" : "false",
rt_present ? "true" : "false", rt_lto_present ? "true" : "false");
for (i = 1; i < argc; i++) {
if (ld_param_cnt >= MAX_PARAM_COUNT)
FATAL(
"Too many command line parameters because of unpacking .a archives, "
"this would need to be done by hand ... sorry! :-(");
if (strcmp(argv[i], "--afl") == 0) {
if (!be_quiet) OKF("afl++ test command line flag detected, exiting.");
exit(0);
}
if (i == gold_pos && !passthrough) {
ld_params[ld_param_cnt++] = alloc_printf("-L%s/../lib", LLVM_BINDIR);
if (!gold_present) {
ld_params[ld_param_cnt++] = "-plugin";
ld_params[ld_param_cnt++] =
alloc_printf("%s/../lib/LLVMgold.so", LLVM_BINDIR);
}
ld_params[ld_param_cnt++] = "--allow-multiple-definition";
if (!inst_present) {
if (instrim)
ld_params[ld_param_cnt++] =
alloc_printf("-mllvm=-load=%s/afl-llvm-lto-instrim.so", afl_path);
else
ld_params[ld_param_cnt++] = alloc_printf(
"-mllvm=-load=%s/afl-llvm-lto-instrumentation.so", afl_path);
}
if (!rt_present)
ld_params[ld_param_cnt++] = alloc_printf("%s/afl-llvm-rt.o", afl_path);
if (!rt_lto_present)
ld_params[ld_param_cnt++] =
alloc_printf("%s/afl-llvm-rt-lto.o", afl_path);
}
ld_params[ld_param_cnt++] = argv[i];
}
ld_params[ld_param_cnt] = NULL;
}
/* Main entry point */
int main(int argc, char **argv) {
s32 pid, i, status;
u8 * ptr;
char thecwd[PATH_MAX];
if ((ptr = getenv("AFL_LD_CALLER")) != NULL) {
FATAL("ld loop detected! Set AFL_REAL_LD!\n");
}
if (isatty(2) && !getenv("AFL_QUIET") && !getenv("AFL_DEBUG")) {
SAYF(cCYA "afl-ld-to" VERSION cRST
" by Marc \"vanHauser\" Heuse <mh@mh-sec.de>\n");
} else
be_quiet = 1;
if (getenv("AFL_DEBUG") != NULL) debug = 1;
if (getenv("AFL_PATH") != NULL) afl_path = getenv("AFL_PATH");
if (getenv("AFL_LD_PASSTHROUGH") != NULL) passthrough = 1;
if (getenv("AFL_REAL_LD") != NULL) real_ld = getenv("AFL_REAL_LD");
if (!afl_path || !*afl_path) afl_path = "/usr/local/lib/afl";
setenv("AFL_LD_CALLER", "1", 1);
if (debug) {
(void)getcwd(thecwd, sizeof(thecwd));
SAYF(cMGN "[D] " cRST "cd \"%s\";", thecwd);
for (i = 0; i < argc; i++)
SAYF(" \"%s\"", argv[i]);
SAYF("\n");
}
if (argc < 2) {
SAYF(
"\n"
"This is a helper application for afl-clang-lto. It is a wrapper "
"around GNU "
"llvm's 'lld',\n"
"executed by the toolchain whenever using "
"afl-clang-lto/afl-clang-lto++.\n"
"You probably don't want to run this program directly but rather pass "
"it as LD parameter to configure scripts\n\n"
"Environment variables:\n"
" AFL_LD_PASSTHROUGH do not link+optimize == no instrumentation\n"
" AFL_REAL_LD point to the real llvm 11 lld if necessary\n"
"\nafl-ld-to was compiled with the fixed real 'ld' of %s and the "
"binary path of %s\n\n",
real_ld, LLVM_BINDIR);
exit(1);
}
edit_params(argc, argv); // here most of the magic happens :-)
if (debug) {
SAYF(cMGN "[D]" cRST " cd \"%s\";", thecwd);
for (i = 0; i < ld_param_cnt; i++)
SAYF(" \"%s\"", ld_params[i]);
SAYF("\n");
}
if (!(pid = fork())) {
if (strlen(real_ld) > 1) execvp(real_ld, (char **)ld_params);
execvp("ld", (char **)ld_params); // fallback
FATAL("Oops, failed to execute 'ld' - check your PATH");
}
if (pid < 0) PFATAL("fork() failed");
if (waitpid(pid, &status, 0) <= 0) PFATAL("waitpid() failed");
if (debug) SAYF(cMGN "[D] " cRST "linker result: %d\n", status);
if (!just_version) {
if (status == 0) {
if (!be_quiet) OKF("Linker was successful");
} else {
SAYF(cLRD "[-] " cRST
"Linker failed, please investigate and send a bug report. Most "
"likely an 'ld' option is incompatible with %s.\n",
AFL_CLANG_FLTO);
}
}
exit(WEXITSTATUS(status));
}

5
src/afl-common.c
View File

@ -61,8 +61,9 @@ char *afl_environment_variables[] = {
"AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES", "AFL_IMPORT_FIRST",
"AFL_INST_LIBS", "AFL_INST_RATIO", "AFL_KEEP_TRACES", "AFL_KEEP_ASSEMBLY",
"AFL_LD_HARD_FAIL", "AFL_LD_LIMIT_MB", "AFL_LD_NO_CALLOC_OVER",
"AFL_LD_PRELOAD", "AFL_LD_VERBOSE", "AFL_LLVM_CMPLOG", "AFL_LLVM_INSTRIM",
"AFL_LLVM_CTX", "AFL_LLVM_INSTRUMENT", "AFL_LLVM_INSTRIM_LOOPHEAD",
"AFL_LD_PASSTHROUGH", "AFL_REAL_LD", "AFL_LD_PRELOAD", "AFL_LD_VERBOSE",
"AFL_LLVM_CMPLOG", "AFL_LLVM_INSTRIM", "AFL_LLVM_CTX",
"AFL_LLVM_INSTRUMENT", "AFL_LLVM_INSTRIM_LOOPHEAD",
"AFL_LLVM_LTO_AUTODICTIONARY", "AFL_LLVM_AUTODICTIONARY",
"AFL_LLVM_SKIPSINGLEBLOCK", "AFL_LLVM_INSTRIM_SKIPSINGLEBLOCK",
"AFL_LLVM_LAF_SPLIT_COMPARES", "AFL_LLVM_LAF_SPLIT_COMPARES_BITW",