add non-x86_64 architecture libfuzzer target support using qemu-user (#600)

This commit is contained in:
bmc-msft
2021-03-03 19:06:50 -05:00
committed by GitHub
parent 92c1d0a7a1
commit 7fc725d012
9 changed files with 493 additions and 5 deletions

View File

@ -327,7 +327,7 @@ jobs:
name: release-artifacts
path: release-artifacts
build-integration-tests-linux:
runs-on: ubuntu-latest
runs-on: ubuntu-18.04
steps:
- uses: actions/checkout@v2
- run: |
@ -363,6 +363,12 @@ jobs:
export AFL_CC_PATH=$PWD/AFL/afl-clang
(cd trivial-crash ; make clean; make CC=$AFL_CC_PATH)
cp -r trivial-crash/fuzz.exe trivial-crash/seeds artifacts/linux-trivial-crash-afl
# libfuzzer cross-compiled to aarch64
mkdir -p artifacts/linux-libfuzzer-aarch64-crosscompile
(cd libfuzzer-aarch64-crosscompile; make)
cp -r libfuzzer-aarch64-crosscompile/fuzz.exe libfuzzer-aarch64-crosscompile/inputs artifacts/linux-libfuzzer-aarch64-crosscompile
- uses: actions/upload-artifact@v2.2.2
with:
name: integration-test-artifacts

View File

@ -0,0 +1,151 @@
# Fuzzing non-X86_64 targets on Azure
Fuzzing non-x86_64 targets using libFuzzer on Azure makes use of two Open
Source capabilities:
2. [libFuzzer](https://www.llvm.org/docs/LibFuzzer.html), which performs the fuzzing
3. [qemu-user](https://qemu.readthedocs.io/en/latest/user/main.html), which provides user-space emulation.
TL/DR: check out our [libfuzzer-aarch64-crosscompile example](../../src/integration-tests/libfuzzer-aarch64-crosscompile/)
## Issues using qemu_user based LibFuzzer in OneFuzz
There are a few notable limitations when using `qemu-user` based libFuzzer targets in OneFuzz.
1. Live reproduction of crashes does not work.
2. The `libfuzzer-coverage` task does not work.
3. Only Linux is supported at this time.
4. Only `aarch64` CPU emulation has been tested. PRs are welcome to support other architectures.
5. Custom setup scripts are not supported, though you can provide your own sysroot using `--sysroot`.
As such, a `libfuzzer qemu_user` template is available, which only uses the `libfuzzer_fuzz` and `libfuzzer_crash_report`. As these issues are resolve, the template will be updated to include the additional tasks.
## Example
Let's build a simple `aarch64` target using GCC as a cross-compiler (See [our example](../../src/integration-tests/libfuzzer-aarch64-crosscompile/)).
1. Make sure you have QEMU and the appropriate cross compiler installed:
```bash
sudo apt update
sudo apt install -y qemu-user g++-aarch64-linux-gnu
```
2. Check out the libFuzzer libraries from `compiler-rt`. Note, GCC requires the use of `pc-guard` for instrumentation, which was removed by the `compiler-rt` project. As such, we need an older version of the library:
```
git clone https://github.com/llvm-mirror/compiler-rt
(cd compiler-rt; git checkout daa6759576548a2f3825faddaa6811cabbfb45eb)
```
3. Build the libFuzzer libraries *without* ASAN:
```
mkdir -p fuzz-libs
(cd fuzz-libs; aarch64-linux-gnu-g++ -c ../compiler-rt/lib/fuzzer/*.cpp)
```
4. Build our target:
```
aarch64-linux-gnu-g++ -pthread -lasan -o fuzz.exe fuzz-libs/*.o fuzz.c -fsanitize=address -fsanitize-coverage=trace-pc
```
5. Verify our target built correctly:
```
ASAN_OPTIONS=:detect_leaks=0 qemu-aarch64 -L /usr/aarch64-linux-gnu ./fuzz.exe -help=1
```
> NOTE: `LSAN` does not work in `qemu-user`, so we need to disable that.
Now we're ready to deploy this target to OneFuzz. Note, if we have custom
libraries or want to run on a different version of linux, we'll need to
provide our own sysroot.
7. Now we can fuzz!
Execute our `fuzz.exe` with `qemu-aarch64` our `inputs` directory:
```
ASAN_OPTIONS=:detect_leaks=0 qemu-aarch64 -L /usr/aarch64-linux-gnu ./fuzz.exe ./inputs
```
In a few seconds, you'll see output that looks something like this:
```
INFO: Seed: 113138795
INFO: 1 files found in ./inputs/ INFO: -max_len is not provided; libFuzzer will not generate inputs larger than 4096 bytes
INFO: seed corpus: files: 1 min: 3b max: 3b total: 3b rss: 314Mb
#2 INITED cov: 5 ft: 5 corp: 1/3b lim: 4 exec/s: 0 rss: 317Mb
#3 NEW cov: 5 ft: 9 corp: 2/6b lim: 4 exec/s: 0 rss: 317Mb L: 3/3 MS: 1 ChangeBit-
#4 NEW cov: 5 ft: 13 corp: 3/9b lim: 4 exec/s: 0 rss: 317Mb L: 3/3 MS: 1 ChangeBit-
#8 NEW cov: 10 ft: 21 corp: 4/13b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 4 ShuffleBytes-ChangeBit-ChangeBinInt-CrossOver-
#9 NEW cov: 10 ft: 26 corp: 5/17b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 1 CopyPart-
#10 NEW cov: 10 ft: 31 corp: 6/21b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 1 ChangeBit-
#11 NEW cov: 10 ft: 36 corp: 7/25b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 1 ShuffleBytes-
#12 NEW cov: 10 ft: 37 corp: 8/28b lim: 4 exec/s: 0 rss: 317Mb L: 3/4 MS: 1 EraseBytes-
#16 NEW cov: 10 ft: 45 corp: 9/32b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 4 CopyPart-CopyPart-CrossOver-ShuffleBytes-
#18 NEW cov: 11 ft: 46 corp: 10/36b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 2 CopyPart-ChangeBit-
#19 NEW cov: 11 ft: 47 corp: 11/40b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 1 ShuffleBytes-
#20 REDUCE cov: 11 ft: 47 corp: 11/39b lim: 4 exec/s: 0 rss: 317Mb L: 2/4 MS: 1 EraseBytes-
#30 NEW cov: 11 ft: 52 corp: 12/43b lim: 4 exec/s: 0 rss: 317Mb L: 4/4 MS: 5 ChangeBit-CrossOver-CrossOver-CrossOver-CrossOver-
#32 NEW cov: 11 ft: 56 corp: 13/45b lim: 4 exec/s: 0 rss: 317Mb L: 2/4 MS: 2 CrossOver-ChangeBit-
#38 REDUCE cov: 11 ft: 56 corp: 13/44b lim: 4 exec/s: 0 rss: 317Mb L: 1/4 MS: 1 EraseBytes-
```
## Launching our example in OneFuzz
These commands launches the a qemu-user based libFuzzer job in OneFuzz. Note, we've added the arguments `--wait_for_running --wait_for_files inputs` such that we can monitor our job until we've seen at least one new input found via fuzzing.
```bash
TARGET_PROJECT=AARCH64
TARGET_NAME=Example
TARGET_BUILD=1
FUZZ_POOL=linux
onefuzz template libfuzzer qemu_user ${TARGET_PROJECT} ${TARGET_NAME} ${TARGET_BUILD} ${FUZZ_POOL} --wait_for_running --wait_for_files inputs
```
When we run this, we'll see output similar to:
```
WARNING:onefuzz:qemu_user jobs are a preview feature and may change in the future
INFO:onefuzz:creating job (runtime: 24 hours)
INFO:onefuzz:created job: fa5b7870-a51b-4f79-924f-2ef11a9830a0
INFO:onefuzz:using container: oft-setup-5346d5f33bc35c3d94cbc70f7815b85e
INFO:onefuzz:using container: oft-inputs-9c31136dc16a5aab8edf7666a614a285
INFO:onefuzz:using container: oft-crashes-9c31136dc16a5aab8edf7666a614a285
INFO:onefuzz:using container: oft-reports-9c31136dc16a5aab8edf7666a614a285
INFO:onefuzz:using container: oft-unique-reports-9c31136dc16a5aab8edf7666a614a285
INFO:onefuzz:using container: oft-no-repro-9c31136dc16a5aab8edf7666a614a285
INFO:onefuzz:uploading target exe `fuzz.exe`
INFO:onefuzz:uploading /tmp/tmp_9_f9kc3/setup.sh
INFO:onefuzz:uploading /tmp/tmp_9_f9kc3/fuzz.exe-wrapper.sh
INFO:onefuzz:creating libfuzzer_fuzz task
INFO:onefuzz:creating libfuzzer_crash_report task
INFO:onefuzz:done creating tasks
- waiting on: libfuzzer_crash_report:init, libfuzzer_fuzz:init
- waiting on: libfuzzer_crash_report:waiting, libfuzzer_fuzz:scheduled
| waiting on: libfuzzer_crash_report:waiting, libfuzzer_fuzz:setting_up
/ waiting on: libfuzzer_crash_report:waiting
| waiting on: libfuzzer_crash_report:scheduled
\ waiting on: libfuzzer_crash_report:setting_up
INFO:onefuzz:tasks started
\ waiting for new files: oft-inputs-9c31136dc16a5aab8edf7666a614a285
INFO:onefuzz:new files found
{
"config": {
"build": "1",
"duration": 24,
"name": "Example",
"project": "AARCH64"
},
"end_time": "2021-02-27T16:41:24+00:00",
"job_id": "fa5b7870-a51b-4f79-924f-2ef11a9830a0",
"state": "enabled",
"task_info": [
{
"state": "stopped",
"task_id": "15cc52b9-b15b-4cf7-9fa7-669db67c8e0b",
"type": "libfuzzer_fuzz"
},
{
"state": "running",
"task_id": "b3466396-7047-46f5-a58d-a21ada881e97",
"type": "libfuzzer_crash_report"
}
],
"user_info": {
"application_id": "e3b350d1-7863-4bd5-a4c0-83e6436c9c09",
"object_id": "232a2ac6-f8fc-4eb3-b427-0c91bbab7eea",
"upn": "example@contoso.com"
}
}
```
## See Also
* [afl-unicorn](https://github.com/Battelle/afl-unicorn) - AFL-based application fuzzing using [Unicorn](https://www.unicorn-engine.org/)
* [TriforceAFL] (https://github.com/nccgroup/TriforceAFL) - AFL-based full-system fuzzing using a fork of [QEMU](https://qemu.readthedocs.io/)

View File

@ -4,6 +4,8 @@
# Licensed under the MIT License.
import os
import tempfile
from enum import Enum
from typing import Dict, List, Optional
from onefuzztypes.enums import OS, ContainerType, TaskDebugFlag, TaskType
@ -17,6 +19,10 @@ from . import JobHelper
LIBFUZZER_MAGIC_STRING = b"ERROR: libFuzzer"
class QemuArch(Enum):
aarch64 = "aarch64"
class Libfuzzer(Command):
""" Pre-defined Libfuzzer job """
@ -488,3 +494,199 @@ class Libfuzzer(Command):
self.logger.info("done creating tasks")
helper.wait()
return helper.job
def qemu_user(
self,
project: str,
name: str,
build: str,
pool_name: PoolName,
*,
arch: QemuArch = QemuArch.aarch64,
target_exe: File = File("fuzz.exe"),
sysroot: Optional[File] = None,
vm_count: int = 1,
inputs: Optional[Directory] = None,
reboot_after_setup: bool = False,
duration: int = 24,
target_workers: Optional[int] = 1,
target_options: Optional[List[str]] = None,
target_env: Optional[Dict[str, str]] = None,
tags: Optional[Dict[str, str]] = None,
wait_for_running: bool = False,
wait_for_files: Optional[List[ContainerType]] = None,
existing_inputs: Optional[Container] = None,
debug: Optional[List[TaskDebugFlag]] = None,
ensemble_sync_delay: Optional[int] = None,
colocate_all_tasks: bool = False,
crash_report_timeout: Optional[int] = 1,
check_retry_count: Optional[int] = 300,
check_fuzzer_help: bool = True,
) -> Optional[Job]:
"""
libfuzzer tasks, wrapped via qemu-user (PREVIEW FEATURE)
"""
self.logger.warning(
"qemu_user jobs are a preview feature and may change in the future"
)
pool = self.onefuzz.pools.get(pool_name)
if pool.os != OS.linux:
raise Exception("libfuzzer qemu-user jobs are only compatible with Linux")
self._check_is_libfuzzer(target_exe)
if target_options is None:
target_options = []
# disable detect_leaks, as this is non-functional on cross-compile targets
if target_env is None:
target_env = {}
target_env["ASAN_OPTIONS"] = (
target_env.get("ASAN_OPTIONS", "") + ":detect_leaks=0"
)
helper = JobHelper(
self.onefuzz,
self.logger,
project,
name,
build,
duration,
pool_name=pool_name,
target_exe=target_exe,
)
helper.add_tags(tags)
helper.define_containers(
ContainerType.setup,
ContainerType.inputs,
ContainerType.crashes,
ContainerType.reports,
ContainerType.unique_reports,
ContainerType.no_repro,
)
if existing_inputs:
self.onefuzz.containers.get(existing_inputs)
helper.containers[ContainerType.inputs] = existing_inputs
else:
helper.define_containers(ContainerType.inputs)
fuzzer_containers = [
(ContainerType.setup, helper.containers[ContainerType.setup]),
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
(ContainerType.inputs, helper.containers[ContainerType.inputs]),
]
helper.create_containers()
target_exe_blob_name = helper.target_exe_blob_name(target_exe, None)
wrapper_name = File(target_exe_blob_name + "-wrapper.sh")
with tempfile.TemporaryDirectory() as tempdir:
if sysroot:
setup_path = File(os.path.join(tempdir, "setup.sh"))
with open(setup_path, "w") as handle:
sysroot_filename = helper.target_exe_blob_name(sysroot, None)
handle.write(
"#!/bin/bash\n"
"set -ex\n"
"sudo apt-get install -y qemu-user g++-aarch64-linux-gnu libasan5-arm64-cross\n"
'cd $(dirname "$(readlink -f "$0")")\n'
"mkdir -p sysroot\n"
"tar -C sysroot -zxvf %s\n" % sysroot_filename
)
wrapper_path = File(os.path.join(tempdir, wrapper_name))
with open(wrapper_path, "w") as handle:
handle.write(
"#!/bin/bash\n"
'SETUP_DIR=$(dirname "$(readlink -f "$0")")\n'
"qemu-%s -L $SETUP_DIR/sysroot $SETUP_DIR/%s $*"
% (arch.name, target_exe_blob_name)
)
upload_files = [setup_path, wrapper_path, sysroot]
else:
setup_path = File(os.path.join(tempdir, "setup.sh"))
with open(setup_path, "w") as handle:
handle.write(
"#!/bin/bash\n"
"set -ex\n"
"sudo apt-get install -y qemu-user g++-aarch64-linux-gnu libasan5-arm64-cross\n"
)
wrapper_path = File(os.path.join(tempdir, wrapper_name))
with open(wrapper_path, "w") as handle:
handle.write(
"#!/bin/bash\n"
'SETUP_DIR=$(dirname "$(readlink -f "$0")")\n'
"qemu-%s -L /usr/%s-linux-gnu $SETUP_DIR/%s $*"
% (arch.name, arch.name, target_exe_blob_name)
)
upload_files = [setup_path, wrapper_path]
helper.upload_setup(None, target_exe, upload_files)
if inputs:
helper.upload_inputs(inputs)
helper.wait_on(wait_for_files, wait_for_running)
self.logger.info("creating libfuzzer_fuzz task")
fuzzer_task = self.onefuzz.tasks.create(
helper.job.job_id,
TaskType.libfuzzer_fuzz,
wrapper_name,
fuzzer_containers,
pool_name=pool_name,
reboot_after_setup=reboot_after_setup,
duration=duration,
vm_count=vm_count,
target_options=target_options,
target_env=target_env,
target_workers=target_workers,
tags=tags,
debug=debug,
ensemble_sync_delay=ensemble_sync_delay,
expect_crash_on_failure=False,
check_fuzzer_help=check_fuzzer_help,
)
report_containers = [
(ContainerType.setup, helper.containers[ContainerType.setup]),
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
(ContainerType.reports, helper.containers[ContainerType.reports]),
(
ContainerType.unique_reports,
helper.containers[ContainerType.unique_reports],
),
(ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
]
self.logger.info("creating libfuzzer_crash_report task")
self.onefuzz.tasks.create(
helper.job.job_id,
TaskType.libfuzzer_crash_report,
wrapper_name,
report_containers,
pool_name=pool_name,
duration=duration,
vm_count=1,
reboot_after_setup=reboot_after_setup,
target_options=target_options,
target_env=target_env,
tags=tags,
prereq_tasks=[fuzzer_task.task_id],
target_timeout=crash_report_timeout,
check_retry_count=check_retry_count,
debug=debug,
colocate=colocate_all_tasks,
expect_crash_on_failure=False,
check_fuzzer_help=check_fuzzer_help,
)
self.logger.info("done creating tasks")
helper.wait()
return helper.job

View File

@ -42,6 +42,7 @@ BUILD = "0"
class TemplateType(Enum):
libfuzzer = "libfuzzer"
libfuzzer_dotnet = "libfuzzer_dotnet"
libfuzzer_qemu_user = "libfuzzer_qemu_user"
afl = "afl"
radamsa = "radamsa"
@ -57,6 +58,7 @@ class Integration(BaseModel):
check_asan_log: Optional[bool] = Field(default=False)
disable_check_debugger: Optional[bool] = Field(default=False)
reboot_after_setup: Optional[bool] = Field(default=False)
test_repro: Optional[bool] = Field(default=True)
TARGETS: Dict[str, Integration] = {
@ -84,6 +86,15 @@ TARGETS: Dict[str, Integration] = {
use_setup=True,
wait_for_files=[ContainerType.inputs, ContainerType.crashes],
),
"linux-libfuzzer-aarch64-crosscompile": Integration(
template=TemplateType.libfuzzer_qemu_user,
os=OS.linux,
target_exe="fuzz.exe",
inputs="inputs",
use_setup=True,
wait_for_files=[ContainerType.inputs, ContainerType.crashes],
test_repro=False,
),
"linux-libfuzzer-rust": Integration(
template=TemplateType.libfuzzer,
os=OS.linux,
@ -237,6 +248,17 @@ class TestOnefuzz:
duration=1,
vm_count=1,
)
elif config.template == TemplateType.libfuzzer_qemu_user:
job = self.of.template.libfuzzer.qemu_user(
self.project,
target,
BUILD,
self.pools[config.os].name,
inputs=inputs,
target_exe=target_exe,
duration=1,
vm_count=1,
)
elif config.template == TemplateType.radamsa:
job = self.of.template.radamsa.basic(
self.project,
@ -407,6 +429,10 @@ class TestOnefuzz:
has_cdb = bool(which("cdb.exe"))
has_gdb = bool(which("gdb"))
for job_id in self.successful_jobs:
if not TARGETS[self.target_jobs[job_id]].test_repro:
self.logger.info("skipping repro for %s", self.target_jobs[job_id])
continue
if self.job_os[job_id] == OS.linux and not has_gdb:
self.logger.warning(
"missing gdb in path, not launching repro: %s",

View File

@ -0,0 +1,3 @@
compiler-rt
fuzz-libs
fuzz.exe

View File

@ -0,0 +1,31 @@
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
all: check
fuzz-libs:
sudo apt update
sudo apt install -y qemu-user g++-aarch64-linux-gnu
git clone https://github.com/llvm-mirror/compiler-rt
# last version that supports pc-guard instrumentation, required by GCC
(cd compiler-rt; git checkout daa6759576548a2f3825faddaa6811cabbfb45eb)
# These *must* be built without ASAN
mkdir -p fuzz-libs
(cd fuzz-libs; aarch64-linux-gnu-g++ -c ../compiler-rt/lib/fuzzer/*.cpp)
fuzz.exe: fuzz-libs fuzz.c
aarch64-linux-gnu-g++ -pthread -lasan -o fuzz.exe fuzz-libs/*.o fuzz.c -fsanitize=address -fsanitize-coverage=trace-pc
check: fuzz.exe
ASAN_OPTIONS=:detect_leaks=0 qemu-aarch64 -L /usr/aarch64-linux-gnu ./fuzz.exe -runs=1
fuzz: check
ASAN_OPTIONS=:detect_leaks=0 qemu-aarch64 -L /usr/aarch64-linux-gnu ./fuzz.exe
.PHONY: check clean fuzz
clean:
rm -rf fuzz.exe fuzz-libs compiler-rt

View File

@ -0,0 +1,64 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <stdlib.h>
extern "C" int LLVMFuzzerTestOneInput(const u_int8_t *data, size_t len) {
int cnt = 0;
if (len < 4) {
return 0;
}
if (data[0] == 'x') { cnt++; }
if (data[1] == 'y') { cnt++; }
if (data[2] == 'z') { cnt++; }
if (cnt >= 3) {
switch (data[3]) {
case '0': {
// segv
int *p = NULL; *p = 123;
break;
}
case '1': {
// stack-buffer-underflow
int* p = &cnt - 32; for (int i = 0; i < 32; i++) { *(p + i) = 0; }
break;
}
case '2': {
// stack-buffer-overflow
int* p = &cnt + 32; for (int i = 0; i < 32; i++) { *(p - i) = 0; }
break;
}
case '3': {
// bad-free
int *p = &cnt; free(p);
break;
}
case '4': {
// double-free
int* p = (int *) malloc(sizeof(int)); free(p); free(p);
break;
}
case '5': {
// heap-use-after-free
int* p = (int *) malloc(sizeof(int)); free(p); *p = 123;
break;
}
case '6': {
// heap-buffer-overflow
int* p = (int *) malloc(8 * sizeof(int)); for (int i = 0; i < 32; i++) { *(p + i) = 0; }
break;
}
case '7': {
// fpe
int x = 0; int y = 123 / x;
break;
}
}
}
return 0;
}

View File

@ -9,7 +9,7 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) {
int cnt = 0;
if (len < 4) {
return 1;
return 0;
}
if (data[0] == 'x') { cnt++; }
@ -40,17 +40,17 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) {
}
case '4': {
// double-free
int* p = malloc(sizeof(int)); free(p); free(p);
int* p = (int *) malloc(sizeof(int)); free(p); free(p);
break;
}
case '5': {
// heap-use-after-free
int* p = malloc(sizeof(int)); free(p); *p = 123;
int* p = (int *) malloc(sizeof(int)); free(p); *p = 123;
break;
}
case '6': {
// heap-buffer-overflow
int* p = malloc(8 * sizeof(int)); for (int i = 0; i < 32; i++) { *(p + i) = 0; }
int* p = (int *) malloc(8 * sizeof(int)); for (int i = 0; i < 32; i++) { *(p + i) = 0; }
break;
}
case '7': {
@ -58,6 +58,10 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) {
int x = 0; int y = 123 / x;
break;
}
case '8': {
abort();
break;
}
}
}