ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-18 04:38:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

handle libfuzzer targets failing without dropping crash files (#108)

Browse Source
This commit is contained in:
bmc-msft
2020-10-07 09:06:03 -04:00
committed by GitHub
parent 752ec83aa5
commit 4071794de1
14 changed files with 170 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
src/agent/onefuzz-agent/Cargo.toml
View File

@ -14,6 +14,7 @@ anyhow = "1.0"
appinsights = "0.1"
async-trait = "0.1"
clap = "2.33"
tempfile = "3.1"
env_logger = "0.7"
futures = "0.3"
hex = "0.4"

2
src/agent/onefuzz-agent/src/debug/cmd.rs
View File

@ -9,6 +9,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
("generic-crash-report", Some(sub)) => crate::debug::generic_crash_report::run(sub)?,
("libfuzzer-coverage", Some(sub)) => crate::debug::libfuzzer_coverage::run(sub)?,
("libfuzzer-crash-report", Some(sub)) => crate::debug::libfuzzer_crash_report::run(sub)?,
("libfuzzer-fuzz", Some(sub)) => crate::debug::libfuzzer_fuzz::run(sub)?,
_ => println!("missing subcommand\nUSAGE : {}", args.usage()),
}
@ -21,4 +22,5 @@ pub fn args() -> App<'static, 'static> {
.subcommand(crate::debug::generic_crash_report::args())
.subcommand(crate::debug::libfuzzer_coverage::args())
.subcommand(crate::debug::libfuzzer_crash_report::args())
.subcommand(crate::debug::libfuzzer_fuzz::args())
}

103
src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs Normal file
View File

@ -0,0 +1,103 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::tasks::{
config::{CommonConfig, SyncedDir},
fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask},
utils::parse_key_value,
};
use anyhow::Result;
use clap::{App, Arg, SubCommand};
use onefuzz::blob::BlobContainerUrl;
use std::{collections::HashMap, path::PathBuf};
use tokio::runtime::Runtime;
use url::Url;
use uuid::Uuid;
async fn run_impl(config: Config) -> Result<()> {
let fuzzer = LibFuzzerFuzzTask::new(config)?;
let result = fuzzer.start_fuzzer_monitor(0, None).await?;
println!("{:#?}", result);
Ok(())
}
pub fn run(args: &clap::ArgMatches) -> Result<()> {
let crashes_dir = value_t!(args, "crashes_dir", String)?;
let inputs_dir = value_t!(args, "inputs_dir", String)?;
let target_exe = value_t!(args, "target_exe", PathBuf)?;
let target_options = args.values_of_lossy("target_options").unwrap_or_default();
let mut target_env = HashMap::new();
for opt in args.values_of_lossy("target_env").unwrap_or_default() {
let (k, v) = parse_key_value(opt)?;
target_env.insert(k, v);
}
let readonly_inputs = None;
let target_workers = Some(1);
let inputs = SyncedDir {
path: inputs_dir.into(),
url: BlobContainerUrl::new(Url::parse("https://contoso.com/inputs")?)?,
};
let crashes = SyncedDir {
path: crashes_dir.into(),
url: BlobContainerUrl::new(Url::parse("https://contoso.com/crashes")?)?,
};
let config = Config {
inputs,
readonly_inputs,
crashes,
target_exe,
target_env,
target_options,
target_workers,
common: CommonConfig {
heartbeat_queue: None,
instrumentation_key: None,
telemetry_key: None,
job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(),
task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(),
},
};
let mut rt = Runtime::new()?;
rt.block_on(async { run_impl(config).await })?;
Ok(())
}
pub fn args() -> App<'static, 'static> {
SubCommand::with_name("libfuzzer-fuzz")
.about("execute a local-only libfuzzer crash report task")
.arg(
Arg::with_name("target_exe")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("target_env")
.long("target_env")
.takes_value(true)
.multiple(true),
)
.arg(
Arg::with_name("target_options")
.long("target_options")
.takes_value(true)
.multiple(true)
.allow_hyphen_values(true)
.help("Supports hyphens. Recommendation: Set target_env first"),
)
.arg(
Arg::with_name("inputs_dir")
.takes_value(true)
.required(true),
)
.arg(
Arg::with_name("crashes_dir")
.takes_value(true)
.required(true),
)
}

1
src/agent/onefuzz-agent/src/debug/mod.rs
View File

@ -5,3 +5,4 @@ pub mod cmd;
pub mod generic_crash_report;
pub mod libfuzzer_coverage;
pub mod libfuzzer_crash_report;
pub mod libfuzzer_fuzz;

90
src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs
View File

@ -9,8 +9,10 @@ use crate::tasks::{
use anyhow::Result;
use futures::{future::try_join_all, stream::StreamExt};
use onefuzz::{
fs::list_files,
libfuzzer::{LibFuzzer, LibFuzzerLine},
monitor::DirectoryMonitor,
process::ExitStatus,
system,
telemetry::{
Event::{new_coverage, new_result, process_stats, runtime_stats},
@ -19,9 +21,11 @@ use onefuzz::{
uploader::BlobUploader,
};
use serde::Deserialize;
use std::{collections::HashMap, path::PathBuf, process::ExitStatus};
use std::{collections::HashMap, path::PathBuf};
use tempfile::tempdir;
use tokio::{
io,
fs::rename,
io::{AsyncBufReadExt, BufReader},
sync::mpsc,
task,
time::{self, Duration},
@ -82,7 +86,7 @@ impl LibFuzzerFuzzTask {
let report_stats = report_runtime_stats(workers as usize, stats_receiver, hb_client);
let fuzzers: Vec<_> = (0..workers)
.map(|id| self.start_fuzzer_monitor(id, stats_sender.clone()))
.map(|id| self.start_fuzzer_monitor(id, Some(&stats_sender)))
.collect();
let fuzzers = try_join_all(fuzzers);
@ -96,22 +100,21 @@ impl LibFuzzerFuzzTask {
//
// A run is one session of continuous fuzzing, terminated by a fuzzing error
// or discovered fault. The monitor restarts the libFuzzer when it exits.
async fn start_fuzzer_monitor(&self, worker_id: u64, stats_sender: StatsSender) -> Result<()> {
pub async fn start_fuzzer_monitor(
&self,
worker_id: u64,
stats_sender: Option<&StatsSender>,
) -> Result<()> {
loop {
let run = self.run_fuzzer(worker_id, stats_sender.clone());
if let Err(err) = run.await {
error!("Fuzzer run failed: {}", err);
}
self.run_fuzzer(worker_id, stats_sender).await?;
}
}
// Fuzz with a libFuzzer until it exits.
//
// While it runs, parse stderr for progress metrics, and report them.
async fn run_fuzzer(&self, worker_id: u64, stats_sender: StatsSender) -> Result<ExitStatus> {
use io::AsyncBufReadExt;
async fn run_fuzzer(&self, worker_id: u64, stats_sender: Option<&StatsSender>) -> Result<()> {
let crash_dir = tempdir()?;
let run_id = Uuid::new_v4();
info!("starting fuzzer run, run_id = {}", run_id);
@ -129,8 +132,7 @@ impl LibFuzzerFuzzTask {
&self.config.target_options,
&self.config.target_env,
);
let mut running =
fuzzer.fuzz(&self.config.crashes.path, &self.config.inputs.path, &inputs)?;
let mut running = fuzzer.fuzz(crash_dir.path(), &self.config.inputs.path, &inputs)?;
let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running.id()));
@ -139,27 +141,42 @@ impl LibFuzzerFuzzTask {
.stderr
.as_mut()
.ok_or_else(|| format_err!("stderr not captured"))?;
let stderr = io::BufReader::new(stderr);
let stderr = BufReader::new(stderr);
stderr
.lines()
.for_each(|line| {
let stats_sender = stats_sender.clone();
async move {
let line = line.map_err(|e| e.into());
if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, line)
{
error!("could not parse fuzzing iteration update: {}", err);
}
let mut libfuzzer_output = Vec::new();
let mut lines = stderr.lines();
while let Some(line) = lines.next_line().await? {
if let Some(stats_sender) = stats_sender {
if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, &line) {
error!("could not parse fuzzing interation update: {}", err);
}
})
.await;
}
libfuzzer_output.push(line);
}
let (exit_status, _) = tokio::join!(running, sys_info);
let exit_status: ExitStatus = exit_status?.into();
Ok(exit_status?)
let files = list_files(crash_dir.path()).await?;
// ignore libfuzzer exiting cleanly without crashing, which could happen via
// -runs=N
if !exit_status.success && files.is_empty() {
bail!(
"libfuzzer exited without generating crashes. status:{} stderr:{:?}",
serde_json::to_string(&exit_status)?,
libfuzzer_output.join("\n")
);
}
for file in &files {
if let Some(filename) = file.file_name() {
let dest = self.config.crashes.path.join(filename);
rename(file, dest).await?;
}
}
Ok(())
}
async fn init_directories(&self) -> Result<()> {
@ -195,9 +212,7 @@ impl LibFuzzerFuzzTask {
async fn monitor_new_corpus(&self) -> Result<()> {
let url = self.config.inputs.url.url();
let dir = self.config.inputs.path.clone();
let mut monitor = DirectoryMonitor::new(dir);
let mut monitor = DirectoryMonitor::new(&self.config.inputs.path);
monitor.start()?;
monitor
@ -247,15 +262,12 @@ impl LibFuzzerFuzzTask {
}
fn try_report_iter_update(
stats_sender: StatsSender,
stats_sender: &StatsSender,
worker_id: u64,
run_id: Uuid,
line: Result<String>,
line: &str,
) -> Result<()> {
let line = line?;
let line = LibFuzzerLine::parse(line)?;
if let Some(line) = line {
if let Some(line) = LibFuzzerLine::parse(line)? {
stats_sender.send(RuntimeStats {
worker_id,
run_id,