From 4071794de1e6ffd0ec0be35e7f4276f7c12e34fb Mon Sep 17 00:00:00 2001 From: bmc-msft <41130664+bmc-msft@users.noreply.github.com> Date: Wed, 7 Oct 2020 09:06:03 -0400 Subject: [PATCH] handle libfuzzer targets failing without dropping crash files (#108) --- src/agent/onefuzz-agent/Cargo.toml | 1 + src/agent/onefuzz-agent/src/debug/cmd.rs | 2 + .../onefuzz-agent/src/debug/libfuzzer_fuzz.rs | 103 ++++++++++++++++++ src/agent/onefuzz-agent/src/debug/mod.rs | 1 + .../src/tasks/fuzz/libfuzzer_fuzz.rs | 90 ++++++++------- .../onefuzz-supervisor/src/coordinator.rs | 2 +- src/agent/onefuzz-supervisor/src/debug.rs | 2 +- src/agent/onefuzz-supervisor/src/main.rs | 1 - src/agent/onefuzz-supervisor/src/scheduler.rs | 2 +- src/agent/onefuzz-supervisor/src/setup.rs | 2 +- src/agent/onefuzz-supervisor/src/worker.rs | 2 +- src/agent/onefuzz/src/lib.rs | 5 +- src/agent/onefuzz/src/libfuzzer.rs | 6 +- .../src/process.rs | 0 14 files changed, 170 insertions(+), 49 deletions(-) create mode 100644 src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs rename src/agent/{onefuzz-supervisor => onefuzz}/src/process.rs (100%) diff --git a/src/agent/onefuzz-agent/Cargo.toml b/src/agent/onefuzz-agent/Cargo.toml index d78c24991..f2700fba9 100644 --- a/src/agent/onefuzz-agent/Cargo.toml +++ b/src/agent/onefuzz-agent/Cargo.toml @@ -14,6 +14,7 @@ anyhow = "1.0" appinsights = "0.1" async-trait = "0.1" clap = "2.33" +tempfile = "3.1" env_logger = "0.7" futures = "0.3" hex = "0.4" diff --git a/src/agent/onefuzz-agent/src/debug/cmd.rs b/src/agent/onefuzz-agent/src/debug/cmd.rs index d565272d8..83a42cf76 100644 --- a/src/agent/onefuzz-agent/src/debug/cmd.rs +++ b/src/agent/onefuzz-agent/src/debug/cmd.rs @@ -9,6 +9,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> { ("generic-crash-report", Some(sub)) => crate::debug::generic_crash_report::run(sub)?, ("libfuzzer-coverage", Some(sub)) => crate::debug::libfuzzer_coverage::run(sub)?, ("libfuzzer-crash-report", Some(sub)) => crate::debug::libfuzzer_crash_report::run(sub)?, + ("libfuzzer-fuzz", Some(sub)) => crate::debug::libfuzzer_fuzz::run(sub)?, _ => println!("missing subcommand\nUSAGE : {}", args.usage()), } @@ -21,4 +22,5 @@ pub fn args() -> App<'static, 'static> { .subcommand(crate::debug::generic_crash_report::args()) .subcommand(crate::debug::libfuzzer_coverage::args()) .subcommand(crate::debug::libfuzzer_crash_report::args()) + .subcommand(crate::debug::libfuzzer_fuzz::args()) } diff --git a/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs new file mode 100644 index 000000000..16864af4f --- /dev/null +++ b/src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs @@ -0,0 +1,103 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::tasks::{ + config::{CommonConfig, SyncedDir}, + fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask}, + utils::parse_key_value, +}; +use anyhow::Result; +use clap::{App, Arg, SubCommand}; +use onefuzz::blob::BlobContainerUrl; +use std::{collections::HashMap, path::PathBuf}; +use tokio::runtime::Runtime; +use url::Url; +use uuid::Uuid; + +async fn run_impl(config: Config) -> Result<()> { + let fuzzer = LibFuzzerFuzzTask::new(config)?; + let result = fuzzer.start_fuzzer_monitor(0, None).await?; + println!("{:#?}", result); + Ok(()) +} + +pub fn run(args: &clap::ArgMatches) -> Result<()> { + let crashes_dir = value_t!(args, "crashes_dir", String)?; + let inputs_dir = value_t!(args, "inputs_dir", String)?; + let target_exe = value_t!(args, "target_exe", PathBuf)?; + let target_options = args.values_of_lossy("target_options").unwrap_or_default(); + let mut target_env = HashMap::new(); + for opt in args.values_of_lossy("target_env").unwrap_or_default() { + let (k, v) = parse_key_value(opt)?; + target_env.insert(k, v); + } + + let readonly_inputs = None; + let target_workers = Some(1); + + let inputs = SyncedDir { + path: inputs_dir.into(), + url: BlobContainerUrl::new(Url::parse("https://contoso.com/inputs")?)?, + }; + + let crashes = SyncedDir { + path: crashes_dir.into(), + url: BlobContainerUrl::new(Url::parse("https://contoso.com/crashes")?)?, + }; + + let config = Config { + inputs, + readonly_inputs, + crashes, + target_exe, + target_env, + target_options, + target_workers, + common: CommonConfig { + heartbeat_queue: None, + instrumentation_key: None, + telemetry_key: None, + job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(), + task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(), + }, + }; + + let mut rt = Runtime::new()?; + rt.block_on(async { run_impl(config).await })?; + + Ok(()) +} + +pub fn args() -> App<'static, 'static> { + SubCommand::with_name("libfuzzer-fuzz") + .about("execute a local-only libfuzzer crash report task") + .arg( + Arg::with_name("target_exe") + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name("target_env") + .long("target_env") + .takes_value(true) + .multiple(true), + ) + .arg( + Arg::with_name("target_options") + .long("target_options") + .takes_value(true) + .multiple(true) + .allow_hyphen_values(true) + .help("Supports hyphens. Recommendation: Set target_env first"), + ) + .arg( + Arg::with_name("inputs_dir") + .takes_value(true) + .required(true), + ) + .arg( + Arg::with_name("crashes_dir") + .takes_value(true) + .required(true), + ) +} diff --git a/src/agent/onefuzz-agent/src/debug/mod.rs b/src/agent/onefuzz-agent/src/debug/mod.rs index 75edd21c9..e698fbf3d 100644 --- a/src/agent/onefuzz-agent/src/debug/mod.rs +++ b/src/agent/onefuzz-agent/src/debug/mod.rs @@ -5,3 +5,4 @@ pub mod cmd; pub mod generic_crash_report; pub mod libfuzzer_coverage; pub mod libfuzzer_crash_report; +pub mod libfuzzer_fuzz; diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index 9b6c760c7..249ad8781 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -9,8 +9,10 @@ use crate::tasks::{ use anyhow::Result; use futures::{future::try_join_all, stream::StreamExt}; use onefuzz::{ + fs::list_files, libfuzzer::{LibFuzzer, LibFuzzerLine}, monitor::DirectoryMonitor, + process::ExitStatus, system, telemetry::{ Event::{new_coverage, new_result, process_stats, runtime_stats}, @@ -19,9 +21,11 @@ use onefuzz::{ uploader::BlobUploader, }; use serde::Deserialize; -use std::{collections::HashMap, path::PathBuf, process::ExitStatus}; +use std::{collections::HashMap, path::PathBuf}; +use tempfile::tempdir; use tokio::{ - io, + fs::rename, + io::{AsyncBufReadExt, BufReader}, sync::mpsc, task, time::{self, Duration}, @@ -82,7 +86,7 @@ impl LibFuzzerFuzzTask { let report_stats = report_runtime_stats(workers as usize, stats_receiver, hb_client); let fuzzers: Vec<_> = (0..workers) - .map(|id| self.start_fuzzer_monitor(id, stats_sender.clone())) + .map(|id| self.start_fuzzer_monitor(id, Some(&stats_sender))) .collect(); let fuzzers = try_join_all(fuzzers); @@ -96,22 +100,21 @@ impl LibFuzzerFuzzTask { // // A run is one session of continuous fuzzing, terminated by a fuzzing error // or discovered fault. The monitor restarts the libFuzzer when it exits. - async fn start_fuzzer_monitor(&self, worker_id: u64, stats_sender: StatsSender) -> Result<()> { + pub async fn start_fuzzer_monitor( + &self, + worker_id: u64, + stats_sender: Option<&StatsSender>, + ) -> Result<()> { loop { - let run = self.run_fuzzer(worker_id, stats_sender.clone()); - - if let Err(err) = run.await { - error!("Fuzzer run failed: {}", err); - } + self.run_fuzzer(worker_id, stats_sender).await?; } } // Fuzz with a libFuzzer until it exits. // // While it runs, parse stderr for progress metrics, and report them. - async fn run_fuzzer(&self, worker_id: u64, stats_sender: StatsSender) -> Result { - use io::AsyncBufReadExt; - + async fn run_fuzzer(&self, worker_id: u64, stats_sender: Option<&StatsSender>) -> Result<()> { + let crash_dir = tempdir()?; let run_id = Uuid::new_v4(); info!("starting fuzzer run, run_id = {}", run_id); @@ -129,8 +132,7 @@ impl LibFuzzerFuzzTask { &self.config.target_options, &self.config.target_env, ); - let mut running = - fuzzer.fuzz(&self.config.crashes.path, &self.config.inputs.path, &inputs)?; + let mut running = fuzzer.fuzz(crash_dir.path(), &self.config.inputs.path, &inputs)?; let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running.id())); @@ -139,27 +141,42 @@ impl LibFuzzerFuzzTask { .stderr .as_mut() .ok_or_else(|| format_err!("stderr not captured"))?; - let stderr = io::BufReader::new(stderr); + let stderr = BufReader::new(stderr); - stderr - .lines() - .for_each(|line| { - let stats_sender = stats_sender.clone(); - - async move { - let line = line.map_err(|e| e.into()); - - if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, line) - { - error!("could not parse fuzzing iteration update: {}", err); - } + let mut libfuzzer_output = Vec::new(); + let mut lines = stderr.lines(); + while let Some(line) = lines.next_line().await? { + if let Some(stats_sender) = stats_sender { + if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, &line) { + error!("could not parse fuzzing interation update: {}", err); } - }) - .await; + } + libfuzzer_output.push(line); + } let (exit_status, _) = tokio::join!(running, sys_info); + let exit_status: ExitStatus = exit_status?.into(); - Ok(exit_status?) + let files = list_files(crash_dir.path()).await?; + + // ignore libfuzzer exiting cleanly without crashing, which could happen via + // -runs=N + if !exit_status.success && files.is_empty() { + bail!( + "libfuzzer exited without generating crashes. status:{} stderr:{:?}", + serde_json::to_string(&exit_status)?, + libfuzzer_output.join("\n") + ); + } + + for file in &files { + if let Some(filename) = file.file_name() { + let dest = self.config.crashes.path.join(filename); + rename(file, dest).await?; + } + } + + Ok(()) } async fn init_directories(&self) -> Result<()> { @@ -195,9 +212,7 @@ impl LibFuzzerFuzzTask { async fn monitor_new_corpus(&self) -> Result<()> { let url = self.config.inputs.url.url(); - let dir = self.config.inputs.path.clone(); - - let mut monitor = DirectoryMonitor::new(dir); + let mut monitor = DirectoryMonitor::new(&self.config.inputs.path); monitor.start()?; monitor @@ -247,15 +262,12 @@ impl LibFuzzerFuzzTask { } fn try_report_iter_update( - stats_sender: StatsSender, + stats_sender: &StatsSender, worker_id: u64, run_id: Uuid, - line: Result, + line: &str, ) -> Result<()> { - let line = line?; - let line = LibFuzzerLine::parse(line)?; - - if let Some(line) = line { + if let Some(line) = LibFuzzerLine::parse(line)? { stats_sender.send(RuntimeStats { worker_id, run_id, diff --git a/src/agent/onefuzz-supervisor/src/coordinator.rs b/src/agent/onefuzz-supervisor/src/coordinator.rs index 5d4e16cee..8ad58c30b 100644 --- a/src/agent/onefuzz-supervisor/src/coordinator.rs +++ b/src/agent/onefuzz-supervisor/src/coordinator.rs @@ -3,13 +3,13 @@ use anyhow::Result; use downcast_rs::Downcast; +use onefuzz::process::Output; use reqwest::{Client, Request, Response, StatusCode}; use serde::Serialize; use uuid::Uuid; use crate::auth::AccessToken; use crate::config::Registration; -use crate::process::Output; use crate::work::{TaskId, WorkSet}; use crate::worker::WorkerEvent; diff --git a/src/agent/onefuzz-supervisor/src/debug.rs b/src/agent/onefuzz-supervisor/src/debug.rs index a83a3cbeb..03fb5ecc1 100644 --- a/src/agent/onefuzz-supervisor/src/debug.rs +++ b/src/agent/onefuzz-supervisor/src/debug.rs @@ -5,12 +5,12 @@ use std::path::PathBuf; use anyhow::Result; use onefuzz::blob::BlobContainerUrl; +use onefuzz::process::ExitStatus; use structopt::StructOpt; use url::Url; use uuid::Uuid; use crate::coordinator::*; -use crate::process::*; use crate::work::*; use crate::worker::*; diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 8d3717cef..2334c81f5 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -27,7 +27,6 @@ pub mod config; pub mod coordinator; pub mod debug; pub mod done; -pub mod process; pub mod reboot; pub mod scheduler; pub mod setup; diff --git a/src/agent/onefuzz-supervisor/src/scheduler.rs b/src/agent/onefuzz-supervisor/src/scheduler.rs index 9f6302c21..62cd21330 100644 --- a/src/agent/onefuzz-supervisor/src/scheduler.rs +++ b/src/agent/onefuzz-supervisor/src/scheduler.rs @@ -4,9 +4,9 @@ use std::fmt; use anyhow::Result; +use onefuzz::process::Output; use crate::coordinator::NodeCommand; -use crate::process::Output; use crate::reboot::RebootContext; use crate::setup::ISetupRunner; use crate::work::*; diff --git a/src/agent/onefuzz-supervisor/src/setup.rs b/src/agent/onefuzz-supervisor/src/setup.rs index 8b6de2af8..e61c0f0c8 100644 --- a/src/agent/onefuzz-supervisor/src/setup.rs +++ b/src/agent/onefuzz-supervisor/src/setup.rs @@ -7,10 +7,10 @@ use std::process::Stdio; use anyhow::Result; use downcast_rs::Downcast; use onefuzz::az_copy; +use onefuzz::process::Output; use tokio::fs; use tokio::process::Command; -use crate::process::Output; use crate::work::*; const SETUP_PATH_ENV: &str = "ONEFUZZ_TARGET_SETUP_PATH"; diff --git a/src/agent/onefuzz-supervisor/src/worker.rs b/src/agent/onefuzz-supervisor/src/worker.rs index 5dd07ed56..085381177 100644 --- a/src/agent/onefuzz-supervisor/src/worker.rs +++ b/src/agent/onefuzz-supervisor/src/worker.rs @@ -5,9 +5,9 @@ use std::process::{Child, Command, Stdio}; use anyhow::Result; use downcast_rs::Downcast; +use onefuzz::process::{ExitStatus, Output}; use tokio::fs; -use crate::process::*; use crate::work::*; #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] diff --git a/src/agent/onefuzz/src/lib.rs b/src/agent/onefuzz/src/lib.rs index f025e614a..447fdf350 100644 --- a/src/agent/onefuzz/src/lib.rs +++ b/src/agent/onefuzz/src/lib.rs @@ -7,6 +7,9 @@ extern crate anyhow; #[macro_use] extern crate lazy_static; +#[macro_use] +extern crate serde; + #[macro_use] pub mod telemetry; @@ -19,9 +22,9 @@ pub mod input_tester; pub mod libfuzzer; pub mod machine_id; pub mod monitor; +pub mod process; pub mod sha256; pub mod system; - #[cfg(target_os = "linux")] pub mod triage; diff --git a/src/agent/onefuzz/src/libfuzzer.rs b/src/agent/onefuzz/src/libfuzzer.rs index 78408606b..d82ee1e05 100644 --- a/src/agent/onefuzz/src/libfuzzer.rs +++ b/src/agent/onefuzz/src/libfuzzer.rs @@ -179,10 +179,10 @@ impl LibFuzzerLine { } } - pub fn parse(line: String) -> Result> { + pub fn parse(line: &str) -> Result> { let re = regex::Regex::new(r"#(\d+)\s*(?:pulse|INITED|NEW|REDUCE).*exec/s: (\d+)")?; - let caps = match re.captures(&line) { + let caps = match re.captures(line) { Some(caps) => caps, None => return Ok(None), }; @@ -190,7 +190,7 @@ impl LibFuzzerLine { let iters = caps[1].parse()?; let execs_sec = caps[2].parse()?; - Ok(Some(Self::new(line, iters, execs_sec))) + Ok(Some(Self::new(line.to_string(), iters, execs_sec))) } pub fn iters(&self) -> u64 { diff --git a/src/agent/onefuzz-supervisor/src/process.rs b/src/agent/onefuzz/src/process.rs similarity index 100% rename from src/agent/onefuzz-supervisor/src/process.rs rename to src/agent/onefuzz/src/process.rs