handle libfuzzer targets failing without dropping crash files (#108)

src/agent/onefuzz-agent/Cargo.toml

@@ -14,6 +14,7 @@ anyhow = "1.0"
 appinsights = "0.1"
 async-trait = "0.1"
 clap = "2.33"
+tempfile = "3.1"
 env_logger = "0.7"
 futures = "0.3"
 hex = "0.4"

src/agent/onefuzz-agent/src/debug/cmd.rs

@@ -9,6 +9,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
         ("generic-crash-report", Some(sub)) => crate::debug::generic_crash_report::run(sub)?,
         ("libfuzzer-coverage", Some(sub)) => crate::debug::libfuzzer_coverage::run(sub)?,
         ("libfuzzer-crash-report", Some(sub)) => crate::debug::libfuzzer_crash_report::run(sub)?,
+        ("libfuzzer-fuzz", Some(sub)) => crate::debug::libfuzzer_fuzz::run(sub)?,
         _ => println!("missing subcommand\nUSAGE : {}", args.usage()),
     }
 
@@ -21,4 +22,5 @@ pub fn args() -> App<'static, 'static> {
         .subcommand(crate::debug::generic_crash_report::args())
         .subcommand(crate::debug::libfuzzer_coverage::args())
         .subcommand(crate::debug::libfuzzer_crash_report::args())
+        .subcommand(crate::debug::libfuzzer_fuzz::args())
 }

src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs

@@ -0,0 +1,103 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use crate::tasks::{
+    config::{CommonConfig, SyncedDir},
+    fuzz::libfuzzer_fuzz::{Config, LibFuzzerFuzzTask},
+    utils::parse_key_value,
+};
+use anyhow::Result;
+use clap::{App, Arg, SubCommand};
+use onefuzz::blob::BlobContainerUrl;
+use std::{collections::HashMap, path::PathBuf};
+use tokio::runtime::Runtime;
+use url::Url;
+use uuid::Uuid;
+
+async fn run_impl(config: Config) -> Result<()> {
+    let fuzzer = LibFuzzerFuzzTask::new(config)?;
+    let result = fuzzer.start_fuzzer_monitor(0, None).await?;
+    println!("{:#?}", result);
+    Ok(())
+}
+
+pub fn run(args: &clap::ArgMatches) -> Result<()> {
+    let crashes_dir = value_t!(args, "crashes_dir", String)?;
+    let inputs_dir = value_t!(args, "inputs_dir", String)?;
+    let target_exe = value_t!(args, "target_exe", PathBuf)?;
+    let target_options = args.values_of_lossy("target_options").unwrap_or_default();
+    let mut target_env = HashMap::new();
+    for opt in args.values_of_lossy("target_env").unwrap_or_default() {
+        let (k, v) = parse_key_value(opt)?;
+        target_env.insert(k, v);
+    }
+
+    let readonly_inputs = None;
+    let target_workers = Some(1);
+
+    let inputs = SyncedDir {
+        path: inputs_dir.into(),
+        url: BlobContainerUrl::new(Url::parse("https://contoso.com/inputs")?)?,
+    };
+
+    let crashes = SyncedDir {
+        path: crashes_dir.into(),
+        url: BlobContainerUrl::new(Url::parse("https://contoso.com/crashes")?)?,
+    };
+
+    let config = Config {
+        inputs,
+        readonly_inputs,
+        crashes,
+        target_exe,
+        target_env,
+        target_options,
+        target_workers,
+        common: CommonConfig {
+            heartbeat_queue: None,
+            instrumentation_key: None,
+            telemetry_key: None,
+            job_id: Uuid::parse_str("00000000-0000-0000-0000-000000000000").unwrap(),
+            task_id: Uuid::parse_str("11111111-1111-1111-1111-111111111111").unwrap(),
+        },
+    };
+
+    let mut rt = Runtime::new()?;
+    rt.block_on(async { run_impl(config).await })?;
+
+    Ok(())
+}
+
+pub fn args() -> App<'static, 'static> {
+    SubCommand::with_name("libfuzzer-fuzz")
+        .about("execute a local-only libfuzzer crash report task")
+        .arg(
+            Arg::with_name("target_exe")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("target_env")
+                .long("target_env")
+                .takes_value(true)
+                .multiple(true),
+        )
+        .arg(
+            Arg::with_name("target_options")
+                .long("target_options")
+                .takes_value(true)
+                .multiple(true)
+                .allow_hyphen_values(true)
+                .help("Supports hyphens.  Recommendation: Set target_env first"),
+        )
+        .arg(
+            Arg::with_name("inputs_dir")
+                .takes_value(true)
+                .required(true),
+        )
+        .arg(
+            Arg::with_name("crashes_dir")
+                .takes_value(true)
+                .required(true),
+        )
+}

src/agent/onefuzz-agent/src/debug/mod.rs

@@ -5,3 +5,4 @@ pub mod cmd;
 pub mod generic_crash_report;
 pub mod libfuzzer_coverage;
 pub mod libfuzzer_crash_report;
+pub mod libfuzzer_fuzz;

src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs

@@ -9,8 +9,10 @@ use crate::tasks::{
 use anyhow::Result;
 use futures::{future::try_join_all, stream::StreamExt};
 use onefuzz::{
+    fs::list_files,
     libfuzzer::{LibFuzzer, LibFuzzerLine},
     monitor::DirectoryMonitor,
+    process::ExitStatus,
     system,
     telemetry::{
         Event::{new_coverage, new_result, process_stats, runtime_stats},
@@ -19,9 +21,11 @@ use onefuzz::{
     uploader::BlobUploader,
 };
 use serde::Deserialize;
-use std::{collections::HashMap, path::PathBuf, process::ExitStatus};
+use std::{collections::HashMap, path::PathBuf};
+use tempfile::tempdir;
 use tokio::{
-    io,
+    fs::rename,
+    io::{AsyncBufReadExt, BufReader},
     sync::mpsc,
     task,
     time::{self, Duration},
@@ -82,7 +86,7 @@ impl LibFuzzerFuzzTask {
         let report_stats = report_runtime_stats(workers as usize, stats_receiver, hb_client);
 
         let fuzzers: Vec<_> = (0..workers)
-            .map(|id| self.start_fuzzer_monitor(id, stats_sender.clone()))
+            .map(|id| self.start_fuzzer_monitor(id, Some(&stats_sender)))
             .collect();
 
         let fuzzers = try_join_all(fuzzers);
@@ -96,22 +100,21 @@ impl LibFuzzerFuzzTask {
     //
     // A run is one session of continuous fuzzing, terminated by a fuzzing error
     // or discovered fault. The monitor restarts the libFuzzer when it exits.
-    async fn start_fuzzer_monitor(&self, worker_id: u64, stats_sender: StatsSender) -> Result<()> {
+    pub async fn start_fuzzer_monitor(
+        &self,
+        worker_id: u64,
+        stats_sender: Option<&StatsSender>,
+    ) -> Result<()> {
         loop {
-            let run = self.run_fuzzer(worker_id, stats_sender.clone());
-
-            if let Err(err) = run.await {
-                error!("Fuzzer run failed: {}", err);
-            }
+            self.run_fuzzer(worker_id, stats_sender).await?;
         }
     }
 
     // Fuzz with a libFuzzer until it exits.
     //
     // While it runs, parse stderr for progress metrics, and report them.
-    async fn run_fuzzer(&self, worker_id: u64, stats_sender: StatsSender) -> Result<ExitStatus> {
-        use io::AsyncBufReadExt;
-
+    async fn run_fuzzer(&self, worker_id: u64, stats_sender: Option<&StatsSender>) -> Result<()> {
+        let crash_dir = tempdir()?;
         let run_id = Uuid::new_v4();
 
         info!("starting fuzzer run, run_id = {}", run_id);
@@ -129,8 +132,7 @@ impl LibFuzzerFuzzTask {
             &self.config.target_options,
             &self.config.target_env,
         );
-        let mut running =
-            fuzzer.fuzz(&self.config.crashes.path, &self.config.inputs.path, &inputs)?;
+        let mut running = fuzzer.fuzz(crash_dir.path(), &self.config.inputs.path, &inputs)?;
 
         let sys_info = task::spawn(report_fuzzer_sys_info(worker_id, run_id, running.id()));
 
@@ -139,27 +141,42 @@ impl LibFuzzerFuzzTask {
             .stderr
             .as_mut()
             .ok_or_else(|| format_err!("stderr not captured"))?;
-        let stderr = io::BufReader::new(stderr);
+        let stderr = BufReader::new(stderr);
 
-        stderr
-            .lines()
-            .for_each(|line| {
-                let stats_sender = stats_sender.clone();
-
-                async move {
-                    let line = line.map_err(|e| e.into());
-
-                    if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, line)
-                    {
-                        error!("could not parse fuzzing iteration update: {}", err);
+        let mut libfuzzer_output = Vec::new();
+        let mut lines = stderr.lines();
+        while let Some(line) = lines.next_line().await? {
+            if let Some(stats_sender) = stats_sender {
+                if let Err(err) = try_report_iter_update(stats_sender, worker_id, run_id, &line) {
+                    error!("could not parse fuzzing interation update: {}", err);
                 }
             }
-            })
-            .await;
+            libfuzzer_output.push(line);
+        }
 
         let (exit_status, _) = tokio::join!(running, sys_info);
+        let exit_status: ExitStatus = exit_status?.into();
 
-        Ok(exit_status?)
+        let files = list_files(crash_dir.path()).await?;
+
+        // ignore libfuzzer exiting cleanly without crashing, which could happen via
+        // -runs=N
+        if !exit_status.success && files.is_empty() {
+            bail!(
+                "libfuzzer exited without generating crashes.  status:{} stderr:{:?}",
+                serde_json::to_string(&exit_status)?,
+                libfuzzer_output.join("\n")
+            );
+        }
+
+        for file in &files {
+            if let Some(filename) = file.file_name() {
+                let dest = self.config.crashes.path.join(filename);
+                rename(file, dest).await?;
+            }
+        }
+
+        Ok(())
     }
 
     async fn init_directories(&self) -> Result<()> {
@@ -195,9 +212,7 @@ impl LibFuzzerFuzzTask {
 
     async fn monitor_new_corpus(&self) -> Result<()> {
         let url = self.config.inputs.url.url();
-        let dir = self.config.inputs.path.clone();
-
-        let mut monitor = DirectoryMonitor::new(dir);
+        let mut monitor = DirectoryMonitor::new(&self.config.inputs.path);
         monitor.start()?;
 
         monitor
@@ -247,15 +262,12 @@ impl LibFuzzerFuzzTask {
 }
 
 fn try_report_iter_update(
-    stats_sender: StatsSender,
+    stats_sender: &StatsSender,
     worker_id: u64,
     run_id: Uuid,
-    line: Result<String>,
+    line: &str,
 ) -> Result<()> {
-    let line = line?;
-    let line = LibFuzzerLine::parse(line)?;
-
-    if let Some(line) = line {
+    if let Some(line) = LibFuzzerLine::parse(line)? {
         stats_sender.send(RuntimeStats {
             worker_id,
             run_id,

src/agent/onefuzz-supervisor/src/coordinator.rs

@@ -3,13 +3,13 @@
 
 use anyhow::Result;
 use downcast_rs::Downcast;
+use onefuzz::process::Output;
 use reqwest::{Client, Request, Response, StatusCode};
 use serde::Serialize;
 use uuid::Uuid;
 
 use crate::auth::AccessToken;
 use crate::config::Registration;
-use crate::process::Output;
 use crate::work::{TaskId, WorkSet};
 use crate::worker::WorkerEvent;
 

src/agent/onefuzz-supervisor/src/debug.rs

@@ -5,12 +5,12 @@ use std::path::PathBuf;
 
 use anyhow::Result;
 use onefuzz::blob::BlobContainerUrl;
+use onefuzz::process::ExitStatus;
 use structopt::StructOpt;
 use url::Url;
 use uuid::Uuid;
 
 use crate::coordinator::*;
-use crate::process::*;
 use crate::work::*;
 use crate::worker::*;
 

src/agent/onefuzz-supervisor/src/main.rs

@@ -27,7 +27,6 @@ pub mod config;
 pub mod coordinator;
 pub mod debug;
 pub mod done;
-pub mod process;
 pub mod reboot;
 pub mod scheduler;
 pub mod setup;

src/agent/onefuzz-supervisor/src/scheduler.rs

@@ -4,9 +4,9 @@
 use std::fmt;
 
 use anyhow::Result;
+use onefuzz::process::Output;
 
 use crate::coordinator::NodeCommand;
-use crate::process::Output;
 use crate::reboot::RebootContext;
 use crate::setup::ISetupRunner;
 use crate::work::*;

src/agent/onefuzz-supervisor/src/setup.rs

@@ -7,10 +7,10 @@ use std::process::Stdio;
 use anyhow::Result;
 use downcast_rs::Downcast;
 use onefuzz::az_copy;
+use onefuzz::process::Output;
 use tokio::fs;
 use tokio::process::Command;
 
-use crate::process::Output;
 use crate::work::*;
 
 const SETUP_PATH_ENV: &str = "ONEFUZZ_TARGET_SETUP_PATH";

src/agent/onefuzz-supervisor/src/worker.rs

@@ -5,9 +5,9 @@ use std::process::{Child, Command, Stdio};
 
 use anyhow::Result;
 use downcast_rs::Downcast;
+use onefuzz::process::{ExitStatus, Output};
 use tokio::fs;
 
-use crate::process::*;
 use crate::work::*;
 
 #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)]

src/agent/onefuzz/src/lib.rs

@@ -7,6 +7,9 @@ extern crate anyhow;
 #[macro_use]
 extern crate lazy_static;
 
+#[macro_use]
+extern crate serde;
+
 #[macro_use]
 pub mod telemetry;
 
@@ -19,9 +22,9 @@ pub mod input_tester;
 pub mod libfuzzer;
 pub mod machine_id;
 pub mod monitor;
+pub mod process;
 pub mod sha256;
 pub mod system;
-
 #[cfg(target_os = "linux")]
 pub mod triage;
 

src/agent/onefuzz/src/libfuzzer.rs

@@ -179,10 +179,10 @@ impl LibFuzzerLine {
         }
     }
 
-    pub fn parse(line: String) -> Result<Option<Self>> {
+    pub fn parse(line: &str) -> Result<Option<Self>> {
         let re = regex::Regex::new(r"#(\d+)\s*(?:pulse|INITED|NEW|REDUCE).*exec/s: (\d+)")?;
 
-        let caps = match re.captures(&line) {
+        let caps = match re.captures(line) {
             Some(caps) => caps,
             None => return Ok(None),
         };
@@ -190,7 +190,7 @@ impl LibFuzzerLine {
         let iters = caps[1].parse()?;
         let execs_sec = caps[2].parse()?;
 
-        Ok(Some(Self::new(line, iters, execs_sec)))
+        Ok(Some(Self::new(line.to_string(), iters, execs_sec)))
     }
 
     pub fn iters(&self) -> u64 {