handle libfuzzer fuzzing non-zero exits better (#381)

When running libfuzzer in 'fuzzing' mode, we expect the following on exit.

If the exit code is zero, crashing input isn't required.  This happens if the user specifies '-runs=N'

If the exit code is non-zero, then crashes are expected.  In practice, there are two causes to non-zero exits.
1. If the binary can't execute for some reason, like a missing prerequisite
2. If the binary _can_ execute, sometimes the sanitizers are put in such a bad place that they are unable to record the input that caused the crash.

This PR enables handling these two non-zero exit cases.

1. Optionally verify the libfuzzer target loads appropriately using `target_exe -help=1`.  This allows failing faster in the common issues, such a missing prerequisite library.
2. Optionally allow non-zero exits without crashes to be a warning, rather than a task failure.

src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs

@@ -48,10 +48,14 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
         target_env.insert(k, v);
     }
 
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
     let config = Config {
         target_exe,
         target_env,
         target_options,
+        check_fuzzer_help,
         input_queue: None,
         readonly_inputs: vec![],
         coverage: SyncedDir {

src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs

@@ -40,12 +40,16 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
     let target_timeout = value_t!(args, "target_timeout", u64).ok();
     let check_retry_count = value_t!(args, "check_retry_count", u64)?;
 
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
     let config = Config {
         target_exe,
         target_env,
         target_options,
         target_timeout,
         check_retry_count,
+        check_fuzzer_help,
         input_queue: None,
         crashes: None,
         reports: None,

src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs

@@ -26,6 +26,12 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
     let inputs_dir = value_t!(args, "inputs_dir", String)?;
     let target_exe = value_t!(args, "target_exe", PathBuf)?;
     let target_options = args.values_of_lossy("target_options").unwrap_or_default();
+
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
+    let expect_crash_on_failure = args.is_present("expect_crash_on_failure");
+
     let mut target_env = HashMap::new();
     for opt in args.values_of_lossy("target_env").unwrap_or_default() {
         let (k, v) = parse_key_value(opt)?;
@@ -56,6 +62,8 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
         target_options,
         target_workers,
         ensemble_sync_delay,
+        check_fuzzer_help,
+        expect_crash_on_failure,
         common: CommonConfig {
             heartbeat_queue: None,
             instrumentation_key: None,
@@ -104,4 +112,9 @@ pub fn args() -> App<'static, 'static> {
                 .takes_value(true)
                 .required(true),
         )
+        .arg(
+            Arg::with_name("expect_crash_on_failure")
+                .takes_value(false)
+                .long("expect_crash_on_failure"),
+        )
 }

src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs

@@ -20,6 +20,9 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
     let unique_inputs = value_t!(args, "unique_inputs", String)?;
     let target_options = args.values_of_lossy("target_options").unwrap_or_default();
 
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
     let mut target_env = HashMap::new();
     for opt in args.values_of_lossy("target_env").unwrap_or_default() {
         let (k, v) = parse_key_value(opt)?;
@@ -30,6 +33,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
         target_exe,
         target_env,
         target_options,
+        check_fuzzer_help,
         input_queue: None,
         inputs: vec![SyncedDir {
             path: inputs.into(),

src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs

@@ -30,14 +30,18 @@
 //!
 //! Versions in parentheses have been tested.
 
-use crate::tasks::coverage::{recorder::CoverageRecorder, total::TotalCoverage};
 use crate::tasks::heartbeat::*;
 use crate::tasks::{config::CommonConfig, generic::input_poller::*};
+use crate::tasks::{
+    coverage::{recorder::CoverageRecorder, total::TotalCoverage},
+    utils::default_bool_true,
+};
 use anyhow::Result;
 use async_trait::async_trait;
 use futures::stream::StreamExt;
 use onefuzz::{
-    fs::list_files, syncdir::SyncedDir, telemetry::Event::coverage_data, telemetry::EventData,
+    fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir, telemetry::Event::coverage_data,
+    telemetry::EventData,
 };
 use reqwest::Url;
 use serde::Deserialize;
@@ -61,6 +65,9 @@ pub struct Config {
     pub readonly_inputs: Vec<SyncedDir>,
     pub coverage: SyncedDir,
 
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
     #[serde(flatten)]
     pub common: CommonConfig,
 }
@@ -91,6 +98,16 @@ impl CoverageTask {
 
     pub async fn run(&mut self) -> Result<()> {
         info!("starting libFuzzer coverage task");
+
+        if self.config.check_fuzzer_help {
+            let target = LibFuzzer::new(
+                &self.config.target_exe,
+                &self.config.target_options,
+                &self.config.target_env,
+            );
+            target.check_help().await?;
+        }
+
         self.config.coverage.init_pull().await?;
         self.process().await
     }

src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs

@@ -1,7 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::tasks::{config::CommonConfig, heartbeat::*, utils};
+use crate::tasks::{
+    config::CommonConfig,
+    heartbeat::*,
+    utils::{self, default_bool_true},
+};
 use anyhow::{Error, Result};
 use futures::stream::StreamExt;
 use onefuzz::{
@@ -23,10 +27,6 @@ use std::{
 };
 use tokio::{fs, process::Command};
 
-fn default_bool_true() -> bool {
-    true
-}
-
 #[derive(Debug, Deserialize, Clone)]
 pub struct GeneratorConfig {
     pub generator_exe: String,

src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender};
+use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender, utils::default_bool_true};
 use anyhow::Result;
 use futures::{future::try_join_all, stream::StreamExt};
 use onefuzz::{
@@ -47,6 +47,12 @@ pub struct Config {
     pub target_workers: Option<u64>,
     pub ensemble_sync_delay: Option<u64>,
 
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
+    #[serde(default = "default_bool_true")]
+    pub expect_crash_on_failure: bool,
+
     #[serde(flatten)]
     pub common: CommonConfig,
 }
@@ -61,6 +67,15 @@ impl LibFuzzerFuzzTask {
     }
 
     pub async fn start(&self) -> Result<()> {
+        if self.config.check_fuzzer_help {
+            let target = LibFuzzer::new(
+                &self.config.target_exe,
+                &self.config.target_options,
+                &self.config.target_env,
+            );
+            target.check_help().await?;
+        }
+
         let workers = self.config.target_workers.unwrap_or_else(|| {
             let cpus = num_cpus::get() as u64;
             u64::max(1, cpus - 1)
@@ -166,14 +181,23 @@ impl LibFuzzerFuzzTask {
 
         let files = list_files(crash_dir.path()).await?;
 
-        // ignore libfuzzer exiting cleanly without crashing, which could happen via
-        // -runs=N
-        if !exit_status.success && files.is_empty() {
-            bail!(
-                "libfuzzer exited without generating crashes.  status:{} stderr:{:?}",
-                serde_json::to_string(&exit_status)?,
-                libfuzzer_output.join("\n")
-            );
+        // If the target exits, crashes are required unless
+        // 1. Exited cleanly (happens with -runs=N)
+        // 2. expect_crash_on_failure is disabled
+        if files.is_empty() && !exit_status.success {
+            if self.config.expect_crash_on_failure {
+                bail!(
+                    "libfuzzer exited without generating crashes.  status:{} stderr:{:?}",
+                    serde_json::to_string(&exit_status)?,
+                    libfuzzer_output.join("\n")
+                );
+            } else {
+                warn!(
+                    "libfuzzer exited without generating crashes, continuing.  status:{} stderr:{:?}",
+                    serde_json::to_string(&exit_status)?,
+                    libfuzzer_output.join("\n")
+                );
+            }
         }
 
         for file in &files {

src/agent/onefuzz-agent/src/tasks/merge/libfuzzer_merge.rs

@@ -1,7 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::tasks::{config::CommonConfig, heartbeat::*, utils};
+use crate::tasks::{
+    config::CommonConfig,
+    heartbeat::*,
+    utils::{self, default_bool_true},
+};
 use anyhow::Result;
 use onefuzz::{
     http::ResponseExt,
@@ -35,11 +39,23 @@ pub struct Config {
     pub unique_inputs: SyncedDir,
     pub preserve_existing_outputs: bool,
 
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
     #[serde(flatten)]
     pub common: CommonConfig,
 }
 
 pub async fn spawn(config: Arc<Config>) -> Result<()> {
+    if config.check_fuzzer_help {
+        let target = LibFuzzer::new(
+            &config.target_exe,
+            &config.target_options,
+            &config.target_env,
+        );
+        target.check_help().await?;
+    }
+
     config.unique_inputs.init().await?;
     if let Some(url) = config.input_queue.clone() {
         loop {

src/agent/onefuzz-agent/src/tasks/report/generic.rs

@@ -6,6 +6,7 @@ use crate::tasks::{
     config::CommonConfig,
     generic::input_poller::{CallbackImpl, InputPoller, Processor},
     heartbeat::*,
+    utils::default_bool_true,
 };
 use anyhow::Result;
 use async_trait::async_trait;
@@ -18,9 +19,6 @@ use std::{
 };
 use storage_queue::Message;
 
-fn default_bool_true() -> bool {
-    true
-}
 #[derive(Debug, Deserialize)]
 pub struct Config {
     pub target_exe: PathBuf,

src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs

@@ -2,7 +2,9 @@
 // Licensed under the MIT License.
 
 use super::crash_report::*;
-use crate::tasks::{config::CommonConfig, generic::input_poller::*, heartbeat::*};
+use crate::tasks::{
+    config::CommonConfig, generic::input_poller::*, heartbeat::*, utils::default_bool_true,
+};
 use anyhow::Result;
 use async_trait::async_trait;
 use onefuzz::{blob::BlobUrl, libfuzzer::LibFuzzer, sha256, syncdir::SyncedDir};
@@ -27,6 +29,10 @@ pub struct Config {
     pub reports: Option<SyncedDir>,
     pub unique_reports: SyncedDir,
     pub no_repro: Option<SyncedDir>,
+
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
     #[serde(default)]
     pub check_retry_count: u64,
 
@@ -50,6 +56,15 @@ impl ReportTask {
     }
 
     pub async fn run(&mut self) -> Result<()> {
+        if self.config.check_fuzzer_help {
+            let target = LibFuzzer::new(
+                &self.config.target_exe,
+                &self.config.target_options,
+                &self.config.target_env,
+            );
+            target.check_help().await?;
+        }
+
         info!("Starting libFuzzer crash report task");
         let mut processor = AsanProcessor::new(self.config.clone()).await?;
 

src/agent/onefuzz-agent/src/tasks/utils.rs

@@ -78,3 +78,7 @@ pub fn parse_key_value(value: String) -> Result<(String, String)> {
 
     Ok((value[..offset].to_string(), value[offset + 1..].to_string()))
 }
+
+pub fn default_bool_true() -> bool {
+    true
+}