handle libfuzzer fuzzing non-zero exits better (#381)

When running libfuzzer in 'fuzzing' mode, we expect the following on exit.

If the exit code is zero, crashing input isn't required.  This happens if the user specifies '-runs=N'

If the exit code is non-zero, then crashes are expected.  In practice, there are two causes to non-zero exits.
1. If the binary can't execute for some reason, like a missing prerequisite
2. If the binary _can_ execute, sometimes the sanitizers are put in such a bad place that they are unable to record the input that caused the crash.

This PR enables handling these two non-zero exit cases.

1. Optionally verify the libfuzzer target loads appropriately using `target_exe -help=1`.  This allows failing faster in the common issues, such a missing prerequisite library.
2. Optionally allow non-zero exits without crashes to be a warning, rather than a task failure.

docs/webhook_events.md

@@ -182,6 +182,14 @@ Each event will be submitted via HTTP POST to the user provided URL.
                     "title": "Check Retry Count",
                     "type": "integer"
                 },
+                "check_fuzzer_help": {
+                    "title": "Check Fuzzer Help",
+                    "type": "boolean"
+                },
+                "expect_crash_on_failure": {
+                    "title": "Expect Crash On Failure",
+                    "type": "boolean"
+                },
                 "rename_output": {
                     "title": "Rename Output",
                     "type": "boolean"
@@ -805,6 +813,14 @@ Each event will be submitted via HTTP POST to the user provided URL.
                     "title": "Check Retry Count",
                     "type": "integer"
                 },
+                "check_fuzzer_help": {
+                    "title": "Check Fuzzer Help",
+                    "type": "boolean"
+                },
+                "expect_crash_on_failure": {
+                    "title": "Expect Crash On Failure",
+                    "type": "boolean"
+                },
                 "rename_output": {
                     "title": "Rename Output",
                     "type": "boolean"

src/agent/onefuzz-agent/src/debug/libfuzzer_coverage.rs

@@ -48,10 +48,14 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
         target_env.insert(k, v);
     }
 
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
     let config = Config {
         target_exe,
         target_env,
         target_options,
+        check_fuzzer_help,
         input_queue: None,
         readonly_inputs: vec![],
         coverage: SyncedDir {

src/agent/onefuzz-agent/src/debug/libfuzzer_crash_report.rs

@@ -40,12 +40,16 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
     let target_timeout = value_t!(args, "target_timeout", u64).ok();
     let check_retry_count = value_t!(args, "check_retry_count", u64)?;
 
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
     let config = Config {
         target_exe,
         target_env,
         target_options,
         target_timeout,
         check_retry_count,
+        check_fuzzer_help,
         input_queue: None,
         crashes: None,
         reports: None,

src/agent/onefuzz-agent/src/debug/libfuzzer_fuzz.rs

@@ -26,6 +26,12 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
     let inputs_dir = value_t!(args, "inputs_dir", String)?;
     let target_exe = value_t!(args, "target_exe", PathBuf)?;
     let target_options = args.values_of_lossy("target_options").unwrap_or_default();
+
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
+    let expect_crash_on_failure = args.is_present("expect_crash_on_failure");
+
     let mut target_env = HashMap::new();
     for opt in args.values_of_lossy("target_env").unwrap_or_default() {
         let (k, v) = parse_key_value(opt)?;
@@ -56,6 +62,8 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
         target_options,
         target_workers,
         ensemble_sync_delay,
+        check_fuzzer_help,
+        expect_crash_on_failure,
         common: CommonConfig {
             heartbeat_queue: None,
             instrumentation_key: None,
@@ -104,4 +112,9 @@ pub fn args() -> App<'static, 'static> {
                 .takes_value(true)
                 .required(true),
         )
+        .arg(
+            Arg::with_name("expect_crash_on_failure")
+                .takes_value(false)
+                .long("expect_crash_on_failure"),
+        )
 }

src/agent/onefuzz-agent/src/debug/libfuzzer_merge.rs

@@ -20,6 +20,9 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
     let unique_inputs = value_t!(args, "unique_inputs", String)?;
     let target_options = args.values_of_lossy("target_options").unwrap_or_default();
 
+    // this happens during setup, not during runtime
+    let check_fuzzer_help = true;
+
     let mut target_env = HashMap::new();
     for opt in args.values_of_lossy("target_env").unwrap_or_default() {
         let (k, v) = parse_key_value(opt)?;
@@ -30,6 +33,7 @@ pub fn run(args: &clap::ArgMatches) -> Result<()> {
         target_exe,
         target_env,
         target_options,
+        check_fuzzer_help,
         input_queue: None,
         inputs: vec![SyncedDir {
             path: inputs.into(),

src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs

@@ -30,14 +30,18 @@
 //!
 //! Versions in parentheses have been tested.
 
-use crate::tasks::coverage::{recorder::CoverageRecorder, total::TotalCoverage};
 use crate::tasks::heartbeat::*;
 use crate::tasks::{config::CommonConfig, generic::input_poller::*};
+use crate::tasks::{
+    coverage::{recorder::CoverageRecorder, total::TotalCoverage},
+    utils::default_bool_true,
+};
 use anyhow::Result;
 use async_trait::async_trait;
 use futures::stream::StreamExt;
 use onefuzz::{
-    fs::list_files, syncdir::SyncedDir, telemetry::Event::coverage_data, telemetry::EventData,
+    fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir, telemetry::Event::coverage_data,
+    telemetry::EventData,
 };
 use reqwest::Url;
 use serde::Deserialize;
@@ -61,6 +65,9 @@ pub struct Config {
     pub readonly_inputs: Vec<SyncedDir>,
     pub coverage: SyncedDir,
 
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
     #[serde(flatten)]
     pub common: CommonConfig,
 }
@@ -91,6 +98,16 @@ impl CoverageTask {
 
     pub async fn run(&mut self) -> Result<()> {
         info!("starting libFuzzer coverage task");
+
+        if self.config.check_fuzzer_help {
+            let target = LibFuzzer::new(
+                &self.config.target_exe,
+                &self.config.target_options,
+                &self.config.target_env,
+            );
+            target.check_help().await?;
+        }
+
         self.config.coverage.init_pull().await?;
         self.process().await
     }

src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs

@@ -1,7 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::tasks::{config::CommonConfig, heartbeat::*, utils};
+use crate::tasks::{
+    config::CommonConfig,
+    heartbeat::*,
+    utils::{self, default_bool_true},
+};
 use anyhow::{Error, Result};
 use futures::stream::StreamExt;
 use onefuzz::{
@@ -23,10 +27,6 @@ use std::{
 };
 use tokio::{fs, process::Command};
 
-fn default_bool_true() -> bool {
-    true
-}
-
 #[derive(Debug, Deserialize, Clone)]
 pub struct GeneratorConfig {
     pub generator_exe: String,

src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender};
+use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender, utils::default_bool_true};
 use anyhow::Result;
 use futures::{future::try_join_all, stream::StreamExt};
 use onefuzz::{
@@ -47,6 +47,12 @@ pub struct Config {
     pub target_workers: Option<u64>,
     pub ensemble_sync_delay: Option<u64>,
 
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
+    #[serde(default = "default_bool_true")]
+    pub expect_crash_on_failure: bool,
+
     #[serde(flatten)]
     pub common: CommonConfig,
 }
@@ -61,6 +67,15 @@ impl LibFuzzerFuzzTask {
     }
 
     pub async fn start(&self) -> Result<()> {
+        if self.config.check_fuzzer_help {
+            let target = LibFuzzer::new(
+                &self.config.target_exe,
+                &self.config.target_options,
+                &self.config.target_env,
+            );
+            target.check_help().await?;
+        }
+
         let workers = self.config.target_workers.unwrap_or_else(|| {
             let cpus = num_cpus::get() as u64;
             u64::max(1, cpus - 1)
@@ -166,14 +181,23 @@ impl LibFuzzerFuzzTask {
 
         let files = list_files(crash_dir.path()).await?;
 
-        // ignore libfuzzer exiting cleanly without crashing, which could happen via
+        // If the target exits, crashes are required unless
-        // -runs=N
+        // 1. Exited cleanly (happens with -runs=N)
-        if !exit_status.success && files.is_empty() {
+        // 2. expect_crash_on_failure is disabled
-            bail!(
+        if files.is_empty() && !exit_status.success {
-                "libfuzzer exited without generating crashes.  status:{} stderr:{:?}",
+            if self.config.expect_crash_on_failure {
-                serde_json::to_string(&exit_status)?,
+                bail!(
-                libfuzzer_output.join("\n")
+                    "libfuzzer exited without generating crashes.  status:{} stderr:{:?}",
-            );
+                    serde_json::to_string(&exit_status)?,
+                    libfuzzer_output.join("\n")
+                );
+            } else {
+                warn!(
+                    "libfuzzer exited without generating crashes, continuing.  status:{} stderr:{:?}",
+                    serde_json::to_string(&exit_status)?,
+                    libfuzzer_output.join("\n")
+                );
+            }
         }
 
         for file in &files {

src/agent/onefuzz-agent/src/tasks/merge/libfuzzer_merge.rs

@@ -1,7 +1,11 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-use crate::tasks::{config::CommonConfig, heartbeat::*, utils};
+use crate::tasks::{
+    config::CommonConfig,
+    heartbeat::*,
+    utils::{self, default_bool_true},
+};
 use anyhow::Result;
 use onefuzz::{
     http::ResponseExt,
@@ -35,11 +39,23 @@ pub struct Config {
     pub unique_inputs: SyncedDir,
     pub preserve_existing_outputs: bool,
 
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
     #[serde(flatten)]
     pub common: CommonConfig,
 }
 
 pub async fn spawn(config: Arc<Config>) -> Result<()> {
+    if config.check_fuzzer_help {
+        let target = LibFuzzer::new(
+            &config.target_exe,
+            &config.target_options,
+            &config.target_env,
+        );
+        target.check_help().await?;
+    }
+
     config.unique_inputs.init().await?;
     if let Some(url) = config.input_queue.clone() {
         loop {

src/agent/onefuzz-agent/src/tasks/report/generic.rs

@@ -6,6 +6,7 @@ use crate::tasks::{
     config::CommonConfig,
     generic::input_poller::{CallbackImpl, InputPoller, Processor},
     heartbeat::*,
+    utils::default_bool_true,
 };
 use anyhow::Result;
 use async_trait::async_trait;
@@ -18,9 +19,6 @@ use std::{
 };
 use storage_queue::Message;
 
-fn default_bool_true() -> bool {
-    true
-}
 #[derive(Debug, Deserialize)]
 pub struct Config {
     pub target_exe: PathBuf,

src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs

@@ -2,7 +2,9 @@
 // Licensed under the MIT License.
 
 use super::crash_report::*;
-use crate::tasks::{config::CommonConfig, generic::input_poller::*, heartbeat::*};
+use crate::tasks::{
+    config::CommonConfig, generic::input_poller::*, heartbeat::*, utils::default_bool_true,
+};
 use anyhow::Result;
 use async_trait::async_trait;
 use onefuzz::{blob::BlobUrl, libfuzzer::LibFuzzer, sha256, syncdir::SyncedDir};
@@ -27,6 +29,10 @@ pub struct Config {
     pub reports: Option<SyncedDir>,
     pub unique_reports: SyncedDir,
     pub no_repro: Option<SyncedDir>,
+
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
     #[serde(default)]
     pub check_retry_count: u64,
 
@@ -50,6 +56,15 @@ impl ReportTask {
     }
 
     pub async fn run(&mut self) -> Result<()> {
+        if self.config.check_fuzzer_help {
+            let target = LibFuzzer::new(
+                &self.config.target_exe,
+                &self.config.target_options,
+                &self.config.target_env,
+            );
+            target.check_help().await?;
+        }
+
         info!("Starting libFuzzer crash report task");
         let mut processor = AsanProcessor::new(self.config.clone()).await?;
 

src/agent/onefuzz-agent/src/tasks/utils.rs

@@ -78,3 +78,7 @@ pub fn parse_key_value(value: String) -> Result<(String, String)> {
 
     Ok((value[..offset].to_string(), value[offset + 1..].to_string()))
 }
+
+pub fn default_bool_true() -> bool {
+    true
+}

src/agent/onefuzz/src/libfuzzer.rs

@@ -40,6 +40,37 @@ impl<'a> LibFuzzer<'a> {
         }
     }
 
+    pub async fn check_help(&self) -> Result<()> {
+        // Verify -help=1 exits with a zero return code, which validates the
+        // libfuzzer works as we expect.
+        let mut cmd = Command::new(&self.exe);
+
+        cmd.kill_on_drop(true)
+            .env_remove("RUST_LOG")
+            .stdin(Stdio::null())
+            .stdout(Stdio::piped())
+            .stderr(Stdio::piped())
+            .arg("-help=1");
+
+        let mut expand = Expand::new();
+        expand.target_exe(&self.exe).target_options(&self.options);
+
+        for (k, v) in self.env {
+            cmd.env(k, expand.evaluate_value(v)?);
+        }
+
+        // Pass custom option arguments.
+        for o in expand.evaluate(self.options)? {
+            cmd.arg(o);
+        }
+
+        let result = cmd.spawn()?.wait_with_output().await?;
+        if !result.status.success() {
+            bail!("fuzzer does not respond to '-help=1'. output:{:?}", result);
+        }
+        Ok(())
+    }
+
     pub fn fuzz(
         &self,
         fault_dir: impl AsRef<Path>,

src/api-service/__app__/onefuzzlib/job_templates/defaults/libfuzzer.py

@@ -217,6 +217,38 @@ libfuzzer_linux = JobTemplate(
                 ),
             ],
         ),
+        UserField(
+            name="check_fuzzer_help",
+            help="Verify fuzzer by checking if it supports -help=1",
+            type=UserFieldType.Bool,
+            default=True,
+            locations=[
+                UserFieldLocation(
+                    op=UserFieldOperation.add,
+                    path="/tasks/0/task/check_fuzzer_help",
+                ),
+                UserFieldLocation(
+                    op=UserFieldOperation.add,
+                    path="/tasks/1/task/check_fuzzer_help",
+                ),
+                UserFieldLocation(
+                    op=UserFieldOperation.add,
+                    path="/tasks/2/task/check_fuzzer_help",
+                ),
+            ],
+        ),
+        UserField(
+            name="expect_crash_on_failure",
+            help="Require crashes upon non-zero exits from libfuzzer",
+            type=UserFieldType.Bool,
+            default=True,
+            locations=[
+                UserFieldLocation(
+                    op=UserFieldOperation.add,
+                    path="/tasks/0/task/expect_crash_on_failure",
+                ),
+            ],
+        ),
         UserField(
             name="reboot_after_setup",
             help=REBOOT_HELP,

src/api-service/__app__/onefuzzlib/tasks/config.py

@@ -317,6 +317,20 @@ def build_task_config(
     if TaskFeature.ensemble_sync_delay in definition.features:
         config.ensemble_sync_delay = task_config.task.ensemble_sync_delay
 
+    if TaskFeature.check_fuzzer_help in definition.features:
+        config.check_fuzzer_help = (
+            task_config.task.check_fuzzer_help
+            if task_config.task.check_fuzzer_help is not None
+            else True
+        )
+
+    if TaskFeature.expect_crash_on_failure in definition.features:
+        config.expect_crash_on_failure = (
+            task_config.task.expect_crash_on_failure
+            if task_config.task.expect_crash_on_failure is not None
+            else True
+        )
+
     return config
 
 

src/api-service/__app__/onefuzzlib/tasks/defs.py

@@ -63,6 +63,8 @@ TASK_DEFINITIONS = {
             TaskFeature.target_options,
             TaskFeature.target_workers,
             TaskFeature.ensemble_sync_delay,
+            TaskFeature.check_fuzzer_help,
+            TaskFeature.expect_crash_on_failure,
         ],
         vm=VmDefinition(compare=Compare.AtLeast, value=1),
         containers=[
@@ -105,6 +107,7 @@ TASK_DEFINITIONS = {
             TaskFeature.target_options,
             TaskFeature.target_timeout,
             TaskFeature.check_retry_count,
+            TaskFeature.check_fuzzer_help,
         ],
         vm=VmDefinition(compare=Compare.AtLeast, value=1),
         containers=[
@@ -146,6 +149,7 @@ TASK_DEFINITIONS = {
             TaskFeature.target_exe,
             TaskFeature.target_env,
             TaskFeature.target_options,
+            TaskFeature.check_fuzzer_help,
         ],
         vm=VmDefinition(compare=Compare.Equal, value=1),
         containers=[
@@ -180,6 +184,7 @@ TASK_DEFINITIONS = {
             TaskFeature.target_exe,
             TaskFeature.target_env,
             TaskFeature.target_options,
+            TaskFeature.check_fuzzer_help,
         ],
         vm=VmDefinition(compare=Compare.Equal, value=1),
         containers=[

src/cli/onefuzz/api.py

@@ -777,6 +777,8 @@ class Tasks(Endpoint):
         check_asan_log: bool = False,
         check_debugger: bool = True,
         check_retry_count: Optional[int] = None,
+        check_fuzzer_help: Optional[bool] = None,
+        expect_crash_on_failure: Optional[bool] = None,
         debug: Optional[List[enums.TaskDebugFlag]] = None,
         duration: int = 24,
         ensemble_sync_delay: Optional[int] = None,
@@ -851,6 +853,8 @@ class Tasks(Endpoint):
                 check_asan_log=check_asan_log,
                 check_debugger=check_debugger,
                 check_retry_count=check_retry_count,
+                check_fuzzer_help=check_fuzzer_help,
+                expect_crash_on_failure=expect_crash_on_failure,
                 duration=duration,
                 ensemble_sync_delay=ensemble_sync_delay,
                 generator_exe=generator_exe,

src/cli/onefuzz/templates/libfuzzer.py

@@ -48,6 +48,8 @@ class Libfuzzer(Command):
         crash_report_timeout: Optional[int] = None,
         debug: Optional[List[TaskDebugFlag]] = None,
         ensemble_sync_delay: Optional[int] = None,
+        check_fuzzer_help: bool = True,
+        expect_crash_on_failure: bool = True,
     ) -> None:
 
         fuzzer_containers = [
@@ -76,6 +78,8 @@ class Libfuzzer(Command):
             tags=tags,
             debug=debug,
             ensemble_sync_delay=ensemble_sync_delay,
+            check_fuzzer_help=check_fuzzer_help,
+            expect_crash_on_failure=expect_crash_on_failure,
         )
 
         coverage_containers = [
@@ -98,6 +102,7 @@ class Libfuzzer(Command):
             tags=tags,
             prereq_tasks=[fuzzer_task.task_id],
             debug=debug,
+            check_fuzzer_help=check_fuzzer_help,
         )
 
         report_containers = [
@@ -124,6 +129,7 @@ class Libfuzzer(Command):
             prereq_tasks=[fuzzer_task.task_id],
             target_timeout=crash_report_timeout,
             check_retry_count=check_retry_count,
+            check_fuzzer_help=check_fuzzer_help,
             debug=debug,
         )
 
@@ -154,6 +160,8 @@ class Libfuzzer(Command):
         notification_config: Optional[NotificationConfig] = None,
         debug: Optional[List[TaskDebugFlag]] = None,
         ensemble_sync_delay: Optional[int] = None,
+        check_fuzzer_help: bool = True,
+        expect_crash_on_failure: bool = True,
     ) -> Optional[Job]:
         """
         Basic libfuzzer job
@@ -229,6 +237,8 @@ class Libfuzzer(Command):
             check_retry_count=check_retry_count,
             debug=debug,
             ensemble_sync_delay=ensemble_sync_delay,
+            check_fuzzer_help=check_fuzzer_help,
+            expect_crash_on_failure=expect_crash_on_failure,
         )
 
         self.logger.info("done creating tasks")
@@ -261,6 +271,7 @@ class Libfuzzer(Command):
         notification_config: Optional[NotificationConfig] = None,
         debug: Optional[List[TaskDebugFlag]] = None,
         preserve_existing_outputs: bool = False,
+        check_fuzzer_help: bool = True,
     ) -> Optional[Job]:
 
         """
@@ -348,6 +359,7 @@ class Libfuzzer(Command):
             check_retry_count=check_retry_count,
             debug=debug,
             preserve_existing_outputs=preserve_existing_outputs,
+            check_fuzzer_help=check_fuzzer_help,
         )
 
         self.logger.info("done creating tasks")

src/pytypes/onefuzztypes/enums.py

@@ -75,6 +75,8 @@ class TaskFeature(Enum):
     check_retry_count = "check_retry_count"
     ensemble_sync_delay = "ensemble_sync_delay"
     preserve_existing_outputs = "preserve_existing_outputs"
+    check_fuzzer_help = "check_fuzzer_help"
+    expect_crash_on_failure = "expect_crash_on_failure"
 
 
 # Permissions for an Azure Blob Storage Container.

src/pytypes/onefuzztypes/models.py

@@ -116,6 +116,8 @@ class TaskDetails(BaseModel):
     check_asan_log: Optional[bool]
     check_debugger: Optional[bool] = Field(default=True)
     check_retry_count: Optional[int]
+    check_fuzzer_help: Optional[bool]
+    expect_crash_on_failure: Optional[bool]
     rename_output: Optional[bool]
     supervisor_exe: Optional[str]
     supervisor_env: Optional[Dict[str, str]]
@@ -310,6 +312,8 @@ class TaskUnitConfig(BaseModel):
     check_asan_log: Optional[bool]
     check_debugger: Optional[bool]
     check_retry_count: Optional[int]
+    check_fuzzer_help: Optional[bool]
+    expect_crash_on_failure: Optional[bool]
     rename_output: Optional[bool]
     generator_exe: Optional[str]
     generator_env: Optional[Dict[str, str]]