Add generic dotnet_crash_report task (#2250)

src/agent/Cargo.lock

@@ -1946,6 +1946,7 @@ dependencies = [
  "onefuzz",
  "onefuzz-telemetry",
  "path-absolutize",
+ "regex",
  "reqwest",
  "reqwest-retry",
  "serde",

src/agent/onefuzz-task/Cargo.toml

@@ -25,6 +25,7 @@ hex = "0.4"
 lazy_static = "1.4"
 log = "0.4"
 num_cpus = "1.13"
+regex = "1.6.0"
 reqwest = { version = "0.11", features = ["json", "stream", "rustls-tls"], default-features=false }
 serde = "1.0"
 serde_json = "1.0"

src/agent/onefuzz-task/src/tasks/config.rs

@@ -87,6 +87,9 @@ pub enum Config {
     #[serde(alias = "dotnet_coverage")]
     DotnetCoverage(coverage::dotnet::Config),
 
+    #[serde(alias = "dotnet_crash_report")]
+    DotnetCrashReport(report::dotnet::generic::Config),
+
     #[cfg(any(target_os = "linux", target_os = "windows"))]
     #[serde(alias = "libfuzzer_dotnet_fuzz")]
     LibFuzzerDotnetFuzz(fuzz::libfuzzer::dotnet::Config),
@@ -140,6 +143,7 @@ impl Config {
             Config::Coverage(c) => &mut c.common,
             #[cfg(any(target_os = "linux", target_os = "windows"))]
             Config::DotnetCoverage(c) => &mut c.common,
+            Config::DotnetCrashReport(c) => &mut c.common,
             Config::LibFuzzerDotnetFuzz(c) => &mut c.common,
             Config::LibFuzzerFuzz(c) => &mut c.common,
             Config::LibFuzzerMerge(c) => &mut c.common,
@@ -160,6 +164,7 @@ impl Config {
             Config::Coverage(c) => &c.common,
             #[cfg(any(target_os = "linux", target_os = "windows"))]
             Config::DotnetCoverage(c) => &c.common,
+            Config::DotnetCrashReport(c) => &c.common,
             Config::LibFuzzerDotnetFuzz(c) => &c.common,
             Config::LibFuzzerFuzz(c) => &c.common,
             Config::LibFuzzerMerge(c) => &c.common,
@@ -180,6 +185,7 @@ impl Config {
             Config::Coverage(_) => "coverage",
             #[cfg(any(target_os = "linux", target_os = "windows"))]
             Config::DotnetCoverage(_) => "dotnet_coverage",
+            Config::DotnetCrashReport(_) => "dotnet_crash_report",
             Config::LibFuzzerDotnetFuzz(_) => "libfuzzer_fuzz",
             Config::LibFuzzerFuzz(_) => "libfuzzer_fuzz",
             Config::LibFuzzerMerge(_) => "libfuzzer_merge",
@@ -231,6 +237,11 @@ impl Config {
                     .run()
                     .await
             }
+            Config::DotnetCrashReport(config) => {
+                report::dotnet::generic::DotnetCrashReportTask::new(config)
+                    .run()
+                    .await
+            }
             Config::LibFuzzerDotnetFuzz(config) => {
                 fuzz::libfuzzer::dotnet::LibFuzzerDotnetFuzzTask::new(config)?
                     .run()

src/agent/onefuzz-task/src/tasks/report/crash_report.rs

@@ -88,6 +88,18 @@ pub enum CrashTestResult {
     NoRepro(Box<NoCrash>),
 }
 
+impl From<CrashReport> for CrashTestResult {
+    fn from(report: CrashReport) -> Self {
+        Self::CrashReport(Box::new(report))
+    }
+}
+
+impl From<NoCrash> for CrashTestResult {
+    fn from(no_crash: NoCrash) -> Self {
+        Self::NoRepro(Box::new(no_crash))
+    }
+}
+
 #[derive(Debug, Deserialize, Serialize)]
 pub struct RegressionReport {
     pub crash_test_result: CrashTestResult,

src/agent/onefuzz-task/src/tasks/report/dotnet.rs

@@ -0,0 +1,5 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+pub mod common;
+pub mod generic;

src/agent/onefuzz-task/src/tasks/report/dotnet/common.rs

@@ -0,0 +1,232 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use std::ffi::OsStr;
+use std::path::{Path, PathBuf};
+use std::process::Output;
+
+use anyhow::Result;
+use tokio::fs;
+use tokio::process::Command;
+use tokio::task::spawn_blocking;
+
+pub async fn collect_exception_info(
+    args: &[impl AsRef<OsStr>],
+    env: impl IntoIterator<Item = (impl AsRef<OsStr>, impl AsRef<OsStr>)>,
+) -> Result<Option<DotnetExceptionInfo>> {
+    let tmp_dir = spawn_blocking(tempfile::tempdir).await??;
+
+    let dump_path = tmp_dir.path().join(DUMP_FILE_NAME);
+
+    let dump = match collect_dump(args, env, &dump_path).await? {
+        Some(dump) => dump,
+        None => {
+            return Ok(None);
+        }
+    };
+
+    let exception = dump.exception().await?;
+
+    // Remove temp dir without blocking.
+    spawn_blocking(move || tmp_dir).await?;
+
+    Ok(exception)
+}
+
+const DUMP_FILE_NAME: &str = "tmp.dmp";
+
+// Assumes `dotnet` >= 6.0.
+//
+// See: https://docs.microsoft.com/en-us/dotnet/core/diagnostics/dumps
+const ENABLE_MINIDUMP_VAR: &str = "DOTNET_DbgEnableMiniDump";
+const MINIDUMP_TYPE_VAR: &str = "DOTNET_DbgMiniDumpType";
+const MINIDUMP_NAME_VAR: &str = "DOTNET_DbgMiniDumpName";
+
+const MINIDUMP_ENABLE: &str = "1";
+const MINIDUMP_TYPE_NORMAL: &str = "1";
+
+// Invoke target with .NET runtime environment vars set to create minidumps.
+//
+// Returns the newly-created dump file, when present.
+async fn collect_dump(
+    args: impl IntoIterator<Item = impl AsRef<OsStr>>,
+    env: impl IntoIterator<Item = (impl AsRef<OsStr>, impl AsRef<OsStr>)>,
+    dump_path: impl AsRef<Path>,
+) -> Result<Option<DotnetDumpFile>> {
+    let dump_path = dump_path.as_ref();
+
+    let mut cmd = Command::new("dotnet");
+    cmd.arg("exec");
+    cmd.args(args);
+
+    cmd.envs(env);
+
+    cmd.env(ENABLE_MINIDUMP_VAR, MINIDUMP_ENABLE);
+    cmd.env(MINIDUMP_TYPE_VAR, MINIDUMP_TYPE_NORMAL);
+    cmd.env(MINIDUMP_NAME_VAR, dump_path);
+
+    let mut child = cmd.spawn()?;
+    let exit_status = child.wait().await?;
+
+    if exit_status.success() {
+        warn!("dotnet target exited normally when attempting to collect minidump");
+    }
+
+    let metadata = fs::metadata(dump_path).await;
+
+    if metadata.is_ok() {
+        // File exists and is readable if metadata is.
+        let dump = DotnetDumpFile::new(dump_path.to_owned());
+
+        Ok(Some(dump))
+    } else {
+        Ok(None)
+    }
+}
+
+pub struct DotnetDumpFile {
+    path: PathBuf,
+}
+
+impl DotnetDumpFile {
+    pub fn new(path: PathBuf) -> Self {
+        Self { path }
+    }
+
+    pub async fn exception(&self) -> Result<Option<DotnetExceptionInfo>> {
+        let output = self.exec_sos_command(SOS_PRINT_EXCEPTION).await?;
+        let text = String::from_utf8_lossy(&output.stdout);
+        let exception = parse_sos_print_exception_output(&text).ok();
+
+        Ok(exception)
+    }
+
+    async fn exec_sos_command(&self, sos_cmd: &str) -> Result<Output> {
+        let mut cmd = Command::new("dotnet");
+
+        // Run `dotnet-analyze` with a single SOS command on startup, then exit
+        // the otherwise-interactive SOS session.
+        let dump_path = self.path.display().to_string();
+        cmd.args(["dump", "analyze", &dump_path, "-c", sos_cmd, "-c", SOS_EXIT]);
+
+        let output = cmd.spawn()?.wait_with_output().await?;
+
+        if !output.status.success() {
+            bail!(
+                "SOS session for command `{}` exited with error {}: {}",
+                sos_cmd,
+                output.status,
+                String::from_utf8_lossy(&output.stdout),
+            );
+        }
+
+        Ok(output)
+    }
+}
+
+pub struct DotnetExceptionInfo {
+    pub exception: String,
+    pub message: String,
+    pub inner_exception: Option<String>,
+    pub call_stack: Vec<String>,
+    pub hresult: String,
+}
+
+pub fn parse_sos_print_exception_output(text: &str) -> Result<DotnetExceptionInfo> {
+    use std::io::*;
+
+    use regex::Regex;
+
+    lazy_static::lazy_static! {
+        pub static ref EXCEPTION_TYPE: Regex = Regex::new(r#"^Exception type:\s+(.+)$"#).unwrap();
+        pub static ref EXCEPTION_MESSAGE: Regex = Regex::new(r#"^Message:\s+(.*)$"#).unwrap();
+        pub static ref INNER_EXCEPTION: Regex = Regex::new(r#"^InnerException:\s+(.*)$"#).unwrap();
+        pub static ref STACK_FRAME: Regex = Regex::new(r#"^\s*([[:xdigit:]]+) ([[:xdigit:]]+) (.+)$"#).unwrap();
+        pub static ref HRESULT: Regex = Regex::new(r#"^HResult:\s+([[:xdigit:]]+)$"#).unwrap();
+    }
+
+    let reader = BufReader::new(text.as_bytes());
+
+    let mut exception: Option<String> = None;
+    let mut message: Option<String> = None;
+    let mut inner_exception: Option<String> = None;
+    let mut call_stack: Vec<String> = vec![];
+    let mut hresult: Option<String> = None;
+
+    for line in reader.lines() {
+        let line = match &line {
+            Ok(line) => line,
+            Err(err) => {
+                warn!("error parsing line: {}", err);
+                continue;
+            }
+        };
+
+        if let Some(captures) = EXCEPTION_TYPE.captures(line) {
+            if let Some(c) = captures.get(1) {
+                exception = Some(c.as_str().to_owned());
+                continue;
+            }
+        }
+
+        if let Some(captures) = EXCEPTION_MESSAGE.captures(line) {
+            if let Some(c) = captures.get(1) {
+                message = Some(c.as_str().to_owned());
+                continue;
+            }
+        }
+
+        if let Some(captures) = INNER_EXCEPTION.captures(line) {
+            if let Some(c) = captures.get(1) {
+                inner_exception = Some(c.as_str().to_owned());
+                continue;
+            }
+        }
+
+        if let Some(captures) = STACK_FRAME.captures(line) {
+            if let Some(c) = captures.get(3) {
+                let frame = c.as_str().to_owned();
+                call_stack.push(frame);
+                continue;
+            }
+        }
+
+        if let Some(captures) = HRESULT.captures(line) {
+            if let Some(c) = captures.get(1) {
+                hresult = Some(c.as_str().to_owned());
+                continue;
+            }
+        }
+    }
+
+    let exception = exception.ok_or_else(|| format_err!("missing exception type"))?;
+    let message = message.ok_or_else(|| format_err!("missing exception message"))?;
+
+    let inner_exception = inner_exception.ok_or_else(|| format_err!("missing inner exception"))?;
+    let inner_exception = if inner_exception == "<none>" {
+        None
+    } else {
+        Some(inner_exception)
+    };
+
+    let hresult = hresult.ok_or_else(|| format_err!("missing exception hresult"))?;
+
+    if call_stack.is_empty() {
+        bail!("missing call_stack");
+    }
+
+    Ok(DotnetExceptionInfo {
+        exception,
+        message,
+        inner_exception,
+        call_stack,
+        hresult,
+    })
+}
+
+// https://docs.microsoft.com/en-us/dotnet/core/diagnostics/dotnet-dump#analyze-sos-commands
+const SOS_EXIT: &str = "exit";
+const SOS_PRINT_EXCEPTION: &str = "printexception -lines";
+
+#[cfg(test)]
+mod tests;

src/agent/onefuzz-task/src/tasks/report/dotnet/common/data/print-exception.stdout

@@ -0,0 +1,19 @@
+Loading core dump: out.dmp ...
+Exception object: 00007f90f00cdea8
+Exception type:   System.IndexOutOfRangeException
+Message:          Index was outside the bounds of the array.
+InnerException:   <none>
+StackTrace (generated):
+    SP               IP               Function
+    00007FFC15077820 00007F91286FC295 System.Private.CoreLib.dll!System.ThrowHelper.ThrowIndexOutOfRangeException()+0x35 [/_/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @ 65]
+    00007FFC15077840 00007F91276B49F9 System.Private.CoreLib.dll!System.ReadOnlySpan`1[[System.Byte, System.Private.CoreLib]].get_Item(Int32)+0x19 [/_/src/libraries/System.Private.CoreLib/src/System/ReadOnlySpan.cs @ 136]
+    00007FFC15077850 00007F91286FC1FF GoodBad.dll!GoodBad.Parser.ParseInput(System.ReadOnlySpan`1<Byte>)+0x39f
+    00007FFC150778F0 00007F91286FBDD2 GoodBad.dll!GoodBad.ParserLibFuzzer.TestOneInput(System.ReadOnlySpan`1<Byte>)+0x82
+    00007FFC15077920 00007F9127E38755 SharpFuzz.dll!SharpFuzz.Fuzzer+LibFuzzer.RunWithoutLibFuzzer(SharpFuzz.ReadOnlySpanAction)+0x135
+    00007FFC15077990 00007F9127E3820B SharpFuzz.dll!SharpFuzz.Fuzzer+LibFuzzer.Run(SharpFuzz.ReadOnlySpanAction)+0x8b
+    00007FFC15077A00 00007F9127E373A1 LibFuzzerSharp.dll!LibFuzzerSharp.Program.TryTestOne[[System.__Canon, System.Private.CoreLib]](System.Reflection.MethodInfo, System.Func`2<System.__Canon,SharpFuzz.ReadOnlySpanAction>)+0xd1
+    00007FFC15077A80 00007F9127E371D2 LibFuzzerSharp.dll!LibFuzzerSharp.Program.TryTestOneSpan(System.Reflection.MethodInfo)+0x112
+    00007FFC15077AD0 00007F9127E2385F LibFuzzerSharp.dll!LibFuzzerSharp.Program.Main(System.String[])+0x39f
+
+StackTraceString: <none>
+HResult: 80131508

src/agent/onefuzz-task/src/tasks/report/dotnet/common/tests.rs

@@ -0,0 +1,28 @@
+use anyhow::Result;
+
+use super::parse_sos_print_exception_output;
+
+const PRINT_EXCEPTION_STDOUT: &str = include_str!("data/print-exception.stdout");
+
+#[test]
+fn test_parse_sos_print_exception() -> Result<()> {
+    let ei = parse_sos_print_exception_output(PRINT_EXCEPTION_STDOUT)?;
+
+    assert_eq!(ei.exception, "System.IndexOutOfRangeException");
+    assert_eq!(ei.message, "Index was outside the bounds of the array.");
+    assert_eq!(ei.inner_exception, None);
+    assert_eq!(ei.call_stack, vec![
+        "System.Private.CoreLib.dll!System.ThrowHelper.ThrowIndexOutOfRangeException()+0x35 [/_/src/libraries/System.Private.CoreLib/src/System/ThrowHelper.cs @ 65]",
+        "System.Private.CoreLib.dll!System.ReadOnlySpan`1[[System.Byte, System.Private.CoreLib]].get_Item(Int32)+0x19 [/_/src/libraries/System.Private.CoreLib/src/System/ReadOnlySpan.cs @ 136]",
+        "GoodBad.dll!GoodBad.Parser.ParseInput(System.ReadOnlySpan`1<Byte>)+0x39f",
+        "GoodBad.dll!GoodBad.ParserLibFuzzer.TestOneInput(System.ReadOnlySpan`1<Byte>)+0x82",
+        "SharpFuzz.dll!SharpFuzz.Fuzzer+LibFuzzer.RunWithoutLibFuzzer(SharpFuzz.ReadOnlySpanAction)+0x135",
+        "SharpFuzz.dll!SharpFuzz.Fuzzer+LibFuzzer.Run(SharpFuzz.ReadOnlySpanAction)+0x8b",
+        "LibFuzzerSharp.dll!LibFuzzerSharp.Program.TryTestOne[[System.__Canon, System.Private.CoreLib]](System.Reflection.MethodInfo, System.Func`2<System.__Canon,SharpFuzz.ReadOnlySpanAction>)+0xd1",
+        "LibFuzzerSharp.dll!LibFuzzerSharp.Program.TryTestOneSpan(System.Reflection.MethodInfo)+0x112",
+        "LibFuzzerSharp.dll!LibFuzzerSharp.Program.Main(System.String[])+0x39f",
+    ]);
+    assert_eq!(ei.hresult, "80131508");
+
+    Ok(())
+}

src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs

@@ -0,0 +1,212 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+use std::{
+    collections::HashMap,
+    path::{Path, PathBuf},
+    sync::Arc,
+};
+
+use anyhow::{Context, Result};
+use async_trait::async_trait;
+use onefuzz::{blob::BlobUrl, sha256, syncdir::SyncedDir};
+use reqwest::Url;
+use serde::Deserialize;
+use storage_queue::{Message, QueueClient};
+
+use crate::tasks::report::crash_report::*;
+use crate::tasks::report::dotnet::common::collect_exception_info;
+use crate::tasks::{
+    config::CommonConfig,
+    generic::input_poller::*,
+    heartbeat::{HeartbeatSender, TaskHeartbeatClient},
+    utils::default_bool_true,
+};
+
+const DOTNET_DUMP_TOOL_NAME: &str = "dotnet-dump";
+
+#[derive(Debug, Deserialize)]
+pub struct Config {
+    pub target_exe: PathBuf,
+    pub target_env: HashMap<String, String>,
+    // TODO:  options are not yet used for crash reporting
+    pub target_options: Vec<String>,
+    pub target_timeout: Option<u64>,
+    pub input_queue: Option<QueueClient>,
+    pub crashes: Option<SyncedDir>,
+    pub reports: Option<SyncedDir>,
+    pub unique_reports: Option<SyncedDir>,
+    pub no_repro: Option<SyncedDir>,
+
+    #[serde(default = "default_bool_true")]
+    pub check_fuzzer_help: bool,
+
+    #[serde(default)]
+    pub check_retry_count: u64,
+
+    #[serde(default)]
+    pub minimized_stack_depth: Option<usize>,
+
+    #[serde(default = "default_bool_true")]
+    pub check_queue: bool,
+
+    #[serde(flatten)]
+    pub common: CommonConfig,
+}
+
+pub struct DotnetCrashReportTask {
+    config: Arc<Config>,
+    pub poller: InputPoller<Message>,
+}
+
+impl DotnetCrashReportTask {
+    pub fn new(config: Config) -> Self {
+        let poller = InputPoller::new("libfuzzer-dotnet-crash-report");
+        let config = Arc::new(config);
+
+        Self { config, poller }
+    }
+
+    pub async fn run(&mut self) -> Result<()> {
+        info!("starting dotnet crash report task");
+
+        if let Some(unique_reports) = &self.config.unique_reports {
+            unique_reports.init().await?;
+        }
+        if let Some(reports) = &self.config.reports {
+            reports.init().await?;
+        }
+        if let Some(no_repro) = &self.config.no_repro {
+            no_repro.init().await?;
+        }
+
+        let mut processor = AsanProcessor::new(self.config.clone()).await?;
+
+        if let Some(crashes) = &self.config.crashes {
+            self.poller.batch_process(&mut processor, crashes).await?;
+        }
+
+        if self.config.check_queue {
+            if let Some(url) = &self.config.input_queue {
+                let callback = CallbackImpl::new(url.clone(), processor)?;
+                self.poller.run(callback).await?;
+            }
+        }
+        Ok(())
+    }
+}
+
+pub struct AsanProcessor {
+    config: Arc<Config>,
+    heartbeat_client: Option<TaskHeartbeatClient>,
+}
+
+impl AsanProcessor {
+    pub async fn new(config: Arc<Config>) -> Result<Self> {
+        let heartbeat_client = config.common.init_heartbeat(None).await?;
+
+        Ok(Self {
+            config,
+            heartbeat_client,
+        })
+    }
+
+    pub async fn test_input(
+        &self,
+        input: &Path,
+        input_url: Option<Url>,
+    ) -> Result<CrashTestResult> {
+        self.heartbeat_client.alive();
+
+        let input_blob = input_url
+            .and_then(|u| BlobUrl::new(u).ok())
+            .map(InputBlob::from);
+
+        let input_sha256 = sha256::digest_file(input).await.with_context(|| {
+            format_err!("unable to sha256 digest input file: {}", input.display())
+        })?;
+
+        let job_id = self.config.common.task_id;
+        let task_id = self.config.common.task_id;
+        let executable = self.config.target_exe.to_owned();
+
+        let mut args = vec![
+            "dotnet".to_owned(),
+            self.config.target_exe.display().to_string(),
+        ];
+        args.extend(self.config.target_options.clone());
+
+        let env = self.config.target_env.clone();
+
+        let crash_test_result = if let Some(exception) = collect_exception_info(&args, env).await? {
+            let call_stack_sha256 = stacktrace_parser::digest_iter(&exception.call_stack, None);
+
+            let crash_report = CrashReport {
+                input_sha256,
+                input_blob,
+                executable,
+                crash_type: exception.exception,
+                crash_site: exception.call_stack[0].clone(),
+                call_stack: exception.call_stack,
+                call_stack_sha256,
+                minimized_stack: None,
+                minimized_stack_sha256: None,
+                minimized_stack_function_names: None,
+                minimized_stack_function_names_sha256: None,
+                minimized_stack_function_lines: None,
+                minimized_stack_function_lines_sha256: None,
+                asan_log: None,
+                task_id,
+                job_id,
+                scariness_score: None,
+                scariness_description: None,
+                onefuzz_version: Some(env!("ONEFUZZ_VERSION").to_owned()),
+                tool_name: Some(DOTNET_DUMP_TOOL_NAME.to_owned()),
+                tool_version: None,
+            };
+
+            crash_report.into()
+        } else {
+            let no_repro = NoCrash {
+                input_sha256,
+                input_blob,
+                executable,
+                job_id,
+                task_id,
+                tries: 1,
+                error: None,
+            };
+
+            no_repro.into()
+        };
+
+        Ok(crash_test_result)
+    }
+}
+
+#[async_trait]
+impl Processor for AsanProcessor {
+    async fn process(&mut self, url: Option<Url>, input: &Path) -> Result<()> {
+        debug!("processing dotnet crash url:{:?} path:{:?}", url, input);
+
+        let crash_test_result = self.test_input(input, url).await?;
+
+        let saved = crash_test_result
+            .save(
+                &self.config.unique_reports,
+                &self.config.reports,
+                &self.config.no_repro,
+            )
+            .await;
+
+        if let Err(err) = saved {
+            error!(
+                "error saving crash test result for input \"{}\": {}",
+                input.display(),
+                err,
+            );
+        }
+
+        Ok(())
+    }
+}

src/agent/onefuzz-task/src/tasks/report/mod.rs

@@ -2,5 +2,6 @@
 // Licensed under the MIT License.
 
 pub mod crash_report;
+pub mod dotnet;
 pub mod generic;
 pub mod libfuzzer_report;

src/agent/stacktrace-parser/src/lib.rs

@@ -281,7 +281,10 @@ pub fn parse_call_stack(text: &str) -> Result<Vec<StackEntry>> {
     asan::parse_asan_call_stack(text)
 }
 
-fn digest_iter(data: impl IntoIterator<Item = impl AsRef<[u8]>>, depth: Option<usize>) -> String {
+pub fn digest_iter(
+    data: impl IntoIterator<Item = impl AsRef<[u8]>>,
+    depth: Option<usize>,
+) -> String {
     let mut ctx = Sha256::new();
 
     if let Some(depth) = depth {