Fixing .NET crash report no-repro (#2642)

The minidump file being generated did not have enough information for SOS to correctly locate the exception object. Switch to “full” minidumps instead.

- Fix a small bug with coverage reporting task ID as job ID.

- I had thought we needed .NET 6 installed as well to run SharpFuzz, but now I'm not so sure. In any case, installing both versions will not hurt (and may help users).

src/ApiService/ApiService/onefuzzlib/Extension.cs

@@ -303,7 +303,7 @@ public class Extensions : IExtensions {
             urlsUpdated.Add(toolsAzCopy);
             urlsUpdated.Add(toolsSetup);
 
-            var toExecuteCmd = $"sh setup.sh {mode.ToString().ToLowerInvariant()}";
+            var toExecuteCmd = $"bash setup.sh {mode.ToString().ToLowerInvariant()}";
             var extensionSettings = JsonSerializer.Serialize(new { CommandToExecute = toExecuteCmd, FileUris = urlsUpdated }, _extensionSerializerOptions);
 
             var extension = new VMExtensionWrapper {

src/agent/onefuzz-task/src/tasks/report/dotnet/common.rs

@@ -27,12 +27,18 @@ pub async fn collect_exception_info(
         }
     };
 
-    let exception = dump.exception().await?;
+    let exception = dump.exception().await;
 
     // Remove temp dir cooperatively.
     spawn_blocking(move || tmp_dir).await?;
 
-    Ok(exception)
+    match exception {
+        Ok(r) => Ok(Some(r)),
+        Err(e) => {
+            error!("unable to extract exception info: {}", e);
+            Ok(None)
+        }
+    }
 }
 
 const DUMP_FILE_NAME: &str = "tmp.dmp";
@@ -43,7 +49,7 @@ const MINIDUMP_TYPE_VAR: &str = "COMPlus_DbgMiniDumpType";
 const MINIDUMP_NAME_VAR: &str = "COMPlus_DbgMiniDumpName";
 
 const MINIDUMP_ENABLE: &str = "1";
-const MINIDUMP_TYPE_HEAP: &str = "2";
+const MINIDUMP_TYPE_FULL: &str = "4";
 
 // Invoke target with .NET runtime environment vars set to create minidumps.
 //
@@ -64,7 +70,7 @@ async fn collect_dump(
 
     // Set `dotnet` environment vars to enable saving minidumps on crash.
     cmd.env(ENABLE_MINIDUMP_VAR, MINIDUMP_ENABLE);
-    cmd.env(MINIDUMP_TYPE_VAR, MINIDUMP_TYPE_HEAP);
+    cmd.env(MINIDUMP_TYPE_VAR, MINIDUMP_TYPE_FULL);
     cmd.env(MINIDUMP_NAME_VAR, dump_path);
 
     let mut child = cmd.spawn()?;
@@ -97,12 +103,10 @@ impl DotnetDumpFile {
         Self { path }
     }
 
-    pub async fn exception(&self) -> Result<Option<DotnetExceptionInfo>> {
+    pub async fn exception(&self) -> Result<DotnetExceptionInfo> {
         let output = self.exec_sos_command(SOS_PRINT_EXCEPTION).await?;
         let text = String::from_utf8_lossy(&output.stdout);
-        let exception = parse_sos_print_exception_output(&text).ok();
+        parse_sos_print_exception_output(&text)
-
-        Ok(exception)
     }
 
     async fn exec_sos_command(&self, sos_cmd: &str) -> Result<Output> {
@@ -209,7 +213,8 @@ pub fn parse_sos_print_exception_output(text: &str) -> Result<DotnetExceptionInf
         }
     }
 
-    let exception = exception.ok_or_else(|| format_err!("missing exception type"))?;
+    let exception =
+        exception.ok_or_else(|| format_err!("missing exception type, output was:\n{}", text))?;
     let message = message.ok_or_else(|| format_err!("missing exception message"))?;
 
     let inner_exception = inner_exception.ok_or_else(|| format_err!("missing inner exception"))?;

src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs

@@ -171,7 +171,7 @@ impl AsanProcessor {
             format_err!("unable to sha256 digest input file: {}", input.display())
         })?;
 
-        let job_id = self.config.common.task_id;
+        let job_id = self.config.common.job_id;
         let task_id = self.config.common.task_id;
 
         let target_exe = self.target_exe().await?;

src/api-service/__app__/onefuzzlib/extension.py

@@ -380,7 +380,7 @@ def agent_config(
                 with_sas,
             ),
         ]
-        to_execute_cmd = "sh setup.sh %s" % (mode.name)
+        to_execute_cmd = "bash setup.sh %s" % (mode.name)
 
         extension = {
             "name": "CustomScript",

src/runtime-tools/linux/setup.sh

@@ -12,7 +12,7 @@ USER_SETUP="/onefuzz/setup/setup.sh"
 TASK_SETUP="/onefuzz/bin/task-setup.sh"
 MANAGED_SETUP="/onefuzz/bin/managed.sh"
 SCALESET_SETUP="/onefuzz/bin/scaleset-setup.sh"
-DOTNET_VERSION="7.0.100"
+DOTNET_VERSIONS=('7.0.100' '6.0.403')
 export DOTNET_ROOT=/onefuzz/tools/dotnet
 export DOTNET_CLI_HOME="$DOTNET_ROOT"
 export ONEFUZZ_ROOT=/onefuzz
@@ -144,16 +144,18 @@ if type apt > /dev/null 2> /dev/null; then
     curl --retry 10 -sSL https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh 2>&1 | logger -s -i -t 'onefuzz-curl-dotnet-install'
     chmod +x dotnet-install.sh
 
-    logger "running dotnet install"
+    for version in "${DOTNET_VERSIONS[@]}"; do
-    /bin/bash ./dotnet-install.sh --version "$DOTNET_VERSION" --install-dir "$DOTNET_ROOT" 2>&1 | logger -s -i -t 'onefuzz-dotnet-setup'
+        logger "running dotnet install $version"
+        /bin/bash ./dotnet-install.sh --version "$version" --install-dir "$DOTNET_ROOT" 2>&1 | logger -s -i -t 'onefuzz-dotnet-setup'
+    done
     rm dotnet-install.sh
 
     logger "install dotnet tools"
     pushd "$DOTNET_ROOT"
     ls -lah 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
-    "$DOTNET_ROOT"/dotnet tool install dotnet-dump --version 6.0.328102 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+    "$DOTNET_ROOT"/dotnet tool install dotnet-dump --version 6.0.351802 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
-    "$DOTNET_ROOT"/dotnet tool install dotnet-coverage --version 17.3.6 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+    "$DOTNET_ROOT"/dotnet tool install dotnet-coverage --version 17.5 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
-    "$DOTNET_ROOT"/dotnet tool install dotnet-sos --version 6.0.328102 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+    "$DOTNET_ROOT"/dotnet tool install dotnet-sos --version 6.0.351802 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
     popd
 fi
 

src/runtime-tools/win64/onefuzz.ps1

@@ -6,7 +6,7 @@ $env:ONEFUZZ_ROOT = "C:\onefuzz"
 $env:ONEFUZZ_TOOLS = "C:\onefuzz\tools"
 $env:LLVM_SYMBOLIZER_PATH = "llvm-symbolizer"
 $env:RUST_LOG = "info"
-$env:DOTNET_VERSION = "7.0.100"
+$env:DOTNET_VERSIONS = "7.0.100;6.0.403"
 # Set a session and machine scoped env var
 $env:DOTNET_ROOT = "c:\onefuzz\tools\dotnet"
 [Environment]::SetEnvironmentVariable("DOTNET_ROOT", $env:DOTNET_ROOT, "Machine")
@@ -19,10 +19,10 @@ function log ($message) {
 }
 
 function Setup-Silent-Notification {
-   # https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/registry-entries-for-silent-process-exit
+  # https://docs.microsoft.com/en-us/windows-hardware/drivers/debugger/registry-entries-for-silent-process-exit
-   log "installing registry key for silent termination notification of onefuzz-agent"
+  log "installing registry key for silent termination notification of onefuzz-agent"
-   reg import c:\onefuzz\tools\win64\onefuzz-silent-exit.reg
+  reg import c:\onefuzz\tools\win64\onefuzz-silent-exit.reg
-   log "done importing registry key"
+  log "done importing registry key"
 }
 
 function Uninstall-OneDrive {
@@ -31,13 +31,13 @@ function Uninstall-OneDrive {
   Unregister-ScheduledTask -TaskName *OneDrive* -Confirm:$false
 
   if (Test-Path $env:windir\SysWOW64\OneDriveSetup.exe) {
-     log "uninstalling onedrive from syswow64"
+    log "uninstalling onedrive from syswow64"
-     Start-Process -FilePath $env:windir\SysWOW64\OneDriveSetup.exe -ArgumentList /uninstall
+    Start-Process -FilePath $env:windir\SysWOW64\OneDriveSetup.exe -ArgumentList /uninstall
   }
 
   if (Test-Path $env:windir\System32\OneDriveSetup.exe) {
-     log "uninstalling onedrive from system32"
+    log "uninstalling onedrive from system32"
-     Start-Process -FilePath $env:windir\System32\OneDriveSetup.exe -ArgumentList /uninstall
+    Start-Process -FilePath $env:windir\System32\OneDriveSetup.exe -ArgumentList /uninstall
   }
 }
 
@@ -173,17 +173,21 @@ function Install-VCRedist {
   log "installing VC Redist: done"
 }
 
-function Install-Dotnet([string]$Version, [string]$InstallDir, [string]$ToolsDir) {
+function Install-Dotnet([string]$Versions, [string]$InstallDir, [string]$ToolsDir) {
-  log "Installing dotnet to ${InstallDir}"
+  $Versions -Split ';' | ForEach-Object {
-  Invoke-WebRequest 'https://dot.net/v1/dotnet-install.ps1' -OutFile 'dotnet-install.ps1'
+    $Version = $_
-  ./dotnet-install.ps1 -Version $Version -InstallDir $InstallDir
+    log "Installing dotnet ${Version} to ${InstallDir}"
-  Remove-Item ./dotnet-install.ps1
+    Invoke-WebRequest 'https://dot.net/v1/dotnet-install.ps1' -OutFile 'dotnet-install.ps1'
-  log "Installing dotnet: done"
+    ./dotnet-install.ps1 -Version $Version -InstallDir $InstallDir
+    Remove-Item ./dotnet-install.ps1
+    log "Installing dotnet ${Version}: done"
+  }
+
   log "Installing dotnet tools to ${ToolsDir}"
   Push-Location $InstallDir
-  ./dotnet.exe tool install dotnet-dump --version 6.0.328102 --tool-path $ToolsDir
+  ./dotnet.exe tool install dotnet-dump --version 6.0.351802 --tool-path $ToolsDir
-  ./dotnet.exe tool install dotnet-coverage --version 17.3.6 --tool-path $ToolsDir
+  ./dotnet.exe tool install dotnet-coverage --version 17.5.0 --tool-path $ToolsDir
-  ./dotnet.exe tool install dotnet-sos --version 6.0.328102 --tool-path $ToolsDir
+  ./dotnet.exe tool install dotnet-sos --version 6.0.351802 --tool-path $ToolsDir
   Pop-Location
   log "Installing dotnet tools: done"
 }

src/runtime-tools/win64/setup.ps1

@@ -69,7 +69,7 @@ function Install-OnefuzzSetup {
   Enable-SSH
   Install-OnBoot
   Install-VCRedist
-  Install-Dotnet -Version $env:DOTNET_VERSION -InstallDir $env:DOTNET_ROOT -ToolsDir $env:ONEFUZZ_TOOLS
+  Install-Dotnet -Version $env:DOTNET_VERSIONS -InstallDir $env:DOTNET_ROOT -ToolsDir $env:ONEFUZZ_TOOLS
   Setup-Silent-Notification
   log "onefuzz: setup done"
 }