diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 07fc1a0d9..25928a008 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -488,6 +488,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -499,6 +507,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -555,6 +567,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -565,8 +580,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -1207,6 +1221,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -1217,8 +1234,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -2381,6 +2397,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -2392,6 +2416,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -2448,6 +2476,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -2458,8 +2489,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -3097,6 +3127,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -3108,6 +3146,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -3164,6 +3206,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -3174,8 +3219,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -3604,6 +3648,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -3615,6 +3667,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -3671,6 +3727,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -3681,8 +3740,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -4054,6 +4112,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -4065,6 +4131,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -4121,6 +4191,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -4131,8 +4204,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -4478,6 +4550,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -4489,6 +4569,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -4559,6 +4643,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -4569,8 +4656,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -4929,6 +5015,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -4940,6 +5034,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -4996,6 +5094,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -5006,8 +5107,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, @@ -6664,6 +6764,14 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Supervisor Options", "type": "array" }, + "target_assembly": { + "title": "Target Assembly", + "type": "string" + }, + "target_class": { + "title": "Target Class", + "type": "string" + }, "target_env": { "additionalProperties": { "type": "string" @@ -6675,6 +6783,10 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Target Exe", "type": "string" }, + "target_method": { + "title": "Target Method", + "type": "string" + }, "target_options": { "items": { "type": "string" @@ -6745,6 +6857,9 @@ If webhook is set to have Event Grid message format then the payload will look a "description": "An enumeration.", "enum": [ "coverage", + "dotnet_coverage", + "dotnet_crash_report", + "libfuzzer_dotnet_fuzz", "libfuzzer_fuzz", "libfuzzer_coverage", "libfuzzer_crash_report", @@ -6755,8 +6870,7 @@ If webhook is set to have Event Grid message format then the payload will look a "generic_merge", "generic_generator", "generic_crash_report", - "generic_regression", - "dotnet_coverage" + "generic_regression" ], "title": "TaskType" }, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index 7019d3e9c..93470dcc9 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -61,6 +61,9 @@ public enum TaskState { public enum TaskType { Coverage, + DotnetCoverage, + DotnetCrashReport, + LibfuzzerDotnetFuzz, LibfuzzerFuzz, LibfuzzerCoverage, LibfuzzerCrashReport, @@ -72,7 +75,6 @@ public enum TaskType { GenericGenerator, GenericCrashReport, GenericRegression, - DotnetCoverage, } public enum Os { @@ -269,7 +271,10 @@ public enum TaskFeature { ReportList, MinimizedStackDepth, CoverageFilter, - TargetMustUseInput + TargetMustUseInput, + TargetAssembly, + TargetClass, + TargetMethod, } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index a1104d678..bd5e42916 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -200,7 +200,10 @@ public record TaskDetails( bool? PreserveExistingOutputs = null, List? ReportList = null, long? MinimizedStackDepth = null, - string? CoverageFilter = null + string? CoverageFilter = null, + string? TargetAssembly = null, + string? TargetClass = null, + string? TargetMethod = null ); public record TaskVm( @@ -919,6 +922,9 @@ public record TaskUnitConfig( public List? ReportList { get; set; } public long? MinimizedStackDepth { get; set; } public string? CoverageFilter { get; set; } + public string? TargetAssembly { get; set; } + public string? TargetClass { get; set; } + public string? TargetMethod { get; set; } // from here forwards are Container definitions. These need to be inline // with TaskDefinitions and ContainerTypes diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs index 77a3a1a66..5c045b34b 100644 --- a/src/ApiService/ApiService/onefuzzlib/Config.cs +++ b/src/ApiService/ApiService/onefuzzlib/Config.cs @@ -262,6 +262,18 @@ public class Config : IConfig { } } + if (definition.Features.Contains(TaskFeature.TargetAssembly)) { + config.TargetAssembly = task.Config.Task.TargetAssembly; + } + + if (definition.Features.Contains(TaskFeature.TargetClass)) { + config.TargetClass = task.Config.Task.TargetClass; + } + + if (definition.Features.Contains(TaskFeature.TargetMethod)) { + config.TargetMethod = task.Config.Task.TargetMethod; + } + return config; } diff --git a/src/ApiService/ApiService/onefuzzlib/Defs.cs b/src/ApiService/ApiService/onefuzzlib/Defs.cs index fe86eddf8..b1692094f 100644 --- a/src/ApiService/ApiService/onefuzzlib/Defs.cs +++ b/src/ApiService/ApiService/onefuzzlib/Defs.cs @@ -35,12 +35,153 @@ public static class Defs { ContainerPermission.List | ContainerPermission.Read | ContainerPermission.Write - )}, MonitorQueue: ContainerType.ReadonlyInputs) }, + { + TaskType.DotnetCoverage, new TaskDefinition( + Features: new[] { + TaskFeature.TargetExe, + TaskFeature.TargetEnv, + TaskFeature.TargetOptions, + TaskFeature.TargetTimeout, + TaskFeature.CoverageFilter, + TaskFeature.TargetMustUseInput, + }, + Vm: new VmDefinition(Compare: Compare.Equal, Value:1), + Containers: new [] { + new ContainerDefinition( + Type:ContainerType.Setup, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + new ContainerDefinition( + Type:ContainerType.ReadonlyInputs, + Compare: Compare.AtLeast, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + new ContainerDefinition( + Type:ContainerType.Coverage, + Compare: Compare.Equal, + Value:1, + Permissions: + ContainerPermission.List | + ContainerPermission.Read | + ContainerPermission.Write - + ), + new ContainerDefinition( + Type:ContainerType.Tools, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + }, + MonitorQueue: ContainerType.ReadonlyInputs) + }, + { + TaskType.DotnetCrashReport, new TaskDefinition( + Features: new[] { + TaskFeature.TargetExe, + TaskFeature.TargetEnv, + TaskFeature.TargetOptions, + TaskFeature.TargetTimeout, + TaskFeature.CheckRetryCount, + TaskFeature.CheckFuzzerHelp, + TaskFeature.MinimizedStackDepth, + }, + Vm: new VmDefinition(Compare: Compare.AtLeast, Value: 1), + Containers: new[] { + new ContainerDefinition( + Type:ContainerType.Setup, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + new ContainerDefinition( + Type:ContainerType.Crashes, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + new ContainerDefinition( + Type:ContainerType.Reports, + Compare: Compare.AtMost, + Value:1, + Permissions: ContainerPermission.Write + ), + new ContainerDefinition( + Type: ContainerType.UniqueReports, + Compare: Compare.AtMost, + Value: 1, + Permissions: ContainerPermission.Write + ), + new ContainerDefinition( + Type: ContainerType.NoRepro, + Compare: Compare.AtMost, + Value: 1, + Permissions: ContainerPermission.Write + ), + new ContainerDefinition( + Type:ContainerType.Tools, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + }, + MonitorQueue: ContainerType.Crashes + ) + }, + { + TaskType.LibfuzzerDotnetFuzz, new TaskDefinition( + Features: new[] { + TaskFeature.TargetExe, + TaskFeature.TargetEnv, + TaskFeature.TargetOptions, + TaskFeature.TargetWorkers, + TaskFeature.EnsembleSyncDelay, + TaskFeature.CheckFuzzerHelp, + TaskFeature.ExpectCrashOnFailure, + TaskFeature.TargetAssembly, + TaskFeature.TargetClass, + TaskFeature.TargetMethod, + }, + Vm: new VmDefinition(Compare: Compare.AtLeast, Value: 1), + Containers: new[] { + new ContainerDefinition( + Type:ContainerType.Setup, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + new ContainerDefinition( + Type:ContainerType.Crashes, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Write + ), + new ContainerDefinition( + Type: ContainerType.Inputs, + Compare: Compare.Equal, + Value: 1, + Permissions: ContainerPermission.Write | ContainerPermission.Read | ContainerPermission.List + ), + new ContainerDefinition( + Type: ContainerType.ReadonlyInputs, + Compare: Compare.AtLeast, + Value: 0, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + new ContainerDefinition( + Type:ContainerType.Tools, + Compare: Compare.Equal, + Value:1, + Permissions: ContainerPermission.Read | ContainerPermission.List + ), + } + )}, { TaskType.GenericAnalysis , new TaskDefinition( Features: new[] { @@ -555,41 +696,5 @@ public static class Defs { ), }) }, - { TaskType.DotnetCoverage , - new TaskDefinition( - Features: new[] { - TaskFeature.TargetExe, - TaskFeature.TargetEnv, - TaskFeature.TargetOptions, - TaskFeature.TargetTimeout, - TaskFeature.CoverageFilter, - TaskFeature.TargetMustUseInput, - }, - Vm: new VmDefinition(Compare: Compare.Equal, Value:1), - Containers: new [] { - new ContainerDefinition( - Type:ContainerType.Setup, - Compare: Compare.Equal, - Value:1, - Permissions: ContainerPermission.Read | ContainerPermission.List - ), - new ContainerDefinition( - Type:ContainerType.ReadonlyInputs, - Compare: Compare.AtLeast, - Value:1, - Permissions: ContainerPermission.Read | ContainerPermission.List - ), - new ContainerDefinition( - Type:ContainerType.Coverage, - Compare: Compare.Equal, - Value:1, - Permissions: - ContainerPermission.List | - ContainerPermission.Read | - ContainerPermission.Write - - )}, - MonitorQueue: ContainerType.ReadonlyInputs) - }, }; } diff --git a/src/agent/onefuzz-task/src/local/cmd.rs b/src/agent/onefuzz-task/src/local/cmd.rs index 0b4b48e64..92c18a313 100644 --- a/src/agent/onefuzz-task/src/local/cmd.rs +++ b/src/agent/onefuzz-task/src/local/cmd.rs @@ -1,13 +1,13 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#[cfg(any(target_os = "linux", target_os = "windows"))] +use crate::local::coverage; use crate::local::{ common::add_common_config, generic_analysis, generic_crash_report, generic_generator, libfuzzer, libfuzzer_crash_report, libfuzzer_fuzz, libfuzzer_merge, libfuzzer_regression, libfuzzer_test_input, radamsa, test_input, tui::TerminalUi, }; -#[cfg(any(target_os = "linux", target_os = "windows"))] -use crate::local::{coverage, dotnet_coverage}; use anyhow::{Context, Result}; use clap::{App, Arg, SubCommand}; use crossterm::tty::IsTty; @@ -23,8 +23,6 @@ enum Commands { Radamsa, #[cfg(any(target_os = "linux", target_os = "windows"))] Coverage, - #[cfg(any(target_os = "linux", target_os = "windows"))] - DotnetCoverage, LibfuzzerFuzz, LibfuzzerMerge, LibfuzzerCrashReport, @@ -61,8 +59,6 @@ pub async fn run(args: clap::ArgMatches<'static>) -> Result<()> { match command { #[cfg(any(target_os = "linux", target_os = "windows"))] Commands::Coverage => coverage::run(&sub_args, event_sender).await, - #[cfg(any(target_os = "linux", target_os = "windows"))] - Commands::DotnetCoverage => dotnet_coverage::run(&sub_args, event_sender).await, Commands::Radamsa => radamsa::run(&sub_args, event_sender).await, Commands::LibfuzzerCrashReport => { libfuzzer_crash_report::run(&sub_args, event_sender).await @@ -121,8 +117,6 @@ pub fn args(name: &str) -> App<'static, 'static> { let app = match subcommand { #[cfg(any(target_os = "linux", target_os = "windows"))] Commands::Coverage => coverage::args(subcommand.into()), - #[cfg(any(target_os = "linux", target_os = "windows"))] - Commands::DotnetCoverage => dotnet_coverage::args(subcommand.into()), Commands::Radamsa => radamsa::args(subcommand.into()), Commands::LibfuzzerCrashReport => libfuzzer_crash_report::args(subcommand.into()), Commands::LibfuzzerFuzz => libfuzzer_fuzz::args(subcommand.into()), diff --git a/src/agent/onefuzz-task/src/local/common.rs b/src/agent/onefuzz-task/src/local/common.rs index 930b3a7af..7472e404c 100644 --- a/src/agent/onefuzz-task/src/local/common.rs +++ b/src/agent/onefuzz-task/src/local/common.rs @@ -29,8 +29,6 @@ pub const CHECK_RETRY_COUNT: &str = "check_retry_count"; pub const DISABLE_CHECK_QUEUE: &str = "disable_check_queue"; pub const UNIQUE_REPORTS_DIR: &str = "unique_reports_dir"; pub const COVERAGE_DIR: &str = "coverage_dir"; -#[cfg(any(target_os = "linux", target_os = "windows"))] -pub const DOTNET_COVERAGE_DIR: &str = "dotnet_coverage_dir"; pub const READONLY_INPUTS: &str = "readonly_inputs_dir"; pub const CHECK_ASAN_LOG: &str = "check_asan_log"; pub const TOOLS_DIR: &str = "tools_dir"; diff --git a/src/agent/onefuzz-task/src/local/dotnet_coverage.rs b/src/agent/onefuzz-task/src/local/dotnet_coverage.rs deleted file mode 100644 index 904f8014d..000000000 --- a/src/agent/onefuzz-task/src/local/dotnet_coverage.rs +++ /dev/null @@ -1,124 +0,0 @@ -// Copyright (c) Microsoft Corporation. -// Licensed under the MIT License. - -use crate::{ - local::common::{ - build_local_context, get_cmd_arg, get_cmd_env, get_cmd_exe, get_synced_dir, - get_synced_dirs, CmdType, SyncCountDirMonitor, UiEvent, COVERAGE_DIR, INPUTS_DIR, - READONLY_INPUTS, TARGET_ENV, TARGET_EXE, TARGET_OPTIONS, TARGET_TIMEOUT, - }, - tasks::{ - config::CommonConfig, - coverage::dotnet::{Config, DotnetCoverageTask}, - }, -}; - -use anyhow::Result; -use clap::{App, Arg, SubCommand}; -use flume::Sender; -use storage_queue::QueueClient; - -pub fn build_shared_args(local_job: bool) -> Vec> { - let mut args = vec![ - Arg::with_name(TARGET_EXE) - .long(TARGET_EXE) - .takes_value(true) - .required(true), - Arg::with_name(TARGET_ENV) - .long(TARGET_ENV) - .takes_value(true) - .multiple(true), - Arg::with_name(TARGET_OPTIONS) - .long(TARGET_OPTIONS) - .default_value("{input}") - .takes_value(true) - .value_delimiter(" ") - .help("Use a quoted string with space separation to denote multiple arguments"), - Arg::with_name(TARGET_TIMEOUT) - .takes_value(true) - .long(TARGET_TIMEOUT), - Arg::with_name(COVERAGE_DIR) - .takes_value(true) - .required(!local_job) - .long(COVERAGE_DIR), - ]; - if local_job { - args.push( - Arg::with_name(INPUTS_DIR) - .long(INPUTS_DIR) - .takes_value(true) - .required(true), - ) - } else { - args.push( - Arg::with_name(READONLY_INPUTS) - .takes_value(true) - .required(true) - .long(READONLY_INPUTS) - .multiple(true), - ) - } - args -} - -pub fn build_coverage_config( - args: &clap::ArgMatches<'_>, - local_job: bool, - input_queue: Option, - common: CommonConfig, - event_sender: Option>, -) -> Result { - let target_exe = get_cmd_exe(CmdType::Target, args)?.into(); - let target_env = get_cmd_env(CmdType::Target, args)?; - let target_options = get_cmd_arg(CmdType::Target, args); - let target_timeout = value_t!(args, TARGET_TIMEOUT, u64).ok(); - - let readonly_inputs = if local_job { - info!("Took inputs_dir"); - vec![ - get_synced_dir(INPUTS_DIR, common.job_id, common.task_id, args)? - .monitor_count(&event_sender)?, - ] - } else { - get_synced_dirs(READONLY_INPUTS, common.job_id, common.task_id, args)? - .into_iter() - .map(|sd| sd.monitor_count(&event_sender)) - .collect::>>()? - }; - - let coverage = get_synced_dir(COVERAGE_DIR, common.job_id, common.task_id, args)? - .monitor_count(&event_sender)?; - - let config = Config { - target_exe, - target_env, - target_options, - target_timeout, - input_queue, - readonly_inputs, - coverage, - common, - }; - - Ok(config) -} - -pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option>) -> Result<()> { - let context = build_local_context(args, true, event_sender.clone())?; - let config = build_coverage_config( - args, - false, - None, - context.common_config.clone(), - event_sender, - )?; - - let mut task = DotnetCoverageTask::new(config); - task.run().await -} - -pub fn args(name: &'static str) -> App<'static, 'static> { - SubCommand::with_name(name) - .about("execute a local-only coverage task") - .args(&build_shared_args(false)) -} diff --git a/src/agent/onefuzz-task/src/local/libfuzzer.rs b/src/agent/onefuzz-task/src/local/libfuzzer.rs index 4be49cb2f..273062589 100644 --- a/src/agent/onefuzz-task/src/local/libfuzzer.rs +++ b/src/agent/onefuzz-task/src/local/libfuzzer.rs @@ -1,6 +1,11 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +#[cfg(any(target_os = "linux", target_os = "windows"))] +use crate::{ + local::{common::COVERAGE_DIR, coverage, coverage::build_shared_args as build_coverage_args}, + tasks::coverage::generic::CoverageTask, +}; use crate::{ local::{ common::{ @@ -20,17 +25,6 @@ use crate::{ regression::libfuzzer::LibFuzzerRegressionTask, report::libfuzzer_report::ReportTask, }, }; -#[cfg(any(target_os = "linux", target_os = "windows"))] -use crate::{ - local::{ - common::{COVERAGE_DIR, DOTNET_COVERAGE_DIR}, - coverage, - coverage::build_shared_args as build_coverage_args, - dotnet_coverage, - }, - tasks::coverage::dotnet::DotnetCoverageTask, - tasks::coverage::generic::CoverageTask, -}; use anyhow::Result; use clap::{App, SubCommand}; use flume::Sender; @@ -78,28 +72,6 @@ pub async fn run(args: &clap::ArgMatches<'_>, event_sender: Option, pub readonly_inputs: Vec, pub coverage: SyncedDir, + pub tools: SyncedDir, #[serde(flatten)] pub common: CommonConfig, @@ -68,6 +69,8 @@ impl DotnetCoverageTask { pub async fn run(&mut self) -> Result<()> { info!("starting dotnet_coverage task"); + + self.config.tools.init_pull().await?; self.config.coverage.init_pull().await?; let dotnet_path = dotnet_path()?; @@ -257,10 +260,38 @@ impl<'a> TaskContext<'a> { Ok(()) } + async fn target_exe(&self) -> Result { + let tools_dir = self.config.tools.local_path.to_string_lossy().into_owned(); + + // Try to expand `target_exe` with support for `{tools_dir}`. + // + // Allows using `LibFuzzerDotnetLoader.exe` from a shared tools container. + let expand = Expand::new().tools_dir(tools_dir); + let expanded = expand.evaluate_value(&self.config.target_exe.to_string_lossy())?; + let expanded_path = Path::new(&expanded); + + // Check if `target_exe` was resolved to an absolute path and an existing file. + // If so, then the user specified a `target_exe` under the `tools` dir. + let is_absolute = expanded_path.is_absolute(); + let file_exists = fs::metadata(&expanded).await.is_ok(); + + if is_absolute && file_exists { + // We have found `target_exe`, so skip `setup`-relative expansion. + return Ok(expanded); + } + + // We haven't yet resolved a local path for `target_exe`. Try the usual + // `setup`-relative interpretation of the configured value of `target_exe`. + let resolved = try_resolve_setup_relative_path(&self.config.common.setup_dir, expanded) + .await? + .to_string_lossy() + .into_owned(); + + Ok(resolved) + } + async fn command_for_input(&self, input: &Path) -> Result { - let target_exe = - try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe) - .await?; + let target_exe = self.target_exe().await?; let expand = Expand::new() .machine_id() @@ -288,7 +319,7 @@ impl<'a> TaskContext<'a> { .arg(format!( "{} {} -- {}", dotnet_path.to_string_lossy(), - self.config.target_exe.canonicalize()?.to_string_lossy(), + &target_exe, target_options.join(" ") )); diff --git a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs index 4f8b4d5ae..ba08e2284 100644 --- a/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs +++ b/src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs @@ -1,36 +1,35 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +use std::path::PathBuf; + use anyhow::Result; use async_trait::async_trait; +use onefuzz::fs::set_executable; use onefuzz::libfuzzer::LibFuzzer; +use onefuzz::syncdir::SyncedDir; use tokio::process::Command; use crate::tasks::fuzz::libfuzzer::common; +use crate::tasks::utils::try_resolve_setup_relative_path; #[cfg(target_os = "linux")] -const LIBFUZZER_DOTNET_PATH: &str = - "/onefuzz/third-party/dotnet-fuzzing-linux/libfuzzer-dotnet/libfuzzer-dotnet"; +const LIBFUZZER_DOTNET_PATH: &str = "libfuzzer-dotnet/libfuzzer-dotnet"; #[cfg(target_os = "windows")] -const LIBFUZZER_DOTNET_PATH: &str = - "/onefuzz/third-party/dotnet-fuzzing-windows/libfuzzer-dotnet/libfuzzer-dotnet.exe"; +const LIBFUZZER_DOTNET_PATH: &str = "libfuzzer-dotnet/libfuzzer-dotnet.exe"; #[cfg(target_os = "linux")] -const LOADER_PATH: &str = - "/onefuzz/third-party/dotnet-fuzzing-linux/LibFuzzerDotnetLoader/LibFuzzerDotnetLoader"; +const LOADER_PATH: &str = "LibFuzzerDotnetLoader/LibFuzzerDotnetLoader"; #[cfg(target_os = "windows")] -const LOADER_PATH: &str = - "/onefuzz/third-party/dotnet-fuzzing-windows/LibFuzzerDotnetLoader/LibFuzzerDotnetLoader.exe"; +const LOADER_PATH: &str = "LibFuzzerDotnetLoader/LibFuzzerDotnetLoader.exe"; #[cfg(target_os = "linux")] -const SHARPFUZZ_PATH: &str = - "/onefuzz/third-party/dotnet-fuzzing-linux/sharpfuzz/SharpFuzz.CommandLine"; +const SHARPFUZZ_PATH: &str = "sharpfuzz/SharpFuzz.CommandLine"; #[cfg(target_os = "windows")] -const SHARPFUZZ_PATH: &str = - "/onefuzz/third-party/dotnet-fuzzing-windows/sharpfuzz/SharpFuzz.CommandLine.exe"; +const SHARPFUZZ_PATH: &str = "sharpfuzz/SharpFuzz.CommandLine.exe"; #[derive(Debug)] pub struct LibFuzzerDotnet; @@ -40,6 +39,21 @@ pub struct LibFuzzerDotnetConfig { pub target_assembly: String, pub target_class: String, pub target_method: String, + pub tools: SyncedDir, +} + +impl LibFuzzerDotnetConfig { + fn libfuzzer_dotnet_path(&self) -> PathBuf { + self.tools.local_path.join(LIBFUZZER_DOTNET_PATH) + } + + fn loader_path(&self) -> PathBuf { + self.tools.local_path.join(LOADER_PATH) + } + + fn sharpfuzz_path(&self) -> PathBuf { + self.tools.local_path.join(SHARPFUZZ_PATH) + } } #[async_trait] @@ -47,12 +61,11 @@ impl common::LibFuzzerType for LibFuzzerDotnet { type Config = LibFuzzerDotnetConfig; async fn from_config(config: &common::Config) -> Result { + let target_assembly = config.target_assembly().await?; + // Configure loader to fuzz user target DLL. let mut env = config.target_env.clone(); - env.insert( - "LIBFUZZER_DOTNET_TARGET_ASSEMBLY".into(), - config.extra.target_assembly.clone(), - ); + env.insert("LIBFUZZER_DOTNET_TARGET_ASSEMBLY".into(), target_assembly); env.insert( "LIBFUZZER_DOTNET_TARGET_CLASS".into(), config.extra.target_class.clone(), @@ -63,10 +76,13 @@ impl common::LibFuzzerType for LibFuzzerDotnet { ); let mut options = config.target_options.clone(); - options.push(format!("--target_path={}", LOADER_PATH)); + options.push(format!( + "--target_path={}", + config.extra.loader_path().display() + )); Ok(LibFuzzer::new( - LIBFUZZER_DOTNET_PATH, + config.extra.libfuzzer_dotnet_path(), options, env, &config.common.setup_dir, @@ -74,18 +90,29 @@ impl common::LibFuzzerType for LibFuzzerDotnet { } async fn extra_setup(config: &common::Config) -> Result<()> { + // Download dotnet fuzzing tools. + config.extra.tools.init_pull().await?; + + // Ensure tools are executable. + set_executable(&config.extra.tools.local_path).await?; + // Use SharpFuzz to statically instrument the target assembly. - let mut cmd = Command::new(SHARPFUZZ_PATH); - cmd.arg(&config.extra.target_assembly); + let mut cmd = Command::new(config.extra.sharpfuzz_path()); - let mut child = cmd.spawn()?; - let status = child.wait().await?; + let target_assembly = config.target_assembly().await?; + cmd.arg(target_assembly); - if !status.success() { + cmd.stdout(std::process::Stdio::piped()); + cmd.stderr(std::process::Stdio::piped()); + + let child = cmd.spawn()?; + let output = child.wait_with_output().await?; + + if !output.status.success() { anyhow::bail!( - "error instrumenting assembly `{}`: {}", + "error instrumenting assembly `{}`: {:?}", config.extra.target_assembly, - status, + output, ); } @@ -93,5 +120,17 @@ impl common::LibFuzzerType for LibFuzzerDotnet { } } +impl common::Config { + async fn target_assembly(&self) -> Result { + let resolved = + try_resolve_setup_relative_path(&self.common.setup_dir, &self.extra.target_assembly) + .await? + .to_string_lossy() + .into_owned(); + + Ok(resolved) + } +} + pub type Config = common::Config; pub type LibFuzzerDotnetFuzzTask = common::LibFuzzerFuzzTask; diff --git a/src/agent/onefuzz-task/src/tasks/report/dotnet/common.rs b/src/agent/onefuzz-task/src/tasks/report/dotnet/common.rs index 0f284c547..106bf017d 100644 --- a/src/agent/onefuzz-task/src/tasks/report/dotnet/common.rs +++ b/src/agent/onefuzz-task/src/tasks/report/dotnet/common.rs @@ -3,7 +3,7 @@ use std::ffi::OsStr; use std::path::{Path, PathBuf}; -use std::process::Output; +use std::process::{Output, Stdio}; use anyhow::Result; use tokio::fs; @@ -14,6 +14,7 @@ pub async fn collect_exception_info( args: &[impl AsRef], env: impl IntoIterator, impl AsRef)>, ) -> Result> { + // Create temp dir cooperatively. let tmp_dir = spawn_blocking(tempfile::tempdir).await??; let dump_path = tmp_dir.path().join(DUMP_FILE_NAME); @@ -21,13 +22,14 @@ pub async fn collect_exception_info( let dump = match collect_dump(args, env, &dump_path).await? { Some(dump) => dump, None => { + warn!("no minidump found, expected at {}", dump_path.display()); return Ok(None); } }; let exception = dump.exception().await?; - // Remove temp dir without blocking. + // Remove temp dir cooperatively. spawn_blocking(move || tmp_dir).await?; Ok(exception) @@ -35,15 +37,13 @@ pub async fn collect_exception_info( const DUMP_FILE_NAME: &str = "tmp.dmp"; -// Assumes `dotnet` >= 6.0. -// // See: https://docs.microsoft.com/en-us/dotnet/core/diagnostics/dumps -const ENABLE_MINIDUMP_VAR: &str = "DOTNET_DbgEnableMiniDump"; -const MINIDUMP_TYPE_VAR: &str = "DOTNET_DbgMiniDumpType"; -const MINIDUMP_NAME_VAR: &str = "DOTNET_DbgMiniDumpName"; +const ENABLE_MINIDUMP_VAR: &str = "COMPlus_DbgEnableMiniDump"; +const MINIDUMP_TYPE_VAR: &str = "COMPlus_DbgMiniDumpType"; +const MINIDUMP_NAME_VAR: &str = "COMPlus_DbgMiniDumpName"; const MINIDUMP_ENABLE: &str = "1"; -const MINIDUMP_TYPE_NORMAL: &str = "1"; +const MINIDUMP_TYPE_HEAP: &str = "2"; // Invoke target with .NET runtime environment vars set to create minidumps. // @@ -55,14 +55,16 @@ async fn collect_dump( ) -> Result> { let dump_path = dump_path.as_ref(); - let mut cmd = Command::new("dotnet"); + let dotnet = dotnet_path()?; + let mut cmd = Command::new(dotnet); cmd.arg("exec"); cmd.args(args); cmd.envs(env); + // Set `dotnet` environment vars to enable saving minidumps on crash. cmd.env(ENABLE_MINIDUMP_VAR, MINIDUMP_ENABLE); - cmd.env(MINIDUMP_TYPE_VAR, MINIDUMP_TYPE_NORMAL); + cmd.env(MINIDUMP_TYPE_VAR, MINIDUMP_TYPE_HEAP); cmd.env(MINIDUMP_NAME_VAR, dump_path); let mut child = cmd.spawn()?; @@ -80,6 +82,8 @@ async fn collect_dump( Ok(Some(dump)) } else { + warn!("target exited nonzero, but no dump file found"); + Ok(None) } } @@ -102,12 +106,17 @@ impl DotnetDumpFile { } async fn exec_sos_command(&self, sos_cmd: &str) -> Result { - let mut cmd = Command::new("dotnet"); + let dotnet_dump = dotnet_dump_path()?; + let mut cmd = Command::new(&dotnet_dump); - // Run `dotnet-analyze` with a single SOS command on startup, then exit - // the otherwise-interactive SOS session. + // Run `dotnet-dump analyze` with a single SOS command on startup, then + // exit the otherwise-interactive SOS session. let dump_path = self.path.display().to_string(); - cmd.args(["dump", "analyze", &dump_path, "-c", sos_cmd, "-c", SOS_EXIT]); + let args = ["analyze", &dump_path, "-c", sos_cmd, "-c", SOS_EXIT]; + cmd.args(args); + + cmd.stderr(Stdio::piped()); + cmd.stdout(Stdio::piped()); let output = cmd.spawn()?.wait_with_output().await?; @@ -228,5 +237,33 @@ pub fn parse_sos_print_exception_output(text: &str) -> Result Result { + let dotnet_root_dir = std::env::var("DOTNET_ROOT")?; + + #[cfg(target_os = "windows")] + let exe_name = "dotnet.exe"; + + #[cfg(not(target_os = "windows"))] + let exe_name = "dotnet"; + + let exe_path = Path::new(&dotnet_root_dir).join(exe_name); + + Ok(exe_path) +} + +fn dotnet_dump_path() -> Result { + let tools_dir = std::env::var("ONEFUZZ_TOOLS")?; + + #[cfg(target_os = "windows")] + let exe_name = "dotnet-dump.exe"; + + #[cfg(not(target_os = "windows"))] + let exe_name = "dotnet-dump"; + + let exe_path = Path::new(&tools_dir).join(exe_name); + + Ok(exe_path) +} + #[cfg(test)] mod tests; diff --git a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs index 751feb5bc..a119f303c 100644 --- a/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs +++ b/src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs @@ -3,16 +3,20 @@ use std::{ collections::HashMap, + env, path::{Path, PathBuf}, sync::Arc, }; use anyhow::{Context, Result}; use async_trait::async_trait; +use onefuzz::expand::Expand; +use onefuzz::fs::set_executable; use onefuzz::{blob::BlobUrl, sha256, syncdir::SyncedDir}; use reqwest::Url; use serde::Deserialize; use storage_queue::{Message, QueueClient}; +use tokio::fs; use crate::tasks::report::crash_report::*; use crate::tasks::report::dotnet::common::collect_exception_info; @@ -37,6 +41,7 @@ pub struct Config { pub reports: Option, pub unique_reports: Option, pub no_repro: Option, + pub tools: SyncedDir, #[serde(default = "default_bool_true")] pub check_fuzzer_help: bool, @@ -70,12 +75,22 @@ impl DotnetCrashReportTask { pub async fn run(&mut self) -> Result<()> { info!("starting dotnet crash report task"); + self.config.tools.init_pull().await?; + + set_executable(&self.config.tools.local_path).await?; + + if let Some(crashes) = &self.config.crashes { + crashes.init().await?; + } + if let Some(unique_reports) = &self.config.unique_reports { unique_reports.init().await?; } + if let Some(reports) = &self.config.reports { reports.init().await?; } + if let Some(no_repro) = &self.config.no_repro { no_repro.init().await?; } @@ -111,6 +126,36 @@ impl AsanProcessor { }) } + async fn target_exe(&self) -> Result { + let tools_dir = self.config.tools.local_path.to_string_lossy().into_owned(); + + // Try to expand `target_exe` with support for `{tools_dir}`. + // + // Allows using `LibFuzzerDotnetLoader.exe` from a shared tools container. + let expand = Expand::new().tools_dir(tools_dir); + let expanded = expand.evaluate_value(&self.config.target_exe.to_string_lossy())?; + let expanded_path = Path::new(&expanded); + + // Check if `target_exe` was resolved to an absolute path and an existing file. + // If so, then the user specified a `target_exe` under the `tools` dir. + let is_absolute = expanded_path.is_absolute(); + let file_exists = fs::metadata(&expanded).await.is_ok(); + + if is_absolute && file_exists { + // We have found `target_exe`, so skip `setup`-relative expansion. + return Ok(expanded); + } + + // We haven't yet resolved a local path for `target_exe`. Try the usual + // `setup`-relative interpretation of the configured value of `target_exe`. + let resolved = try_resolve_setup_relative_path(&self.config.common.setup_dir, expanded) + .await? + .to_string_lossy() + .into_owned(); + + Ok(resolved) + } + pub async fn test_input( &self, input: &Path, @@ -128,57 +173,72 @@ impl AsanProcessor { let job_id = self.config.common.task_id; let task_id = self.config.common.task_id; - let executable = - try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe) - .await?; - let mut args = vec!["dotnet".to_owned(), executable.display().to_string()]; + let target_exe = self.target_exe().await?; + let executable = PathBuf::from(&target_exe); + + let mut args = vec![target_exe]; args.extend(self.config.target_options.clone()); - let env = self.config.target_env.clone(); + let expand = Expand::new() + .input_path(input) + .setup_dir(&self.config.common.setup_dir); + let expanded_args = expand.evaluate(&args)?; - let crash_test_result = if let Some(exception) = collect_exception_info(&args, env).await? { - let call_stack_sha256 = stacktrace_parser::digest_iter(&exception.call_stack, None); + let env = { + let mut new = HashMap::new(); - let crash_report = CrashReport { - input_sha256, - input_blob, - executable, - crash_type: exception.exception, - crash_site: exception.call_stack[0].clone(), - call_stack: exception.call_stack, - call_stack_sha256, - minimized_stack: None, - minimized_stack_sha256: None, - minimized_stack_function_names: None, - minimized_stack_function_names_sha256: None, - minimized_stack_function_lines: None, - minimized_stack_function_lines_sha256: None, - asan_log: None, - task_id, - job_id, - scariness_score: None, - scariness_description: None, - onefuzz_version: Some(env!("ONEFUZZ_VERSION").to_owned()), - tool_name: Some(DOTNET_DUMP_TOOL_NAME.to_owned()), - tool_version: None, - }; + for (k, v) in &self.config.target_env { + let ev = expand.evaluate_value(v)?; + new.insert(k, ev); + } - crash_report.into() - } else { - let no_repro = NoCrash { - input_sha256, - input_blob, - executable, - job_id, - task_id, - tries: 1, - error: None, - }; - - no_repro.into() + new }; + let crash_test_result = + if let Some(exception) = collect_exception_info(&expanded_args, env).await? { + let call_stack_sha256 = stacktrace_parser::digest_iter(&exception.call_stack, None); + + let crash_report = CrashReport { + input_sha256, + input_blob, + executable, + crash_type: exception.exception, + crash_site: exception.call_stack[0].clone(), + call_stack: exception.call_stack, + call_stack_sha256, + minimized_stack: None, + minimized_stack_sha256: None, + minimized_stack_function_names: None, + minimized_stack_function_names_sha256: None, + minimized_stack_function_lines: None, + minimized_stack_function_lines_sha256: None, + asan_log: None, + task_id, + job_id, + scariness_score: None, + scariness_description: None, + onefuzz_version: Some(env!("ONEFUZZ_VERSION").to_owned()), + tool_name: Some(DOTNET_DUMP_TOOL_NAME.to_owned()), + tool_version: None, + }; + + crash_report.into() + } else { + let no_repro = NoCrash { + input_sha256, + input_blob, + executable, + job_id, + task_id, + tries: 1, + error: None, + }; + + no_repro.into() + }; + Ok(crash_test_result) } } diff --git a/src/api-service/__app__/onefuzzlib/tasks/config.py b/src/api-service/__app__/onefuzzlib/tasks/config.py index 6cfe4a922..2c60c2842 100644 --- a/src/api-service/__app__/onefuzzlib/tasks/config.py +++ b/src/api-service/__app__/onefuzzlib/tasks/config.py @@ -440,6 +440,15 @@ def build_task_config(job: Job, task: Task) -> TaskUnitConfig: if coverage_filter is not None: config.coverage_filter = coverage_filter + if TaskFeature.target_assembly in definition.features: + config.target_assembly = task_config.task.target_assembly + + if TaskFeature.target_class in definition.features: + config.target_class = task_config.task.target_class + + if TaskFeature.target_method in definition.features: + config.target_method = task_config.task.target_method + return config diff --git a/src/api-service/__app__/onefuzzlib/tasks/defs.py b/src/api-service/__app__/onefuzzlib/tasks/defs.py index b2d674a8d..e9b2e36ee 100644 --- a/src/api-service/__app__/onefuzzlib/tasks/defs.py +++ b/src/api-service/__app__/onefuzzlib/tasks/defs.py @@ -50,6 +50,152 @@ TASK_DEFINITIONS = { ], monitor_queue=ContainerType.readonly_inputs, ), + TaskType.dotnet_coverage: TaskDefinition( + features=[ + TaskFeature.target_exe, + TaskFeature.target_env, + TaskFeature.target_options, + TaskFeature.target_timeout, + TaskFeature.coverage_filter, + TaskFeature.target_must_use_input, + ], + vm=VmDefinition(compare=Compare.Equal, value=1), + containers=[ + ContainerDefinition( + type=ContainerType.setup, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.readonly_inputs, + compare=Compare.AtLeast, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.coverage, + compare=Compare.Equal, + value=1, + permissions=[ + ContainerPermission.List, + ContainerPermission.Read, + ContainerPermission.Write, + ], + ), + ContainerDefinition( + type=ContainerType.tools, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ], + monitor_queue=ContainerType.readonly_inputs, + ), + TaskType.dotnet_crash_report: TaskDefinition( + features=[ + TaskFeature.target_exe, + TaskFeature.target_env, + TaskFeature.target_options, + TaskFeature.target_timeout, + TaskFeature.check_asan_log, + TaskFeature.check_debugger, + TaskFeature.check_retry_count, + TaskFeature.minimized_stack_depth, + ], + vm=VmDefinition(compare=Compare.AtLeast, value=1), + containers=[ + ContainerDefinition( + type=ContainerType.setup, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.crashes, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Write], + ), + ContainerDefinition( + type=ContainerType.unique_reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Write], + ), + ContainerDefinition( + type=ContainerType.no_repro, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Write], + ), + ContainerDefinition( + type=ContainerType.tools, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ], + monitor_queue=ContainerType.crashes, + ), + TaskType.libfuzzer_dotnet_fuzz: TaskDefinition( + features=[ + TaskFeature.target_exe, + TaskFeature.target_env, + TaskFeature.target_options, + TaskFeature.target_workers, + TaskFeature.ensemble_sync_delay, + TaskFeature.check_fuzzer_help, + TaskFeature.expect_crash_on_failure, + TaskFeature.target_assembly, + TaskFeature.target_class, + TaskFeature.target_method, + ], + vm=VmDefinition(compare=Compare.AtLeast, value=1), + containers=[ + ContainerDefinition( + type=ContainerType.setup, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.crashes, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Write], + ), + ContainerDefinition( + type=ContainerType.inputs, + compare=Compare.Equal, + value=1, + permissions=[ + ContainerPermission.Write, + ContainerPermission.Read, + ContainerPermission.List, + ], + ), + ContainerDefinition( + type=ContainerType.readonly_inputs, + compare=Compare.AtLeast, + value=0, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.tools, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ], + monitor_queue=None, + ), TaskType.generic_analysis: TaskDefinition( features=[ TaskFeature.target_exe, @@ -558,40 +704,4 @@ TASK_DEFINITIONS = { ), ], ), - TaskType.dotnet_coverage: TaskDefinition( - features=[ - TaskFeature.target_exe, - TaskFeature.target_env, - TaskFeature.target_options, - TaskFeature.target_timeout, - TaskFeature.coverage_filter, - TaskFeature.target_must_use_input, - ], - vm=VmDefinition(compare=Compare.Equal, value=1), - containers=[ - ContainerDefinition( - type=ContainerType.setup, - compare=Compare.Equal, - value=1, - permissions=[ContainerPermission.Read, ContainerPermission.List], - ), - ContainerDefinition( - type=ContainerType.readonly_inputs, - compare=Compare.AtLeast, - value=1, - permissions=[ContainerPermission.Read, ContainerPermission.List], - ), - ContainerDefinition( - type=ContainerType.coverage, - compare=Compare.Equal, - value=1, - permissions=[ - ContainerPermission.List, - ContainerPermission.Read, - ContainerPermission.Write, - ], - ), - ], - monitor_queue=ContainerType.readonly_inputs, - ), } diff --git a/src/ci/dotnet-fuzzing-tools.ps1 b/src/ci/dotnet-fuzzing-tools.ps1 index 9922627bd..6f0e13e08 100644 --- a/src/ci/dotnet-fuzzing-tools.ps1 +++ b/src/ci/dotnet-fuzzing-tools.ps1 @@ -5,7 +5,7 @@ $SHARPFUZZ_REPO = 'https://github.com/Metalnem/sharpfuzz' $SHARPFUZZ_COMMIT = 'v2.0.0' $LIBFUZZER_DOTNET_REPO = 'https://github.com/Metalnem/libfuzzer-dotnet' -$LIBFUZZER_DOTNET_COMMIT = '55d84f84b3540c864371e855c2a5ecb728865d97' +$LIBFUZZER_DOTNET_COMMIT = 'ed148633f8df078cb2b0ba0ca30166aa72f1de90' # Script below assumes an absolute path. $ARTIFACTS = "${env:GITHUB_WORKSPACE}/artifacts/third-party/dotnet-fuzzing-windows" @@ -20,12 +20,12 @@ mkdir $ARTIFACTS/sharpfuzz git clone $SHARPFUZZ_REPO sharpfuzz pushd sharpfuzz git checkout $SHARPFUZZ_COMMIT -dotnet publish src/SharpFuzz.CommandLine -f net6.0 -c Release -o $ARTIFACTS/sharpfuzz --sc -r win10-x64 +dotnet publish src/SharpFuzz.CommandLine -f net6.0 -c Release -o $ARTIFACTS/sharpfuzz --self-contained -r win10-x64 popd # Build SharpFuzz and our dynamic loader harness for `libfuzzer-dotnet`. pushd src/agent/LibFuzzerDotnetLoader -dotnet publish . -c Release -o $ARTIFACTS/LibFuzzerDotnetLoader --sc -r win10-x64 -p:PublishSingleFile=true +dotnet publish . -c Release -o $ARTIFACTS/LibFuzzerDotnetLoader --sc -r win10-x64 popd # Build `libfuzzer-dotnet`. diff --git a/src/ci/dotnet-fuzzing-tools.sh b/src/ci/dotnet-fuzzing-tools.sh index 2f2e65627..a3599cdbf 100755 --- a/src/ci/dotnet-fuzzing-tools.sh +++ b/src/ci/dotnet-fuzzing-tools.sh @@ -8,7 +8,7 @@ export SHARPFUZZ_REPO='https://github.com/Metalnem/sharpfuzz' export SHARPFUZZ_COMMIT='v2.0.0' export LIBFUZZER_DOTNET_REPO='https://github.com/Metalnem/libfuzzer-dotnet' -export LIBFUZZER_DOTNET_COMMIT='55d84f84b3540c864371e855c2a5ecb728865d97' +export LIBFUZZER_DOTNET_COMMIT='ed148633f8df078cb2b0ba0ca30166aa72f1de90' # Script below assumes an absolute path. export ARTIFACTS="${GITHUB_WORKSPACE}/artifacts/third-party/dotnet-fuzzing-linux" @@ -29,12 +29,12 @@ sudo apt-get install -y dotnet-sdk-6.0 git clone $SHARPFUZZ_REPO sharpfuzz pushd sharpfuzz git checkout $SHARPFUZZ_COMMIT -dotnet publish src/SharpFuzz.CommandLine -f net6.0 -c Release -o $ARTIFACTS/sharpfuzz --sc -r linux-x64 +dotnet publish src/SharpFuzz.CommandLine -f net6.0 -c Release -o $ARTIFACTS/sharpfuzz --self-contained -r linux-x64 popd # Build SharpFuzz and our dynamic loader harness for `libfuzzer-dotnet`. pushd src/agent/LibFuzzerDotnetLoader -dotnet publish . -c Release -o $ARTIFACTS/LibFuzzerDotnetLoader --sc -r linux-x64 -p:PublishSingleFile=true +dotnet publish . -c Release -o $ARTIFACTS/LibFuzzerDotnetLoader --sc -r linux-x64 popd # Build `libfuzzer-dotnet`. diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 17be517f3..d2e61a68c 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -902,6 +902,9 @@ class Tasks(Endpoint): target_options_merge: bool = False, target_timeout: Optional[int] = None, target_workers: Optional[int] = None, + target_assembly: Optional[str] = None, + target_class: Optional[str] = None, + target_method: Optional[str] = None, vm_count: int = 1, preserve_existing_outputs: bool = False, colocate: bool = False, @@ -975,6 +978,9 @@ class Tasks(Endpoint): target_options_merge=target_options_merge, target_timeout=target_timeout, target_workers=target_workers, + target_assembly=target_assembly, + target_class=target_class, + target_method=target_method, type=task_type, wait_for_files=task_wait_for_files, report_list=report_list, diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index a3c0621a1..049596f85 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -18,6 +18,12 @@ from . import JobHelper LIBFUZZER_MAGIC_STRING = b"ERROR: libFuzzer" +# The loader DLL is managed, but links platform-specific code. Task VMs must pull the +# tools container that matches their platform (which will contain the correct DLL). +LIBFUZZER_DOTNET_LOADER_PATH = ( + "{tools_dir}/LibFuzzerDotnetLoader/LibFuzzerDotnetLoader.dll" +) + class QemuArch(Enum): aarch64 = "aarch64" @@ -536,7 +542,7 @@ class Libfuzzer(Command): pool = self.onefuzz.pools.get(pool_name) if pool.os != OS.linux: - raise Exception("libfuzzer-dotnet jobs are only compatable on linux") + raise Exception("libfuzzer-dotnet jobs are only compatible on linux") target_exe = File(os.path.join(setup_dir, harness)) if not os.path.exists(target_exe): @@ -544,7 +550,7 @@ class Libfuzzer(Command): assembly_path = os.path.join(setup_dir, target_harness) if not os.path.exists(assembly_path): - raise Exception(f"missing assembly: {assembly_path}") + raise Exception(f"missing assembly: {target_harness}") self._check_is_libfuzzer(target_exe) if target_options is None: @@ -622,6 +628,207 @@ class Libfuzzer(Command): helper.wait() return helper.job + def dotnet_dll( + self, + project: str, + name: str, + build: str, + pool_name: PoolName, + *, + setup_dir: Directory, + target_dll: File, + target_class: str, + target_method: str, + vm_count: int = 1, + inputs: Optional[Directory] = None, + reboot_after_setup: bool = False, + duration: int = 24, + target_workers: Optional[int] = None, + fuzzing_target_options: Optional[List[str]] = None, + target_env: Optional[Dict[str, str]] = None, + target_timeout: Optional[int] = None, + check_retry_count: Optional[int] = None, + tags: Optional[Dict[str, str]] = None, + wait_for_running: bool = False, + wait_for_files: Optional[List[ContainerType]] = None, + existing_inputs: Optional[Container] = None, + debug: Optional[List[TaskDebugFlag]] = None, + ensemble_sync_delay: Optional[int] = None, + colocate_all_tasks: bool = False, + colocate_secondary_tasks: bool = True, + expect_crash_on_failure: bool = False, + ) -> Optional[Job]: + pool = self.onefuzz.pools.get(pool_name) + + # We _must_ proactively specify the OS based on pool. + # + # This is because managed DLLs are always (Windows-native) PE files, so the job + # helper's platform guess (based on the file type of `target_exe`) will always + # evaluate to `OS.windows`. In the case of true Linux `libfuzzer dotnet_dll` jobs, + # this leads to a client- side validation error when the helper checks the nominal + # target OS against the pool OS. + platform = pool.os + + helper = JobHelper( + self.onefuzz, + self.logger, + project, + name, + build, + duration, + pool_name=pool_name, + target_exe=target_dll, + platform=platform, + ) + + target_dll_blob_name = helper.setup_relative_blob_name(target_dll, setup_dir) + + target_env = target_env or {} + + # Set target environment variables for `LibFuzzerDotnetLoader`. + target_env["LIBFUZZER_DOTNET_TARGET_ASSEMBLY"] = ( + "{setup_dir}/" + target_dll_blob_name + ) + target_env["LIBFUZZER_DOTNET_TARGET_CLASS"] = target_class + target_env["LIBFUZZER_DOTNET_TARGET_METHOD"] = target_method + + helper.add_tags(tags) + helper.define_containers( + ContainerType.setup, + ContainerType.inputs, + ContainerType.crashes, + ContainerType.coverage, + ContainerType.reports, + ContainerType.unique_reports, + ContainerType.no_repro, + ) + + containers = helper.containers + + if existing_inputs: + self.onefuzz.containers.get(existing_inputs) + helper.containers[ContainerType.inputs] = existing_inputs + else: + helper.define_containers(ContainerType.inputs) + + # Assumes that `libfuzzer-dotnet` and supporting tools were uploaded upon deployment. + fuzzer_tools_container = Container( + "dotnet-fuzzing-linux" if platform == OS.linux else "dotnet-fuzzing-windows" + ) + + fuzzer_containers = [ + (ContainerType.setup, containers[ContainerType.setup]), + (ContainerType.crashes, containers[ContainerType.crashes]), + (ContainerType.inputs, containers[ContainerType.inputs]), + (ContainerType.tools, fuzzer_tools_container), + ] + + helper.create_containers() + + helper.upload_setup(setup_dir, target_dll) + + if inputs: + helper.upload_inputs(inputs) + + helper.wait_on(wait_for_files, wait_for_running) + + fuzzer_task = self.onefuzz.tasks.create( + helper.job.job_id, + TaskType.libfuzzer_dotnet_fuzz, + target_dll_blob_name, # Not used + fuzzer_containers, + pool_name=pool_name, + reboot_after_setup=reboot_after_setup, + duration=duration, + vm_count=vm_count, + target_options=fuzzing_target_options, + target_env=target_env, + target_assembly=target_dll_blob_name, + target_class=target_class, + target_method=target_method, + target_workers=target_workers, + tags=tags, + debug=debug, + ensemble_sync_delay=ensemble_sync_delay, + expect_crash_on_failure=expect_crash_on_failure, + check_fuzzer_help=False, + ) + + # Ensure the fuzzing task starts before we schedule the coverage and + # crash reporting tasks (which are useless without it). + prereq_tasks = [fuzzer_task.task_id] + + # Target options for the .NET harness produced by SharpFuzz, when _not_ + # invoked as a child process of `libfuzzer-dotnet`. This harness has a + # `main()` function with one argument: the path to an input test case. + sharpfuzz_harness_target_options = ["{input}"] + + # Set the path to the `LibFuzzerDotnetLoader` DLL. + # + # This provides a `main()` function that dynamically loads a target DLL + # passed via environment variables. This is assumed to be installed on + # the VMs. + libfuzzer_dotnet_loader_dll = LIBFUZZER_DOTNET_LOADER_PATH + + coverage_containers = [ + (ContainerType.setup, containers[ContainerType.setup]), + (ContainerType.coverage, containers[ContainerType.coverage]), + (ContainerType.readonly_inputs, containers[ContainerType.inputs]), + (ContainerType.tools, fuzzer_tools_container), + ] + + self.logger.info("creating `dotnet_coverage` task") + self.onefuzz.tasks.create( + helper.job.job_id, + TaskType.dotnet_coverage, + libfuzzer_dotnet_loader_dll, + coverage_containers, + pool_name=pool_name, + duration=duration, + vm_count=1, + reboot_after_setup=reboot_after_setup, + target_options=sharpfuzz_harness_target_options, + target_env=target_env, + target_timeout=target_timeout, + tags=tags, + prereq_tasks=prereq_tasks, + debug=debug, + colocate=colocate_all_tasks or colocate_secondary_tasks, + ) + + report_containers = [ + (ContainerType.setup, containers[ContainerType.setup]), + (ContainerType.crashes, containers[ContainerType.crashes]), + (ContainerType.reports, containers[ContainerType.reports]), + (ContainerType.unique_reports, containers[ContainerType.unique_reports]), + (ContainerType.no_repro, containers[ContainerType.no_repro]), + (ContainerType.tools, fuzzer_tools_container), + ] + + self.logger.info("creating `dotnet_crash_report` task") + self.onefuzz.tasks.create( + helper.job.job_id, + TaskType.dotnet_crash_report, + libfuzzer_dotnet_loader_dll, + report_containers, + pool_name=pool_name, + duration=duration, + vm_count=1, + reboot_after_setup=reboot_after_setup, + target_options=sharpfuzz_harness_target_options, + target_env=target_env, + tags=tags, + prereq_tasks=prereq_tasks, + target_timeout=target_timeout, + check_retry_count=check_retry_count, + debug=debug, + colocate=colocate_all_tasks or colocate_secondary_tasks, + ) + + self.logger.info("done creating tasks") + helper.wait() + return helper.job + def qemu_user( self, project: str, diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index 106e4ec59..f05c24585 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -52,6 +52,7 @@ class TaskTestState(Enum): class TemplateType(Enum): libfuzzer = "libfuzzer" libfuzzer_dotnet = "libfuzzer_dotnet" + libfuzzer_dotnet_dll = "libfuzzer_dotnet_dll" libfuzzer_qemu_user = "libfuzzer_qemu_user" afl = "afl" radamsa = "radamsa" @@ -76,6 +77,9 @@ class Integration(BaseModel): test_repro: Optional[bool] = Field(default=True) target_options: Optional[List[str]] inject_fake_regression: bool = Field(default=False) + target_class: Optional[str] + target_method: Optional[str] + setup_dir: Optional[str] TARGETS: Dict[str, Integration] = { @@ -148,6 +152,23 @@ TARGETS: Dict[str, Integration] = { wait_for_files={ContainerType.inputs: 2, ContainerType.crashes: 1}, test_repro=False, ), + "linux-libfuzzer-dotnet-dll": Integration( + template=TemplateType.libfuzzer_dotnet_dll, + os=OS.linux, + setup_dir="GoodBadDotnet", + target_exe="GoodBadDotnet/GoodBad.dll", + target_options=["-max_len=4", "-only_ascii=1", "-seed=1"], + target_class="GoodBad.Fuzzer", + target_method="TestInput", + use_setup=True, + wait_for_files={ + ContainerType.inputs: 2, + ContainerType.coverage: 1, + ContainerType.crashes: 1, + ContainerType.unique_reports: 1, + }, + test_repro=False, + ), "linux-libfuzzer-aarch64-crosscompile": Integration( template=TemplateType.libfuzzer_qemu_user, os=OS.linux, @@ -216,6 +237,23 @@ TARGETS: Dict[str, Integration] = { }, use_setup=True, ), + "windows-libfuzzer-dotnet-dll": Integration( + template=TemplateType.libfuzzer_dotnet_dll, + os=OS.windows, + setup_dir="GoodBadDotnet", + target_exe="GoodBadDotnet/GoodBad.dll", + target_options=["-max_len=4", "-only_ascii=1", "-seed=1"], + target_class="GoodBad.Fuzzer", + target_method="TestInput", + use_setup=True, + wait_for_files={ + ContainerType.inputs: 2, + ContainerType.coverage: 1, + ContainerType.crashes: 1, + ContainerType.unique_reports: 1, + }, + test_repro=False, + ), "windows-trivial-crash": Integration( template=TemplateType.radamsa, os=OS.windows, @@ -315,7 +353,11 @@ class TestOnefuzz: self.logger.info("launching: %s", target) - setup = Directory(os.path.join(path, target)) if config.use_setup else None + if config.setup_dir is None: + setup = Directory(os.path.join(path, target)) if config.use_setup else None + else: + setup = config.setup_dir + target_exe = File(os.path.join(path, target, config.target_exe)) inputs = ( Directory(os.path.join(path, target, config.inputs)) @@ -356,6 +398,28 @@ class TestOnefuzz: vm_count=1, target_options=config.target_options, ) + elif config.template == TemplateType.libfuzzer_dotnet_dll: + if setup is None: + raise Exception("setup required for libfuzzer_dotnet_dll") + if config.target_class is None: + raise Exception("target_class required for libfuzzer_dotnet_dll") + if config.target_method is None: + raise Exception("target_method required for libfuzzer_dotnet_dll") + + job = self.of.template.libfuzzer.dotnet_dll( + self.project, + target, + BUILD, + pools[config.os].name, + target_dll=config.target_exe, + inputs=inputs, + setup_dir=setup, + duration=duration, + vm_count=1, + fuzzing_target_options=config.target_options, + target_class=config.target_class, + target_method=config.target_method, + ) elif config.template == TemplateType.libfuzzer_qemu_user: job = self.of.template.libfuzzer.qemu_user( self.project, diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index ed2ebe9bf..8269c37e7 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -82,6 +82,9 @@ class TaskFeature(Enum): minimized_stack_depth = "minimized_stack_depth" coverage_filter = "coverage_filter" target_must_use_input = "target_must_use_input" + target_assembly = "target_assembly" + target_class = "target_class" + target_method = "target_method" # Permissions for an Azure Blob Storage Container. @@ -150,6 +153,9 @@ class TaskState(Enum): class TaskType(Enum): coverage = "coverage" + dotnet_coverage = "dotnet_coverage" + dotnet_crash_report = "dotnet_crash_report" + libfuzzer_dotnet_fuzz = "libfuzzer_dotnet_fuzz" libfuzzer_fuzz = "libfuzzer_fuzz" # Deprecated, kept for deserialization of old task data. @@ -165,8 +171,6 @@ class TaskType(Enum): generic_crash_report = "generic_crash_report" generic_regression = "generic_regression" - dotnet_coverage = "dotnet_coverage" - class VmState(Enum): init = "init" diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 72c99d259..702492caa 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -163,6 +163,9 @@ class TaskDetails(BaseModel): report_list: Optional[List[str]] minimized_stack_depth: Optional[int] coverage_filter: Optional[str] + target_assembly: Optional[str] + target_class: Optional[str] + target_method: Optional[str] class TaskPool(BaseModel): @@ -378,6 +381,9 @@ class TaskUnitConfig(BaseModel): report_list: Optional[List[str]] minimized_stack_depth: Optional[int] coverage_filter: Optional[str] + target_assembly: Optional[str] + target_class: Optional[str] + target_method: Optional[str] # from here forwards are Container definitions. These need to be inline # with TaskDefinitions and ContainerTypes