diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e07cc2f15..65d77589a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -358,6 +358,10 @@ jobs: (cd libfuzzer ; make ) cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer + mkdir -p artifacts/linux-libfuzzer-regression + (cd libfuzzer-regression ; make ) + cp -r libfuzzer-regression/broken.exe libfuzzer-regression/fixed.exe libfuzzer-regression/seeds artifacts/linux-libfuzzer-regression + mkdir -p artifacts/linux-trivial-crash (cd trivial-crash ; make ) cp -r trivial-crash/fuzz.exe trivial-crash/seeds artifacts/linux-trivial-crash diff --git a/docs/webhook_events.md b/docs/webhook_events.md index dc7e00bb2..c1d5bc607 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -37,6 +37,7 @@ Each event will be submitted via HTTP POST to the user provided URL. * [proxy_created](#proxy_created) * [proxy_deleted](#proxy_deleted) * [proxy_failed](#proxy_failed) +* [regression_reported](#regression_reported) * [scaleset_created](#scaleset_created) * [scaleset_deleted](#scaleset_deleted) * [scaleset_failed](#scaleset_failed) @@ -168,7 +169,6 @@ Each event will be submitted via HTTP POST to the user provided URL. } }, "required": [ - "input_blob", "executable", "crash_type", "crash_site", @@ -475,11 +475,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -1030,6 +1032,261 @@ Each event will be submitted via HTTP POST to the user provided URL. } ``` +### regression_reported + +#### Example + +```json +{ + "container": "container-name", + "filename": "example.json", + "regression_report": { + "crash_test_result": { + "crash_report": { + "asan_log": "example asan log", + "call_stack": [ + "#0 line", + "#1 line", + "#2 line" + ], + "call_stack_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "crash_site": "example crash site", + "crash_type": "example crash report type", + "executable": "fuzz.exe", + "input_blob": { + "account": "contoso-storage-account", + "container": "crashes", + "name": "input.txt" + }, + "input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "job_id": "00000000-0000-0000-0000-000000000000", + "scariness_description": "example-scariness", + "scariness_score": 10, + "task_id": "00000000-0000-0000-0000-000000000000" + } + }, + "original_crash_test_result": { + "crash_report": { + "asan_log": "example asan log", + "call_stack": [ + "#0 line", + "#1 line", + "#2 line" + ], + "call_stack_sha256": "0000000000000000000000000000000000000000000000000000000000000000", + "crash_site": "example crash site", + "crash_type": "example crash report type", + "executable": "fuzz.exe", + "input_blob": { + "account": "contoso-storage-account", + "container": "crashes", + "name": "input.txt" + }, + "input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + "job_id": "00000000-0000-0000-0000-000000000000", + "scariness_description": "example-scariness", + "scariness_score": 10, + "task_id": "00000000-0000-0000-0000-000000000000" + } + } + } +} +``` + +#### Schema + +```json +{ + "additionalProperties": false, + "definitions": { + "BlobRef": { + "properties": { + "account": { + "title": "Account", + "type": "string" + }, + "container": { + "title": "Container", + "type": "string" + }, + "name": { + "title": "Name", + "type": "string" + } + }, + "required": [ + "account", + "container", + "name" + ], + "title": "BlobRef", + "type": "object" + }, + "CrashTestResult": { + "properties": { + "crash_report": { + "$ref": "#/definitions/Report" + }, + "no_repro": { + "$ref": "#/definitions/NoReproReport" + } + }, + "title": "CrashTestResult", + "type": "object" + }, + "NoReproReport": { + "properties": { + "error": { + "title": "Error", + "type": "string" + }, + "executable": { + "title": "Executable", + "type": "string" + }, + "input_blob": { + "$ref": "#/definitions/BlobRef" + }, + "input_sha256": { + "title": "Input Sha256", + "type": "string" + }, + "job_id": { + "format": "uuid", + "title": "Job Id", + "type": "string" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "tries": { + "title": "Tries", + "type": "integer" + } + }, + "required": [ + "input_sha256", + "executable", + "task_id", + "job_id", + "tries" + ], + "title": "NoReproReport", + "type": "object" + }, + "RegressionReport": { + "properties": { + "crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + }, + "original_crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + } + }, + "required": [ + "crash_test_result" + ], + "title": "RegressionReport", + "type": "object" + }, + "Report": { + "properties": { + "asan_log": { + "title": "Asan Log", + "type": "string" + }, + "call_stack": { + "items": { + "type": "string" + }, + "title": "Call Stack", + "type": "array" + }, + "call_stack_sha256": { + "title": "Call Stack Sha256", + "type": "string" + }, + "crash_site": { + "title": "Crash Site", + "type": "string" + }, + "crash_type": { + "title": "Crash Type", + "type": "string" + }, + "executable": { + "title": "Executable", + "type": "string" + }, + "input_blob": { + "$ref": "#/definitions/BlobRef" + }, + "input_sha256": { + "title": "Input Sha256", + "type": "string" + }, + "input_url": { + "title": "Input Url", + "type": "string" + }, + "job_id": { + "format": "uuid", + "title": "Job Id", + "type": "string" + }, + "scariness_description": { + "title": "Scariness Description", + "type": "string" + }, + "scariness_score": { + "title": "Scariness Score", + "type": "integer" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + } + }, + "required": [ + "executable", + "crash_type", + "crash_site", + "call_stack", + "call_stack_sha256", + "input_sha256", + "task_id", + "job_id" + ], + "title": "Report", + "type": "object" + } + }, + "properties": { + "container": { + "title": "Container", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "regression_report": { + "$ref": "#/definitions/RegressionReport" + } + }, + "required": [ + "regression_report", + "container", + "filename" + ], + "title": "EventRegressionReported", + "type": "object" +} +``` + ### scaleset_created #### Example @@ -1283,7 +1540,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -1456,6 +1714,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -1554,11 +1819,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -1715,7 +1982,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -1936,6 +2204,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -2034,11 +2309,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -2188,7 +2465,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -2361,6 +2639,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -2459,11 +2744,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -2587,7 +2874,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -2760,6 +3048,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -2872,11 +3167,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -3013,7 +3310,8 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, @@ -3186,6 +3484,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -3284,11 +3589,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -3468,10 +3775,23 @@ Each event will be submitted via HTTP POST to the user provided URL. "setup", "tools", "unique_inputs", - "unique_reports" + "unique_reports", + "regression_reports" ], "title": "ContainerType" }, + "CrashTestResult": { + "properties": { + "crash_report": { + "$ref": "#/definitions/Report" + }, + "no_repro": { + "$ref": "#/definitions/NoReproReport" + } + }, + "title": "CrashTestResult", + "type": "object" + }, "Error": { "properties": { "code": { @@ -3821,6 +4141,29 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "EventProxyFailed", "type": "object" }, + "EventRegressionReported": { + "additionalProperties": false, + "properties": { + "container": { + "title": "Container", + "type": "string" + }, + "filename": { + "title": "Filename", + "type": "string" + }, + "regression_report": { + "$ref": "#/definitions/RegressionReport" + } + }, + "required": [ + "regression_report", + "container", + "filename" + ], + "title": "EventRegressionReported", + "type": "object" + }, "EventScalesetCreated": { "additionalProperties": false, "properties": { @@ -4074,6 +4417,7 @@ Each event will be submitted via HTTP POST to the user provided URL. "task_state_updated", "task_stopped", "crash_reported", + "regression_reported", "file_added", "task_heartbeat", "node_heartbeat" @@ -4129,6 +4473,48 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "JobTaskStopped", "type": "object" }, + "NoReproReport": { + "properties": { + "error": { + "title": "Error", + "type": "string" + }, + "executable": { + "title": "Executable", + "type": "string" + }, + "input_blob": { + "$ref": "#/definitions/BlobRef" + }, + "input_sha256": { + "title": "Input Sha256", + "type": "string" + }, + "job_id": { + "format": "uuid", + "title": "Job Id", + "type": "string" + }, + "task_id": { + "format": "uuid", + "title": "Task Id", + "type": "string" + }, + "tries": { + "title": "Tries", + "type": "integer" + } + }, + "required": [ + "input_sha256", + "executable", + "task_id", + "job_id", + "tries" + ], + "title": "NoReproReport", + "type": "object" + }, "NodeState": { "description": "An enumeration.", "enum": [ @@ -4152,6 +4538,21 @@ Each event will be submitted via HTTP POST to the user provided URL. ], "title": "OS" }, + "RegressionReport": { + "properties": { + "crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + }, + "original_crash_test_result": { + "$ref": "#/definitions/CrashTestResult" + } + }, + "required": [ + "crash_test_result" + ], + "title": "RegressionReport", + "type": "object" + }, "Report": { "properties": { "asan_log": { @@ -4212,7 +4613,6 @@ Each event will be submitted via HTTP POST to the user provided URL. } }, "required": [ - "input_blob", "executable", "crash_type", "crash_site", @@ -4394,6 +4794,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "title": "Rename Output", "type": "boolean" }, + "report_list": { + "items": { + "type": "string" + }, + "title": "Report List", + "type": "array" + }, "stats_file": { "title": "Stats File", "type": "string" @@ -4506,11 +4913,13 @@ Each event will be submitted via HTTP POST to the user provided URL. "libfuzzer_coverage", "libfuzzer_crash_report", "libfuzzer_merge", + "libfuzzer_regression", "generic_analysis", "generic_supervisor", "generic_merge", "generic_generator", - "generic_crash_report" + "generic_crash_report", + "generic_regression" ], "title": "TaskType" }, @@ -4638,6 +5047,9 @@ Each event will be submitted via HTTP POST to the user provided URL. { "$ref": "#/definitions/EventCrashReported" }, + { + "$ref": "#/definitions/EventRegressionReported" + }, { "$ref": "#/definitions/EventFileAdded" } diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index c70d34a07..4fbec9c79 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -2,7 +2,11 @@ // Licensed under the MIT License. #![allow(clippy::large_enum_variant)] -use crate::tasks::{analysis, coverage, fuzz, heartbeat::*, merge, report}; +use crate::tasks::{ + analysis, coverage, fuzz, + heartbeat::{init_task_heartbeat, TaskHeartbeatClient}, + merge, regression, report, +}; use anyhow::Result; use onefuzz::machine_id::{get_machine_id, get_scaleset_name}; use onefuzz_telemetry::{ @@ -70,6 +74,9 @@ pub enum Config { #[serde(alias = "libfuzzer_coverage")] LibFuzzerCoverage(coverage::libfuzzer_coverage::Config), + #[serde(alias = "libfuzzer_regression")] + LibFuzzerRegression(regression::libfuzzer::Config), + #[serde(alias = "generic_analysis")] GenericAnalysis(analysis::generic::Config), @@ -84,6 +91,9 @@ pub enum Config { #[serde(alias = "generic_crash_report")] GenericReport(report::generic::Config), + + #[serde(alias = "generic_regression")] + GenericRegression(regression::generic::Config), } impl Config { @@ -104,11 +114,13 @@ impl Config { Config::LibFuzzerMerge(c) => &mut c.common, Config::LibFuzzerReport(c) => &mut c.common, Config::LibFuzzerCoverage(c) => &mut c.common, + Config::LibFuzzerRegression(c) => &mut c.common, Config::GenericAnalysis(c) => &mut c.common, Config::GenericMerge(c) => &mut c.common, Config::GenericReport(c) => &mut c.common, Config::GenericSupervisor(c) => &mut c.common, Config::GenericGenerator(c) => &mut c.common, + Config::GenericRegression(c) => &mut c.common, } } @@ -118,11 +130,13 @@ impl Config { Config::LibFuzzerMerge(c) => &c.common, Config::LibFuzzerReport(c) => &c.common, Config::LibFuzzerCoverage(c) => &c.common, + Config::LibFuzzerRegression(c) => &c.common, Config::GenericAnalysis(c) => &c.common, Config::GenericMerge(c) => &c.common, Config::GenericReport(c) => &c.common, Config::GenericSupervisor(c) => &c.common, Config::GenericGenerator(c) => &c.common, + Config::GenericRegression(c) => &c.common, } } @@ -132,11 +146,13 @@ impl Config { Config::LibFuzzerMerge(_) => "libfuzzer_merge", Config::LibFuzzerReport(_) => "libfuzzer_crash_report", Config::LibFuzzerCoverage(_) => "libfuzzer_coverage", + Config::LibFuzzerRegression(_) => "libfuzzer_regression", Config::GenericAnalysis(_) => "generic_analysis", Config::GenericMerge(_) => "generic_merge", Config::GenericReport(_) => "generic_crash_report", Config::GenericSupervisor(_) => "generic_supervisor", Config::GenericGenerator(_) => "generic_generator", + Config::GenericRegression(_) => "generic_regression", }; match self { @@ -193,6 +209,16 @@ impl Config { Config::GenericReport(config) => { report::generic::ReportTask::new(config).managed_run().await } + Config::GenericRegression(config) => { + regression::generic::GenericRegressionTask::new(config) + .run() + .await + } + Config::LibFuzzerRegression(config) => { + regression::libfuzzer::LibFuzzerRegressionTask::new(config) + .run() + .await + } } } } diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index 9f6dc94a6..058dd6790 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -3,7 +3,7 @@ use crate::tasks::{ config::CommonConfig, - heartbeat::*, + heartbeat::{HeartbeatSender, TaskHeartbeatClient}, utils::{self, default_bool_true}, }; use anyhow::{Context, Result}; diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs index a597602bf..f24f3db91 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs @@ -4,7 +4,7 @@ #![allow(clippy::too_many_arguments)] use crate::tasks::{ config::{CommonConfig, ContainerType}, - heartbeat::*, + heartbeat::{HeartbeatSender, TaskHeartbeatClient}, report::crash_report::monitor_reports, stats::common::{monitor_stats, StatsFormat}, utils::CheckNotify, diff --git a/src/agent/onefuzz-agent/src/tasks/merge/libfuzzer_merge.rs b/src/agent/onefuzz-agent/src/tasks/merge/libfuzzer_merge.rs index c6b2e0eee..3f57dfb47 100644 --- a/src/agent/onefuzz-agent/src/tasks/merge/libfuzzer_merge.rs +++ b/src/agent/onefuzz-agent/src/tasks/merge/libfuzzer_merge.rs @@ -3,7 +3,7 @@ use crate::tasks::{ config::CommonConfig, - heartbeat::*, + heartbeat::HeartbeatSender, utils::{self, default_bool_true}, }; use anyhow::Result; diff --git a/src/agent/onefuzz-agent/src/tasks/mod.rs b/src/agent/onefuzz-agent/src/tasks/mod.rs index 799aee043..425956c65 100644 --- a/src/agent/onefuzz-agent/src/tasks/mod.rs +++ b/src/agent/onefuzz-agent/src/tasks/mod.rs @@ -8,6 +8,7 @@ pub mod fuzz; pub mod generic; pub mod heartbeat; pub mod merge; +pub mod regression; pub mod report; pub mod stats; pub mod utils; diff --git a/src/agent/onefuzz-agent/src/tasks/regression/common.rs b/src/agent/onefuzz-agent/src/tasks/regression/common.rs new file mode 100644 index 000000000..49b1c4dee --- /dev/null +++ b/src/agent/onefuzz-agent/src/tasks/regression/common.rs @@ -0,0 +1,162 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::tasks::{ + heartbeat::{HeartbeatSender, TaskHeartbeatClient}, + report::crash_report::{parse_report_file, CrashTestResult, RegressionReport}, +}; +use anyhow::{Context, Result}; +use async_trait::async_trait; +use onefuzz::syncdir::SyncedDir; +use reqwest::Url; +use std::path::PathBuf; + +/// Defines implementation-provided callbacks for all implementers of regression tasks. +/// +/// Shared regression task behavior is implemented in this module. +#[async_trait] +pub trait RegressionHandler { + /// Test the provided input and generate a crash result + /// * `input` - path to the input to test + /// * `input_url` - input url + async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result; +} + +/// Runs the regression task +pub async fn run( + heartbeat_client: Option, + regression_reports: &SyncedDir, + crashes: &SyncedDir, + report_dirs: &[&SyncedDir], + report_list: &Option>, + readonly_inputs: &Option, + handler: &impl RegressionHandler, +) -> Result<()> { + info!("Starting generic regression task"); + handle_crash_reports( + handler, + crashes, + report_dirs, + report_list, + ®ression_reports, + &heartbeat_client, + ) + .await?; + + if let Some(readonly_inputs) = &readonly_inputs { + handle_inputs( + handler, + readonly_inputs, + ®ression_reports, + &heartbeat_client, + ) + .await?; + } + + Ok(()) +} + +/// Run the regression on the files in the 'inputs' location +/// * `handler` - regression handler +/// * `readonly_inputs` - location of the input files +/// * `regression_reports` - where reports should be saved +/// * `heartbeat_client` - heartbeat client +pub async fn handle_inputs( + handler: &impl RegressionHandler, + readonly_inputs: &SyncedDir, + regression_reports: &SyncedDir, + heartbeat_client: &Option, +) -> Result<()> { + readonly_inputs.init_pull().await?; + let mut input_files = tokio::fs::read_dir(&readonly_inputs.path).await?; + while let Some(file) = input_files.next_entry().await? { + heartbeat_client.alive(); + + let file_path = file.path(); + if !file_path.is_file() { + continue; + } + + let file_name = file_path + .file_name() + .ok_or_else(|| format_err!("missing filename"))? + .to_string_lossy() + .to_string(); + + let input_url = readonly_inputs.url.url().join(&file_name)?; + + let crash_test_result = handler.get_crash_result(file_path, input_url).await?; + RegressionReport { + crash_test_result, + original_crash_test_result: None, + } + .save(None, regression_reports) + .await? + } + + Ok(()) +} + +pub async fn handle_crash_reports( + handler: &impl RegressionHandler, + crashes: &SyncedDir, + report_dirs: &[&SyncedDir], + report_list: &Option>, + regression_reports: &SyncedDir, + heartbeat_client: &Option, +) -> Result<()> { + // without crash report containers, skip this method + if report_dirs.is_empty() { + return Ok(()); + } + + crashes.init_pull().await?; + + for possible_dir in report_dirs { + possible_dir.init_pull().await?; + + let mut report_files = tokio::fs::read_dir(&possible_dir.path).await?; + while let Some(file) = report_files.next_entry().await? { + heartbeat_client.alive(); + let file_path = file.path(); + if !file_path.is_file() { + continue; + } + + let file_name = file_path + .file_name() + .ok_or_else(|| format_err!("missing filename"))? + .to_string_lossy() + .to_string(); + + if let Some(report_list) = &report_list { + if !report_list.contains(&file_name) { + continue; + } + } + + let original_crash_test_result = parse_report_file(file.path()) + .await + .with_context(|| format!("unable to parse crash report: {}", file_name))?; + + let input_blob = match &original_crash_test_result { + CrashTestResult::CrashReport(x) => x.input_blob.clone(), + CrashTestResult::NoRepro(x) => x.input_blob.clone(), + } + .ok_or_else(|| format_err!("crash report is missing input blob: {}", file_name))?; + + let input_url = crashes.url.blob(&input_blob.name).url(); + let input = crashes.path.join(&input_blob.name); + let crash_test_result = handler.get_crash_result(input, input_url).await?; + + RegressionReport { + crash_test_result, + original_crash_test_result: Some(original_crash_test_result), + } + .save(Some(file_name), regression_reports) + .await? + } + } + + Ok(()) +} diff --git a/src/agent/onefuzz-agent/src/tasks/regression/generic.rs b/src/agent/onefuzz-agent/src/tasks/regression/generic.rs new file mode 100644 index 000000000..b90656877 --- /dev/null +++ b/src/agent/onefuzz-agent/src/tasks/regression/generic.rs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::tasks::{ + config::CommonConfig, + report::{crash_report::CrashTestResult, generic}, + utils::default_bool_true, +}; +use anyhow::Result; +use async_trait::async_trait; +use onefuzz::syncdir::SyncedDir; +use reqwest::Url; +use serde::Deserialize; +use std::{collections::HashMap, path::PathBuf}; + +use super::common::{self, RegressionHandler}; + +#[derive(Debug, Deserialize)] +pub struct Config { + pub target_exe: PathBuf, + + #[serde(default)] + pub target_options: Vec, + + #[serde(default)] + pub target_env: HashMap, + + pub target_timeout: Option, + + pub crashes: SyncedDir, + pub regression_reports: SyncedDir, + pub report_list: Option>, + pub reports: Option, + pub unique_reports: Option, + pub no_repro: Option, + pub readonly_inputs: Option, + + #[serde(default)] + pub check_asan_log: bool, + #[serde(default = "default_bool_true")] + pub check_debugger: bool, + #[serde(default)] + pub check_retry_count: u64, + + #[serde(default)] + pub minimized_stack_depth: Option, + + #[serde(flatten)] + pub common: CommonConfig, +} + +pub struct GenericRegressionTask { + config: Config, +} + +#[async_trait] +impl RegressionHandler for GenericRegressionTask { + async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result { + let args = generic::TestInputArgs { + input_url: Some(input_url), + input: &input, + target_exe: &self.config.target_exe, + target_options: &self.config.target_options, + target_env: &self.config.target_env, + setup_dir: &self.config.common.setup_dir, + task_id: self.config.common.task_id, + job_id: self.config.common.job_id, + target_timeout: self.config.target_timeout, + check_retry_count: self.config.check_retry_count, + check_asan_log: self.config.check_asan_log, + check_debugger: self.config.check_debugger, + minimized_stack_depth: self.config.minimized_stack_depth, + }; + generic::test_input(args).await + } +} + +impl GenericRegressionTask { + pub fn new(config: Config) -> Self { + Self { config } + } + + pub async fn run(&self) -> Result<()> { + info!("Starting generic regression task"); + let heartbeat_client = self.config.common.init_heartbeat().await?; + + let mut report_dirs = vec![]; + for dir in &[ + &self.config.reports, + &self.config.unique_reports, + &self.config.no_repro, + ] { + if let Some(dir) = dir { + report_dirs.push(dir); + } + } + common::run( + heartbeat_client, + &self.config.regression_reports, + &self.config.crashes, + &report_dirs, + &self.config.report_list, + &self.config.readonly_inputs, + self, + ) + .await?; + Ok(()) + } +} diff --git a/src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs b/src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs new file mode 100644 index 000000000..e6cb26bb1 --- /dev/null +++ b/src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs @@ -0,0 +1,107 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +use crate::tasks::{ + config::CommonConfig, + report::{crash_report::CrashTestResult, libfuzzer_report}, + utils::default_bool_true, +}; + +use anyhow::Result; +use reqwest::Url; + +use super::common::{self, RegressionHandler}; +use async_trait::async_trait; +use onefuzz::syncdir::SyncedDir; +use serde::Deserialize; +use std::{collections::HashMap, path::PathBuf}; + +#[derive(Debug, Deserialize)] +pub struct Config { + pub target_exe: PathBuf, + + #[serde(default)] + pub target_options: Vec, + + #[serde(default)] + pub target_env: HashMap, + + pub target_timeout: Option, + + pub crashes: SyncedDir, + pub regression_reports: SyncedDir, + pub report_list: Option>, + pub unique_reports: Option, + pub reports: Option, + pub no_repro: Option, + pub readonly_inputs: Option, + + #[serde(default = "default_bool_true")] + pub check_fuzzer_help: bool, + #[serde(default)] + pub check_retry_count: u64, + + #[serde(default)] + pub minimized_stack_depth: Option, + + #[serde(flatten)] + pub common: CommonConfig, +} + +pub struct LibFuzzerRegressionTask { + config: Config, +} + +#[async_trait] +impl RegressionHandler for LibFuzzerRegressionTask { + async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result { + let args = libfuzzer_report::TestInputArgs { + input_url: Some(input_url), + input: &input, + target_exe: &self.config.target_exe, + target_options: &self.config.target_options, + target_env: &self.config.target_env, + setup_dir: &self.config.common.setup_dir, + task_id: self.config.common.task_id, + job_id: self.config.common.job_id, + target_timeout: self.config.target_timeout, + check_retry_count: self.config.check_retry_count, + minimized_stack_depth: self.config.minimized_stack_depth, + }; + libfuzzer_report::test_input(args).await + } +} + +impl LibFuzzerRegressionTask { + pub fn new(config: Config) -> Self { + Self { config } + } + + pub async fn run(&self) -> Result<()> { + info!("Starting libfuzzer regression task"); + + let mut report_dirs = vec![]; + for dir in &[ + &self.config.reports, + &self.config.unique_reports, + &self.config.no_repro, + ] { + if let Some(dir) = dir { + report_dirs.push(dir); + } + } + + let heartbeat_client = self.config.common.init_heartbeat().await?; + common::run( + heartbeat_client, + &self.config.regression_reports, + &self.config.crashes, + &report_dirs, + &self.config.report_list, + &self.config.readonly_inputs, + self, + ) + .await?; + Ok(()) + } +} diff --git a/src/agent/onefuzz-agent/src/tasks/regression/mod.rs b/src/agent/onefuzz-agent/src/tasks/regression/mod.rs new file mode 100644 index 000000000..821fb601e --- /dev/null +++ b/src/agent/onefuzz-agent/src/tasks/regression/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +pub mod common; +pub mod generic; +pub mod libfuzzer; diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index a72d3b7be..6f3f907c5 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -5,7 +5,10 @@ use anyhow::{Context, Result}; use futures::StreamExt; use onefuzz::{blob::BlobUrl, monitor::DirectoryMonitor, syncdir::SyncedDir}; use onefuzz_telemetry::{ - Event::{new_report, new_unable_to_reproduce, new_unique_report}, + Event::{ + new_report, new_unable_to_reproduce, new_unique_report, regression_report, + regression_unable_to_reproduce, + }, EventData, }; use serde::{Deserialize, Serialize}; @@ -46,6 +49,7 @@ pub struct CrashReport { pub job_id: Uuid, pub scariness_score: Option, + pub scariness_description: Option, } @@ -62,11 +66,42 @@ pub struct NoCrash { } #[derive(Debug, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] pub enum CrashTestResult { CrashReport(CrashReport), NoRepro(NoCrash), } +#[derive(Debug, Deserialize, Serialize)] +pub struct RegressionReport { + pub crash_test_result: CrashTestResult, + pub original_crash_test_result: Option, +} + +impl RegressionReport { + pub async fn save( + self, + report_name: Option, + regression_reports: &SyncedDir, + ) -> Result<()> { + let (event, name) = match &self.crash_test_result { + CrashTestResult::CrashReport(report) => { + let name = report_name.unwrap_or_else(|| report.unique_blob_name()); + (regression_report, name) + } + CrashTestResult::NoRepro(report) => { + let name = report_name.unwrap_or_else(|| report.blob_name()); + (regression_unable_to_reproduce, name) + } + }; + + if upload_or_save_local(&self, &name, regression_reports).await? { + event!(event; EventData::Path = name); + } + Ok(()) + } +} + async fn upload_or_save_local( report: &T, dest_name: &str, @@ -76,6 +111,10 @@ async fn upload_or_save_local( } impl CrashTestResult { + /// Saves the crash result as a crash report + /// * `unique_reports` - location to save the deduplicated report if the bug was reproduced + /// * `reports` - location to save the report if the bug was reproduced + /// * `no_repro` - location to save the report if the bug was not reproduced pub async fn save( &self, unique_reports: &Option, @@ -113,7 +152,7 @@ impl CrashTestResult { } } -#[derive(Debug, Deserialize, Serialize)] +#[derive(Debug, Deserialize, Serialize, Clone)] pub struct InputBlob { pub account: Option, pub container: Option, @@ -188,7 +227,7 @@ impl NoCrash { } } -async fn parse_report_file(path: PathBuf) -> Result { +pub async fn parse_report_file(path: PathBuf) -> Result { let raw = std::fs::read_to_string(&path) .with_context(|| format_err!("unable to open crash report: {}", path.display()))?; diff --git a/src/agent/onefuzz-agent/src/tasks/report/generic.rs b/src/agent/onefuzz-agent/src/tasks/report/generic.rs index b1ce3a868..2515c50a9 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/generic.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/generic.rs @@ -5,7 +5,7 @@ use super::crash_report::{CrashReport, CrashTestResult, InputBlob, NoCrash}; use crate::tasks::{ config::CommonConfig, generic::input_poller::{CallbackImpl, InputPoller, Processor}, - heartbeat::*, + heartbeat::{HeartbeatSender, TaskHeartbeatClient}, utils::default_bool_true, }; use anyhow::Result; @@ -18,6 +18,7 @@ use std::{ path::{Path, PathBuf}, }; use storage_queue::{Message, QueueClient}; +use uuid::Uuid; #[derive(Debug, Deserialize)] pub struct Config { @@ -86,27 +87,103 @@ impl ReportTask { } } +pub struct TestInputArgs<'a> { + pub input_url: Option, + pub input: &'a Path, + pub target_exe: &'a Path, + pub target_options: &'a [String], + pub target_env: &'a HashMap, + pub setup_dir: &'a Path, + pub task_id: Uuid, + pub job_id: Uuid, + pub target_timeout: Option, + pub check_retry_count: u64, + pub check_asan_log: bool, + pub check_debugger: bool, + pub minimized_stack_depth: Option, +} + +pub async fn test_input(args: TestInputArgs<'_>) -> Result { + let tester = Tester::new( + args.setup_dir, + args.target_exe, + args.target_options, + args.target_env, + ) + .check_asan_log(args.check_asan_log) + .check_debugger(args.check_debugger) + .check_retry_count(args.check_retry_count) + .set_optional(args.target_timeout, |tester, timeout| { + tester.timeout(timeout) + }); + + let input_sha256 = sha256::digest_file(args.input).await?; + let task_id = args.task_id; + let job_id = args.job_id; + let input_blob = args + .input_url + .and_then(|u| BlobUrl::new(u).ok()) + .map(InputBlob::from); + + let test_report = tester.test_input(args.input).await?; + + if let Some(crash_log) = test_report.asan_log { + let crash_report = CrashReport::new( + crash_log, + task_id, + job_id, + args.target_exe, + input_blob, + input_sha256, + args.minimized_stack_depth, + ); + Ok(CrashTestResult::CrashReport(crash_report)) + } else if let Some(crash) = test_report.crash { + let call_stack_sha256 = sha256::digest_iter(&crash.call_stack); + let crash_report = CrashReport { + input_blob, + input_sha256, + executable: PathBuf::from(args.target_exe), + call_stack: crash.call_stack, + crash_type: crash.crash_type, + crash_site: crash.crash_site, + call_stack_sha256, + asan_log: None, + scariness_score: None, + scariness_description: None, + task_id, + job_id, + minimized_stack: vec![], + minimized_stack_sha256: None, + minimized_stack_function_names: vec![], + minimized_stack_function_names_sha256: None, + }; + + Ok(CrashTestResult::CrashReport(crash_report)) + } else { + let no_repro = NoCrash { + input_blob, + input_sha256, + executable: PathBuf::from(args.target_exe), + task_id, + job_id, + tries: 1 + args.check_retry_count, + error: test_report.error.map(|e| format!("{}", e)), + }; + + Ok(CrashTestResult::NoRepro(no_repro)) + } +} + pub struct GenericReportProcessor<'a> { config: &'a Config, - tester: Tester<'a>, heartbeat_client: Option, } impl<'a> GenericReportProcessor<'a> { pub fn new(config: &'a Config, heartbeat_client: Option) -> Self { - let tester = Tester::new( - &config.common.setup_dir, - &config.target_exe, - &config.target_options, - &config.target_env, - ) - .check_asan_log(config.check_asan_log) - .check_debugger(config.check_debugger) - .check_retry_count(config.check_retry_count); - Self { config, - tester, heartbeat_client, } } @@ -117,56 +194,25 @@ impl<'a> GenericReportProcessor<'a> { input: &Path, ) -> Result { self.heartbeat_client.alive(); - let input_sha256 = sha256::digest_file(input).await?; - let task_id = self.config.common.task_id; - let job_id = self.config.common.job_id; - let input_blob = match input_url { - Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)), - None => None, + + let args = TestInputArgs { + input_url, + input, + target_exe: &self.config.target_exe, + target_options: &self.config.target_options, + target_env: &self.config.target_env, + setup_dir: &self.config.common.setup_dir, + task_id: self.config.common.task_id, + job_id: self.config.common.job_id, + target_timeout: self.config.target_timeout, + check_retry_count: self.config.check_retry_count, + check_asan_log: self.config.check_asan_log, + check_debugger: self.config.check_debugger, + minimized_stack_depth: self.config.minimized_stack_depth, }; + let result = test_input(args).await?; - let test_report = self.tester.test_input(input).await?; - - if let Some(asan_log) = test_report.asan_log { - let crash_report = CrashReport::new( - asan_log, - task_id, - job_id, - &self.config.target_exe, - input_blob, - input_sha256, - self.config.minimized_stack_depth, - ); - Ok(CrashTestResult::CrashReport(crash_report)) - } else if let Some(crash) = test_report.crash { - let call_stack_sha256 = sha256::digest_iter(&crash.call_stack); - let crash_report = CrashReport { - input_blob, - input_sha256, - executable: PathBuf::from(&self.config.target_exe), - call_stack: crash.call_stack, - crash_type: crash.crash_type, - crash_site: crash.crash_site, - call_stack_sha256, - task_id, - job_id, - ..Default::default() - }; - - Ok(CrashTestResult::CrashReport(crash_report)) - } else { - let no_repro = NoCrash { - input_blob, - input_sha256, - executable: PathBuf::from(&self.config.target_exe), - task_id, - job_id, - tries: 1 + self.config.check_retry_count, - error: test_report.error.map(|e| format!("{}", e)), - }; - - Ok(CrashTestResult::NoRepro(no_repro)) - } + Ok(result) } } diff --git a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs index 2617d7949..dad7d1334 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs @@ -3,7 +3,10 @@ use super::crash_report::*; use crate::tasks::{ - config::CommonConfig, generic::input_poller::*, heartbeat::*, utils::default_bool_true, + config::CommonConfig, + generic::input_poller::*, + heartbeat::{HeartbeatSender, TaskHeartbeatClient}, + utils::default_bool_true, }; use anyhow::{Context, Result}; use async_trait::async_trait; @@ -75,7 +78,7 @@ impl ReportTask { let mut processor = AsanProcessor::new(self.config.clone()).await?; if let Some(crashes) = &self.config.crashes { - self.poller.batch_process(&mut processor, crashes).await?; + self.poller.batch_process(&mut processor, &crashes).await?; } if self.config.check_queue { @@ -88,6 +91,72 @@ impl ReportTask { } } +pub struct TestInputArgs<'a> { + pub input_url: Option, + pub input: &'a Path, + pub target_exe: &'a Path, + pub target_options: &'a [String], + pub target_env: &'a HashMap, + pub setup_dir: &'a Path, + pub task_id: uuid::Uuid, + pub job_id: uuid::Uuid, + pub target_timeout: Option, + pub check_retry_count: u64, + pub minimized_stack_depth: Option, +} + +pub async fn test_input(args: TestInputArgs<'_>) -> Result { + let fuzzer = LibFuzzer::new( + args.target_exe, + args.target_options, + args.target_env, + args.setup_dir, + ); + + let task_id = args.task_id; + let job_id = args.job_id; + let input_blob = args + .input_url + .and_then(|u| BlobUrl::new(u).ok()) + .map(InputBlob::from); + let input = args.input; + let input_sha256 = sha256::digest_file(args.input) + .await + .with_context(|| format_err!("unable to sha256 digest input file: {}", input.display()))?; + + let test_report = fuzzer + .repro(args.input, args.target_timeout, args.check_retry_count) + .await?; + + match test_report.asan_log { + Some(crash_log) => { + let crash_report = CrashReport::new( + crash_log, + task_id, + job_id, + args.target_exe, + input_blob, + input_sha256, + args.minimized_stack_depth, + ); + Ok(CrashTestResult::CrashReport(crash_report)) + } + None => { + let no_repro = NoCrash { + input_blob, + input_sha256, + executable: PathBuf::from(&args.target_exe), + task_id, + job_id, + tries: 1 + args.check_retry_count, + error: test_report.error.map(|e| format!("{}", e)), + }; + + Ok(CrashTestResult::NoRepro(no_repro)) + } + } +} + pub struct AsanProcessor { config: Arc, heartbeat_client: Option, @@ -109,58 +178,22 @@ impl AsanProcessor { input: &Path, ) -> Result { self.heartbeat_client.alive(); - let fuzzer = LibFuzzer::new( - &self.config.target_exe, - &self.config.target_options, - &self.config.target_env, - &self.config.common.setup_dir, - ); - - let task_id = self.config.common.task_id; - let job_id = self.config.common.job_id; - let input_blob = match input_url { - Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)), - None => None, + let args = TestInputArgs { + input_url, + input, + target_exe: &self.config.target_exe, + target_options: &self.config.target_options, + target_env: &self.config.target_env, + setup_dir: &self.config.common.setup_dir, + task_id: self.config.common.task_id, + job_id: self.config.common.job_id, + target_timeout: self.config.target_timeout, + check_retry_count: self.config.check_retry_count, + minimized_stack_depth: self.config.minimized_stack_depth, }; - let input_sha256 = sha256::digest_file(input).await.with_context(|| { - format_err!("unable to sha256 digest input file: {}", input.display()) - })?; + let result = test_input(args).await?; - let test_report = fuzzer - .repro( - input, - self.config.target_timeout, - self.config.check_retry_count, - ) - .await?; - - match test_report.asan_log { - Some(asan_log) => { - let crash_report = CrashReport::new( - asan_log, - task_id, - job_id, - &self.config.target_exe, - input_blob, - input_sha256, - self.config.minimized_stack_depth, - ); - Ok(CrashTestResult::CrashReport(crash_report)) - } - None => { - let no_repro = NoCrash { - input_blob, - input_sha256, - executable: PathBuf::from(&self.config.target_exe), - task_id, - job_id, - tries: 1 + self.config.check_retry_count, - error: test_report.error.map(|e| format!("{}", e)), - }; - - Ok(CrashTestResult::NoRepro(no_repro)) - } - } + Ok(result) } } diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 1f7666302..f70e70731 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -14,8 +14,8 @@ extern crate onefuzz_telemetry; extern crate onefuzz; use crate::{ - config::StaticConfig, coordinator::StateUpdateEvent, heartbeat::*, work::WorkSet, - worker::WorkerEvent, + config::StaticConfig, coordinator::StateUpdateEvent, heartbeat::init_agent_heartbeat, + work::WorkSet, worker::WorkerEvent, }; use std::path::PathBuf; diff --git a/src/agent/onefuzz-telemetry/src/lib.rs b/src/agent/onefuzz-telemetry/src/lib.rs index 1dffd5220..63cfd83a6 100644 --- a/src/agent/onefuzz-telemetry/src/lib.rs +++ b/src/agent/onefuzz-telemetry/src/lib.rs @@ -95,6 +95,8 @@ pub enum Event { new_report, new_unique_report, new_unable_to_reproduce, + regression_report, + regression_unable_to_reproduce, } impl Event { @@ -109,6 +111,8 @@ impl Event { Self::new_report => "new_report", Self::new_unique_report => "new_unique_report", Self::new_unable_to_reproduce => "new_unable_to_reproduce", + Self::regression_report => "regression_report", + Self::regression_unable_to_reproduce => "regression_unable_to_reproduce", } } } diff --git a/src/agent/onefuzz/src/blob/url.rs b/src/agent/onefuzz/src/blob/url.rs index 1dd2c2783..949048b7a 100644 --- a/src/agent/onefuzz/src/blob/url.rs +++ b/src/agent/onefuzz/src/blob/url.rs @@ -25,6 +25,15 @@ impl BlobUrl { bail!("Invalid blob URL: {}", url) } + pub fn from_blob_info(account: &str, container: &str, name: &str) -> Result { + // format https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#resource-uri-syntax + let url = Url::parse(&format!( + "https://{}.blob.core.windows.net/{}/{}", + account, container, name + ))?; + Self::new(url) + } + pub fn parse(url: impl AsRef) -> Result { let url = Url::parse(url.as_ref())?; diff --git a/src/api-service/__app__/onefuzzlib/extension.py b/src/api-service/__app__/onefuzzlib/extension.py index b00142115..b49fde70d 100644 --- a/src/api-service/__app__/onefuzzlib/extension.py +++ b/src/api-service/__app__/onefuzzlib/extension.py @@ -271,6 +271,9 @@ def repro_extensions( if report is None: raise Exception("invalid report: %s" % repro_config) + if report.input_blob is None: + raise Exception("unable to perform reproduction without an input blob") + commands = [] if setup_container: commands += [ diff --git a/src/api-service/__app__/onefuzzlib/notifications/ado.py b/src/api-service/__app__/onefuzzlib/notifications/ado.py index ee6b90c39..3d9e67f9d 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/ado.py +++ b/src/api-service/__app__/onefuzzlib/notifications/ado.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import logging -from typing import Iterator, List, Optional +from typing import Iterator, List, Optional, Union from azure.devops.connection import Connection from azure.devops.credentials import BasicAuthentication @@ -24,7 +24,7 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import ( WorkItemTrackingClient, ) from memoization import cached -from onefuzztypes.models import ADOTemplate, Report +from onefuzztypes.models import ADOTemplate, RegressionReport, Report from onefuzztypes.primitives import Container from ..secrets import get_secret_string_value @@ -51,7 +51,11 @@ def get_valid_fields( class ADO: def __init__( - self, container: Container, filename: str, config: ADOTemplate, report: Report + self, + container: Container, + filename: str, + config: ADOTemplate, + report: Report, ): self.config = config self.renderer = Render(container, filename, report) @@ -203,8 +207,20 @@ class ADO: def notify_ado( - config: ADOTemplate, container: Container, filename: str, report: Report + config: ADOTemplate, + container: Container, + filename: str, + report: Union[Report, RegressionReport], ) -> None: + if isinstance(report, RegressionReport): + logging.info( + "ado integration does not support regression reports. " + "container:%s filename:%s", + container, + filename, + ) + return + logging.info( "notify ado: job_id:%s task_id:%s container:%s filename:%s", report.job_id, diff --git a/src/api-service/__app__/onefuzzlib/notifications/github_issues.py b/src/api-service/__app__/onefuzzlib/notifications/github_issues.py index de048b2e4..e17f94d6a 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/github_issues.py +++ b/src/api-service/__app__/onefuzzlib/notifications/github_issues.py @@ -4,13 +4,18 @@ # Licensed under the MIT License. import logging -from typing import List, Optional +from typing import List, Optional, Union from github3 import login from github3.exceptions import GitHubException from github3.issues import Issue from onefuzztypes.enums import GithubIssueSearchMatch -from onefuzztypes.models import GithubAuth, GithubIssueTemplate, Report +from onefuzztypes.models import ( + GithubAuth, + GithubIssueTemplate, + RegressionReport, + Report, +) from onefuzztypes.primitives import Container from ..secrets import get_secret_obj @@ -107,10 +112,18 @@ def github_issue( config: GithubIssueTemplate, container: Container, filename: str, - report: Optional[Report], + report: Optional[Union[Report, RegressionReport]], ) -> None: if report is None: return + if isinstance(report, RegressionReport): + logging.info( + "github issue integration does not support regression reports. " + "container:%s filename:%s", + container, + filename, + ) + return try: handler = GithubIssue(config, container, filename, report) diff --git a/src/api-service/__app__/onefuzzlib/notifications/main.py b/src/api-service/__app__/onefuzzlib/notifications/main.py index 64c7e963a..50a90be58 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/main.py +++ b/src/api-service/__app__/onefuzzlib/notifications/main.py @@ -10,12 +10,18 @@ from uuid import UUID from memoization import cached from onefuzztypes import models from onefuzztypes.enums import ErrorCode, TaskState -from onefuzztypes.events import EventCrashReported, EventFileAdded +from onefuzztypes.events import ( + EventCrashReported, + EventFileAdded, + EventRegressionReported, +) from onefuzztypes.models import ( ADOTemplate, Error, GithubIssueTemplate, NotificationTemplate, + RegressionReport, + Report, Result, TeamsTemplate, ) @@ -26,7 +32,7 @@ from ..azure.queue import send_message from ..azure.storage import StorageType from ..events import send_event from ..orm import ORMMixin -from ..reports import get_report +from ..reports import get_report_or_regression from ..tasks.config import get_input_container_queues from ..tasks.main import Task from .ado import notify_ado @@ -102,7 +108,7 @@ def get_queue_tasks() -> Sequence[Tuple[Task, Sequence[str]]]: def new_files(container: Container, filename: str) -> None: - report = get_report(container, filename) + report = get_report_or_regression(container, filename) notifications = get_notifications(container) if notifications: @@ -134,9 +140,15 @@ def new_files(container: Container, filename: str) -> None: ) send_message(task.task_id, bytes(url, "utf-8"), StorageType.corpus) - if report: + if isinstance(report, Report): send_event( EventCrashReported(report=report, container=container, filename=filename) ) + elif isinstance(report, RegressionReport): + send_event( + EventRegressionReported( + regression_report=report, container=container, filename=filename + ) + ) else: send_event(EventFileAdded(container=container, filename=filename)) diff --git a/src/api-service/__app__/onefuzzlib/notifications/teams.py b/src/api-service/__app__/onefuzzlib/notifications/teams.py index 542f704ec..f8ed06840 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/teams.py +++ b/src/api-service/__app__/onefuzzlib/notifications/teams.py @@ -4,10 +4,10 @@ # Licensed under the MIT License. import logging -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Union import requests -from onefuzztypes.models import Report, TeamsTemplate +from onefuzztypes.models import RegressionReport, Report, TeamsTemplate from onefuzztypes.primitives import Container from ..azure.containers import auth_download_url @@ -54,12 +54,15 @@ def send_teams_webhook( def notify_teams( - config: TeamsTemplate, container: Container, filename: str, report: Optional[Report] + config: TeamsTemplate, + container: Container, + filename: str, + report: Optional[Union[Report, RegressionReport]], ) -> None: text = None facts: List[Dict[str, str]] = [] - if report: + if isinstance(report, Report): task = Task.get(report.job_id, report.task_id) if not task: logging.error( diff --git a/src/api-service/__app__/onefuzzlib/reports.py b/src/api-service/__app__/onefuzzlib/reports.py index 9c628b044..0a996ea8b 100644 --- a/src/api-service/__app__/onefuzzlib/reports.py +++ b/src/api-service/__app__/onefuzzlib/reports.py @@ -8,7 +8,7 @@ import logging from typing import Optional, Union from memoization import cached -from onefuzztypes.models import Report +from onefuzztypes.models import RegressionReport, Report from onefuzztypes.primitives import Container from pydantic import ValidationError @@ -16,17 +16,16 @@ from .azure.containers import get_blob from .azure.storage import StorageType -def parse_report( +def parse_report_or_regression( content: Union[str, bytes], file_path: Optional[str] = None -) -> Optional[Report]: +) -> Optional[Union[Report, RegressionReport]]: if isinstance(content, bytes): try: content = content.decode() except UnicodeDecodeError as err: logging.error( - "unable to parse report (%s): unicode decode of report failed - %s", - file_path, - err, + f"unable to parse report ({file_path}): " + f"unicode decode of report failed - {err}" ) return None @@ -34,22 +33,31 @@ def parse_report( data = json.loads(content) except json.decoder.JSONDecodeError as err: logging.error( - "unable to parse report (%s): json decoding failed - %s", file_path, err + f"unable to parse report ({file_path}): json decoding failed - {err}" ) return None + regression_err = None try: - entry = Report.parse_obj(data) + return RegressionReport.parse_obj(data) except ValidationError as err: - logging.error("unable to parse report (%s): %s", file_path, err) - return None + regression_err = err - return entry + try: + return Report.parse_obj(data) + except ValidationError as err: + logging.error( + f"unable to parse report ({file_path}) as a report or regression. " + f"regression error: {regression_err} report error: {err}" + ) + return None # cache the last 1000 reports @cached(max_size=1000) -def get_report(container: Container, filename: str) -> Optional[Report]: +def get_report_or_regression( + container: Container, filename: str +) -> Optional[Union[Report, RegressionReport]]: file_path = "/".join([container, filename]) if not filename.endswith(".json"): logging.error("get_report invalid extension: %s", file_path) @@ -60,4 +68,11 @@ def get_report(container: Container, filename: str) -> Optional[Report]: logging.error("get_report invalid blob: %s", file_path) return None - return parse_report(blob, file_path=file_path) + return parse_report_or_regression(blob, file_path=file_path) + + +def get_report(container: Container, filename: str) -> Optional[Report]: + result = get_report_or_regression(container, filename) + if isinstance(result, Report): + return result + return None diff --git a/src/api-service/__app__/onefuzzlib/repro.py b/src/api-service/__app__/onefuzzlib/repro.py index c6ea9e21e..38fcbebc1 100644 --- a/src/api-service/__app__/onefuzzlib/repro.py +++ b/src/api-service/__app__/onefuzzlib/repro.py @@ -164,6 +164,12 @@ class Repro(BASE_REPRO, ORMMixin): if report is None: return Error(code=ErrorCode.VM_CREATE_FAILED, errors=["missing report"]) + if report.input_blob is None: + return Error( + code=ErrorCode.VM_CREATE_FAILED, + errors=["unable to perform repro for crash reports without inputs"], + ) + files = {} if task.os == OS.windows: diff --git a/src/api-service/__app__/onefuzzlib/tasks/config.py b/src/api-service/__app__/onefuzzlib/tasks/config.py index da5480856..f973bd001 100644 --- a/src/api-service/__app__/onefuzzlib/tasks/config.py +++ b/src/api-service/__app__/onefuzzlib/tasks/config.py @@ -348,6 +348,9 @@ def build_task_config( else True ) + if TaskFeature.report_list in definition.features: + config.report_list = task_config.task.report_list + if TaskFeature.expect_crash_on_failure in definition.features: config.expect_crash_on_failure = ( task_config.task.expect_crash_on_failure diff --git a/src/api-service/__app__/onefuzzlib/tasks/defs.py b/src/api-service/__app__/onefuzzlib/tasks/defs.py index e6cae4d74..619922e1e 100644 --- a/src/api-service/__app__/onefuzzlib/tasks/defs.py +++ b/src/api-service/__app__/onefuzzlib/tasks/defs.py @@ -414,4 +414,131 @@ TASK_DEFINITIONS = { ], monitor_queue=ContainerType.crashes, ), + TaskType.generic_regression: TaskDefinition( + features=[ + TaskFeature.target_exe, + TaskFeature.target_env, + TaskFeature.target_options, + TaskFeature.target_timeout, + TaskFeature.check_asan_log, + TaskFeature.check_debugger, + TaskFeature.check_retry_count, + TaskFeature.report_list, + ], + vm=VmDefinition(compare=Compare.AtLeast, value=1), + containers=[ + ContainerDefinition( + type=ContainerType.setup, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.regression_reports, + compare=Compare.Equal, + value=1, + permissions=[ + ContainerPermission.Write, + ContainerPermission.Read, + ContainerPermission.List, + ], + ), + ContainerDefinition( + type=ContainerType.crashes, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.unique_reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.no_repro, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.readonly_inputs, + compare=Compare.AtMost, + value=1, + permissions=[ + ContainerPermission.Read, + ContainerPermission.List, + ], + ), + ], + ), + TaskType.libfuzzer_regression: TaskDefinition( + features=[ + TaskFeature.target_exe, + TaskFeature.target_env, + TaskFeature.target_options, + TaskFeature.target_timeout, + TaskFeature.check_fuzzer_help, + TaskFeature.check_retry_count, + TaskFeature.report_list, + ], + vm=VmDefinition(compare=Compare.AtLeast, value=1), + containers=[ + ContainerDefinition( + type=ContainerType.setup, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.regression_reports, + compare=Compare.Equal, + value=1, + permissions=[ + ContainerPermission.Write, + ContainerPermission.Read, + ContainerPermission.List, + ], + ), + ContainerDefinition( + type=ContainerType.crashes, + compare=Compare.Equal, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.unique_reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.reports, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.no_repro, + compare=Compare.AtMost, + value=1, + permissions=[ContainerPermission.Read, ContainerPermission.List], + ), + ContainerDefinition( + type=ContainerType.readonly_inputs, + compare=Compare.AtMost, + value=1, + permissions=[ + ContainerPermission.Read, + ContainerPermission.List, + ], + ), + ], + ), } diff --git a/src/api-service/tests/test_report_parse.py b/src/api-service/tests/test_report_parse.py index c7d282c77..de9ce76f1 100755 --- a/src/api-service/tests/test_report_parse.py +++ b/src/api-service/tests/test_report_parse.py @@ -10,7 +10,7 @@ from pathlib import Path from onefuzztypes.models import Report -from __app__.onefuzzlib.reports import parse_report +from __app__.onefuzzlib.reports import parse_report_or_regression class TestReportParse(unittest.TestCase): @@ -20,16 +20,15 @@ class TestReportParse(unittest.TestCase): data = json.load(handle) invalid = {"unused_field_1": 3} - report = parse_report(json.dumps(data)) + report = parse_report_or_regression(json.dumps(data)) self.assertIsInstance(report, Report) with self.assertLogs(level="ERROR"): - self.assertIsNone(parse_report('"invalid"')) + self.assertIsNone(parse_report_or_regression('"invalid"')) with self.assertLogs(level="WARNING") as logs: - self.assertIsNone(parse_report(json.dumps(invalid))) - - self.assertTrue(any(["unable to parse report" in x for x in logs.output])) + self.assertIsNone(parse_report_or_regression(json.dumps(invalid))) + self.assertTrue(any(["unable to parse report" in x for x in logs.output])) if __name__ == "__main__": diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 1d4b74e54..99733c29b 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -174,11 +174,13 @@ class Files(Endpoint): sas = self.onefuzz.containers.get(container).sas_url return ContainerWrapper(sas) - def list(self, container: primitives.Container) -> models.Files: + def list( + self, container: primitives.Container, prefix: Optional[str] = None + ) -> models.Files: """ Get a list of files in a container """ self.logger.debug("listing files in container: %s", container) client = self._get_client(container) - return models.Files(files=client.list_blobs()) + return models.Files(files=client.list_blobs(name_starts_with=prefix)) def delete(self, container: primitives.Container, filename: str) -> None: """ delete a file from a container """ @@ -845,6 +847,7 @@ class Tasks(Endpoint): vm_count: int = 1, preserve_existing_outputs: bool = False, colocate: bool = False, + report_list: Optional[List[str]] = None, ) -> models.Task: """ Create a task @@ -907,6 +910,8 @@ class Tasks(Endpoint): target_workers=target_workers, type=task_type, wait_for_files=task_wait_for_files, + report_list=report_list, + preserve_existing_outputs=preserve_existing_outputs, ), ) diff --git a/src/cli/onefuzz/cli.py b/src/cli/onefuzz/cli.py index 56f0ed866..4e557c650 100644 --- a/src/cli/onefuzz/cli.py +++ b/src/cli/onefuzz/cli.py @@ -505,6 +505,14 @@ def output(result: Any, output_format: str, expression: Optional[Any]) -> None: print(result, flush=True) +def log_exception(args: argparse.Namespace, err: Exception) -> None: + if args.verbose > 0: + entry = traceback.format_exc() + for x in entry.split("\n"): + LOGGER.error("traceback: %s", x) + LOGGER.error("command failed: %s", " ".join([str(x) for x in err.args])) + + def execute_api(api: Any, api_types: List[Any], version: str) -> int: builder = Builder(api_types) builder.add_version(version) @@ -545,11 +553,7 @@ def execute_api(api: Any, api_types: List[Any], version: str) -> int: try: result = call_func(args.func, args) except Exception as err: - if args.verbose > 0: - entry = traceback.format_exc() - for x in entry.split("\n"): - LOGGER.error("traceback: %s", x) - LOGGER.error("command failed: %s", " ".join([str(x) for x in err.args])) + log_exception(args, err) return 1 output(result, args.format, expression) diff --git a/src/cli/onefuzz/job_templates/job_monitor.py b/src/cli/onefuzz/job_templates/job_monitor.py index fc1345c8b..f0bb11f70 100644 --- a/src/cli/onefuzz/job_templates/job_monitor.py +++ b/src/cli/onefuzz/job_templates/job_monitor.py @@ -78,11 +78,21 @@ class JobMonitor: None, ) + def is_stopped(self) -> Tuple[bool, str, Any]: + tasks = self.onefuzz.tasks.list(job_id=self.job.job_id) + waiting = [ + "%s:%s" % (x.config.task.type.name, x.state.name) + for x in tasks + if x.state != TaskState.stopped + ] + return (not waiting, "waiting on: %s" % ", ".join(sorted(waiting)), None) + def wait( self, *, wait_for_running: Optional[bool] = False, wait_for_files: Optional[Dict[Container, int]] = None, + wait_for_stopped: Optional[bool] = False, ) -> None: if wait_for_running: wait(self.is_running) @@ -92,3 +102,7 @@ class JobMonitor: self.containers = wait_for_files wait(self.has_files) self.onefuzz.logger.info("new files found") + + if wait_for_stopped: + wait(self.is_stopped) + self.onefuzz.logger.info("tasks stopped") diff --git a/src/cli/onefuzz/template.py b/src/cli/onefuzz/template.py index d35aef454..2e15e7b2b 100644 --- a/src/cli/onefuzz/template.py +++ b/src/cli/onefuzz/template.py @@ -13,6 +13,7 @@ from .templates.afl import AFL from .templates.libfuzzer import Libfuzzer from .templates.ossfuzz import OssFuzz from .templates.radamsa import Radamsa +from .templates.regression import Regression class Template(Command): @@ -24,6 +25,7 @@ class Template(Command): self.afl = AFL(onefuzz, logger) self.radamsa = Radamsa(onefuzz, logger) self.ossfuzz = OssFuzz(onefuzz, logger) + self.regression = Regression(onefuzz, logger) def stop( self, diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index 976ce4777..e2b548d00 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -7,9 +7,10 @@ import json import os import tempfile import zipfile -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple +from uuid import uuid4 -from onefuzztypes.enums import OS, ContainerType +from onefuzztypes.enums import OS, ContainerType, TaskState from onefuzztypes.models import Job, NotificationConfig from onefuzztypes.primitives import Container, Directory, File @@ -39,6 +40,12 @@ def _build_container_name( build=build, platform=platform.name, ) + elif container_type == ContainerType.regression_reports: + guid = onefuzz.utils.namespaced_guid( + project, + name, + build=build, + ) else: guid = onefuzz.utils.namespaced_guid(project, name) @@ -87,6 +94,7 @@ class JobHelper: ) self.wait_for_running: bool = False + self.wait_for_stopped: bool = False self.containers: Dict[ContainerType, Container] = {} self.tags: Dict[str, str] = {"project": project, "name": name, "build": build} if job is None: @@ -116,6 +124,15 @@ class JobHelper: self.platform, ) + def get_unique_container_name(self, container_type: ContainerType) -> Container: + return Container( + "oft-%s-%s" + % ( + container_type.name.replace("_", "-"), + uuid4().hex, + ) + ) + def create_containers(self) -> None: for (container_type, container_name) in self.containers.items(): self.logger.info("using container: %s", container_name) @@ -123,6 +140,9 @@ class JobHelper: container_name, metadata={"container_type": container_type.name} ) + def delete_container(self, container_name: Container) -> None: + self.onefuzz.containers.delete(container_name) + def setup_notifications(self, config: Optional[NotificationConfig]) -> None: if not config: return @@ -233,9 +253,58 @@ class JobHelper: } self.wait_for_running = wait_for_running + def check_current_job(self) -> Job: + job = self.onefuzz.jobs.get(self.job.job_id) + if job.state in ["stopped", "stopping"]: + raise StoppedEarly("job unexpectedly stopped early") + + errors = [] + for task in self.onefuzz.tasks.list(job_id=self.job.job_id): + if task.state in ["stopped", "stopping"]: + if task.error: + errors.append("%s: %s" % (task.config.task.type, task.error)) + else: + errors.append("%s" % task.config.task.type) + + if errors: + raise StoppedEarly("tasks stopped unexpectedly.\n%s" % "\n".join(errors)) + return job + + def get_waiting(self) -> List[str]: + tasks = self.onefuzz.tasks.list(job_id=self.job.job_id) + waiting = [ + "%s:%s" % (x.config.task.type.name, x.state.name) + for x in tasks + if x.state not in TaskState.has_started() + ] + return waiting + + def is_running(self) -> Tuple[bool, str, Any]: + waiting = self.get_waiting() + return (not waiting, "waiting on: %s" % ", ".join(sorted(waiting)), None) + + def has_files(self) -> Tuple[bool, str, Any]: + self.check_current_job() + + new = { + x: len(self.onefuzz.containers.files.list(x).files) + for x in self.to_monitor.keys() + } + + for container in new: + if new[container] > self.to_monitor[container]: + del self.to_monitor[container] + return ( + not self.to_monitor, + "waiting for new files: %s" % ", ".join(self.to_monitor.keys()), + None, + ) + def wait(self) -> None: JobMonitor(self.onefuzz, self.job).wait( - wait_for_running=self.wait_for_running, wait_for_files=self.to_monitor + wait_for_running=self.wait_for_running, + wait_for_files=self.to_monitor, + wait_for_stopped=self.wait_for_stopped, ) def target_exe_blob_name( diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index 8b2485a35..63013c5cc 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -61,6 +61,36 @@ class Libfuzzer(Command): expect_crash_on_failure: bool = True, ) -> None: + regression_containers = [ + (ContainerType.setup, containers[ContainerType.setup]), + (ContainerType.crashes, containers[ContainerType.crashes]), + (ContainerType.unique_reports, containers[ContainerType.unique_reports]), + ( + ContainerType.regression_reports, + containers[ContainerType.regression_reports], + ), + ] + + self.logger.info("creating libfuzzer_regression task") + regression_task = self.onefuzz.tasks.create( + job.job_id, + TaskType.libfuzzer_regression, + target_exe, + regression_containers, + pool_name=pool_name, + duration=duration, + vm_count=1, + reboot_after_setup=reboot_after_setup, + target_options=target_options, + target_env=target_env, + tags=tags, + target_timeout=crash_report_timeout, + check_retry_count=check_retry_count, + check_fuzzer_help=check_fuzzer_help, + debug=debug, + colocate=colocate_all_tasks or colocate_secondary_tasks, + ) + fuzzer_containers = [ (ContainerType.setup, containers[ContainerType.setup]), (ContainerType.crashes, containers[ContainerType.crashes]), @@ -92,7 +122,7 @@ class Libfuzzer(Command): expect_crash_on_failure=expect_crash_on_failure, ) - prereq_tasks = [fuzzer_task.task_id] + prereq_tasks = [fuzzer_task.task_id, regression_task.task_id] coverage_containers = [ (ContainerType.setup, containers[ContainerType.setup]), @@ -219,6 +249,7 @@ class Libfuzzer(Command): ContainerType.no_repro, ContainerType.coverage, ContainerType.unique_inputs, + ContainerType.regression_reports, ) if existing_inputs: diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py new file mode 100644 index 000000000..d13d9a31d --- /dev/null +++ b/src/cli/onefuzz/templates/regression.py @@ -0,0 +1,284 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. + +import json +from typing import Dict, List, Optional + +from onefuzztypes.enums import ContainerType, TaskDebugFlag, TaskType +from onefuzztypes.models import NotificationConfig, RegressionReport +from onefuzztypes.primitives import Container, Directory, File, PoolName + +from onefuzz.api import Command + +from . import JobHelper + + +class Regression(Command): + """ Regression job """ + + def _check_regression(self, container: Container, file: File) -> bool: + content = self.onefuzz.containers.files.get(Container(container), file) + as_str = content.decode() + as_obj = json.loads(as_str) + report = RegressionReport.parse_obj(as_obj) + + if report.crash_test_result.crash_report is not None: + return True + + if report.crash_test_result.no_repro is not None: + return False + + raise Exception("invalid crash report") + + def generic( + self, + project: str, + name: str, + build: str, + pool_name: PoolName, + *, + reports: Optional[List[str]] = None, + crashes: Optional[List[File]] = None, + target_exe: File = File("fuzz.exe"), + tags: Optional[Dict[str, str]] = None, + notification_config: Optional[NotificationConfig] = None, + target_env: Optional[Dict[str, str]] = None, + setup_dir: Optional[Directory] = None, + reboot_after_setup: bool = False, + target_options: Optional[List[str]] = None, + dryrun: bool = False, + duration: int = 24, + crash_report_timeout: Optional[int] = None, + debug: Optional[List[TaskDebugFlag]] = None, + check_retry_count: Optional[int] = None, + check_fuzzer_help: bool = True, + delete_input_container: bool = True, + check_regressions: bool = False, + ) -> None: + """ + generic regression task + + :param File crashes: Specify crashing input files to check in the regression task + :param str reports: Specify specific report names to verify in the regression task + :param bool check_regressions: Specify if exceptions should be thrown on finding crash regressions + :param bool delete_input_container: Specify wether or not to delete the input container + """ + + self._create_job( + TaskType.generic_regression, + project, + name, + build, + pool_name, + crashes=crashes, + reports=reports, + target_exe=target_exe, + tags=tags, + notification_config=notification_config, + target_env=target_env, + setup_dir=setup_dir, + reboot_after_setup=reboot_after_setup, + target_options=target_options, + dryrun=dryrun, + duration=duration, + crash_report_timeout=crash_report_timeout, + debug=debug, + check_retry_count=check_retry_count, + check_fuzzer_help=check_fuzzer_help, + delete_input_container=delete_input_container, + check_regressions=check_regressions, + ) + + def libfuzzer( + self, + project: str, + name: str, + build: str, + pool_name: PoolName, + *, + reports: Optional[List[str]] = None, + crashes: Optional[List[File]] = None, + target_exe: File = File("fuzz.exe"), + tags: Optional[Dict[str, str]] = None, + notification_config: Optional[NotificationConfig] = None, + target_env: Optional[Dict[str, str]] = None, + setup_dir: Optional[Directory] = None, + reboot_after_setup: bool = False, + target_options: Optional[List[str]] = None, + dryrun: bool = False, + duration: int = 24, + crash_report_timeout: Optional[int] = None, + debug: Optional[List[TaskDebugFlag]] = None, + check_retry_count: Optional[int] = None, + check_fuzzer_help: bool = True, + delete_input_container: bool = True, + check_regressions: bool = False, + ) -> None: + + """ + libfuzzer regression task + + :param File crashes: Specify crashing input files to check in the regression task + :param str reports: Specify specific report names to verify in the regression task + :param bool check_regressions: Specify if exceptions should be thrown on finding crash regressions + :param bool delete_input_container: Specify wether or not to delete the input container + """ + + self._create_job( + TaskType.libfuzzer_regression, + project, + name, + build, + pool_name, + crashes=crashes, + reports=reports, + target_exe=target_exe, + tags=tags, + notification_config=notification_config, + target_env=target_env, + setup_dir=setup_dir, + reboot_after_setup=reboot_after_setup, + target_options=target_options, + dryrun=dryrun, + duration=duration, + crash_report_timeout=crash_report_timeout, + debug=debug, + check_retry_count=check_retry_count, + check_fuzzer_help=check_fuzzer_help, + delete_input_container=delete_input_container, + check_regressions=check_regressions, + ) + + def _create_job( + self, + task_type: TaskType, + project: str, + name: str, + build: str, + pool_name: PoolName, + *, + crashes: Optional[List[File]] = None, + reports: Optional[List[str]] = None, + target_exe: File = File("fuzz.exe"), + tags: Optional[Dict[str, str]] = None, + notification_config: Optional[NotificationConfig] = None, + target_env: Optional[Dict[str, str]] = None, + setup_dir: Optional[Directory] = None, + reboot_after_setup: bool = False, + target_options: Optional[List[str]] = None, + dryrun: bool = False, + duration: int = 24, + crash_report_timeout: Optional[int] = None, + debug: Optional[List[TaskDebugFlag]] = None, + check_retry_count: Optional[int] = None, + check_fuzzer_help: bool = True, + delete_input_container: bool = True, + check_regressions: bool = False, + ) -> None: + + if dryrun: + return None + + self.logger.info("creating regression task from template") + + helper = JobHelper( + self.onefuzz, + self.logger, + project, + name, + build, + duration, + pool_name=pool_name, + target_exe=target_exe, + ) + + helper.define_containers( + ContainerType.setup, + ContainerType.crashes, + ContainerType.reports, + ContainerType.no_repro, + ContainerType.unique_reports, + ContainerType.regression_reports, + ) + + containers = [ + (ContainerType.setup, helper.containers[ContainerType.setup]), + (ContainerType.crashes, helper.containers[ContainerType.crashes]), + (ContainerType.reports, helper.containers[ContainerType.reports]), + (ContainerType.no_repro, helper.containers[ContainerType.no_repro]), + ( + ContainerType.unique_reports, + helper.containers[ContainerType.unique_reports], + ), + ( + ContainerType.regression_reports, + helper.containers[ContainerType.regression_reports], + ), + ] + + if crashes: + helper.containers[ + ContainerType.readonly_inputs + ] = helper.get_unique_container_name(ContainerType.readonly_inputs) + containers.append( + ( + ContainerType.readonly_inputs, + helper.containers[ContainerType.readonly_inputs], + ) + ) + + helper.create_containers() + if crashes: + for file in crashes: + self.onefuzz.containers.files.upload_file( + helper.containers[ContainerType.readonly_inputs], file + ) + + helper.setup_notifications(notification_config) + + helper.upload_setup(setup_dir, target_exe) + target_exe_blob_name = helper.target_exe_blob_name(target_exe, setup_dir) + + self.logger.info("creating regression task") + task = self.onefuzz.tasks.create( + helper.job.job_id, + task_type, + target_exe_blob_name, + containers, + pool_name=pool_name, + duration=duration, + vm_count=1, + reboot_after_setup=reboot_after_setup, + target_options=target_options, + target_env=target_env, + tags=tags, + target_timeout=crash_report_timeout, + check_retry_count=check_retry_count, + debug=debug, + check_fuzzer_help=check_fuzzer_help, + report_list=reports, + ) + helper.wait_for_stopped = check_regressions + + self.logger.info("done creating tasks") + helper.wait() + + if check_regressions: + task = self.onefuzz.tasks.get(task.task_id) + if task.error: + raise Exception("task failed: %s", task.error) + + container = helper.containers[ContainerType.regression_reports] + for filename in self.onefuzz.containers.files.list(container).files: + self.logger.info("checking file: %s", filename) + if self._check_regression(container, File(filename)): + raise Exception(f"regression identified: {filename}") + self.logger.info("no regressions") + + if ( + delete_input_container + and ContainerType.readonly_inputs in helper.containers + ): + helper.delete_container(helper.containers[ContainerType.readonly_inputs]) diff --git a/src/integration-tests/check-regression.py b/src/integration-tests/check-regression.py new file mode 100755 index 000000000..2ae0d7672 --- /dev/null +++ b/src/integration-tests/check-regression.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# +# Copyright (c) Microsoft Corporation. +# Licensed under the MIT License. +# + +import json +import logging +import os +import sys +import time +from typing import Optional +from uuid import UUID, uuid4 + +from onefuzz.api import Command, Onefuzz +from onefuzz.cli import execute_api +from onefuzztypes.enums import OS, ContainerType, TaskState, TaskType +from onefuzztypes.models import Job, RegressionReport +from onefuzztypes.primitives import Container, Directory, File, PoolName + + +class Run(Command): + def cleanup(self, test_id: UUID): + for pool in self.onefuzz.pools.list(): + if str(test_id) in pool.name: + self.onefuzz.pools.shutdown(pool.name, now=True) + + self.onefuzz.template.stop( + str(test_id), "linux-libfuzzer", build=None, delete_containers=True + ) + + def _wait_for_regression_task(self, job: Job) -> None: + while True: + self.logger.info("waiting for regression task to finish") + for task in self.onefuzz.jobs.tasks.list(job.job_id): + if task.config.task.type not in [ + TaskType.libfuzzer_regression, + TaskType.generic_regression, + ]: + continue + if task.state != TaskState.stopped: + continue + return + time.sleep(10) + + def _check_regression(self, job: Job) -> bool: + # get the regression reports containers for the job + results = self.onefuzz.jobs.containers.list( + job.job_id, ContainerType.regression_reports + ) + + # expect one and only one regression report container + if len(results) != 1: + raise Exception(f"unexpected regression containers: {results}") + container = list(results.keys())[0] + + # expect one and only one file in the container + if len(results[container]) != 1: + raise Exception(f"unexpected regression container output: {results}") + file = results[container][0] + + # get the regression report + content = self.onefuzz.containers.files.get(Container(container), file) + as_str = content.decode() + as_obj = json.loads(as_str) + report = RegressionReport.parse_obj(as_obj) + + if report.crash_test_result.crash_report is not None: + self.logger.info("regression report has crash report") + return True + + if report.crash_test_result.no_repro is not None: + self.logger.info("regression report has no-repro") + return False + + raise Exception(f"unexpected report: {report}") + + def _run_job( + self, test_id: UUID, pool: PoolName, target: str, exe: File, build: int + ) -> Job: + if build == 1: + wait_for_files = [ContainerType.unique_reports] + else: + wait_for_files = [ContainerType.regression_reports] + job = self.onefuzz.template.libfuzzer.basic( + str(test_id), + target, + str(build), + pool, + target_exe=exe, + duration=1, + vm_count=1, + wait_for_files=wait_for_files, + ) + if job is None: + raise Exception(f"invalid job: {target} {build}") + + if build > 1: + self._wait_for_regression_task(job) + self.onefuzz.template.stop(str(test_id), target, str(build)) + return job + + def _run(self, target_os: OS, test_id: UUID, base: Directory, target: str) -> None: + pool = PoolName(f"{target}-{target_os.name}-{test_id}") + self.onefuzz.pools.create(pool, target_os) + self.onefuzz.scalesets.create(pool, 5) + broken = File(os.path.join(base, target, "broken.exe")) + fixed = File(os.path.join(base, target, "fixed.exe")) + + self.logger.info("starting first build") + self._run_job(test_id, pool, target, broken, 1) + + self.logger.info("starting second build") + job = self._run_job(test_id, pool, target, fixed, 2) + if self._check_regression(job): + raise Exception("fixed binary should be a no repro") + + self.logger.info("starting third build") + job = self._run_job(test_id, pool, target, broken, 3) + if not self._check_regression(job): + raise Exception("broken binary should be a crash report") + + self.onefuzz.pools.shutdown(pool, now=True) + + def test( + self, + samples: Directory, + *, + endpoint: Optional[str] = None, + ): + test_id = uuid4() + self.logger.info(f"launch test {test_id}") + self.onefuzz.__setup__(endpoint=endpoint) + error: Optional[Exception] = None + try: + self._run(OS.linux, test_id, samples, "linux-libfuzzer-regression") + except Exception as err: + error = err + except KeyboardInterrupt: + self.logger.warning("interruptted") + finally: + self.logger.info("cleaning up tests") + self.cleanup(test_id) + + if error: + raise error + + +def main() -> int: + return execute_api( + Run(Onefuzz(), logging.getLogger("regression")), [Command], "0.0.1" + ) + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/src/integration-tests/git-bisect/README.md b/src/integration-tests/git-bisect/README.md new file mode 100644 index 000000000..eabf5f0bc --- /dev/null +++ b/src/integration-tests/git-bisect/README.md @@ -0,0 +1,21 @@ +# git-bisect regression source + +This assumes you have a working clang with libfuzzer, bash, and git. + +This makes a git repo `test` with 9 commits. Each commit after the first adds a bug. + +* `commit 0` has no bugs. +* `commit 1` will additionally cause an abort if the input is `1`. +* `commit 2` will additionally cause an abort if the input is `2`. +* `commit 3` will additionally cause an abort if the input is `3`. +* etc. + +This directory provides exemplar scripts that demonstrate how to perform + `git bisect` with libfuzzer. + + * [run-local.sh](run-local.sh) builds & runs the libfuzzer target locally. It uses [src/bisect-local.sh](src/bisect-local.sh) as the `git bisect run` command. + * [run-onefuzz.sh](run-onefuzz.sh) builds the libfuzzer target locally, but uses OneFuzz to run the regression tasks. It uses [src/bisect-onefuzz.sh](src/bisect-onefuzz.sh) as the `git bisect run` command. + +With each project having their own unique paradigm for building, this model +allows plugging OneFuzz as a `bisect` command in whatever fashion your +project requires. \ No newline at end of file diff --git a/src/integration-tests/git-bisect/build.sh b/src/integration-tests/git-bisect/build.sh new file mode 100755 index 000000000..756b89fbd --- /dev/null +++ b/src/integration-tests/git-bisect/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -e + +rm -rf test + +git init test +(cd test; git config user.name "Example"; git config user.email example@contoso.com) +(cp src/Makefile test; cd test; git add Makefile) +for i in $(seq 0 8); do + cp src/fuzz.c test/fuzz.c + for j in $(seq $i 8); do + if [ $i != $j ]; then + sed -i /TEST$j/d test/fuzz.c + fi + done + (cd test; git add fuzz.c; git commit -m "commit $i") +done \ No newline at end of file diff --git a/src/integration-tests/git-bisect/run-local.sh b/src/integration-tests/git-bisect/run-local.sh new file mode 100755 index 000000000..3d641d259 --- /dev/null +++ b/src/integration-tests/git-bisect/run-local.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +# build our git repo with our samples in `test` +# (note, we don't care about the output of this script) +./build.sh 2>/dev/null > /dev/null + +# create our crashing input +echo -n '3' > test/test.txt + +cd test + +# start the bisect, looking from HEAD backwards 8 commits +git bisect start HEAD HEAD~8 -- +git bisect run ../src/bisect-local.sh test.txt +git bisect reset \ No newline at end of file diff --git a/src/integration-tests/git-bisect/run-onefuzz.sh b/src/integration-tests/git-bisect/run-onefuzz.sh new file mode 100755 index 000000000..7c761cadc --- /dev/null +++ b/src/integration-tests/git-bisect/run-onefuzz.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e + +# build our git repo with our samples in `test` +# (note, we don't care about the output of this script) +./build.sh 2>/dev/null > /dev/null + +# create our crashing input +echo -n '3' > test/test.txt + +cd test + +# start the bisect, looking from HEAD backwards 8 commits +git bisect start HEAD HEAD~8 -- +git bisect run ../src/bisect-onefuzz.sh test.txt +git bisect reset \ No newline at end of file diff --git a/src/integration-tests/git-bisect/src/Makefile b/src/integration-tests/git-bisect/src/Makefile new file mode 100644 index 000000000..f01ebbf5c --- /dev/null +++ b/src/integration-tests/git-bisect/src/Makefile @@ -0,0 +1,13 @@ +CC=clang + +CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address + +all: fuzz.exe + +fuzz.exe: fuzz.c + $(CC) $(CFLAGS) fuzz.c -o fuzz.exe + +.PHONY: clean + +clean: + @rm -f fuzz.exe diff --git a/src/integration-tests/git-bisect/src/bisect-local.sh b/src/integration-tests/git-bisect/src/bisect-local.sh new file mode 100755 index 000000000..69d43001f --- /dev/null +++ b/src/integration-tests/git-bisect/src/bisect-local.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -ex + +make clean +make +./fuzz.exe $* \ No newline at end of file diff --git a/src/integration-tests/git-bisect/src/bisect-onefuzz.sh b/src/integration-tests/git-bisect/src/bisect-onefuzz.sh new file mode 100755 index 000000000..2f7ba2969 --- /dev/null +++ b/src/integration-tests/git-bisect/src/bisect-onefuzz.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e + +PROJECT=${PROJECT:-regression-test} +TARGET=${TARGET:-$(uuidgen)} +BUILD=regression-$(git rev-parse HEAD) +POOL=${ONEFUZZ_POOL:-linux} + +make clean +make +onefuzz template regression libfuzzer ${PROJECT} ${TARGET} ${BUILD} ${POOL} --check_regressions --delete_input_container --reports --crashes $* \ No newline at end of file diff --git a/src/integration-tests/git-bisect/src/fuzz.c b/src/integration-tests/git-bisect/src/fuzz.c new file mode 100644 index 000000000..53c2cd1cb --- /dev/null +++ b/src/integration-tests/git-bisect/src/fuzz.c @@ -0,0 +1,13 @@ +#include +int LLVMFuzzerTestOneInput(char *data, size_t len) { + if (len != 1) { return 0; } + if (data[0] == '1') { abort(); } // TEST1 + if (data[0] == '2') { abort(); } // TEST2 + if (data[0] == '3') { abort(); } // TEST3 + if (data[0] == '4') { abort(); } // TEST4 + if (data[0] == '5') { abort(); } // TEST5 + if (data[0] == '6') { abort(); } // TEST6 + if (data[0] == '7') { abort(); } // TEST7 + if (data[0] == '8') { abort(); } // TEST8 + return 0; +} diff --git a/src/integration-tests/libfuzzer-regression/Makefile b/src/integration-tests/libfuzzer-regression/Makefile new file mode 100644 index 000000000..b8a0f6e84 --- /dev/null +++ b/src/integration-tests/libfuzzer-regression/Makefile @@ -0,0 +1,16 @@ +CC=clang + +CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address + +all: broken.exe fixed.exe + +broken.exe: simple.c + $(CC) $(CFLAGS) simple.c -o broken.exe + +fixed.exe: simple.c + $(CC) $(CFLAGS) simple.c -o fixed.exe -DFIXED + +.PHONY: clean + +clean: + rm -f broken.exe fixed.exe diff --git a/src/integration-tests/libfuzzer-regression/seeds/good.txt b/src/integration-tests/libfuzzer-regression/seeds/good.txt new file mode 100644 index 000000000..12799ccbe --- /dev/null +++ b/src/integration-tests/libfuzzer-regression/seeds/good.txt @@ -0,0 +1 @@ +good diff --git a/src/integration-tests/libfuzzer-regression/simple.c b/src/integration-tests/libfuzzer-regression/simple.c new file mode 100644 index 000000000..8b02c742d --- /dev/null +++ b/src/integration-tests/libfuzzer-regression/simple.c @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include + + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) { + int cnt = 0; + + if (len < 3) { + return 0; + } + + if (data[0] == 'x') { cnt++; } + if (data[1] == 'y') { cnt++; } + if (data[2] == 'z') { cnt++; } + +#ifndef FIXED + if (cnt >= 3) { + abort(); + } +#endif + + return 0; +} diff --git a/src/pytypes/extra/generate-docs.py b/src/pytypes/extra/generate-docs.py index f551812be..c2c39c446 100755 --- a/src/pytypes/extra/generate-docs.py +++ b/src/pytypes/extra/generate-docs.py @@ -31,6 +31,7 @@ from onefuzztypes.events import ( EventProxyCreated, EventProxyDeleted, EventProxyFailed, + EventRegressionReported, EventScalesetCreated, EventScalesetDeleted, EventScalesetFailed, @@ -45,8 +46,10 @@ from onefuzztypes.events import ( ) from onefuzztypes.models import ( BlobRef, + CrashTestResult, Error, JobConfig, + RegressionReport, Report, TaskConfig, TaskContainers, @@ -87,6 +90,24 @@ def main() -> None: ], tags={}, ) + report = Report( + input_blob=BlobRef( + account="contoso-storage-account", + container=Container("crashes"), + name="input.txt", + ), + executable="fuzz.exe", + crash_type="example crash report type", + crash_site="example crash site", + call_stack=["#0 line", "#1 line", "#2 line"], + call_stack_sha256=ZERO_SHA256, + input_sha256=EMPTY_SHA256, + asan_log="example asan log", + task_id=UUID(int=0), + job_id=UUID(int=0), + scariness_score=10, + scariness_description="example-scariness", + ) examples: List[Event] = [ EventPing(ping_id=UUID(int=0)), EventTaskCreated( @@ -193,27 +214,18 @@ def main() -> None: pool_name=PoolName("example"), state=NodeState.setting_up, ), + EventRegressionReported( + regression_report=RegressionReport( + crash_test_result=CrashTestResult(crash_report=report), + original_crash_test_result=CrashTestResult(crash_report=report), + ), + container=Container("container-name"), + filename="example.json", + ), EventCrashReported( container=Container("container-name"), filename="example.json", - report=Report( - input_blob=BlobRef( - account="contoso-storage-account", - container=Container("crashes"), - name="input.txt", - ), - executable="fuzz.exe", - crash_type="example crash report type", - crash_site="example crash site", - call_stack=["#0 line", "#1 line", "#2 line"], - call_stack_sha256=ZERO_SHA256, - input_sha256=EMPTY_SHA256, - asan_log="example asan log", - task_id=UUID(int=0), - job_id=UUID(int=0), - scariness_score=10, - scariness_description="example-scariness", - ), + report=report, ), EventFileAdded(container=Container("container-name"), filename="example.txt"), EventNodeHeartbeat(machine_id=UUID(int=0), pool_name=PoolName("example")), diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index 869e0dd1c..bb856ee85 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -78,6 +78,7 @@ class TaskFeature(Enum): preserve_existing_outputs = "preserve_existing_outputs" check_fuzzer_help = "check_fuzzer_help" expect_crash_on_failure = "expect_crash_on_failure" + report_list = "report_list" # Permissions for an Azure Blob Storage Container. @@ -149,11 +150,13 @@ class TaskType(Enum): libfuzzer_coverage = "libfuzzer_coverage" libfuzzer_crash_report = "libfuzzer_crash_report" libfuzzer_merge = "libfuzzer_merge" + libfuzzer_regression = "libfuzzer_regression" generic_analysis = "generic_analysis" generic_supervisor = "generic_supervisor" generic_merge = "generic_merge" generic_generator = "generic_generator" generic_crash_report = "generic_crash_report" + generic_regression = "generic_regression" class VmState(Enum): @@ -207,6 +210,7 @@ class ContainerType(Enum): tools = "tools" unique_inputs = "unique_inputs" unique_reports = "unique_reports" + regression_reports = "regression_reports" @classmethod def reset_defaults(cls) -> List["ContainerType"]: @@ -219,8 +223,9 @@ class ContainerType(Enum): cls.readonly_inputs, cls.reports, cls.setup, - cls.unique_reports, cls.unique_inputs, + cls.unique_reports, + cls.regression_reports, ] @classmethod diff --git a/src/pytypes/onefuzztypes/events.py b/src/pytypes/onefuzztypes/events.py index fc78334a3..9e43e3296 100644 --- a/src/pytypes/onefuzztypes/events.py +++ b/src/pytypes/onefuzztypes/events.py @@ -11,7 +11,15 @@ from uuid import UUID, uuid4 from pydantic import BaseModel, Extra, Field from .enums import OS, Architecture, NodeState, TaskState, TaskType -from .models import AutoScaleConfig, Error, JobConfig, Report, TaskConfig, UserInfo +from .models import ( + AutoScaleConfig, + Error, + JobConfig, + RegressionReport, + Report, + TaskConfig, + UserInfo, +) from .primitives import Container, PoolName, Region from .responses import BaseResponse @@ -156,6 +164,12 @@ class EventCrashReported(BaseEvent): filename: str +class EventRegressionReported(BaseEvent): + regression_report: RegressionReport + container: Container + filename: str + + class EventFileAdded(BaseEvent): container: Container filename: str @@ -183,6 +197,7 @@ Event = Union[ EventTaskStopped, EventTaskHeartbeat, EventCrashReported, + EventRegressionReported, EventFileAdded, ] @@ -207,6 +222,7 @@ class EventType(Enum): task_state_updated = "task_state_updated" task_stopped = "task_stopped" crash_reported = "crash_reported" + regression_reported = "regression_reported" file_added = "file_added" task_heartbeat = "task_heartbeat" node_heartbeat = "node_heartbeat" @@ -234,6 +250,7 @@ EventTypeMap = { EventType.task_heartbeat: EventTaskHeartbeat, EventType.task_stopped: EventTaskStopped, EventType.crash_reported: EventCrashReported, + EventType.regression_reported: EventRegressionReported, EventType.file_added: EventFileAdded, } diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index f6a4996c0..414373138 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -170,6 +170,7 @@ class TaskDetails(BaseModel): target_timeout: Optional[int] ensemble_sync_delay: Optional[int] preserve_existing_outputs: Optional[bool] + report_list: Optional[List[str]] @validator("check_retry_count", allow_reuse=True) def validate_check_retry_count(cls, value: int) -> int: @@ -237,7 +238,7 @@ class BlobRef(BaseModel): class Report(BaseModel): input_url: Optional[str] - input_blob: BlobRef + input_blob: Optional[BlobRef] executable: str crash_type: str crash_site: str @@ -251,6 +252,26 @@ class Report(BaseModel): scariness_description: Optional[str] +class NoReproReport(BaseModel): + input_sha256: str + input_blob: Optional[BlobRef] + executable: str + task_id: UUID + job_id: UUID + tries: int + error: Optional[str] + + +class CrashTestResult(BaseModel): + crash_report: Optional[Report] + no_repro: Optional[NoReproReport] + + +class RegressionReport(BaseModel): + crash_test_result: CrashTestResult + original_crash_test_result: Optional[CrashTestResult] + + class ADODuplicateTemplate(BaseModel): increment: List[str] comment: Optional[str] @@ -377,6 +398,7 @@ class TaskUnitConfig(BaseModel): stats_file: Optional[str] stats_format: Optional[StatsFormat] ensemble_sync_delay: Optional[int] + report_list: Optional[List[str]] # from here forwards are Container definitions. These need to be inline # with TaskDefinitions and ContainerTypes @@ -390,6 +412,7 @@ class TaskUnitConfig(BaseModel): tools: CONTAINER_DEF unique_inputs: CONTAINER_DEF unique_reports: CONTAINER_DEF + regression_reports: CONTAINER_DEF class Forward(BaseModel):