ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-11 09:41:37 +00:00
Code Issues Packages Projects Releases Wiki Activity

add regression testing tasks (#664)

Browse Source
This commit is contained in:
bmc-msft 2021-03-18 15:37:19 -04:00 committed by GitHub
parent 34b2a739cb
commit 6e60a8cf10
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
50 changed files with 2141 additions and 203 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.github/workflows/ci.yml vendored
View File

@ -358,6 +358,10 @@ jobs:
(cd libfuzzer ; make )
cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer
mkdir -p artifacts/linux-libfuzzer-regression
(cd libfuzzer-regression ; make )
cp -r libfuzzer-regression/broken.exe libfuzzer-regression/fixed.exe libfuzzer-regression/seeds artifacts/linux-libfuzzer-regression
mkdir -p artifacts/linux-trivial-crash
(cd trivial-crash ; make )
cp -r trivial-crash/fuzz.exe trivial-crash/seeds artifacts/linux-trivial-crash

442
docs/webhook_events.md
View File

@ -37,6 +37,7 @@ Each event will be submitted via HTTP POST to the user provided URL.
* [proxy_created](#proxy_created)
* [proxy_deleted](#proxy_deleted)
* [proxy_failed](#proxy_failed)
* [regression_reported](#regression_reported)
* [scaleset_created](#scaleset_created)
* [scaleset_deleted](#scaleset_deleted)
* [scaleset_failed](#scaleset_failed)
@ -168,7 +169,6 @@ Each event will be submitted via HTTP POST to the user provided URL.
}
},
"required": [
"input_blob",
"executable",
"crash_type",
"crash_site",
@ -475,11 +475,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"libfuzzer_regression",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
"generic_crash_report",
"generic_regression"
],
"title": "TaskType"
},
@ -1030,6 +1032,261 @@ Each event will be submitted via HTTP POST to the user provided URL.
}
```
### regression_reported
#### Example
```json
{
"container": "container-name",
"filename": "example.json",
"regression_report": {
"crash_test_result": {
"crash_report": {
"asan_log": "example asan log",
"call_stack": [
"#0 line",
"#1 line",
"#2 line"
],
"call_stack_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
"crash_site": "example crash site",
"crash_type": "example crash report type",
"executable": "fuzz.exe",
"input_blob": {
"account": "contoso-storage-account",
"container": "crashes",
"name": "input.txt"
},
"input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"job_id": "00000000-0000-0000-0000-000000000000",
"scariness_description": "example-scariness",
"scariness_score": 10,
"task_id": "00000000-0000-0000-0000-000000000000"
}
},
"original_crash_test_result": {
"crash_report": {
"asan_log": "example asan log",
"call_stack": [
"#0 line",
"#1 line",
"#2 line"
],
"call_stack_sha256": "0000000000000000000000000000000000000000000000000000000000000000",
"crash_site": "example crash site",
"crash_type": "example crash report type",
"executable": "fuzz.exe",
"input_blob": {
"account": "contoso-storage-account",
"container": "crashes",
"name": "input.txt"
},
"input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"job_id": "00000000-0000-0000-0000-000000000000",
"scariness_description": "example-scariness",
"scariness_score": 10,
"task_id": "00000000-0000-0000-0000-000000000000"
}
}
}
}
```
#### Schema
```json
{
"additionalProperties": false,
"definitions": {
"BlobRef": {
"properties": {
"account": {
"title": "Account",
"type": "string"
},
"container": {
"title": "Container",
"type": "string"
},
"name": {
"title": "Name",
"type": "string"
}
},
"required": [
"account",
"container",
"name"
],
"title": "BlobRef",
"type": "object"
},
"CrashTestResult": {
"properties": {
"crash_report": {
"$ref": "#/definitions/Report"
},
"no_repro": {
"$ref": "#/definitions/NoReproReport"
}
},
"title": "CrashTestResult",
"type": "object"
},
"NoReproReport": {
"properties": {
"error": {
"title": "Error",
"type": "string"
},
"executable": {
"title": "Executable",
"type": "string"
},
"input_blob": {
"$ref": "#/definitions/BlobRef"
},
"input_sha256": {
"title": "Input Sha256",
"type": "string"
},
"job_id": {
"format": "uuid",
"title": "Job Id",
"type": "string"
},
"task_id": {
"format": "uuid",
"title": "Task Id",
"type": "string"
},
"tries": {
"title": "Tries",
"type": "integer"
}
},
"required": [
"input_sha256",
"executable",
"task_id",
"job_id",
"tries"
],
"title": "NoReproReport",
"type": "object"
},
"RegressionReport": {
"properties": {
"crash_test_result": {
"$ref": "#/definitions/CrashTestResult"
},
"original_crash_test_result": {
"$ref": "#/definitions/CrashTestResult"
}
},
"required": [
"crash_test_result"
],
"title": "RegressionReport",
"type": "object"
},
"Report": {
"properties": {
"asan_log": {
"title": "Asan Log",
"type": "string"
},
"call_stack": {
"items": {
"type": "string"
},
"title": "Call Stack",
"type": "array"
},
"call_stack_sha256": {
"title": "Call Stack Sha256",
"type": "string"
},
"crash_site": {
"title": "Crash Site",
"type": "string"
},
"crash_type": {
"title": "Crash Type",
"type": "string"
},
"executable": {
"title": "Executable",
"type": "string"
},
"input_blob": {
"$ref": "#/definitions/BlobRef"
},
"input_sha256": {
"title": "Input Sha256",
"type": "string"
},
"input_url": {
"title": "Input Url",
"type": "string"
},
"job_id": {
"format": "uuid",
"title": "Job Id",
"type": "string"
},
"scariness_description": {
"title": "Scariness Description",
"type": "string"
},
"scariness_score": {
"title": "Scariness Score",
"type": "integer"
},
"task_id": {
"format": "uuid",
"title": "Task Id",
"type": "string"
}
},
"required": [
"executable",
"crash_type",
"crash_site",
"call_stack",
"call_stack_sha256",
"input_sha256",
"task_id",
"job_id"
],
"title": "Report",
"type": "object"
}
},
"properties": {
"container": {
"title": "Container",
"type": "string"
},
"filename": {
"title": "Filename",
"type": "string"
},
"regression_report": {
"$ref": "#/definitions/RegressionReport"
}
},
"required": [
"regression_report",
"container",
"filename"
],
"title": "EventRegressionReported",
"type": "object"
}
```
### scaleset_created
#### Example
@ -1283,7 +1540,8 @@ Each event will be submitted via HTTP POST to the user provided URL.
"setup",
"tools",
"unique_inputs",
"unique_reports"
"unique_reports",
"regression_reports"
],
"title": "ContainerType"
},
@ -1456,6 +1714,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Rename Output",
"type": "boolean"
},
"report_list": {
"items": {
"type": "string"
},
"title": "Report List",
"type": "array"
},
"stats_file": {
"title": "Stats File",
"type": "string"
@ -1554,11 +1819,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"libfuzzer_regression",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
"generic_crash_report",
"generic_regression"
],
"title": "TaskType"
},
@ -1715,7 +1982,8 @@ Each event will be submitted via HTTP POST to the user provided URL.
"setup",
"tools",
"unique_inputs",
"unique_reports"
"unique_reports",
"regression_reports"
],
"title": "ContainerType"
},
@ -1936,6 +2204,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Rename Output",
"type": "boolean"
},
"report_list": {
"items": {
"type": "string"
},
"title": "Report List",
"type": "array"
},
"stats_file": {
"title": "Stats File",
"type": "string"
@ -2034,11 +2309,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"libfuzzer_regression",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
"generic_crash_report",
"generic_regression"
],
"title": "TaskType"
},
@ -2188,7 +2465,8 @@ Each event will be submitted via HTTP POST to the user provided URL.
"setup",
"tools",
"unique_inputs",
"unique_reports"
"unique_reports",
"regression_reports"
],
"title": "ContainerType"
},
@ -2361,6 +2639,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Rename Output",
"type": "boolean"
},
"report_list": {
"items": {
"type": "string"
},
"title": "Report List",
"type": "array"
},
"stats_file": {
"title": "Stats File",
"type": "string"
@ -2459,11 +2744,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"libfuzzer_regression",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
"generic_crash_report",
"generic_regression"
],
"title": "TaskType"
},
@ -2587,7 +2874,8 @@ Each event will be submitted via HTTP POST to the user provided URL.
"setup",
"tools",
"unique_inputs",
"unique_reports"
"unique_reports",
"regression_reports"
],
"title": "ContainerType"
},
@ -2760,6 +3048,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Rename Output",
"type": "boolean"
},
"report_list": {
"items": {
"type": "string"
},
"title": "Report List",
"type": "array"
},
"stats_file": {
"title": "Stats File",
"type": "string"
@ -2872,11 +3167,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"libfuzzer_regression",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
"generic_crash_report",
"generic_regression"
],
"title": "TaskType"
},
@ -3013,7 +3310,8 @@ Each event will be submitted via HTTP POST to the user provided URL.
"setup",
"tools",
"unique_inputs",
"unique_reports"
"unique_reports",
"regression_reports"
],
"title": "ContainerType"
},
@ -3186,6 +3484,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Rename Output",
"type": "boolean"
},
"report_list": {
"items": {
"type": "string"
},
"title": "Report List",
"type": "array"
},
"stats_file": {
"title": "Stats File",
"type": "string"
@ -3284,11 +3589,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"libfuzzer_regression",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
"generic_crash_report",
"generic_regression"
],
"title": "TaskType"
},
@ -3468,10 +3775,23 @@ Each event will be submitted via HTTP POST to the user provided URL.
"setup",
"tools",
"unique_inputs",
"unique_reports"
"unique_reports",
"regression_reports"
],
"title": "ContainerType"
},
"CrashTestResult": {
"properties": {
"crash_report": {
"$ref": "#/definitions/Report"
},
"no_repro": {
"$ref": "#/definitions/NoReproReport"
}
},
"title": "CrashTestResult",
"type": "object"
},
"Error": {
"properties": {
"code": {
@ -3821,6 +4141,29 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "EventProxyFailed",
"type": "object"
},
"EventRegressionReported": {
"additionalProperties": false,
"properties": {
"container": {
"title": "Container",
"type": "string"
},
"filename": {
"title": "Filename",
"type": "string"
},
"regression_report": {
"$ref": "#/definitions/RegressionReport"
}
},
"required": [
"regression_report",
"container",
"filename"
],
"title": "EventRegressionReported",
"type": "object"
},
"EventScalesetCreated": {
"additionalProperties": false,
"properties": {
@ -4074,6 +4417,7 @@ Each event will be submitted via HTTP POST to the user provided URL.
"task_state_updated",
"task_stopped",
"crash_reported",
"regression_reported",
"file_added",
"task_heartbeat",
"node_heartbeat"
@ -4129,6 +4473,48 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "JobTaskStopped",
"type": "object"
},
"NoReproReport": {
"properties": {
"error": {
"title": "Error",
"type": "string"
},
"executable": {
"title": "Executable",
"type": "string"
},
"input_blob": {
"$ref": "#/definitions/BlobRef"
},
"input_sha256": {
"title": "Input Sha256",
"type": "string"
},
"job_id": {
"format": "uuid",
"title": "Job Id",
"type": "string"
},
"task_id": {
"format": "uuid",
"title": "Task Id",
"type": "string"
},
"tries": {
"title": "Tries",
"type": "integer"
}
},
"required": [
"input_sha256",
"executable",
"task_id",
"job_id",
"tries"
],
"title": "NoReproReport",
"type": "object"
},
"NodeState": {
"description": "An enumeration.",
"enum": [
@ -4152,6 +4538,21 @@ Each event will be submitted via HTTP POST to the user provided URL.
],
"title": "OS"
},
"RegressionReport": {
"properties": {
"crash_test_result": {
"$ref": "#/definitions/CrashTestResult"
},
"original_crash_test_result": {
"$ref": "#/definitions/CrashTestResult"
}
},
"required": [
"crash_test_result"
],
"title": "RegressionReport",
"type": "object"
},
"Report": {
"properties": {
"asan_log": {
@ -4212,7 +4613,6 @@ Each event will be submitted via HTTP POST to the user provided URL.
}
},
"required": [
"input_blob",
"executable",
"crash_type",
"crash_site",
@ -4394,6 +4794,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"title": "Rename Output",
"type": "boolean"
},
"report_list": {
"items": {
"type": "string"
},
"title": "Report List",
"type": "array"
},
"stats_file": {
"title": "Stats File",
"type": "string"
@ -4506,11 +4913,13 @@ Each event will be submitted via HTTP POST to the user provided URL.
"libfuzzer_coverage",
"libfuzzer_crash_report",
"libfuzzer_merge",
"libfuzzer_regression",
"generic_analysis",
"generic_supervisor",
"generic_merge",
"generic_generator",
"generic_crash_report"
"generic_crash_report",
"generic_regression"
],
"title": "TaskType"
},
@ -4638,6 +5047,9 @@ Each event will be submitted via HTTP POST to the user provided URL.
{
"$ref": "#/definitions/EventCrashReported"
},
{
"$ref": "#/definitions/EventRegressionReported"
},
{
"$ref": "#/definitions/EventFileAdded"
}

28
src/agent/onefuzz-agent/src/tasks/config.rs
View File

@ -2,7 +2,11 @@
// Licensed under the MIT License.
#![allow(clippy::large_enum_variant)]
use crate::tasks::{analysis, coverage, fuzz, heartbeat::*, merge, report};
use crate::tasks::{
analysis, coverage, fuzz,
heartbeat::{init_task_heartbeat, TaskHeartbeatClient},
merge, regression, report,
};
use anyhow::Result;
use onefuzz::machine_id::{get_machine_id, get_scaleset_name};
use onefuzz_telemetry::{
@ -70,6 +74,9 @@ pub enum Config {
#[serde(alias = "libfuzzer_coverage")]
LibFuzzerCoverage(coverage::libfuzzer_coverage::Config),
#[serde(alias = "libfuzzer_regression")]
LibFuzzerRegression(regression::libfuzzer::Config),
#[serde(alias = "generic_analysis")]
GenericAnalysis(analysis::generic::Config),
@ -84,6 +91,9 @@ pub enum Config {
#[serde(alias = "generic_crash_report")]
GenericReport(report::generic::Config),
#[serde(alias = "generic_regression")]
GenericRegression(regression::generic::Config),
}
impl Config {
@ -104,11 +114,13 @@ impl Config {
Config::LibFuzzerMerge(c) => &mut c.common,
Config::LibFuzzerReport(c) => &mut c.common,
Config::LibFuzzerCoverage(c) => &mut c.common,
Config::LibFuzzerRegression(c) => &mut c.common,
Config::GenericAnalysis(c) => &mut c.common,
Config::GenericMerge(c) => &mut c.common,
Config::GenericReport(c) => &mut c.common,
Config::GenericSupervisor(c) => &mut c.common,
Config::GenericGenerator(c) => &mut c.common,
Config::GenericRegression(c) => &mut c.common,
}
}
@ -118,11 +130,13 @@ impl Config {
Config::LibFuzzerMerge(c) => &c.common,
Config::LibFuzzerReport(c) => &c.common,
Config::LibFuzzerCoverage(c) => &c.common,
Config::LibFuzzerRegression(c) => &c.common,
Config::GenericAnalysis(c) => &c.common,
Config::GenericMerge(c) => &c.common,
Config::GenericReport(c) => &c.common,
Config::GenericSupervisor(c) => &c.common,
Config::GenericGenerator(c) => &c.common,
Config::GenericRegression(c) => &c.common,
}
}
@ -132,11 +146,13 @@ impl Config {
Config::LibFuzzerMerge(_) => "libfuzzer_merge",
Config::LibFuzzerReport(_) => "libfuzzer_crash_report",
Config::LibFuzzerCoverage(_) => "libfuzzer_coverage",
Config::LibFuzzerRegression(_) => "libfuzzer_regression",
Config::GenericAnalysis(_) => "generic_analysis",
Config::GenericMerge(_) => "generic_merge",
Config::GenericReport(_) => "generic_crash_report",
Config::GenericSupervisor(_) => "generic_supervisor",
Config::GenericGenerator(_) => "generic_generator",
Config::GenericRegression(_) => "generic_regression",
};
match self {
@ -193,6 +209,16 @@ impl Config {
Config::GenericReport(config) => {
report::generic::ReportTask::new(config).managed_run().await
}
Config::GenericRegression(config) => {
regression::generic::GenericRegressionTask::new(config)
.run()
.await
}
Config::LibFuzzerRegression(config) => {
regression::libfuzzer::LibFuzzerRegressionTask::new(config)
.run()
.await
}
}
}
}

2
src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs
View File

@ -3,7 +3,7 @@
use crate::tasks::{
config::CommonConfig,
heartbeat::*,
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::{self, default_bool_true},
};
use anyhow::{Context, Result};

2
src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs
View File

@ -4,7 +4,7 @@
#![allow(clippy::too_many_arguments)]
use crate::tasks::{
config::{CommonConfig, ContainerType},
heartbeat::*,
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
report::crash_report::monitor_reports,
stats::common::{monitor_stats, StatsFormat},
utils::CheckNotify,

2
src/agent/onefuzz-agent/src/tasks/merge/libfuzzer_merge.rs
View File

@ -3,7 +3,7 @@
use crate::tasks::{
config::CommonConfig,
heartbeat::*,
heartbeat::HeartbeatSender,
utils::{self, default_bool_true},
};
use anyhow::Result;

1
src/agent/onefuzz-agent/src/tasks/mod.rs
View File

@ -8,6 +8,7 @@ pub mod fuzz;
pub mod generic;
pub mod heartbeat;
pub mod merge;
pub mod regression;
pub mod report;
pub mod stats;
pub mod utils;

162
src/agent/onefuzz-agent/src/tasks/regression/common.rs Normal file
View File

@ -0,0 +1,162 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::tasks::{
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
report::crash_report::{parse_report_file, CrashTestResult, RegressionReport},
};
use anyhow::{Context, Result};
use async_trait::async_trait;
use onefuzz::syncdir::SyncedDir;
use reqwest::Url;
use std::path::PathBuf;
/// Defines implementation-provided callbacks for all implementers of regression tasks.
///
/// Shared regression task behavior is implemented in this module.
#[async_trait]
pub trait RegressionHandler {
/// Test the provided input and generate a crash result
/// * `input` - path to the input to test
/// * `input_url` - input url
async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result<CrashTestResult>;
}
/// Runs the regression task
pub async fn run(
heartbeat_client: Option<TaskHeartbeatClient>,
regression_reports: &SyncedDir,
crashes: &SyncedDir,
report_dirs: &[&SyncedDir],
report_list: &Option<Vec<String>>,
readonly_inputs: &Option<SyncedDir>,
handler: &impl RegressionHandler,
) -> Result<()> {
info!("Starting generic regression task");
handle_crash_reports(
handler,
crashes,
report_dirs,
report_list,
&regression_reports,
&heartbeat_client,
)
.await?;
if let Some(readonly_inputs) = &readonly_inputs {
handle_inputs(
handler,
readonly_inputs,
&regression_reports,
&heartbeat_client,
)
.await?;
}
Ok(())
}
/// Run the regression on the files in the 'inputs' location
/// * `handler` - regression handler
/// * `readonly_inputs` - location of the input files
/// * `regression_reports` - where reports should be saved
/// * `heartbeat_client` - heartbeat client
pub async fn handle_inputs(
handler: &impl RegressionHandler,
readonly_inputs: &SyncedDir,
regression_reports: &SyncedDir,
heartbeat_client: &Option<TaskHeartbeatClient>,
) -> Result<()> {
readonly_inputs.init_pull().await?;
let mut input_files = tokio::fs::read_dir(&readonly_inputs.path).await?;
while let Some(file) = input_files.next_entry().await? {
heartbeat_client.alive();
let file_path = file.path();
if !file_path.is_file() {
continue;
}
let file_name = file_path
.file_name()
.ok_or_else(|| format_err!("missing filename"))?
.to_string_lossy()
.to_string();
let input_url = readonly_inputs.url.url().join(&file_name)?;
let crash_test_result = handler.get_crash_result(file_path, input_url).await?;
RegressionReport {
crash_test_result,
original_crash_test_result: None,
}
.save(None, regression_reports)
.await?
}
Ok(())
}
pub async fn handle_crash_reports(
handler: &impl RegressionHandler,
crashes: &SyncedDir,
report_dirs: &[&SyncedDir],
report_list: &Option<Vec<String>>,
regression_reports: &SyncedDir,
heartbeat_client: &Option<TaskHeartbeatClient>,
) -> Result<()> {
// without crash report containers, skip this method
if report_dirs.is_empty() {
return Ok(());
}
crashes.init_pull().await?;
for possible_dir in report_dirs {
possible_dir.init_pull().await?;
let mut report_files = tokio::fs::read_dir(&possible_dir.path).await?;
while let Some(file) = report_files.next_entry().await? {
heartbeat_client.alive();
let file_path = file.path();
if !file_path.is_file() {
continue;
}
let file_name = file_path
.file_name()
.ok_or_else(|| format_err!("missing filename"))?
.to_string_lossy()
.to_string();
if let Some(report_list) = &report_list {
if !report_list.contains(&file_name) {
continue;
}
}
let original_crash_test_result = parse_report_file(file.path())
.await
.with_context(|| format!("unable to parse crash report: {}", file_name))?;
let input_blob = match &original_crash_test_result {
CrashTestResult::CrashReport(x) => x.input_blob.clone(),
CrashTestResult::NoRepro(x) => x.input_blob.clone(),
}
.ok_or_else(|| format_err!("crash report is missing input blob: {}", file_name))?;
let input_url = crashes.url.blob(&input_blob.name).url();
let input = crashes.path.join(&input_blob.name);
let crash_test_result = handler.get_crash_result(input, input_url).await?;
RegressionReport {
crash_test_result,
original_crash_test_result: Some(original_crash_test_result),
}
.save(Some(file_name), regression_reports)
.await?
}
}
Ok(())
}

109
src/agent/onefuzz-agent/src/tasks/regression/generic.rs Normal file
View File

@ -0,0 +1,109 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::tasks::{
config::CommonConfig,
report::{crash_report::CrashTestResult, generic},
utils::default_bool_true,
};
use anyhow::Result;
use async_trait::async_trait;
use onefuzz::syncdir::SyncedDir;
use reqwest::Url;
use serde::Deserialize;
use std::{collections::HashMap, path::PathBuf};
use super::common::{self, RegressionHandler};
#[derive(Debug, Deserialize)]
pub struct Config {
pub target_exe: PathBuf,
#[serde(default)]
pub target_options: Vec<String>,
#[serde(default)]
pub target_env: HashMap<String, String>,
pub target_timeout: Option<u64>,
pub crashes: SyncedDir,
pub regression_reports: SyncedDir,
pub report_list: Option<Vec<String>>,
pub reports: Option<SyncedDir>,
pub unique_reports: Option<SyncedDir>,
pub no_repro: Option<SyncedDir>,
pub readonly_inputs: Option<SyncedDir>,
#[serde(default)]
pub check_asan_log: bool,
#[serde(default = "default_bool_true")]
pub check_debugger: bool,
#[serde(default)]
pub check_retry_count: u64,
#[serde(default)]
pub minimized_stack_depth: Option<usize>,
#[serde(flatten)]
pub common: CommonConfig,
}
pub struct GenericRegressionTask {
config: Config,
}
#[async_trait]
impl RegressionHandler for GenericRegressionTask {
async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result<CrashTestResult> {
let args = generic::TestInputArgs {
input_url: Some(input_url),
input: &input,
target_exe: &self.config.target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,
task_id: self.config.common.task_id,
job_id: self.config.common.job_id,
target_timeout: self.config.target_timeout,
check_retry_count: self.config.check_retry_count,
check_asan_log: self.config.check_asan_log,
check_debugger: self.config.check_debugger,
minimized_stack_depth: self.config.minimized_stack_depth,
};
generic::test_input(args).await
}
}
impl GenericRegressionTask {
pub fn new(config: Config) -> Self {
Self { config }
}
pub async fn run(&self) -> Result<()> {
info!("Starting generic regression task");
let heartbeat_client = self.config.common.init_heartbeat().await?;
let mut report_dirs = vec![];
for dir in &[
&self.config.reports,
&self.config.unique_reports,
&self.config.no_repro,
] {
if let Some(dir) = dir {
report_dirs.push(dir);
}
}
common::run(
heartbeat_client,
&self.config.regression_reports,
&self.config.crashes,
&report_dirs,
&self.config.report_list,
&self.config.readonly_inputs,
self,
)
.await?;
Ok(())
}
}

107
src/agent/onefuzz-agent/src/tasks/regression/libfuzzer.rs Normal file
View File

@ -0,0 +1,107 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::tasks::{
config::CommonConfig,
report::{crash_report::CrashTestResult, libfuzzer_report},
utils::default_bool_true,
};
use anyhow::Result;
use reqwest::Url;
use super::common::{self, RegressionHandler};
use async_trait::async_trait;
use onefuzz::syncdir::SyncedDir;
use serde::Deserialize;
use std::{collections::HashMap, path::PathBuf};
#[derive(Debug, Deserialize)]
pub struct Config {
pub target_exe: PathBuf,
#[serde(default)]
pub target_options: Vec<String>,
#[serde(default)]
pub target_env: HashMap<String, String>,
pub target_timeout: Option<u64>,
pub crashes: SyncedDir,
pub regression_reports: SyncedDir,
pub report_list: Option<Vec<String>>,
pub unique_reports: Option<SyncedDir>,
pub reports: Option<SyncedDir>,
pub no_repro: Option<SyncedDir>,
pub readonly_inputs: Option<SyncedDir>,
#[serde(default = "default_bool_true")]
pub check_fuzzer_help: bool,
#[serde(default)]
pub check_retry_count: u64,
#[serde(default)]
pub minimized_stack_depth: Option<usize>,
#[serde(flatten)]
pub common: CommonConfig,
}
pub struct LibFuzzerRegressionTask {
config: Config,
}
#[async_trait]
impl RegressionHandler for LibFuzzerRegressionTask {
async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result<CrashTestResult> {
let args = libfuzzer_report::TestInputArgs {
input_url: Some(input_url),
input: &input,
target_exe: &self.config.target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,
task_id: self.config.common.task_id,
job_id: self.config.common.job_id,
target_timeout: self.config.target_timeout,
check_retry_count: self.config.check_retry_count,
minimized_stack_depth: self.config.minimized_stack_depth,
};
libfuzzer_report::test_input(args).await
}
}
impl LibFuzzerRegressionTask {
pub fn new(config: Config) -> Self {
Self { config }
}
pub async fn run(&self) -> Result<()> {
info!("Starting libfuzzer regression task");
let mut report_dirs = vec![];
for dir in &[
&self.config.reports,
&self.config.unique_reports,
&self.config.no_repro,
] {
if let Some(dir) = dir {
report_dirs.push(dir);
}
}
let heartbeat_client = self.config.common.init_heartbeat().await?;
common::run(
heartbeat_client,
&self.config.regression_reports,
&self.config.crashes,
&report_dirs,
&self.config.report_list,
&self.config.readonly_inputs,
self,
)
.await?;
Ok(())
}
}

6
src/agent/onefuzz-agent/src/tasks/regression/mod.rs Normal file
View File

@ -0,0 +1,6 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
pub mod common;
pub mod generic;
pub mod libfuzzer;

45
src/agent/onefuzz-agent/src/tasks/report/crash_report.rs
View File

@ -5,7 +5,10 @@ use anyhow::{Context, Result};
use futures::StreamExt;
use onefuzz::{blob::BlobUrl, monitor::DirectoryMonitor, syncdir::SyncedDir};
use onefuzz_telemetry::{
Event::{new_report, new_unable_to_reproduce, new_unique_report},
Event::{
new_report, new_unable_to_reproduce, new_unique_report, regression_report,
regression_unable_to_reproduce,
},
EventData,
};
use serde::{Deserialize, Serialize};
@ -46,6 +49,7 @@ pub struct CrashReport {
pub job_id: Uuid,
pub scariness_score: Option<u32>,
pub scariness_description: Option<String>,
}
@ -62,11 +66,42 @@ pub struct NoCrash {
}
#[derive(Debug, Deserialize, Serialize)]
#[serde(rename_all = "snake_case")]
pub enum CrashTestResult {
CrashReport(CrashReport),
NoRepro(NoCrash),
}
#[derive(Debug, Deserialize, Serialize)]
pub struct RegressionReport {
pub crash_test_result: CrashTestResult,
pub original_crash_test_result: Option<CrashTestResult>,
}
impl RegressionReport {
pub async fn save(
self,
report_name: Option<String>,
regression_reports: &SyncedDir,
) -> Result<()> {
let (event, name) = match &self.crash_test_result {
CrashTestResult::CrashReport(report) => {
let name = report_name.unwrap_or_else(|| report.unique_blob_name());
(regression_report, name)
}
CrashTestResult::NoRepro(report) => {
let name = report_name.unwrap_or_else(|| report.blob_name());
(regression_unable_to_reproduce, name)
}
};
if upload_or_save_local(&self, &name, regression_reports).await? {
event!(event; EventData::Path = name);
}
Ok(())
}
}
async fn upload_or_save_local<T: Serialize>(
report: &T,
dest_name: &str,
@ -76,6 +111,10 @@ async fn upload_or_save_local<T: Serialize>(
}
impl CrashTestResult {
/// Saves the crash result as a crash report
/// * `unique_reports` - location to save the deduplicated report if the bug was reproduced
/// * `reports` - location to save the report if the bug was reproduced
/// * `no_repro` - location to save the report if the bug was not reproduced
pub async fn save(
&self,
unique_reports: &Option<SyncedDir>,
@ -113,7 +152,7 @@ impl CrashTestResult {
}
}
#[derive(Debug, Deserialize, Serialize)]
#[derive(Debug, Deserialize, Serialize, Clone)]
pub struct InputBlob {
pub account: Option<String>,
pub container: Option<String>,
@ -188,7 +227,7 @@ impl NoCrash {
}
}
async fn parse_report_file(path: PathBuf) -> Result<CrashTestResult> {
pub async fn parse_report_file(path: PathBuf) -> Result<CrashTestResult> {
let raw = std::fs::read_to_string(&path)
.with_context(|| format_err!("unable to open crash report: {}", path.display()))?;

168
src/agent/onefuzz-agent/src/tasks/report/generic.rs
View File

@ -5,7 +5,7 @@ use super::crash_report::{CrashReport, CrashTestResult, InputBlob, NoCrash};
use crate::tasks::{
config::CommonConfig,
generic::input_poller::{CallbackImpl, InputPoller, Processor},
heartbeat::*,
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::default_bool_true,
};
use anyhow::Result;
@ -18,6 +18,7 @@ use std::{
path::{Path, PathBuf},
};
use storage_queue::{Message, QueueClient};
use uuid::Uuid;
#[derive(Debug, Deserialize)]
pub struct Config {
@ -86,27 +87,103 @@ impl ReportTask {
}
}
pub struct TestInputArgs<'a> {
pub input_url: Option<Url>,
pub input: &'a Path,
pub target_exe: &'a Path,
pub target_options: &'a [String],
pub target_env: &'a HashMap<String, String>,
pub setup_dir: &'a Path,
pub task_id: Uuid,
pub job_id: Uuid,
pub target_timeout: Option<u64>,
pub check_retry_count: u64,
pub check_asan_log: bool,
pub check_debugger: bool,
pub minimized_stack_depth: Option<usize>,
}
pub async fn test_input(args: TestInputArgs<'_>) -> Result<CrashTestResult> {
let tester = Tester::new(
args.setup_dir,
args.target_exe,
args.target_options,
args.target_env,
)
.check_asan_log(args.check_asan_log)
.check_debugger(args.check_debugger)
.check_retry_count(args.check_retry_count)
.set_optional(args.target_timeout, |tester, timeout| {
tester.timeout(timeout)
});
let input_sha256 = sha256::digest_file(args.input).await?;
let task_id = args.task_id;
let job_id = args.job_id;
let input_blob = args
.input_url
.and_then(|u| BlobUrl::new(u).ok())
.map(InputBlob::from);
let test_report = tester.test_input(args.input).await?;
if let Some(crash_log) = test_report.asan_log {
let crash_report = CrashReport::new(
crash_log,
task_id,
job_id,
args.target_exe,
input_blob,
input_sha256,
args.minimized_stack_depth,
);
Ok(CrashTestResult::CrashReport(crash_report))
} else if let Some(crash) = test_report.crash {
let call_stack_sha256 = sha256::digest_iter(&crash.call_stack);
let crash_report = CrashReport {
input_blob,
input_sha256,
executable: PathBuf::from(args.target_exe),
call_stack: crash.call_stack,
crash_type: crash.crash_type,
crash_site: crash.crash_site,
call_stack_sha256,
asan_log: None,
scariness_score: None,
scariness_description: None,
task_id,
job_id,
minimized_stack: vec![],
minimized_stack_sha256: None,
minimized_stack_function_names: vec![],
minimized_stack_function_names_sha256: None,
};
Ok(CrashTestResult::CrashReport(crash_report))
} else {
let no_repro = NoCrash {
input_blob,
input_sha256,
executable: PathBuf::from(args.target_exe),
task_id,
job_id,
tries: 1 + args.check_retry_count,
error: test_report.error.map(|e| format!("{}", e)),
};
Ok(CrashTestResult::NoRepro(no_repro))
}
}
pub struct GenericReportProcessor<'a> {
config: &'a Config,
tester: Tester<'a>,
heartbeat_client: Option<TaskHeartbeatClient>,
}
impl<'a> GenericReportProcessor<'a> {
pub fn new(config: &'a Config, heartbeat_client: Option<TaskHeartbeatClient>) -> Self {
let tester = Tester::new(
&config.common.setup_dir,
&config.target_exe,
&config.target_options,
&config.target_env,
)
.check_asan_log(config.check_asan_log)
.check_debugger(config.check_debugger)
.check_retry_count(config.check_retry_count);
Self {
config,
tester,
heartbeat_client,
}
}
@ -117,56 +194,25 @@ impl<'a> GenericReportProcessor<'a> {
input: &Path,
) -> Result<CrashTestResult> {
self.heartbeat_client.alive();
let input_sha256 = sha256::digest_file(input).await?;
let task_id = self.config.common.task_id;
let job_id = self.config.common.job_id;
let input_blob = match input_url {
Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)),
None => None,
let args = TestInputArgs {
input_url,
input,
target_exe: &self.config.target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,
task_id: self.config.common.task_id,
job_id: self.config.common.job_id,
target_timeout: self.config.target_timeout,
check_retry_count: self.config.check_retry_count,
check_asan_log: self.config.check_asan_log,
check_debugger: self.config.check_debugger,
minimized_stack_depth: self.config.minimized_stack_depth,
};
let result = test_input(args).await?;
let test_report = self.tester.test_input(input).await?;
if let Some(asan_log) = test_report.asan_log {
let crash_report = CrashReport::new(
asan_log,
task_id,
job_id,
&self.config.target_exe,
input_blob,
input_sha256,
self.config.minimized_stack_depth,
);
Ok(CrashTestResult::CrashReport(crash_report))
} else if let Some(crash) = test_report.crash {
let call_stack_sha256 = sha256::digest_iter(&crash.call_stack);
let crash_report = CrashReport {
input_blob,
input_sha256,
executable: PathBuf::from(&self.config.target_exe),
call_stack: crash.call_stack,
crash_type: crash.crash_type,
crash_site: crash.crash_site,
call_stack_sha256,
task_id,
job_id,
..Default::default()
};
Ok(CrashTestResult::CrashReport(crash_report))
} else {
let no_repro = NoCrash {
input_blob,
input_sha256,
executable: PathBuf::from(&self.config.target_exe),
task_id,
job_id,
tries: 1 + self.config.check_retry_count,
error: test_report.error.map(|e| format!("{}", e)),
};
Ok(CrashTestResult::NoRepro(no_repro))
}
Ok(result)
}
}

137
src/agent/onefuzz-agent/src/tasks/report/libfuzzer_report.rs
View File

@ -3,7 +3,10 @@
use super::crash_report::*;
use crate::tasks::{
config::CommonConfig, generic::input_poller::*, heartbeat::*, utils::default_bool_true,
config::CommonConfig,
generic::input_poller::*,
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::default_bool_true,
};
use anyhow::{Context, Result};
use async_trait::async_trait;
@ -75,7 +78,7 @@ impl ReportTask {
let mut processor = AsanProcessor::new(self.config.clone()).await?;
if let Some(crashes) = &self.config.crashes {
self.poller.batch_process(&mut processor, crashes).await?;
self.poller.batch_process(&mut processor, &crashes).await?;
}
if self.config.check_queue {
@ -88,6 +91,72 @@ impl ReportTask {
}
}
pub struct TestInputArgs<'a> {
pub input_url: Option<Url>,
pub input: &'a Path,
pub target_exe: &'a Path,
pub target_options: &'a [String],
pub target_env: &'a HashMap<String, String>,
pub setup_dir: &'a Path,
pub task_id: uuid::Uuid,
pub job_id: uuid::Uuid,
pub target_timeout: Option<u64>,
pub check_retry_count: u64,
pub minimized_stack_depth: Option<usize>,
}
pub async fn test_input(args: TestInputArgs<'_>) -> Result<CrashTestResult> {
let fuzzer = LibFuzzer::new(
args.target_exe,
args.target_options,
args.target_env,
args.setup_dir,
);
let task_id = args.task_id;
let job_id = args.job_id;
let input_blob = args
.input_url
.and_then(|u| BlobUrl::new(u).ok())
.map(InputBlob::from);
let input = args.input;
let input_sha256 = sha256::digest_file(args.input)
.await
.with_context(|| format_err!("unable to sha256 digest input file: {}", input.display()))?;
let test_report = fuzzer
.repro(args.input, args.target_timeout, args.check_retry_count)
.await?;
match test_report.asan_log {
Some(crash_log) => {
let crash_report = CrashReport::new(
crash_log,
task_id,
job_id,
args.target_exe,
input_blob,
input_sha256,
args.minimized_stack_depth,
);
Ok(CrashTestResult::CrashReport(crash_report))
}
None => {
let no_repro = NoCrash {
input_blob,
input_sha256,
executable: PathBuf::from(&args.target_exe),
task_id,
job_id,
tries: 1 + args.check_retry_count,
error: test_report.error.map(|e| format!("{}", e)),
};
Ok(CrashTestResult::NoRepro(no_repro))
}
}
}
pub struct AsanProcessor {
config: Arc<Config>,
heartbeat_client: Option<TaskHeartbeatClient>,
@ -109,58 +178,22 @@ impl AsanProcessor {
input: &Path,
) -> Result<CrashTestResult> {
self.heartbeat_client.alive();
let fuzzer = LibFuzzer::new(
&self.config.target_exe,
&self.config.target_options,
&self.config.target_env,
&self.config.common.setup_dir,
);
let task_id = self.config.common.task_id;
let job_id = self.config.common.job_id;
let input_blob = match input_url {
Some(x) => Some(InputBlob::from(BlobUrl::new(x)?)),
None => None,
let args = TestInputArgs {
input_url,
input,
target_exe: &self.config.target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,
task_id: self.config.common.task_id,
job_id: self.config.common.job_id,
target_timeout: self.config.target_timeout,
check_retry_count: self.config.check_retry_count,
minimized_stack_depth: self.config.minimized_stack_depth,
};
let input_sha256 = sha256::digest_file(input).await.with_context(|| {
format_err!("unable to sha256 digest input file: {}", input.display())
})?;
let result = test_input(args).await?;
let test_report = fuzzer
.repro(
input,
self.config.target_timeout,
self.config.check_retry_count,
)
.await?;
match test_report.asan_log {
Some(asan_log) => {
let crash_report = CrashReport::new(
asan_log,
task_id,
job_id,
&self.config.target_exe,
input_blob,
input_sha256,
self.config.minimized_stack_depth,
);
Ok(CrashTestResult::CrashReport(crash_report))
}
None => {
let no_repro = NoCrash {
input_blob,
input_sha256,
executable: PathBuf::from(&self.config.target_exe),
task_id,
job_id,
tries: 1 + self.config.check_retry_count,
error: test_report.error.map(|e| format!("{}", e)),
};
Ok(CrashTestResult::NoRepro(no_repro))
}
}
Ok(result)
}
}

4
src/agent/onefuzz-supervisor/src/main.rs
View File

@ -14,8 +14,8 @@ extern crate onefuzz_telemetry;
extern crate onefuzz;
use crate::{
config::StaticConfig, coordinator::StateUpdateEvent, heartbeat::*, work::WorkSet,
worker::WorkerEvent,
config::StaticConfig, coordinator::StateUpdateEvent, heartbeat::init_agent_heartbeat,
work::WorkSet, worker::WorkerEvent,
};
use std::path::PathBuf;

4
src/agent/onefuzz-telemetry/src/lib.rs
View File

@ -95,6 +95,8 @@ pub enum Event {
new_report,
new_unique_report,
new_unable_to_reproduce,
regression_report,
regression_unable_to_reproduce,
}
impl Event {
@ -109,6 +111,8 @@ impl Event {
Self::new_report => "new_report",
Self::new_unique_report => "new_unique_report",
Self::new_unable_to_reproduce => "new_unable_to_reproduce",
Self::regression_report => "regression_report",
Self::regression_unable_to_reproduce => "regression_unable_to_reproduce",
}
}
}

9
src/agent/onefuzz/src/blob/url.rs
View File

@ -25,6 +25,15 @@ impl BlobUrl {
bail!("Invalid blob URL: {}", url)
}
pub fn from_blob_info(account: &str, container: &str, name: &str) -> Result<Self> {
// format https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#resource-uri-syntax
let url = Url::parse(&format!(
"https://{}.blob.core.windows.net/{}/{}",
account, container, name
))?;
Self::new(url)
}
pub fn parse(url: impl AsRef<str>) -> Result<Self> {
let url = Url::parse(url.as_ref())?;

3
src/api-service/__app__/onefuzzlib/extension.py
View File

@ -271,6 +271,9 @@ def repro_extensions(
if report is None:
raise Exception("invalid report: %s" % repro_config)
if report.input_blob is None:
raise Exception("unable to perform reproduction without an input blob")
commands = []
if setup_container:
commands += [

24
src/api-service/__app__/onefuzzlib/notifications/ado.py
View File

@ -4,7 +4,7 @@
# Licensed under the MIT License.
import logging
from typing import Iterator, List, Optional
from typing import Iterator, List, Optional, Union
from azure.devops.connection import Connection
from azure.devops.credentials import BasicAuthentication
@ -24,7 +24,7 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import (
WorkItemTrackingClient,
)
from memoization import cached
from onefuzztypes.models import ADOTemplate, Report
from onefuzztypes.models import ADOTemplate, RegressionReport, Report
from onefuzztypes.primitives import Container
from ..secrets import get_secret_string_value
@ -51,7 +51,11 @@ def get_valid_fields(
class ADO:
def __init__(
self, container: Container, filename: str, config: ADOTemplate, report: Report
self,
container: Container,
filename: str,
config: ADOTemplate,
report: Report,
):
self.config = config
self.renderer = Render(container, filename, report)
@ -203,8 +207,20 @@ class ADO:
def notify_ado(
config: ADOTemplate, container: Container, filename: str, report: Report
config: ADOTemplate,
container: Container,
filename: str,
report: Union[Report, RegressionReport],
) -> None:
if isinstance(report, RegressionReport):
logging.info(
"ado integration does not support regression reports. "
"container:%s filename:%s",
container,
filename,
)
return
logging.info(
"notify ado: job_id:%s task_id:%s container:%s filename:%s",
report.job_id,

19
src/api-service/__app__/onefuzzlib/notifications/github_issues.py
View File

@ -4,13 +4,18 @@
# Licensed under the MIT License.
import logging
from typing import List, Optional
from typing import List, Optional, Union
from github3 import login
from github3.exceptions import GitHubException
from github3.issues import Issue
from onefuzztypes.enums import GithubIssueSearchMatch
from onefuzztypes.models import GithubAuth, GithubIssueTemplate, Report
from onefuzztypes.models import (
GithubAuth,
GithubIssueTemplate,
RegressionReport,
Report,
)
from onefuzztypes.primitives import Container
from ..secrets import get_secret_obj
@ -107,10 +112,18 @@ def github_issue(
config: GithubIssueTemplate,
container: Container,
filename: str,
report: Optional[Report],
report: Optional[Union[Report, RegressionReport]],
) -> None:
if report is None:
return
if isinstance(report, RegressionReport):
logging.info(
"github issue integration does not support regression reports. "
"container:%s filename:%s",
container,
filename,
)
return
try:
handler = GithubIssue(config, container, filename, report)

20
src/api-service/__app__/onefuzzlib/notifications/main.py
View File

@ -10,12 +10,18 @@ from uuid import UUID
from memoization import cached
from onefuzztypes import models
from onefuzztypes.enums import ErrorCode, TaskState
from onefuzztypes.events import EventCrashReported, EventFileAdded
from onefuzztypes.events import (
EventCrashReported,
EventFileAdded,
EventRegressionReported,
)
from onefuzztypes.models import (
ADOTemplate,
Error,
GithubIssueTemplate,
NotificationTemplate,
RegressionReport,
Report,
Result,
TeamsTemplate,
)
@ -26,7 +32,7 @@ from ..azure.queue import send_message
from ..azure.storage import StorageType
from ..events import send_event
from ..orm import ORMMixin
from ..reports import get_report
from ..reports import get_report_or_regression
from ..tasks.config import get_input_container_queues
from ..tasks.main import Task
from .ado import notify_ado
@ -102,7 +108,7 @@ def get_queue_tasks() -> Sequence[Tuple[Task, Sequence[str]]]:
def new_files(container: Container, filename: str) -> None:
report = get_report(container, filename)
report = get_report_or_regression(container, filename)
notifications = get_notifications(container)
if notifications:
@ -134,9 +140,15 @@ def new_files(container: Container, filename: str) -> None:
)
send_message(task.task_id, bytes(url, "utf-8"), StorageType.corpus)
if report:
if isinstance(report, Report):
send_event(
EventCrashReported(report=report, container=container, filename=filename)
)
elif isinstance(report, RegressionReport):
send_event(
EventRegressionReported(
regression_report=report, container=container, filename=filename
)
)
else:
send_event(EventFileAdded(container=container, filename=filename))

11
src/api-service/__app__/onefuzzlib/notifications/teams.py
View File

@ -4,10 +4,10 @@
# Licensed under the MIT License.
import logging
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Union
import requests
from onefuzztypes.models import Report, TeamsTemplate
from onefuzztypes.models import RegressionReport, Report, TeamsTemplate
from onefuzztypes.primitives import Container
from ..azure.containers import auth_download_url
@ -54,12 +54,15 @@ def send_teams_webhook(
def notify_teams(
config: TeamsTemplate, container: Container, filename: str, report: Optional[Report]
config: TeamsTemplate,
container: Container,
filename: str,
report: Optional[Union[Report, RegressionReport]],
) -> None:
text = None
facts: List[Dict[str, str]] = []
if report:
if isinstance(report, Report):
task = Task.get(report.job_id, report.task_id)
if not task:
logging.error(

41
src/api-service/__app__/onefuzzlib/reports.py
View File

@ -8,7 +8,7 @@ import logging
from typing import Optional, Union
from memoization import cached
from onefuzztypes.models import Report
from onefuzztypes.models import RegressionReport, Report
from onefuzztypes.primitives import Container
from pydantic import ValidationError
@ -16,17 +16,16 @@ from .azure.containers import get_blob
from .azure.storage import StorageType
def parse_report(
def parse_report_or_regression(
content: Union[str, bytes], file_path: Optional[str] = None
) -> Optional[Report]:
) -> Optional[Union[Report, RegressionReport]]:
if isinstance(content, bytes):
try:
content = content.decode()
except UnicodeDecodeError as err:
logging.error(
"unable to parse report (%s): unicode decode of report failed - %s",
file_path,
err,
f"unable to parse report ({file_path}): "
f"unicode decode of report failed - {err}"
)
return None
@ -34,22 +33,31 @@ def parse_report(
data = json.loads(content)
except json.decoder.JSONDecodeError as err:
logging.error(
"unable to parse report (%s): json decoding failed - %s", file_path, err
f"unable to parse report ({file_path}): json decoding failed - {err}"
)
return None
regression_err = None
try:
entry = Report.parse_obj(data)
return RegressionReport.parse_obj(data)
except ValidationError as err:
logging.error("unable to parse report (%s): %s", file_path, err)
return None
regression_err = err
return entry
try:
return Report.parse_obj(data)
except ValidationError as err:
logging.error(
f"unable to parse report ({file_path}) as a report or regression. "
f"regression error: {regression_err} report error: {err}"
)
return None
# cache the last 1000 reports
@cached(max_size=1000)
def get_report(container: Container, filename: str) -> Optional[Report]:
def get_report_or_regression(
container: Container, filename: str
) -> Optional[Union[Report, RegressionReport]]:
file_path = "/".join([container, filename])
if not filename.endswith(".json"):
logging.error("get_report invalid extension: %s", file_path)
@ -60,4 +68,11 @@ def get_report(container: Container, filename: str) -> Optional[Report]:
logging.error("get_report invalid blob: %s", file_path)
return None
return parse_report(blob, file_path=file_path)
return parse_report_or_regression(blob, file_path=file_path)
def get_report(container: Container, filename: str) -> Optional[Report]:
result = get_report_or_regression(container, filename)
if isinstance(result, Report):
return result
return None

6
src/api-service/__app__/onefuzzlib/repro.py
View File

@ -164,6 +164,12 @@ class Repro(BASE_REPRO, ORMMixin):
if report is None:
return Error(code=ErrorCode.VM_CREATE_FAILED, errors=["missing report"])
if report.input_blob is None:
return Error(
code=ErrorCode.VM_CREATE_FAILED,
errors=["unable to perform repro for crash reports without inputs"],
)
files = {}
if task.os == OS.windows:

3
src/api-service/__app__/onefuzzlib/tasks/config.py
View File

@ -348,6 +348,9 @@ def build_task_config(
else True
)
if TaskFeature.report_list in definition.features:
config.report_list = task_config.task.report_list
if TaskFeature.expect_crash_on_failure in definition.features:
config.expect_crash_on_failure = (
task_config.task.expect_crash_on_failure

127
src/api-service/__app__/onefuzzlib/tasks/defs.py
View File

@ -414,4 +414,131 @@ TASK_DEFINITIONS = {
],
monitor_queue=ContainerType.crashes,
),
TaskType.generic_regression: TaskDefinition(
features=[
TaskFeature.target_exe,
TaskFeature.target_env,
TaskFeature.target_options,
TaskFeature.target_timeout,
TaskFeature.check_asan_log,
TaskFeature.check_debugger,
TaskFeature.check_retry_count,
TaskFeature.report_list,
],
vm=VmDefinition(compare=Compare.AtLeast, value=1),
containers=[
ContainerDefinition(
type=ContainerType.setup,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.regression_reports,
compare=Compare.Equal,
value=1,
permissions=[
ContainerPermission.Write,
ContainerPermission.Read,
ContainerPermission.List,
],
),
ContainerDefinition(
type=ContainerType.crashes,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.unique_reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.no_repro,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.readonly_inputs,
compare=Compare.AtMost,
value=1,
permissions=[
ContainerPermission.Read,
ContainerPermission.List,
],
),
],
),
TaskType.libfuzzer_regression: TaskDefinition(
features=[
TaskFeature.target_exe,
TaskFeature.target_env,
TaskFeature.target_options,
TaskFeature.target_timeout,
TaskFeature.check_fuzzer_help,
TaskFeature.check_retry_count,
TaskFeature.report_list,
],
vm=VmDefinition(compare=Compare.AtLeast, value=1),
containers=[
ContainerDefinition(
type=ContainerType.setup,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.regression_reports,
compare=Compare.Equal,
value=1,
permissions=[
ContainerPermission.Write,
ContainerPermission.Read,
ContainerPermission.List,
],
),
ContainerDefinition(
type=ContainerType.crashes,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.unique_reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.no_repro,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.readonly_inputs,
compare=Compare.AtMost,
value=1,
permissions=[
ContainerPermission.Read,
ContainerPermission.List,
],
),
],
),
}

11
src/api-service/tests/test_report_parse.py
View File

@ -10,7 +10,7 @@ from pathlib import Path
from onefuzztypes.models import Report
from __app__.onefuzzlib.reports import parse_report
from __app__.onefuzzlib.reports import parse_report_or_regression
class TestReportParse(unittest.TestCase):
@ -20,16 +20,15 @@ class TestReportParse(unittest.TestCase):
data = json.load(handle)
invalid = {"unused_field_1": 3}
report = parse_report(json.dumps(data))
report = parse_report_or_regression(json.dumps(data))
self.assertIsInstance(report, Report)
with self.assertLogs(level="ERROR"):
self.assertIsNone(parse_report('"invalid"'))
self.assertIsNone(parse_report_or_regression('"invalid"'))
with self.assertLogs(level="WARNING") as logs:
self.assertIsNone(parse_report(json.dumps(invalid)))
self.assertTrue(any(["unable to parse report" in x for x in logs.output]))
self.assertIsNone(parse_report_or_regression(json.dumps(invalid)))
self.assertTrue(any(["unable to parse report" in x for x in logs.output]))
if __name__ == "__main__":

9
src/cli/onefuzz/api.py
View File

@ -174,11 +174,13 @@ class Files(Endpoint):
sas = self.onefuzz.containers.get(container).sas_url
return ContainerWrapper(sas)
def list(self, container: primitives.Container) -> models.Files:
def list(
self, container: primitives.Container, prefix: Optional[str] = None
) -> models.Files:
""" Get a list of files in a container """
self.logger.debug("listing files in container: %s", container)
client = self._get_client(container)
return models.Files(files=client.list_blobs())
return models.Files(files=client.list_blobs(name_starts_with=prefix))
def delete(self, container: primitives.Container, filename: str) -> None:
""" delete a file from a container """
@ -845,6 +847,7 @@ class Tasks(Endpoint):
vm_count: int = 1,
preserve_existing_outputs: bool = False,
colocate: bool = False,
report_list: Optional[List[str]] = None,
) -> models.Task:
"""
Create a task
@ -907,6 +910,8 @@ class Tasks(Endpoint):
target_workers=target_workers,
type=task_type,
wait_for_files=task_wait_for_files,
report_list=report_list,
preserve_existing_outputs=preserve_existing_outputs,
),
)

14
src/cli/onefuzz/cli.py
View File

@ -505,6 +505,14 @@ def output(result: Any, output_format: str, expression: Optional[Any]) -> None:
print(result, flush=True)
def log_exception(args: argparse.Namespace, err: Exception) -> None:
if args.verbose > 0:
entry = traceback.format_exc()
for x in entry.split("\n"):
LOGGER.error("traceback: %s", x)
LOGGER.error("command failed: %s", " ".join([str(x) for x in err.args]))
def execute_api(api: Any, api_types: List[Any], version: str) -> int:
builder = Builder(api_types)
builder.add_version(version)
@ -545,11 +553,7 @@ def execute_api(api: Any, api_types: List[Any], version: str) -> int:
try:
result = call_func(args.func, args)
except Exception as err:
if args.verbose > 0:
entry = traceback.format_exc()
for x in entry.split("\n"):
LOGGER.error("traceback: %s", x)
LOGGER.error("command failed: %s", " ".join([str(x) for x in err.args]))
log_exception(args, err)
return 1
output(result, args.format, expression)

14
src/cli/onefuzz/job_templates/job_monitor.py
View File

@ -78,11 +78,21 @@ class JobMonitor:
None,
)
def is_stopped(self) -> Tuple[bool, str, Any]:
tasks = self.onefuzz.tasks.list(job_id=self.job.job_id)
waiting = [
"%s:%s" % (x.config.task.type.name, x.state.name)
for x in tasks
if x.state != TaskState.stopped
]
return (not waiting, "waiting on: %s" % ", ".join(sorted(waiting)), None)
def wait(
self,
*,
wait_for_running: Optional[bool] = False,
wait_for_files: Optional[Dict[Container, int]] = None,
wait_for_stopped: Optional[bool] = False,
) -> None:
if wait_for_running:
wait(self.is_running)
@ -92,3 +102,7 @@ class JobMonitor:
self.containers = wait_for_files
wait(self.has_files)
self.onefuzz.logger.info("new files found")
if wait_for_stopped:
wait(self.is_stopped)
self.onefuzz.logger.info("tasks stopped")

2
src/cli/onefuzz/template.py
View File

@ -13,6 +13,7 @@ from .templates.afl import AFL
from .templates.libfuzzer import Libfuzzer
from .templates.ossfuzz import OssFuzz
from .templates.radamsa import Radamsa
from .templates.regression import Regression
class Template(Command):
@ -24,6 +25,7 @@ class Template(Command):
self.afl = AFL(onefuzz, logger)
self.radamsa = Radamsa(onefuzz, logger)
self.ossfuzz = OssFuzz(onefuzz, logger)
self.regression = Regression(onefuzz, logger)
def stop(
self,

75
src/cli/onefuzz/templates/__init__.py
View File

@ -7,9 +7,10 @@ import json
import os
import tempfile
import zipfile
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Tuple
from uuid import uuid4
from onefuzztypes.enums import OS, ContainerType
from onefuzztypes.enums import OS, ContainerType, TaskState
from onefuzztypes.models import Job, NotificationConfig
from onefuzztypes.primitives import Container, Directory, File
@ -39,6 +40,12 @@ def _build_container_name(
build=build,
platform=platform.name,
)
elif container_type == ContainerType.regression_reports:
guid = onefuzz.utils.namespaced_guid(
project,
name,
build=build,
)
else:
guid = onefuzz.utils.namespaced_guid(project, name)
@ -87,6 +94,7 @@ class JobHelper:
)
self.wait_for_running: bool = False
self.wait_for_stopped: bool = False
self.containers: Dict[ContainerType, Container] = {}
self.tags: Dict[str, str] = {"project": project, "name": name, "build": build}
if job is None:
@ -116,6 +124,15 @@ class JobHelper:
self.platform,
)
def get_unique_container_name(self, container_type: ContainerType) -> Container:
return Container(
"oft-%s-%s"
% (
container_type.name.replace("_", "-"),
uuid4().hex,
)
)
def create_containers(self) -> None:
for (container_type, container_name) in self.containers.items():
self.logger.info("using container: %s", container_name)
@ -123,6 +140,9 @@ class JobHelper:
container_name, metadata={"container_type": container_type.name}
)
def delete_container(self, container_name: Container) -> None:
self.onefuzz.containers.delete(container_name)
def setup_notifications(self, config: Optional[NotificationConfig]) -> None:
if not config:
return
@ -233,9 +253,58 @@ class JobHelper:
}
self.wait_for_running = wait_for_running
def check_current_job(self) -> Job:
job = self.onefuzz.jobs.get(self.job.job_id)
if job.state in ["stopped", "stopping"]:
raise StoppedEarly("job unexpectedly stopped early")
errors = []
for task in self.onefuzz.tasks.list(job_id=self.job.job_id):
if task.state in ["stopped", "stopping"]:
if task.error:
errors.append("%s: %s" % (task.config.task.type, task.error))
else:
errors.append("%s" % task.config.task.type)
if errors:
raise StoppedEarly("tasks stopped unexpectedly.\n%s" % "\n".join(errors))
return job
def get_waiting(self) -> List[str]:
tasks = self.onefuzz.tasks.list(job_id=self.job.job_id)
waiting = [
"%s:%s" % (x.config.task.type.name, x.state.name)
for x in tasks
if x.state not in TaskState.has_started()
]
return waiting
def is_running(self) -> Tuple[bool, str, Any]:
waiting = self.get_waiting()
return (not waiting, "waiting on: %s" % ", ".join(sorted(waiting)), None)
def has_files(self) -> Tuple[bool, str, Any]:
self.check_current_job()
new = {
x: len(self.onefuzz.containers.files.list(x).files)
for x in self.to_monitor.keys()
}
for container in new:
if new[container] > self.to_monitor[container]:
del self.to_monitor[container]
return (
not self.to_monitor,
"waiting for new files: %s" % ", ".join(self.to_monitor.keys()),
None,
)
def wait(self) -> None:
JobMonitor(self.onefuzz, self.job).wait(
wait_for_running=self.wait_for_running, wait_for_files=self.to_monitor
wait_for_running=self.wait_for_running,
wait_for_files=self.to_monitor,
wait_for_stopped=self.wait_for_stopped,
)
def target_exe_blob_name(

33
src/cli/onefuzz/templates/libfuzzer.py
View File

@ -61,6 +61,36 @@ class Libfuzzer(Command):
expect_crash_on_failure: bool = True,
) -> None:
regression_containers = [
(ContainerType.setup, containers[ContainerType.setup]),
(ContainerType.crashes, containers[ContainerType.crashes]),
(ContainerType.unique_reports, containers[ContainerType.unique_reports]),
(
ContainerType.regression_reports,
containers[ContainerType.regression_reports],
),
]
self.logger.info("creating libfuzzer_regression task")
regression_task = self.onefuzz.tasks.create(
job.job_id,
TaskType.libfuzzer_regression,
target_exe,
regression_containers,
pool_name=pool_name,
duration=duration,
vm_count=1,
reboot_after_setup=reboot_after_setup,
target_options=target_options,
target_env=target_env,
tags=tags,
target_timeout=crash_report_timeout,
check_retry_count=check_retry_count,
check_fuzzer_help=check_fuzzer_help,
debug=debug,
colocate=colocate_all_tasks or colocate_secondary_tasks,
)
fuzzer_containers = [
(ContainerType.setup, containers[ContainerType.setup]),
(ContainerType.crashes, containers[ContainerType.crashes]),
@ -92,7 +122,7 @@ class Libfuzzer(Command):
expect_crash_on_failure=expect_crash_on_failure,
)
prereq_tasks = [fuzzer_task.task_id]
prereq_tasks = [fuzzer_task.task_id, regression_task.task_id]
coverage_containers = [
(ContainerType.setup, containers[ContainerType.setup]),
@ -219,6 +249,7 @@ class Libfuzzer(Command):
ContainerType.no_repro,
ContainerType.coverage,
ContainerType.unique_inputs,
ContainerType.regression_reports,
)
if existing_inputs:

284
src/cli/onefuzz/templates/regression.py Normal file
View File

@ -0,0 +1,284 @@
#!/usr/bin/env python
#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
import json
from typing import Dict, List, Optional
from onefuzztypes.enums import ContainerType, TaskDebugFlag, TaskType
from onefuzztypes.models import NotificationConfig, RegressionReport
from onefuzztypes.primitives import Container, Directory, File, PoolName
from onefuzz.api import Command
from . import JobHelper
class Regression(Command):
""" Regression job """
def _check_regression(self, container: Container, file: File) -> bool:
content = self.onefuzz.containers.files.get(Container(container), file)
as_str = content.decode()
as_obj = json.loads(as_str)
report = RegressionReport.parse_obj(as_obj)
if report.crash_test_result.crash_report is not None:
return True
if report.crash_test_result.no_repro is not None:
return False
raise Exception("invalid crash report")
def generic(
self,
project: str,
name: str,
build: str,
pool_name: PoolName,
*,
reports: Optional[List[str]] = None,
crashes: Optional[List[File]] = None,
target_exe: File = File("fuzz.exe"),
tags: Optional[Dict[str, str]] = None,
notification_config: Optional[NotificationConfig] = None,
target_env: Optional[Dict[str, str]] = None,
setup_dir: Optional[Directory] = None,
reboot_after_setup: bool = False,
target_options: Optional[List[str]] = None,
dryrun: bool = False,
duration: int = 24,
crash_report_timeout: Optional[int] = None,
debug: Optional[List[TaskDebugFlag]] = None,
check_retry_count: Optional[int] = None,
check_fuzzer_help: bool = True,
delete_input_container: bool = True,
check_regressions: bool = False,
) -> None:
"""
generic regression task
:param File crashes: Specify crashing input files to check in the regression task
:param str reports: Specify specific report names to verify in the regression task
:param bool check_regressions: Specify if exceptions should be thrown on finding crash regressions
:param bool delete_input_container: Specify wether or not to delete the input container
"""
self._create_job(
TaskType.generic_regression,
project,
name,
build,
pool_name,
crashes=crashes,
reports=reports,
target_exe=target_exe,
tags=tags,
notification_config=notification_config,
target_env=target_env,
setup_dir=setup_dir,
reboot_after_setup=reboot_after_setup,
target_options=target_options,
dryrun=dryrun,
duration=duration,
crash_report_timeout=crash_report_timeout,
debug=debug,
check_retry_count=check_retry_count,
check_fuzzer_help=check_fuzzer_help,
delete_input_container=delete_input_container,
check_regressions=check_regressions,
)
def libfuzzer(
self,
project: str,
name: str,
build: str,
pool_name: PoolName,
*,
reports: Optional[List[str]] = None,
crashes: Optional[List[File]] = None,
target_exe: File = File("fuzz.exe"),
tags: Optional[Dict[str, str]] = None,
notification_config: Optional[NotificationConfig] = None,
target_env: Optional[Dict[str, str]] = None,
setup_dir: Optional[Directory] = None,
reboot_after_setup: bool = False,
target_options: Optional[List[str]] = None,
dryrun: bool = False,
duration: int = 24,
crash_report_timeout: Optional[int] = None,
debug: Optional[List[TaskDebugFlag]] = None,
check_retry_count: Optional[int] = None,
check_fuzzer_help: bool = True,
delete_input_container: bool = True,
check_regressions: bool = False,
) -> None:
"""
libfuzzer regression task
:param File crashes: Specify crashing input files to check in the regression task
:param str reports: Specify specific report names to verify in the regression task
:param bool check_regressions: Specify if exceptions should be thrown on finding crash regressions
:param bool delete_input_container: Specify wether or not to delete the input container
"""
self._create_job(
TaskType.libfuzzer_regression,
project,
name,
build,
pool_name,
crashes=crashes,
reports=reports,
target_exe=target_exe,
tags=tags,
notification_config=notification_config,
target_env=target_env,
setup_dir=setup_dir,
reboot_after_setup=reboot_after_setup,
target_options=target_options,
dryrun=dryrun,
duration=duration,
crash_report_timeout=crash_report_timeout,
debug=debug,
check_retry_count=check_retry_count,
check_fuzzer_help=check_fuzzer_help,
delete_input_container=delete_input_container,
check_regressions=check_regressions,
)
def _create_job(
self,
task_type: TaskType,
project: str,
name: str,
build: str,
pool_name: PoolName,
*,
crashes: Optional[List[File]] = None,
reports: Optional[List[str]] = None,
target_exe: File = File("fuzz.exe"),
tags: Optional[Dict[str, str]] = None,
notification_config: Optional[NotificationConfig] = None,
target_env: Optional[Dict[str, str]] = None,
setup_dir: Optional[Directory] = None,
reboot_after_setup: bool = False,
target_options: Optional[List[str]] = None,
dryrun: bool = False,
duration: int = 24,
crash_report_timeout: Optional[int] = None,
debug: Optional[List[TaskDebugFlag]] = None,
check_retry_count: Optional[int] = None,
check_fuzzer_help: bool = True,
delete_input_container: bool = True,
check_regressions: bool = False,
) -> None:
if dryrun:
return None
self.logger.info("creating regression task from template")
helper = JobHelper(
self.onefuzz,
self.logger,
project,
name,
build,
duration,
pool_name=pool_name,
target_exe=target_exe,
)
helper.define_containers(
ContainerType.setup,
ContainerType.crashes,
ContainerType.reports,
ContainerType.no_repro,
ContainerType.unique_reports,
ContainerType.regression_reports,
)
containers = [
(ContainerType.setup, helper.containers[ContainerType.setup]),
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
(ContainerType.reports, helper.containers[ContainerType.reports]),
(ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
(
ContainerType.unique_reports,
helper.containers[ContainerType.unique_reports],
),
(
ContainerType.regression_reports,
helper.containers[ContainerType.regression_reports],
),
]
if crashes:
helper.containers[
ContainerType.readonly_inputs
] = helper.get_unique_container_name(ContainerType.readonly_inputs)
containers.append(
(
ContainerType.readonly_inputs,
helper.containers[ContainerType.readonly_inputs],
)
)
helper.create_containers()
if crashes:
for file in crashes:
self.onefuzz.containers.files.upload_file(
helper.containers[ContainerType.readonly_inputs], file
)
helper.setup_notifications(notification_config)
helper.upload_setup(setup_dir, target_exe)
target_exe_blob_name = helper.target_exe_blob_name(target_exe, setup_dir)
self.logger.info("creating regression task")
task = self.onefuzz.tasks.create(
helper.job.job_id,
task_type,
target_exe_blob_name,
containers,
pool_name=pool_name,
duration=duration,
vm_count=1,
reboot_after_setup=reboot_after_setup,
target_options=target_options,
target_env=target_env,
tags=tags,
target_timeout=crash_report_timeout,
check_retry_count=check_retry_count,
debug=debug,
check_fuzzer_help=check_fuzzer_help,
report_list=reports,
)
helper.wait_for_stopped = check_regressions
self.logger.info("done creating tasks")
helper.wait()
if check_regressions:
task = self.onefuzz.tasks.get(task.task_id)
if task.error:
raise Exception("task failed: %s", task.error)
container = helper.containers[ContainerType.regression_reports]
for filename in self.onefuzz.containers.files.list(container).files:
self.logger.info("checking file: %s", filename)
if self._check_regression(container, File(filename)):
raise Exception(f"regression identified: {filename}")
self.logger.info("no regressions")
if (
delete_input_container
and ContainerType.readonly_inputs in helper.containers
):
helper.delete_container(helper.containers[ContainerType.readonly_inputs])

156
src/integration-tests/check-regression.py Executable file
View File

@ -0,0 +1,156 @@
#!/usr/bin/env python
#
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
#
import json
import logging
import os
import sys
import time
from typing import Optional
from uuid import UUID, uuid4
from onefuzz.api import Command, Onefuzz
from onefuzz.cli import execute_api
from onefuzztypes.enums import OS, ContainerType, TaskState, TaskType
from onefuzztypes.models import Job, RegressionReport
from onefuzztypes.primitives import Container, Directory, File, PoolName
class Run(Command):
def cleanup(self, test_id: UUID):
for pool in self.onefuzz.pools.list():
if str(test_id) in pool.name:
self.onefuzz.pools.shutdown(pool.name, now=True)
self.onefuzz.template.stop(
str(test_id), "linux-libfuzzer", build=None, delete_containers=True
)
def _wait_for_regression_task(self, job: Job) -> None:
while True:
self.logger.info("waiting for regression task to finish")
for task in self.onefuzz.jobs.tasks.list(job.job_id):
if task.config.task.type not in [
TaskType.libfuzzer_regression,
TaskType.generic_regression,
]:
continue
if task.state != TaskState.stopped:
continue
return
time.sleep(10)
def _check_regression(self, job: Job) -> bool:
# get the regression reports containers for the job
results = self.onefuzz.jobs.containers.list(
job.job_id, ContainerType.regression_reports
)
# expect one and only one regression report container
if len(results) != 1:
raise Exception(f"unexpected regression containers: {results}")
container = list(results.keys())[0]
# expect one and only one file in the container
if len(results[container]) != 1:
raise Exception(f"unexpected regression container output: {results}")
file = results[container][0]
# get the regression report
content = self.onefuzz.containers.files.get(Container(container), file)
as_str = content.decode()
as_obj = json.loads(as_str)
report = RegressionReport.parse_obj(as_obj)
if report.crash_test_result.crash_report is not None:
self.logger.info("regression report has crash report")
return True
if report.crash_test_result.no_repro is not None:
self.logger.info("regression report has no-repro")
return False
raise Exception(f"unexpected report: {report}")
def _run_job(
self, test_id: UUID, pool: PoolName, target: str, exe: File, build: int
) -> Job:
if build == 1:
wait_for_files = [ContainerType.unique_reports]
else:
wait_for_files = [ContainerType.regression_reports]
job = self.onefuzz.template.libfuzzer.basic(
str(test_id),
target,
str(build),
pool,
target_exe=exe,
duration=1,
vm_count=1,
wait_for_files=wait_for_files,
)
if job is None:
raise Exception(f"invalid job: {target} {build}")
if build > 1:
self._wait_for_regression_task(job)
self.onefuzz.template.stop(str(test_id), target, str(build))
return job
def _run(self, target_os: OS, test_id: UUID, base: Directory, target: str) -> None:
pool = PoolName(f"{target}-{target_os.name}-{test_id}")
self.onefuzz.pools.create(pool, target_os)
self.onefuzz.scalesets.create(pool, 5)
broken = File(os.path.join(base, target, "broken.exe"))
fixed = File(os.path.join(base, target, "fixed.exe"))
self.logger.info("starting first build")
self._run_job(test_id, pool, target, broken, 1)
self.logger.info("starting second build")
job = self._run_job(test_id, pool, target, fixed, 2)
if self._check_regression(job):
raise Exception("fixed binary should be a no repro")
self.logger.info("starting third build")
job = self._run_job(test_id, pool, target, broken, 3)
if not self._check_regression(job):
raise Exception("broken binary should be a crash report")
self.onefuzz.pools.shutdown(pool, now=True)
def test(
self,
samples: Directory,
*,
endpoint: Optional[str] = None,
):
test_id = uuid4()
self.logger.info(f"launch test {test_id}")
self.onefuzz.__setup__(endpoint=endpoint)
error: Optional[Exception] = None
try:
self._run(OS.linux, test_id, samples, "linux-libfuzzer-regression")
except Exception as err:
error = err
except KeyboardInterrupt:
self.logger.warning("interruptted")
finally:
self.logger.info("cleaning up tests")
self.cleanup(test_id)
if error:
raise error
def main() -> int:
return execute_api(
Run(Onefuzz(), logging.getLogger("regression")), [Command], "0.0.1"
)
if __name__ == "__main__":
sys.exit(main())

21
src/integration-tests/git-bisect/README.md Normal file
View File

@ -0,0 +1,21 @@
# git-bisect regression source
This assumes you have a working clang with libfuzzer, bash, and git.
This makes a git repo `test` with 9 commits. Each commit after the first adds a bug.
* `commit 0` has no bugs.
* `commit 1` will additionally cause an abort if the input is `1`.
* `commit 2` will additionally cause an abort if the input is `2`.
* `commit 3` will additionally cause an abort if the input is `3`.
* etc.
This directory provides exemplar scripts that demonstrate how to perform
`git bisect` with libfuzzer.
* [run-local.sh](run-local.sh) builds & runs the libfuzzer target locally. It uses [src/bisect-local.sh](src/bisect-local.sh) as the `git bisect run` command.
* [run-onefuzz.sh](run-onefuzz.sh) builds the libfuzzer target locally, but uses OneFuzz to run the regression tasks. It uses [src/bisect-onefuzz.sh](src/bisect-onefuzz.sh) as the `git bisect run` command.
With each project having their own unique paradigm for building, this model
allows plugging OneFuzz as a `bisect` command in whatever fashion your
project requires.

18
src/integration-tests/git-bisect/build.sh Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash
set -e
rm -rf test
git init test
(cd test; git config user.name "Example"; git config user.email example@contoso.com)
(cp src/Makefile test; cd test; git add Makefile)
for i in $(seq 0 8); do
cp src/fuzz.c test/fuzz.c
for j in $(seq $i 8); do
if [ $i != $j ]; then
sed -i /TEST$j/d test/fuzz.c
fi
done
(cd test; git add fuzz.c; git commit -m "commit $i")
done

17
src/integration-tests/git-bisect/run-local.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
set -e
# build our git repo with our samples in `test`
# (note, we don't care about the output of this script)
./build.sh 2>/dev/null > /dev/null
# create our crashing input
echo -n '3' > test/test.txt
cd test
# start the bisect, looking from HEAD backwards 8 commits
git bisect start HEAD HEAD~8 --
git bisect run ../src/bisect-local.sh test.txt
git bisect reset

17
src/integration-tests/git-bisect/run-onefuzz.sh Executable file
View File

@ -0,0 +1,17 @@
#!/bin/bash
set -e
# build our git repo with our samples in `test`
# (note, we don't care about the output of this script)
./build.sh 2>/dev/null > /dev/null
# create our crashing input
echo -n '3' > test/test.txt
cd test
# start the bisect, looking from HEAD backwards 8 commits
git bisect start HEAD HEAD~8 --
git bisect run ../src/bisect-onefuzz.sh test.txt
git bisect reset

13
src/integration-tests/git-bisect/src/Makefile Normal file
View File

@ -0,0 +1,13 @@
CC=clang
CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address
all: fuzz.exe
fuzz.exe: fuzz.c
$(CC) $(CFLAGS) fuzz.c -o fuzz.exe
.PHONY: clean
clean:
@rm -f fuzz.exe

7
src/integration-tests/git-bisect/src/bisect-local.sh Executable file
View File

@ -0,0 +1,7 @@
#!/bin/bash
set -ex
make clean
make
./fuzz.exe $*

12
src/integration-tests/git-bisect/src/bisect-onefuzz.sh Executable file
View File

@ -0,0 +1,12 @@
#!/bin/bash
set -e
PROJECT=${PROJECT:-regression-test}
TARGET=${TARGET:-$(uuidgen)}
BUILD=regression-$(git rev-parse HEAD)
POOL=${ONEFUZZ_POOL:-linux}
make clean
make
onefuzz template regression libfuzzer ${PROJECT} ${TARGET} ${BUILD} ${POOL} --check_regressions --delete_input_container --reports --crashes $*

13
src/integration-tests/git-bisect/src/fuzz.c Normal file
View File

@ -0,0 +1,13 @@
#include <stdlib.h>
int LLVMFuzzerTestOneInput(char *data, size_t len) {
if (len != 1) { return 0; }
if (data[0] == '1') { abort(); } // TEST1
if (data[0] == '2') { abort(); } // TEST2
if (data[0] == '3') { abort(); } // TEST3
if (data[0] == '4') { abort(); } // TEST4
if (data[0] == '5') { abort(); } // TEST5
if (data[0] == '6') { abort(); } // TEST6
if (data[0] == '7') { abort(); } // TEST7
if (data[0] == '8') { abort(); } // TEST8
return 0;
}

16
src/integration-tests/libfuzzer-regression/Makefile Normal file
View File

@ -0,0 +1,16 @@
CC=clang
CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address
all: broken.exe fixed.exe
broken.exe: simple.c
$(CC) $(CFLAGS) simple.c -o broken.exe
fixed.exe: simple.c
$(CC) $(CFLAGS) simple.c -o fixed.exe -DFIXED
.PHONY: clean
clean:
rm -f broken.exe fixed.exe

1
src/integration-tests/libfuzzer-regression/seeds/good.txt Normal file
View File

@ -0,0 +1 @@
good

26
src/integration-tests/libfuzzer-regression/simple.c Normal file
View File

@ -0,0 +1,26 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <stdint.h>
#include <stdlib.h>
int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) {
int cnt = 0;
if (len < 3) {
return 0;
}
if (data[0] == 'x') { cnt++; }
if (data[1] == 'y') { cnt++; }
if (data[2] == 'z') { cnt++; }
#ifndef FIXED
if (cnt >= 3) {
abort();
}
#endif
return 0;
}

48
src/pytypes/extra/generate-docs.py
View File

@ -31,6 +31,7 @@ from onefuzztypes.events import (
EventProxyCreated,
EventProxyDeleted,
EventProxyFailed,
EventRegressionReported,
EventScalesetCreated,
EventScalesetDeleted,
EventScalesetFailed,
@ -45,8 +46,10 @@ from onefuzztypes.events import (
)
from onefuzztypes.models import (
BlobRef,
CrashTestResult,
Error,
JobConfig,
RegressionReport,
Report,
TaskConfig,
TaskContainers,
@ -87,6 +90,24 @@ def main() -> None:
],
tags={},
)
report = Report(
input_blob=BlobRef(
account="contoso-storage-account",
container=Container("crashes"),
name="input.txt",
),
executable="fuzz.exe",
crash_type="example crash report type",
crash_site="example crash site",
call_stack=["#0 line", "#1 line", "#2 line"],
call_stack_sha256=ZERO_SHA256,
input_sha256=EMPTY_SHA256,
asan_log="example asan log",
task_id=UUID(int=0),
job_id=UUID(int=0),
scariness_score=10,
scariness_description="example-scariness",
)
examples: List[Event] = [
EventPing(ping_id=UUID(int=0)),
EventTaskCreated(
@ -193,27 +214,18 @@ def main() -> None:
pool_name=PoolName("example"),
state=NodeState.setting_up,
),
EventRegressionReported(
regression_report=RegressionReport(
crash_test_result=CrashTestResult(crash_report=report),
original_crash_test_result=CrashTestResult(crash_report=report),
),
container=Container("container-name"),
filename="example.json",
),
EventCrashReported(
container=Container("container-name"),
filename="example.json",
report=Report(
input_blob=BlobRef(
account="contoso-storage-account",
container=Container("crashes"),
name="input.txt",
),
executable="fuzz.exe",
crash_type="example crash report type",
crash_site="example crash site",
call_stack=["#0 line", "#1 line", "#2 line"],
call_stack_sha256=ZERO_SHA256,
input_sha256=EMPTY_SHA256,
asan_log="example asan log",
task_id=UUID(int=0),
job_id=UUID(int=0),
scariness_score=10,
scariness_description="example-scariness",
),
report=report,
),
EventFileAdded(container=Container("container-name"), filename="example.txt"),
EventNodeHeartbeat(machine_id=UUID(int=0), pool_name=PoolName("example")),

7
src/pytypes/onefuzztypes/enums.py
View File

@ -78,6 +78,7 @@ class TaskFeature(Enum):
preserve_existing_outputs = "preserve_existing_outputs"
check_fuzzer_help = "check_fuzzer_help"
expect_crash_on_failure = "expect_crash_on_failure"
report_list = "report_list"
# Permissions for an Azure Blob Storage Container.
@ -149,11 +150,13 @@ class TaskType(Enum):
libfuzzer_coverage = "libfuzzer_coverage"
libfuzzer_crash_report = "libfuzzer_crash_report"
libfuzzer_merge = "libfuzzer_merge"
libfuzzer_regression = "libfuzzer_regression"
generic_analysis = "generic_analysis"
generic_supervisor = "generic_supervisor"
generic_merge = "generic_merge"
generic_generator = "generic_generator"
generic_crash_report = "generic_crash_report"
generic_regression = "generic_regression"
class VmState(Enum):
@ -207,6 +210,7 @@ class ContainerType(Enum):
tools = "tools"
unique_inputs = "unique_inputs"
unique_reports = "unique_reports"
regression_reports = "regression_reports"
@classmethod
def reset_defaults(cls) -> List["ContainerType"]:
@ -219,8 +223,9 @@ class ContainerType(Enum):
cls.readonly_inputs,
cls.reports,
cls.setup,
cls.unique_reports,
cls.unique_inputs,
cls.unique_reports,
cls.regression_reports,
]
@classmethod

19
src/pytypes/onefuzztypes/events.py
View File

@ -11,7 +11,15 @@ from uuid import UUID, uuid4
from pydantic import BaseModel, Extra, Field
from .enums import OS, Architecture, NodeState, TaskState, TaskType
from .models import AutoScaleConfig, Error, JobConfig, Report, TaskConfig, UserInfo
from .models import (
AutoScaleConfig,
Error,
JobConfig,
RegressionReport,
Report,
TaskConfig,
UserInfo,
)
from .primitives import Container, PoolName, Region
from .responses import BaseResponse
@ -156,6 +164,12 @@ class EventCrashReported(BaseEvent):
filename: str
class EventRegressionReported(BaseEvent):
regression_report: RegressionReport
container: Container
filename: str
class EventFileAdded(BaseEvent):
container: Container
filename: str
@ -183,6 +197,7 @@ Event = Union[
EventTaskStopped,
EventTaskHeartbeat,
EventCrashReported,
EventRegressionReported,
EventFileAdded,
]
@ -207,6 +222,7 @@ class EventType(Enum):
task_state_updated = "task_state_updated"
task_stopped = "task_stopped"
crash_reported = "crash_reported"
regression_reported = "regression_reported"
file_added = "file_added"
task_heartbeat = "task_heartbeat"
node_heartbeat = "node_heartbeat"
@ -234,6 +250,7 @@ EventTypeMap = {
EventType.task_heartbeat: EventTaskHeartbeat,
EventType.task_stopped: EventTaskStopped,
EventType.crash_reported: EventCrashReported,
EventType.regression_reported: EventRegressionReported,
EventType.file_added: EventFileAdded,
}

25
src/pytypes/onefuzztypes/models.py
View File

@ -170,6 +170,7 @@ class TaskDetails(BaseModel):
target_timeout: Optional[int]
ensemble_sync_delay: Optional[int]
preserve_existing_outputs: Optional[bool]
report_list: Optional[List[str]]
@validator("check_retry_count", allow_reuse=True)
def validate_check_retry_count(cls, value: int) -> int:
@ -237,7 +238,7 @@ class BlobRef(BaseModel):
class Report(BaseModel):
input_url: Optional[str]
input_blob: BlobRef
input_blob: Optional[BlobRef]
executable: str
crash_type: str
crash_site: str
@ -251,6 +252,26 @@ class Report(BaseModel):
scariness_description: Optional[str]
class NoReproReport(BaseModel):
input_sha256: str
input_blob: Optional[BlobRef]
executable: str
task_id: UUID
job_id: UUID
tries: int
error: Optional[str]
class CrashTestResult(BaseModel):
crash_report: Optional[Report]
no_repro: Optional[NoReproReport]
class RegressionReport(BaseModel):
crash_test_result: CrashTestResult
original_crash_test_result: Optional[CrashTestResult]
class ADODuplicateTemplate(BaseModel):
increment: List[str]
comment: Optional[str]
@ -377,6 +398,7 @@ class TaskUnitConfig(BaseModel):
stats_file: Optional[str]
stats_format: Optional[StatsFormat]
ensemble_sync_delay: Optional[int]
report_list: Optional[List[str]]
# from here forwards are Container definitions. These need to be inline
# with TaskDefinitions and ContainerTypes
@ -390,6 +412,7 @@ class TaskUnitConfig(BaseModel):
tools: CONTAINER_DEF
unique_inputs: CONTAINER_DEF
unique_reports: CONTAINER_DEF
regression_reports: CONTAINER_DEF
class Forward(BaseModel):