ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-19 04:58:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Prefix target_exe with setup dir at use sites (#2405)

Browse Source
This commit is contained in:
Joe Ranweiler
2022-09-29 13:47:04 -07:00
committed by GitHub
parent 0c4cd5414d
commit 41f973184e
18 changed files with 264 additions and 52 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
src/ApiService/ApiService/onefuzzlib/Config.cs
View File

@ -160,11 +160,11 @@ public class Config : IConfig {
}
if (definition.Features.Contains(TaskFeature.TargetExe)) {
config.TargetExe = $"setup/{task.Config.Task.TargetExe}";
config.TargetExe = task.Config.Task.TargetExe;
}
if (definition.Features.Contains(TaskFeature.TargetExeOptional) && config.TargetExe != null) {
config.TargetExe = $"setup/{task.Config.Task.TargetExe}";
config.TargetExe = task.Config.Task.TargetExe;
}
if (definition.Features.Contains(TaskFeature.TargetEnv)) {
@ -257,9 +257,8 @@ public class Config : IConfig {
}
if (definition.Features.Contains(TaskFeature.CoverageFilter)) {
var coverageFilter = task.Config.Task.CoverageFilter;
if (coverageFilter != null) {
config.CoverageFilter = $"setup/{coverageFilter}";
if (task.Config.Task.CoverageFilter != null) {
config.CoverageFilter = task.Config.Task.CoverageFilter;
}
}

6
src/agent/onefuzz-task/src/tasks/analysis/generic.rs
View File

@ -3,6 +3,7 @@
use crate::tasks::{
config::CommonConfig, heartbeat::HeartbeatSender, report::crash_report::monitor_reports,
utils::try_resolve_setup_relative_path,
};
use anyhow::{Context, Result};
use onefuzz::{az_copy, blob::url::BlobUrl};
@ -194,11 +195,14 @@ pub async fn run_tool(
config: &Config,
reports_dir: &Option<PathBuf>,
) -> Result<()> {
let target_exe =
try_resolve_setup_relative_path(&config.common.setup_dir, &config.target_exe).await?;
let expand = Expand::new()
.machine_id()
.await?
.input_path(&input)
.target_exe(&config.target_exe)
.target_exe(&target_exe)
.target_options(&config.target_options)
.analyzer_exe(&config.analyzer_exe)
.analyzer_options(&config.analyzer_options)

7
src/agent/onefuzz-task/src/tasks/coverage/dotnet.rs
View File

@ -26,6 +26,7 @@ use crate::tasks::{
coverage::COBERTURA_COVERAGE_FILE,
generic::input_poller::{CallbackImpl, InputPoller, Processor},
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::try_resolve_setup_relative_path,
};
const MAX_COVERAGE_RECORDING_ATTEMPTS: usize = 2;
@ -257,13 +258,17 @@ impl<'a> TaskContext<'a> {
}
async fn command_for_input(&self, input: &Path) -> Result<Command> {
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let expand = Expand::new()
.machine_id()
.await?
.input_path(input)
.job_id(&self.config.common.job_id)
.setup_dir(&self.config.common.setup_dir)
.target_exe(&self.config.target_exe)
.target_exe(&target_exe)
.target_options(&self.config.target_options)
.task_id(&self.config.common.task_id);

23
src/agent/onefuzz-task/src/tasks/coverage/generic.rs
View File

@ -27,6 +27,7 @@ use url::Url;
use crate::tasks::config::CommonConfig;
use crate::tasks::generic::input_poller::{CallbackImpl, InputPoller, Processor};
use crate::tasks::heartbeat::{HeartbeatSender, TaskHeartbeatClient};
use crate::tasks::utils::{resolve_setup_relative_path, try_resolve_setup_relative_path};
use super::COBERTURA_COVERAGE_FILE;
@ -138,9 +139,17 @@ impl CoverageTask {
return Ok(CmdFilter::default());
};
// Ensure users can locate the filter relative to the setup container.
let expand = Expand::new().setup_dir(&self.config.common.setup_dir);
let filter_path = expand.evaluate_value(raw_filter_path)?;
let resolved =
resolve_setup_relative_path(&self.config.common.setup_dir, raw_filter_path).await?;
let filter_path = if let Some(path) = resolved {
path
} else {
error!(
"unable to resolve setup-relative coverage filter path: {}",
raw_filter_path
);
return Ok(CmdFilter::default());
};
let data = fs::read(&filter_path).await?;
let def: CmdFilterDef = serde_json::from_slice(&data)?;
@ -275,17 +284,21 @@ impl<'a> TaskContext<'a> {
}
async fn command_for_input(&self, input: &Path) -> Result<Command> {
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let expand = Expand::new()
.machine_id()
.await?
.input_path(input)
.job_id(&self.config.common.job_id)
.setup_dir(&self.config.common.setup_dir)
.target_exe(&self.config.target_exe)
.target_exe(&target_exe)
.target_options(&self.config.target_options)
.task_id(&self.config.common.task_id);
let mut cmd = Command::new(&self.config.target_exe);
let mut cmd = Command::new(&target_exe);
let target_options = expand.evaluate(&self.config.target_options)?;
cmd.args(target_options);

8
src/agent/onefuzz-task/src/tasks/fuzz/generator.rs
View File

@ -4,7 +4,7 @@
use crate::tasks::{
config::CommonConfig,
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::{self, default_bool_true},
utils::{self, default_bool_true, try_resolve_setup_relative_path},
};
use anyhow::{Context, Result};
use onefuzz::{
@ -93,9 +93,13 @@ impl GeneratorTask {
}
async fn fuzzing_loop(&self, heartbeat_client: Option<TaskHeartbeatClient>) -> Result<()> {
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let tester = Tester::new(
&self.config.common.setup_dir,
&self.config.target_exe,
&target_exe,
&self.config.target_options,
&self.config.target_env,
)

11
src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/common.rs
View File

@ -51,7 +51,7 @@ pub trait LibFuzzerType: Send + Sync {
///
/// This may include things like setting special environment variables, or overriding
/// the defaults or values of some command arguments.
fn from_config(config: &Config<Self>) -> LibFuzzer;
async fn from_config(config: &Config<Self>) -> Result<LibFuzzer>;
/// Perform any environmental setup common to all targets of this fuzzer type.
///
@ -141,7 +141,7 @@ where
directories.append(&mut dirs);
}
let fuzzer = L::from_config(&self.config);
let fuzzer = L::from_config(&self.config).await?;
fuzzer
.verify(self.config.check_fuzzer_help, Some(directories))
@ -236,12 +236,7 @@ where
.for_each(|d| inputs.push(&d.local_path));
}
let fuzzer = LibFuzzer::new(
&self.config.target_exe,
self.config.target_options.clone(),
self.config.target_env.clone(),
&self.config.common.setup_dir,
);
let fuzzer = L::from_config(&self.config).await?;
let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs).await?;
let notify = Arc::new(Notify::new());

6
src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/dotnet.rs
View File

@ -46,7 +46,7 @@ pub struct LibFuzzerDotnetConfig {
impl common::LibFuzzerType for LibFuzzerDotnet {
type Config = LibFuzzerDotnetConfig;
fn from_config(config: &common::Config<Self>) -> LibFuzzer {
async fn from_config(config: &common::Config<Self>) -> Result<LibFuzzer> {
// Configure loader to fuzz user target DLL.
let mut env = config.target_env.clone();
env.insert(
@ -65,12 +65,12 @@ impl common::LibFuzzerType for LibFuzzerDotnet {
let mut options = config.target_options.clone();
options.push(format!("--target_path={}", LOADER_PATH));
LibFuzzer::new(
Ok(LibFuzzer::new(
LIBFUZZER_DOTNET_PATH,
options,
env,
&config.common.setup_dir,
)
))
}
async fn extra_setup(config: &common::Config<Self>) -> Result<()> {

13
src/agent/onefuzz-task/src/tasks/fuzz/libfuzzer/generic.rs
View File

@ -1,10 +1,12 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use anyhow::Result;
use async_trait::async_trait;
use onefuzz::libfuzzer::LibFuzzer;
use crate::tasks::fuzz::libfuzzer::common;
use crate::tasks::utils::try_resolve_setup_relative_path;
/// Generic LibFuzzer with no special extra configuration.
///
@ -17,13 +19,16 @@ pub struct GenericLibFuzzer;
impl common::LibFuzzerType for GenericLibFuzzer {
type Config = ();
fn from_config(config: &common::Config<Self>) -> LibFuzzer {
LibFuzzer::new(
&config.target_exe,
async fn from_config(config: &common::Config<Self>) -> Result<LibFuzzer> {
let target_exe =
try_resolve_setup_relative_path(&config.common.setup_dir, &config.target_exe).await?;
Ok(LibFuzzer::new(
&target_exe,
config.target_options.clone(),
config.target_env.clone(),
&config.common.setup_dir,
)
))
}
}

10
src/agent/onefuzz-task/src/tasks/fuzz/supervisor.rs
View File

@ -7,7 +7,7 @@ use crate::tasks::{
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
report::crash_report::monitor_reports,
stats::common::{monitor_stats, StatsFormat},
utils::CheckNotify,
utils::{try_resolve_setup_relative_path, CheckNotify},
};
use anyhow::{Context, Error, Result};
use onefuzz::{
@ -198,6 +198,12 @@ async fn start_supervisor(
inputs: &SyncedDir,
reports_dir: PathBuf,
) -> Result<Child> {
let target_exe = if let Some(target_exe) = &config.target_exe {
Some(try_resolve_setup_relative_path(&config.common.setup_dir, target_exe).await?)
} else {
None
};
let expand = Expand::new()
.machine_id()
.await?
@ -216,7 +222,7 @@ async fn start_supervisor(
.set_optional_ref(&config.coverage, |expand, coverage| {
expand.coverage_dir(&coverage.local_path)
})
.set_optional_ref(&config.target_exe, |expand, target_exe| {
.set_optional_ref(&target_exe, |expand, target_exe| {
expand.target_exe(target_exe)
})
.set_optional_ref(&config.supervisor_input_marker, |expand, input_marker| {

11
src/agent/onefuzz-task/src/tasks/merge/generic.rs
View File

@ -1,7 +1,11 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.
use crate::tasks::{config::CommonConfig, heartbeat::HeartbeatSender, utils};
use crate::tasks::{
config::CommonConfig,
heartbeat::HeartbeatSender,
utils::{self, try_resolve_setup_relative_path},
};
use anyhow::{Context, Result};
use onefuzz::{
expand::Expand, fs::set_executable, http::ResponseExt, jitter::delay_with_jitter,
@ -123,6 +127,9 @@ async fn try_delete_blob(input_url: Url) -> Result<()> {
}
async fn merge(config: &Config, output_dir: impl AsRef<Path>) -> Result<()> {
let target_exe =
try_resolve_setup_relative_path(&config.common.setup_dir, &config.target_exe).await?;
let expand = Expand::new()
.machine_id()
.await?
@ -132,7 +139,7 @@ async fn merge(config: &Config, output_dir: impl AsRef<Path>) -> Result<()> {
.supervisor_exe(&config.supervisor_exe)
.supervisor_options(&config.supervisor_options)
.generated_inputs(output_dir)
.target_exe(&config.target_exe)
.target_exe(&target_exe)
.setup_dir(&config.common.setup_dir)
.tools_dir(&config.tools.local_path)
.job_id(&config.common.job_id)

8
src/agent/onefuzz-task/src/tasks/regression/generic.rs
View File

@ -4,7 +4,7 @@
use crate::tasks::{
config::CommonConfig,
report::{crash_report::CrashTestResult, generic},
utils::default_bool_true,
utils::{default_bool_true, try_resolve_setup_relative_path},
};
use anyhow::Result;
use async_trait::async_trait;
@ -56,10 +56,14 @@ pub struct GenericRegressionTask {
#[async_trait]
impl RegressionHandler for GenericRegressionTask {
async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result<CrashTestResult> {
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let args = generic::TestInputArgs {
input_url: Some(input_url),
input: &input,
target_exe: &self.config.target_exe,
target_exe: &target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,

8
src/agent/onefuzz-task/src/tasks/regression/libfuzzer.rs
View File

@ -4,7 +4,7 @@
use crate::tasks::{
config::CommonConfig,
report::{crash_report::CrashTestResult, libfuzzer_report},
utils::default_bool_true,
utils::{default_bool_true, try_resolve_setup_relative_path},
};
use anyhow::{Context, Result};
@ -55,10 +55,14 @@ pub struct LibFuzzerRegressionTask {
#[async_trait]
impl RegressionHandler for LibFuzzerRegressionTask {
async fn get_crash_result(&self, input: PathBuf, input_url: Url) -> Result<CrashTestResult> {
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let args = libfuzzer_report::TestInputArgs {
input_url: Some(input_url),
input: &input,
target_exe: &self.config.target_exe,
target_exe: &target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,

11
src/agent/onefuzz-task/src/tasks/report/dotnet/generic.rs
View File

@ -20,7 +20,7 @@ use crate::tasks::{
config::CommonConfig,
generic::input_poller::*,
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::default_bool_true,
utils::{default_bool_true, try_resolve_setup_relative_path},
};
const DOTNET_DUMP_TOOL_NAME: &str = "dotnet-dump";
@ -128,12 +128,11 @@ impl AsanProcessor {
let job_id = self.config.common.task_id;
let task_id = self.config.common.task_id;
let executable = self.config.target_exe.to_owned();
let executable =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let mut args = vec![
"dotnet".to_owned(),
self.config.target_exe.display().to_string(),
];
let mut args = vec!["dotnet".to_owned(), executable.display().to_string()];
args.extend(self.config.target_options.clone());
let env = self.config.target_env.clone();

8
src/agent/onefuzz-task/src/tasks/report/generic.rs
View File

@ -6,7 +6,7 @@ use crate::tasks::{
config::CommonConfig,
generic::input_poller::{CallbackImpl, InputPoller, Processor},
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::default_bool_true,
utils::{default_bool_true, try_resolve_setup_relative_path},
};
use anyhow::{Context, Result};
use async_trait::async_trait;
@ -193,10 +193,14 @@ impl<'a> GenericReportProcessor<'a> {
) -> Result<CrashTestResult> {
self.heartbeat_client.alive();
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let args = TestInputArgs {
input_url,
input,
target_exe: &self.config.target_exe,
target_exe: &target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,

15
src/agent/onefuzz-task/src/tasks/report/libfuzzer_report.rs
View File

@ -6,7 +6,7 @@ use crate::tasks::{
config::CommonConfig,
generic::input_poller::*,
heartbeat::{HeartbeatSender, TaskHeartbeatClient},
utils::default_bool_true,
utils::{default_bool_true, try_resolve_setup_relative_path},
};
use anyhow::{Context, Result};
use async_trait::async_trait;
@ -65,8 +65,12 @@ impl ReportTask {
}
pub async fn verify(&self) -> Result<()> {
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let fuzzer = LibFuzzer::new(
&self.config.target_exe,
&target_exe,
self.config.target_options.clone(),
self.config.target_env.clone(),
&self.config.common.setup_dir,
@ -194,10 +198,15 @@ impl AsanProcessor {
input: &Path,
) -> Result<CrashTestResult> {
self.heartbeat_client.alive();
let target_exe =
try_resolve_setup_relative_path(&self.config.common.setup_dir, &self.config.target_exe)
.await?;
let args = TestInputArgs {
input_url,
input,
target_exe: &self.config.target_exe,
target_exe: &target_exe,
target_options: &self.config.target_options,
target_env: &self.config.target_env,
setup_dir: &self.config.common.setup_dir,

154
src/agent/onefuzz-task/src/tasks/utils.rs
View File

@ -96,3 +96,157 @@ pub fn parse_key_value(value: String) -> Result<(String, String)> {
pub fn default_bool_true() -> bool {
true
}
/// Try to resolve an ambiguous setup-relative subpath, returning an error if not found.
pub async fn try_resolve_setup_relative_path(
setup_dir: impl AsRef<Path>,
subpath: impl AsRef<Path>,
) -> Result<PathBuf> {
let setup_dir = setup_dir.as_ref();
let subpath = subpath.as_ref();
resolve_setup_relative_path(setup_dir, subpath)
.await?
.ok_or_else(|| {
anyhow::format_err!(
"unable to resolve subpath `{}` under setup dir `{}`",
subpath.display(),
setup_dir.display()
)
})
}
/// Try to resolve an ambiguous setup-relative subpath, returning `None` if not found.
pub async fn resolve_setup_relative_path(
setup_dir: impl AsRef<Path>,
subpath: impl AsRef<Path>,
) -> Result<Option<PathBuf>> {
let setup_dir = setup_dir.as_ref();
let subpath = subpath.as_ref();
// Case: non-legacy `subpath`, relativized to `setup_dir`.
//
// Even if `subpath` is prefixed by `setup`, it is because it truly names a file in a
// non-root subdirectory like `{setup_dir}/setup`.
{
let resolved = setup_dir.join(subpath);
if exists(&resolved).await {
return Ok(Some(resolved));
}
}
// Case: non-legacy `subpath` that uses the `{setup_dir}` placeholder variable.
//
// Note that we do not do full expansion, just a form of restricted find/replace.
{
let subpath = subpath.strip_prefix("{setup_dir}").unwrap_or(subpath);
let resolved = setup_dir.join(subpath);
if exists(&resolved).await {
return Ok(Some(resolved));
}
}
// Case: legacy `subpath`.
//
// The `setup_dir`-relativized `subpath` was prefixed server-side with the hardcoded
// relative path `setup`. We only expect to see this after upgrading deployments that
// have pending tasks with legacy configs.
{
let subpath = subpath.strip_prefix("setup").unwrap_or(subpath);
let resolved = setup_dir.join(subpath);
if exists(&resolved).await {
return Ok(Some(resolved));
}
}
Ok(None)
}
async fn exists(path: impl AsRef<Path>) -> bool {
fs::metadata(path).await.is_ok()
}
#[cfg(test)]
mod tests {
use std::path::Path;
use anyhow::Result;
use tempfile::TempDir;
use super::*;
fn init_setup_dir(actual: impl AsRef<Path>) -> Result<TempDir> {
let actual = actual.as_ref();
let setup_dir = TempDir::new()?;
// If the actual file is nested in any subdirectories of `setup`, ensure that the
// intermediates exist.
if let Some(parent) = actual.parent() {
let intermediate = setup_dir.path().join(parent);
std::fs::create_dir_all(intermediate)?;
}
// Create an (empty) file on-disk for the file being referenced.
std::fs::write(setup_dir.path().join(actual), "")?;
Ok(setup_dir)
}
async fn test_case(relative: impl AsRef<Path>, given: impl AsRef<Path>) -> Result<()> {
let relative = relative.as_ref();
let setup_dir = init_setup_dir(relative)?;
let expected = setup_dir.path().join(relative);
let resolved = resolve_setup_relative_path(setup_dir.path(), given).await?;
assert_eq!(Some(expected), resolved);
Ok(())
}
#[tokio::test]
async fn test_resolve_setup_relative_path_root() -> Result<()> {
const RELATIVE: &str = "fuzz.exe";
test_case(RELATIVE, "fuzz.exe").await?;
test_case(RELATIVE, "setup/fuzz.exe").await?;
test_case(RELATIVE, "{setup_dir}/fuzz.exe").await?;
Ok(())
}
#[tokio::test]
async fn test_resolve_setup_relative_path_nested() -> Result<()> {
const RELATIVE: &str = "x/fuzz.exe";
test_case(RELATIVE, "x/fuzz.exe").await?;
test_case(RELATIVE, "setup/x/fuzz.exe").await?;
test_case(RELATIVE, "{setup_dir}/x/fuzz.exe").await?;
Ok(())
}
#[tokio::test]
async fn test_resolve_setup_relative_path_double_nested() -> Result<()> {
const RELATIVE: &str = "x/y/fuzz.exe";
test_case(RELATIVE, "x/y/fuzz.exe").await?;
test_case(RELATIVE, "setup/x/y/fuzz.exe").await?;
test_case(RELATIVE, "{setup_dir}/x/y/fuzz.exe").await?;
Ok(())
}
#[tokio::test]
async fn test_resolve_setup_relative_path_nested_setup() -> Result<()> {
const RELATIVE: &str = "setup/fuzz.exe";
test_case(RELATIVE, "setup/fuzz.exe").await?;
test_case(RELATIVE, "setup/setup/fuzz.exe").await?;
test_case(RELATIVE, "{setup_dir}/setup/fuzz.exe").await?;
Ok(())
}
}

6
src/api-service/__app__/onefuzzlib/tasks/config.py
View File

@ -348,13 +348,13 @@ def build_task_config(job: Job, task: Task) -> TaskUnitConfig:
config.supervisor_input_marker = task_config.task.supervisor_input_marker
if TaskFeature.target_exe in definition.features:
config.target_exe = "setup/%s" % task_config.task.target_exe
config.target_exe = task_config.task.target_exe
if (
TaskFeature.target_exe_optional in definition.features
and task_config.task.target_exe
):
config.target_exe = "setup/%s" % task_config.task.target_exe
config.target_exe = task_config.task.target_exe
if TaskFeature.target_env in definition.features:
config.target_env = task_config.task.target_env or EMPTY_DICT
@ -438,7 +438,7 @@ def build_task_config(job: Job, task: Task) -> TaskUnitConfig:
coverage_filter = task_config.task.coverage_filter
if coverage_filter is not None:
config.coverage_filter = "setup/%s" % coverage_filter
config.coverage_filter = coverage_filter
return config

2
src/cli/onefuzz/templates/libfuzzer.py
View File

@ -550,7 +550,7 @@ class Libfuzzer(Command):
if target_options is None:
target_options = []
target_options = [
"--target_path={setup_dir}/" f"{target_harness}"
"--target_path={setup_dir}/" + "{target_harness}"
] + target_options
helper = JobHelper(