libfuzzer fuzzing task perf improvements (#941)

- resuse the regex to parse the output of libfuzzer
- added a cancellation notification to report_fuzzer_sys_info.
   ~~The code seems to be actively waiting this function and consuming some cpu time~~
   The notification allows us to reduce the time waiting for the fuzzing loop to terminate. 

## Summary of the Pull Request

_What is this about?_

## PR Checklist
* [ ] Applies to work item: #xxx
* [ ] CLA signed. If not, go over [here](https://cla.opensource.microsoft.com/microsoft/onefuzz) and sign the CLI.
* [ ] Tests added/passed
* [ ] Requires documentation to be updated
* [ ] I've discussed this with core contributors already. If not checked, I'm ready to accept this work might be rejected in favor of a different grand plan. Issue number where discussion took place: #xxx

## Info on Pull Request

_What does this include?_

## Validation Steps Performed

_How does someone test & validate?_
This commit is contained in:
Cheick Keita
2021-06-03 17:14:15 -07:00
committed by GitHub
parent 2c72bd590f
commit 6b7906dfb1
2 changed files with 32 additions and 13 deletions

View File

@ -17,11 +17,12 @@ use onefuzz_telemetry::{
EventData, EventData,
}; };
use serde::Deserialize; use serde::Deserialize;
use std::{collections::HashMap, path::PathBuf}; use std::{collections::HashMap, path::PathBuf, sync::Arc};
use tempfile::{tempdir_in, TempDir}; use tempfile::{tempdir_in, TempDir};
use tokio::{ use tokio::{
io::{AsyncBufReadExt, BufReader}, io::{AsyncBufReadExt, BufReader},
sync::mpsc, select,
sync::{mpsc, Notify},
task, task,
time::{sleep, Duration}, time::{sleep, Duration},
}; };
@ -212,11 +213,12 @@ impl LibFuzzerFuzzTask {
); );
let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?; let mut running = fuzzer.fuzz(crash_dir.path(), local_inputs, &inputs)?;
let running_id = running.id(); let running_id = running.id();
let notify = Arc::new(Notify::new());
let sys_info = task::spawn(report_fuzzer_sys_info( let sys_info = task::spawn(report_fuzzer_sys_info(
worker_id, worker_id,
run_id, run_id,
running_id.unwrap_or_default(), running_id.unwrap_or_default(),
notify.clone(),
)); ));
// Splitting borrow. // Splitting borrow.
@ -227,7 +229,6 @@ impl LibFuzzerFuzzTask {
let mut stderr = BufReader::new(stderr); let mut stderr = BufReader::new(stderr);
let mut libfuzzer_output: ArrayDeque<[_; LOGS_BUFFER_SIZE], Wrapping> = ArrayDeque::new(); let mut libfuzzer_output: ArrayDeque<[_; LOGS_BUFFER_SIZE], Wrapping> = ArrayDeque::new();
loop { loop {
let mut buf = vec![]; let mut buf = vec![];
let bytes_read = stderr.read_until(b'\n', &mut buf).await?; let bytes_read = stderr.read_until(b'\n', &mut buf).await?;
@ -243,7 +244,10 @@ impl LibFuzzerFuzzTask {
libfuzzer_output.push_back(line); libfuzzer_output.push_back(line);
} }
let (exit_status, _) = tokio::join!(running.wait(), sys_info); let exit_status = running.wait().await;
notify.notify_one();
let _ = sys_info.await;
let exit_status: ExitStatus = exit_status?.into(); let exit_status: ExitStatus = exit_status?.into();
let files = list_files(crash_dir.path()).await?; let files = list_files(crash_dir.path()).await?;
@ -323,11 +327,23 @@ fn try_report_iter_update(
Ok(()) Ok(())
} }
async fn report_fuzzer_sys_info(worker_id: usize, run_id: Uuid, fuzzer_pid: u32) -> Result<()> { async fn report_fuzzer_sys_info(
worker_id: usize,
run_id: Uuid,
fuzzer_pid: u32,
cancellation: Arc<Notify>,
) -> Result<()> {
// Allow for sampling CPU usage. // Allow for sampling CPU usage.
sleep(PROC_INFO_COLLECTION_DELAY).await; let mut period = tokio::time::interval_at(
tokio::time::Instant::now() + PROC_INFO_COLLECTION_DELAY,
PROC_INFO_PERIOD,
);
loop { loop {
select! {
() = cancellation.notified() => break,
_ = period.tick() => (),
}
// process doesn't exist // process doesn't exist
if !system::refresh_process(fuzzer_pid)? { if !system::refresh_process(fuzzer_pid)? {
break; break;
@ -348,8 +364,6 @@ async fn report_fuzzer_sys_info(worker_id: usize, run_id: Uuid, fuzzer_pid: u32)
// The process no longer exists. // The process no longer exists.
break; break;
} }
sleep(PROC_INFO_PERIOD).await;
} }
Ok(()) Ok(())

View File

@ -21,6 +21,13 @@ use tokio::process::{Child, Command};
const DEFAULT_MAX_TOTAL_SECONDS: i32 = 10 * 60; const DEFAULT_MAX_TOTAL_SECONDS: i32 = 10 * 60;
use lazy_static::lazy_static;
lazy_static! {
static ref LIBFUZZERLINEREGEX: regex::Regex =
regex::Regex::new(r"#(\d+)\s*(?:pulse|INITED|NEW|REDUCE).*exec/s: (\d+)").unwrap();
}
#[derive(Debug)] #[derive(Debug)]
pub struct LibFuzzerMergeOutput { pub struct LibFuzzerMergeOutput {
pub added_files_count: i32, pub added_files_count: i32,
@ -304,9 +311,7 @@ impl LibFuzzerLine {
} }
pub fn parse(line: &str) -> Result<Option<Self>> { pub fn parse(line: &str) -> Result<Option<Self>> {
let re = regex::Regex::new(r"#(\d+)\s*(?:pulse|INITED|NEW|REDUCE).*exec/s: (\d+)")?; let caps = match LIBFUZZERLINEREGEX.captures(line) {
let caps = match re.captures(line) {
Some(caps) => caps, Some(caps) => caps,
None => return Ok(None), None => return Ok(None),
}; };