diff --git a/docs/telemetry.md b/docs/telemetry.md index 134a17a73..7ebe74cd3 100644 --- a/docs/telemetry.md +++ b/docs/telemetry.md @@ -33,7 +33,7 @@ on-premise hardware, third-party clouds, etc). All telemetry is gathered from two places, the agents that run within fuzzing nodes and the service API running in the Azure Functions instance. -1. The rust library [onefuzz::telemetry](../src/agent/onefuzz/src/telemetry.rs) +1. The rust library [onefuzz::telemetry](../src/agent/onefuzz-telemetry/src/lib.rs) provides a detailed set of telemetry types, as well as the function `can_share`, which gates if a given telemetry field should be sent to the Microsoft central telemetry instance. diff --git a/src/agent/Cargo.lock b/src/agent/Cargo.lock index 38af5efc6..b4378533f 100644 --- a/src/agent/Cargo.lock +++ b/src/agent/Cargo.lock @@ -1556,6 +1556,7 @@ dependencies = [ "log", "nix 0.19.1", "notify", + "onefuzz-telemetry", "pete", "proc-maps", "process_control", @@ -1597,6 +1598,7 @@ dependencies = [ "log", "num_cpus", "onefuzz", + "onefuzz-telemetry", "reqwest", "reqwest-retry", "serde", @@ -1622,6 +1624,7 @@ dependencies = [ "futures", "log", "onefuzz", + "onefuzz-telemetry", "reqwest", "reqwest-retry", "serde", @@ -1634,6 +1637,16 @@ dependencies = [ "uuid", ] +[[package]] +name = "onefuzz-telemetry" +version = "0.1.0" +dependencies = [ + "appinsights", + "log", + "tokio", + "uuid", +] + [[package]] name = "opaque-debug" version = "0.3.0" diff --git a/src/agent/Cargo.toml b/src/agent/Cargo.toml index b2e876cd0..be2b8b119 100644 --- a/src/agent/Cargo.toml +++ b/src/agent/Cargo.toml @@ -7,6 +7,7 @@ members = [ "onefuzz", "onefuzz-agent", "onefuzz-supervisor", + "onefuzz-telemetry", "reqwest-retry", "storage-queue", "win-util", diff --git a/src/agent/onefuzz-agent/Cargo.toml b/src/agent/onefuzz-agent/Cargo.toml index b2f5fedc2..d7fa60580 100644 --- a/src/agent/onefuzz-agent/Cargo.toml +++ b/src/agent/onefuzz-agent/Cargo.toml @@ -31,6 +31,7 @@ uuid = { version = "0.8", features = ["serde", "v4"] } onefuzz = { path = "../onefuzz" } storage-queue = { path = "../storage-queue" } reqwest-retry = { path = "../reqwest-retry" } +onefuzz-telemetry = { path = "../onefuzz-telemetry" } [dev-dependencies] tempfile = "3.1" diff --git a/src/agent/onefuzz-agent/src/main.rs b/src/agent/onefuzz-agent/src/main.rs index 0ad9c284f..28e5ec867 100644 --- a/src/agent/onefuzz-agent/src/main.rs +++ b/src/agent/onefuzz-agent/src/main.rs @@ -3,12 +3,11 @@ #[macro_use] extern crate anyhow; - -#[macro_use] -extern crate onefuzz; - #[macro_use] extern crate clap; +#[macro_use] +extern crate onefuzz_telemetry; +extern crate onefuzz; use anyhow::Result; use clap::{App, ArgMatches, SubCommand}; diff --git a/src/agent/onefuzz-agent/src/managed/cmd.rs b/src/agent/onefuzz-agent/src/managed/cmd.rs index 4cd49b41b..fe379a4f7 100644 --- a/src/agent/onefuzz-agent/src/managed/cmd.rs +++ b/src/agent/onefuzz-agent/src/managed/cmd.rs @@ -4,7 +4,6 @@ use crate::tasks::config::{CommonConfig, Config}; use anyhow::Result; use clap::{App, Arg, SubCommand}; -use onefuzz::telemetry; use std::path::PathBuf; pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { @@ -19,12 +18,12 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> { error!("error running task: {}", err); } - telemetry::try_flush_and_close(); + onefuzz_telemetry::try_flush_and_close(); result } fn init_telemetry(config: &CommonConfig) { - telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key); + onefuzz_telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key); } pub fn args(name: &str) -> App<'static, 'static> { diff --git a/src/agent/onefuzz-agent/src/tasks/config.rs b/src/agent/onefuzz-agent/src/tasks/config.rs index 9849e9a56..46437a617 100644 --- a/src/agent/onefuzz-agent/src/tasks/config.rs +++ b/src/agent/onefuzz-agent/src/tasks/config.rs @@ -4,10 +4,8 @@ #![allow(clippy::large_enum_variant)] use crate::tasks::{analysis, coverage, fuzz, heartbeat::*, merge, report}; use anyhow::Result; -use onefuzz::{ - machine_id::{get_machine_id, get_scaleset_name}, - telemetry::{self, Event::task_start, EventData}, -}; +use onefuzz::machine_id::{get_machine_id, get_scaleset_name}; +use onefuzz_telemetry::{self as telemetry, Event::task_start, EventData}; use reqwest::Url; use serde::{self, Deserialize}; use std::path::PathBuf; diff --git a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs index 2d1c287a2..ac00f1102 100644 --- a/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs +++ b/src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs @@ -39,10 +39,8 @@ use crate::tasks::{ use anyhow::{Context, Result}; use async_trait::async_trait; use futures::stream::StreamExt; -use onefuzz::{ - fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir, telemetry::Event::coverage_data, - telemetry::EventData, -}; +use onefuzz::{fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir}; +use onefuzz_telemetry::{Event::coverage_data, EventData}; use reqwest::Url; use serde::Deserialize; use std::collections::HashMap; diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs index be7de2d91..d0ff1084b 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs @@ -15,8 +15,8 @@ use onefuzz::{ process::monitor_process, sha256, syncdir::{continuous_sync, SyncOperation::Pull, SyncedDir}, - telemetry::Event::new_result, }; +use onefuzz_telemetry::Event::new_result; use serde::Deserialize; use std::collections::HashMap; use std::{ diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs index f7f652af3..2db87bc30 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs @@ -10,10 +10,10 @@ use onefuzz::{ process::ExitStatus, syncdir::{continuous_sync, SyncOperation::Pull, SyncedDir}, system, - telemetry::{ - Event::{new_coverage, new_result, process_stats, runtime_stats}, - EventData, - }, +}; +use onefuzz_telemetry::{ + Event::{new_coverage, new_result, process_stats, runtime_stats}, + EventData, }; use serde::Deserialize; use std::{collections::HashMap, path::PathBuf}; diff --git a/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs b/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs index fa3513fd5..5b31221c3 100644 --- a/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs +++ b/src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs @@ -16,8 +16,8 @@ use onefuzz::{ jitter::delay_with_jitter, process::monitor_process, syncdir::{SyncOperation::Pull, SyncedDir}, - telemetry::Event::{new_coverage, new_result}, }; +use onefuzz_telemetry::Event::{new_coverage, new_result}; use serde::Deserialize; use std::{ collections::HashMap, diff --git a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs index 4bb993daf..50175eea2 100644 --- a/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs +++ b/src/agent/onefuzz-agent/src/tasks/report/crash_report.rs @@ -9,10 +9,10 @@ use onefuzz::{ fs::exists, monitor::DirectoryMonitor, syncdir::SyncedDir, - telemetry::{ - Event::{new_report, new_unable_to_reproduce, new_unique_report}, - EventData, - }, +}; +use onefuzz_telemetry::{ + Event::{new_report, new_unable_to_reproduce, new_unique_report}, + EventData, }; use reqwest::{StatusCode, Url}; use reqwest_retry::SendRetry; diff --git a/src/agent/onefuzz-agent/src/tasks/stats/afl.rs b/src/agent/onefuzz-agent/src/tasks/stats/afl.rs index 194a25e71..cf84efab0 100644 --- a/src/agent/onefuzz-agent/src/tasks/stats/afl.rs +++ b/src/agent/onefuzz-agent/src/tasks/stats/afl.rs @@ -2,7 +2,7 @@ // Licensed under the MIT License. use anyhow::{Context, Error, Result}; -use onefuzz::telemetry::EventData; +use onefuzz_telemetry::EventData; use std::path::Path; use tokio::io::AsyncBufReadExt; diff --git a/src/agent/onefuzz-agent/src/tasks/stats/common.rs b/src/agent/onefuzz-agent/src/tasks/stats/common.rs index ea9ca4b29..962940ca1 100644 --- a/src/agent/onefuzz-agent/src/tasks/stats/common.rs +++ b/src/agent/onefuzz-agent/src/tasks/stats/common.rs @@ -3,10 +3,8 @@ use super::afl; use anyhow::{Error, Result}; -use onefuzz::{ - jitter::delay_with_jitter, - telemetry::{track_event, Event::runtime_stats}, -}; +use onefuzz::jitter::delay_with_jitter; +use onefuzz_telemetry::{track_event, Event::runtime_stats}; use serde::Deserialize; pub const STATS_DELAY: std::time::Duration = std::time::Duration::from_secs(30); diff --git a/src/agent/onefuzz-supervisor/Cargo.toml b/src/agent/onefuzz-supervisor/Cargo.toml index 4cba6945c..1b50191f0 100644 --- a/src/agent/onefuzz-supervisor/Cargo.toml +++ b/src/agent/onefuzz-supervisor/Cargo.toml @@ -25,6 +25,7 @@ url = { version = "2.1.1", features = ["serde"] } uuid = { version = "0.8.1", features = ["serde", "v4"] } clap = "2.33" reqwest-retry = { path = "../reqwest-retry" } +onefuzz-telemetry = { path = "../onefuzz-telemetry" } [target.'cfg(target_os = "linux")'.dependencies] users = "0.11" \ No newline at end of file diff --git a/src/agent/onefuzz-supervisor/src/main.rs b/src/agent/onefuzz-supervisor/src/main.rs index 63d6c2501..e62875be1 100644 --- a/src/agent/onefuzz-supervisor/src/main.rs +++ b/src/agent/onefuzz-supervisor/src/main.rs @@ -6,13 +6,14 @@ extern crate async_trait; #[macro_use] extern crate downcast_rs; #[macro_use] -extern crate onefuzz; -#[macro_use] extern crate serde; #[macro_use] extern crate clap; #[macro_use] extern crate anyhow; +#[macro_use] +extern crate onefuzz_telemetry; +extern crate onefuzz; use crate::{ config::StaticConfig, coordinator::StateUpdateEvent, heartbeat::*, work::WorkSet, @@ -24,8 +25,8 @@ use anyhow::Result; use onefuzz::{ machine_id::{get_machine_id, get_scaleset_name}, process::ExitStatus, - telemetry::{self, EventData}, }; +use onefuzz_telemetry::{self as telemetry, EventData}; use structopt::StructOpt; pub mod agent; diff --git a/src/agent/onefuzz-telemetry/Cargo.toml b/src/agent/onefuzz-telemetry/Cargo.toml new file mode 100644 index 000000000..18cb6dd8d --- /dev/null +++ b/src/agent/onefuzz-telemetry/Cargo.toml @@ -0,0 +1,14 @@ +[package] +name = "onefuzz-telemetry" +version = "0.1.0" +authors = ["fuzzing@microsoft.com"] +edition = "2018" +license = "MIT" + +[dependencies] +appinsights = "0.1.4" +log = "0.4" +uuid = { version = "0.8", features = ["serde", "v4"] } + +[dev-dependencies] +tokio = { version = "0.2" } diff --git a/src/agent/onefuzz/src/telemetry.rs b/src/agent/onefuzz-telemetry/src/lib.rs similarity index 89% rename from src/agent/onefuzz/src/telemetry.rs rename to src/agent/onefuzz-telemetry/src/lib.rs index f00d2818d..f821eeccd 100644 --- a/src/agent/onefuzz/src/telemetry.rs +++ b/src/agent/onefuzz-telemetry/src/lib.rs @@ -4,6 +4,8 @@ use std::sync::{LockResult, RwLockReadGuard, RwLockWriteGuard}; use uuid::Uuid; +pub use appinsights::telemetry::SeverityLevel::{Critical, Error, Information, Verbose, Warning}; + pub type TelemetryClient = appinsights::TelemetryClient; pub enum ClientType { Instance, @@ -347,6 +349,28 @@ pub fn track_event(event: Event, properties: Vec) { local_log_event(&event, &properties); } +pub fn to_log_level(level: &appinsights::telemetry::SeverityLevel) -> log::Level { + match level { + Verbose => log::Level::Debug, + Information => log::Level::Info, + Warning => log::Level::Warn, + Error => log::Level::Error, + Critical => log::Level::Error, + } +} + +pub fn should_log(level: &appinsights::telemetry::SeverityLevel) -> bool { + to_log_level(level) <= log::max_level() +} + +pub fn log_message(level: appinsights::telemetry::SeverityLevel, msg: String) { + let log_level = to_log_level(&level); + log::log!(log_level, "{}", msg); + if let Some(client) = client(ClientType::Instance) { + client.track_trace(msg, level); + } +} + #[macro_export] macro_rules! event { ($name: expr ; $($k: path = $v: expr),*) => {{ @@ -357,33 +381,15 @@ macro_rules! event { })*; - $crate::telemetry::track_event($name, events); + onefuzz_telemetry::track_event($name, events); }}; } #[macro_export] macro_rules! log { ($level: expr, $msg: expr) => {{ - use appinsights::telemetry::SeverityLevel::{ - Critical, Error, Information, Verbose, Warning, - }; - - let log_level = match $level { - Verbose => log::Level::Debug, - Information => log::Level::Info, - Warning => log::Level::Warn, - Error => log::Level::Error, - Critical => log::Level::Error, - }; - - // while log::log will filter based on log level, the telemetry - // client does *not*. - if log_level <= log::max_level() { - log::log!(log_level, "{}", $msg.to_string()); - if let Some(client) = $crate::telemetry::client($crate::telemetry::ClientType::Instance) - { - client.track_trace($msg, $level); - } + if onefuzz_telemetry::should_log(&$level) { + onefuzz_telemetry::log_message($level, $msg.to_string()); } }}; } @@ -392,7 +398,7 @@ macro_rules! log { macro_rules! verbose { ($($tt: tt)*) => {{ let msg = format!($($tt)*); - $crate::log!(Verbose, msg); + onefuzz_telemetry::log!(onefuzz_telemetry::Verbose, msg); }} } @@ -400,7 +406,7 @@ macro_rules! verbose { macro_rules! info { ($($tt: tt)*) => {{ let msg = format!($($tt)*); - $crate::log!(Information, msg); + onefuzz_telemetry::log!(onefuzz_telemetry::Information, msg); }} } @@ -408,7 +414,7 @@ macro_rules! info { macro_rules! warn { ($($tt: tt)*) => {{ let msg = format!($($tt)*); - $crate::log!(Warning, msg); + onefuzz_telemetry::log!(onefuzz_telemetry::Warning, msg); }} } @@ -416,7 +422,7 @@ macro_rules! warn { macro_rules! error { ($($tt: tt)*) => {{ let msg = format!($($tt)*); - $crate::log!(Error, msg); + onefuzz_telemetry::log!(onefuzz_telemetry::Error, msg); }} } @@ -424,14 +430,14 @@ macro_rules! error { macro_rules! critical { ($($tt: tt)*) => {{ let msg = format!($($tt)*); - $crate::log!(Critical, msg); + onefuzz_telemetry::log!(onefuzz_telemetry::Critical, msg); }} } #[macro_export] macro_rules! metric { ($name: expr, $value: expr) => {{ - let client = $crate::telemetry::client($crate::telemetry::ClientType::Instance); + let client = onefuzz_telemetry::client(onefuzz_telemetry::ClientType::Instance); client.track_metric($name.into(), $value); }}; } diff --git a/src/agent/onefuzz/Cargo.toml b/src/agent/onefuzz/Cargo.toml index ac7e4e808..c6eead339 100644 --- a/src/agent/onefuzz/Cargo.toml +++ b/src/agent/onefuzz/Cargo.toml @@ -38,6 +38,7 @@ strum_macros = "0.19" tempfile = "3.1" process_control = "2.0" reqwest-retry = { path = "../reqwest-retry"} +onefuzz-telemetry = { path = "../onefuzz-telemetry"} [target.'cfg(target_os = "windows")'.dependencies] winreg = "0.7" diff --git a/src/agent/onefuzz/src/lib.rs b/src/agent/onefuzz/src/lib.rs index 242bf24da..768d45c06 100644 --- a/src/agent/onefuzz/src/lib.rs +++ b/src/agent/onefuzz/src/lib.rs @@ -11,7 +11,7 @@ extern crate lazy_static; extern crate serde; #[macro_use] -pub mod telemetry; +extern crate onefuzz_telemetry; pub mod asan; pub mod az_copy; diff --git a/src/agent/onefuzz/src/syncdir.rs b/src/agent/onefuzz/src/syncdir.rs index b9e2cf7e5..3ed88db24 100644 --- a/src/agent/onefuzz/src/syncdir.rs +++ b/src/agent/onefuzz/src/syncdir.rs @@ -2,15 +2,12 @@ // Licensed under the MIT License. use crate::{ - az_copy, - blob::BlobContainerUrl, - jitter::delay_with_jitter, - monitor::DirectoryMonitor, - telemetry::{Event, EventData}, + az_copy, blob::BlobContainerUrl, jitter::delay_with_jitter, monitor::DirectoryMonitor, uploader::BlobUploader, }; use anyhow::{Context, Result}; use futures::stream::StreamExt; +use onefuzz_telemetry::{Event, EventData}; use std::{path::PathBuf, str, time::Duration}; use tokio::fs;