split telemetry into it's own crate (#501)

Splits out telemetry crate such that it can be reused by other components (specifically the proxy-manager) easily.

docs/telemetry.md

@@ -33,7 +33,7 @@ on-premise hardware, third-party clouds, etc).
 All telemetry is gathered from two places, the agents that run within fuzzing
 nodes and the service API running in the Azure Functions instance.
 
-1. The rust library [onefuzz::telemetry](../src/agent/onefuzz/src/telemetry.rs)
+1. The rust library [onefuzz::telemetry](../src/agent/onefuzz-telemetry/src/lib.rs)
    provides a detailed set of telemetry types, as well as the function
    `can_share`, which gates if a given telemetry field should be sent to the
    Microsoft central telemetry instance.

src/agent/Cargo.lock

@@ -1556,6 +1556,7 @@ dependencies = [
  "log",
  "nix 0.19.1",
  "notify",
+ "onefuzz-telemetry",
  "pete",
  "proc-maps",
  "process_control",
@@ -1597,6 +1598,7 @@ dependencies = [
  "log",
  "num_cpus",
  "onefuzz",
+ "onefuzz-telemetry",
  "reqwest",
  "reqwest-retry",
  "serde",
@@ -1622,6 +1624,7 @@ dependencies = [
  "futures",
  "log",
  "onefuzz",
+ "onefuzz-telemetry",
  "reqwest",
  "reqwest-retry",
  "serde",
@@ -1634,6 +1637,16 @@ dependencies = [
  "uuid",
 ]
 
+[[package]]
+name = "onefuzz-telemetry"
+version = "0.1.0"
+dependencies = [
+ "appinsights",
+ "log",
+ "tokio",
+ "uuid",
+]
+
 [[package]]
 name = "opaque-debug"
 version = "0.3.0"

src/agent/Cargo.toml

@@ -7,6 +7,7 @@ members = [
     "onefuzz",
     "onefuzz-agent",
     "onefuzz-supervisor",
+    "onefuzz-telemetry",
     "reqwest-retry",
     "storage-queue",
     "win-util",

src/agent/onefuzz-agent/Cargo.toml

@@ -31,6 +31,7 @@ uuid = { version = "0.8", features = ["serde", "v4"] }
 onefuzz = { path = "../onefuzz" }
 storage-queue = { path = "../storage-queue" }
 reqwest-retry = { path = "../reqwest-retry" }
+onefuzz-telemetry = { path = "../onefuzz-telemetry" }
 
 [dev-dependencies]
 tempfile = "3.1"

src/agent/onefuzz-agent/src/main.rs

@@ -3,12 +3,11 @@
 
 #[macro_use]
 extern crate anyhow;
-
-#[macro_use]
-extern crate onefuzz;
-
 #[macro_use]
 extern crate clap;
+#[macro_use]
+extern crate onefuzz_telemetry;
+extern crate onefuzz;
 
 use anyhow::Result;
 use clap::{App, ArgMatches, SubCommand};

src/agent/onefuzz-agent/src/managed/cmd.rs

@@ -4,7 +4,6 @@
 use crate::tasks::config::{CommonConfig, Config};
 use anyhow::Result;
 use clap::{App, Arg, SubCommand};
-use onefuzz::telemetry;
 use std::path::PathBuf;
 
 pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
@@ -19,12 +18,12 @@ pub async fn run(args: &clap::ArgMatches<'_>) -> Result<()> {
         error!("error running task: {}", err);
     }
 
-    telemetry::try_flush_and_close();
+    onefuzz_telemetry::try_flush_and_close();
     result
 }
 
 fn init_telemetry(config: &CommonConfig) {
-    telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key);
+    onefuzz_telemetry::set_appinsights_clients(config.instrumentation_key, config.telemetry_key);
 }
 
 pub fn args(name: &str) -> App<'static, 'static> {

src/agent/onefuzz-agent/src/tasks/config.rs

@@ -4,10 +4,8 @@
 #![allow(clippy::large_enum_variant)]
 use crate::tasks::{analysis, coverage, fuzz, heartbeat::*, merge, report};
 use anyhow::Result;
-use onefuzz::{
+use onefuzz::machine_id::{get_machine_id, get_scaleset_name};
-    machine_id::{get_machine_id, get_scaleset_name},
+use onefuzz_telemetry::{self as telemetry, Event::task_start, EventData};
-    telemetry::{self, Event::task_start, EventData},
-};
 use reqwest::Url;
 use serde::{self, Deserialize};
 use std::path::PathBuf;

src/agent/onefuzz-agent/src/tasks/coverage/libfuzzer_coverage.rs

@@ -39,10 +39,8 @@ use crate::tasks::{
 use anyhow::{Context, Result};
 use async_trait::async_trait;
 use futures::stream::StreamExt;
-use onefuzz::{
+use onefuzz::{fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir};
-    fs::list_files, libfuzzer::LibFuzzer, syncdir::SyncedDir, telemetry::Event::coverage_data,
+use onefuzz_telemetry::{Event::coverage_data, EventData};
-    telemetry::EventData,
-};
 use reqwest::Url;
 use serde::Deserialize;
 use std::collections::HashMap;

src/agent/onefuzz-agent/src/tasks/fuzz/generator.rs

@@ -15,8 +15,8 @@ use onefuzz::{
     process::monitor_process,
     sha256,
     syncdir::{continuous_sync, SyncOperation::Pull, SyncedDir},
-    telemetry::Event::new_result,
 };
+use onefuzz_telemetry::Event::new_result;
 use serde::Deserialize;
 use std::collections::HashMap;
 use std::{

src/agent/onefuzz-agent/src/tasks/fuzz/libfuzzer_fuzz.rs

@@ -10,10 +10,10 @@ use onefuzz::{
     process::ExitStatus,
     syncdir::{continuous_sync, SyncOperation::Pull, SyncedDir},
     system,
-    telemetry::{
+};
+use onefuzz_telemetry::{
     Event::{new_coverage, new_result, process_stats, runtime_stats},
     EventData,
-    },
 };
 use serde::Deserialize;
 use std::{collections::HashMap, path::PathBuf};

src/agent/onefuzz-agent/src/tasks/fuzz/supervisor.rs

@@ -16,8 +16,8 @@ use onefuzz::{
     jitter::delay_with_jitter,
     process::monitor_process,
     syncdir::{SyncOperation::Pull, SyncedDir},
-    telemetry::Event::{new_coverage, new_result},
 };
+use onefuzz_telemetry::Event::{new_coverage, new_result};
 use serde::Deserialize;
 use std::{
     collections::HashMap,

src/agent/onefuzz-agent/src/tasks/report/crash_report.rs

@@ -9,10 +9,10 @@ use onefuzz::{
     fs::exists,
     monitor::DirectoryMonitor,
     syncdir::SyncedDir,
-    telemetry::{
+};
+use onefuzz_telemetry::{
     Event::{new_report, new_unable_to_reproduce, new_unique_report},
     EventData,
-    },
 };
 use reqwest::{StatusCode, Url};
 use reqwest_retry::SendRetry;

src/agent/onefuzz-agent/src/tasks/stats/afl.rs

@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 use anyhow::{Context, Error, Result};
-use onefuzz::telemetry::EventData;
+use onefuzz_telemetry::EventData;
 use std::path::Path;
 use tokio::io::AsyncBufReadExt;
 

src/agent/onefuzz-agent/src/tasks/stats/common.rs

@@ -3,10 +3,8 @@
 
 use super::afl;
 use anyhow::{Error, Result};
-use onefuzz::{
+use onefuzz::jitter::delay_with_jitter;
-    jitter::delay_with_jitter,
+use onefuzz_telemetry::{track_event, Event::runtime_stats};
-    telemetry::{track_event, Event::runtime_stats},
-};
 use serde::Deserialize;
 pub const STATS_DELAY: std::time::Duration = std::time::Duration::from_secs(30);
 

src/agent/onefuzz-supervisor/Cargo.toml

@@ -25,6 +25,7 @@ url = { version = "2.1.1", features = ["serde"] }
 uuid = { version = "0.8.1", features = ["serde", "v4"] }
 clap = "2.33"
 reqwest-retry = { path = "../reqwest-retry" }
+onefuzz-telemetry = { path = "../onefuzz-telemetry" }
 
 [target.'cfg(target_os = "linux")'.dependencies]
 users = "0.11"

src/agent/onefuzz-supervisor/src/main.rs

@@ -6,13 +6,14 @@ extern crate async_trait;
 #[macro_use]
 extern crate downcast_rs;
 #[macro_use]
-extern crate onefuzz;
-#[macro_use]
 extern crate serde;
 #[macro_use]
 extern crate clap;
 #[macro_use]
 extern crate anyhow;
+#[macro_use]
+extern crate onefuzz_telemetry;
+extern crate onefuzz;
 
 use crate::{
     config::StaticConfig, coordinator::StateUpdateEvent, heartbeat::*, work::WorkSet,
@@ -24,8 +25,8 @@ use anyhow::Result;
 use onefuzz::{
     machine_id::{get_machine_id, get_scaleset_name},
     process::ExitStatus,
-    telemetry::{self, EventData},
 };
+use onefuzz_telemetry::{self as telemetry, EventData};
 use structopt::StructOpt;
 
 pub mod agent;

src/agent/onefuzz-telemetry/Cargo.toml

@@ -0,0 +1,14 @@
+[package]
+name = "onefuzz-telemetry"
+version = "0.1.0"
+authors = ["fuzzing@microsoft.com"]
+edition = "2018"
+license = "MIT"
+
+[dependencies]
+appinsights = "0.1.4"
+log = "0.4"
+uuid = { version = "0.8", features = ["serde", "v4"] }
+
+[dev-dependencies]
+tokio = { version = "0.2" }

src/agent/onefuzz-telemetry/src/lib.rs

@@ -4,6 +4,8 @@
 use std::sync::{LockResult, RwLockReadGuard, RwLockWriteGuard};
 use uuid::Uuid;
 
+pub use appinsights::telemetry::SeverityLevel::{Critical, Error, Information, Verbose, Warning};
+
 pub type TelemetryClient = appinsights::TelemetryClient<appinsights::InMemoryChannel>;
 pub enum ClientType {
     Instance,
@@ -347,6 +349,28 @@ pub fn track_event(event: Event, properties: Vec<EventData>) {
     local_log_event(&event, &properties);
 }
 
+pub fn to_log_level(level: &appinsights::telemetry::SeverityLevel) -> log::Level {
+    match level {
+        Verbose => log::Level::Debug,
+        Information => log::Level::Info,
+        Warning => log::Level::Warn,
+        Error => log::Level::Error,
+        Critical => log::Level::Error,
+    }
+}
+
+pub fn should_log(level: &appinsights::telemetry::SeverityLevel) -> bool {
+    to_log_level(level) <= log::max_level()
+}
+
+pub fn log_message(level: appinsights::telemetry::SeverityLevel, msg: String) {
+    let log_level = to_log_level(&level);
+    log::log!(log_level, "{}", msg);
+    if let Some(client) = client(ClientType::Instance) {
+        client.track_trace(msg, level);
+    }
+}
+
 #[macro_export]
 macro_rules! event {
     ($name: expr ; $($k: path = $v: expr),*) => {{
@@ -357,33 +381,15 @@ macro_rules! event {
 
         })*;
 
-        $crate::telemetry::track_event($name, events);
+        onefuzz_telemetry::track_event($name, events);
     }};
 }
 
 #[macro_export]
 macro_rules! log {
     ($level: expr, $msg: expr) => {{
-        use appinsights::telemetry::SeverityLevel::{
+        if onefuzz_telemetry::should_log(&$level) {
-            Critical, Error, Information, Verbose, Warning,
+            onefuzz_telemetry::log_message($level, $msg.to_string());
-        };
-
-        let log_level = match $level {
-            Verbose => log::Level::Debug,
-            Information => log::Level::Info,
-            Warning => log::Level::Warn,
-            Error => log::Level::Error,
-            Critical => log::Level::Error,
-        };
-
-        // while log::log will filter based on log level, the telemetry
-        // client does *not*.
-        if log_level <= log::max_level() {
-            log::log!(log_level, "{}", $msg.to_string());
-            if let Some(client) = $crate::telemetry::client($crate::telemetry::ClientType::Instance)
-            {
-                client.track_trace($msg, $level);
-            }
         }
     }};
 }
@@ -392,7 +398,7 @@ macro_rules! log {
 macro_rules! verbose {
     ($($tt: tt)*) => {{
         let msg = format!($($tt)*);
-        $crate::log!(Verbose, msg);
+        onefuzz_telemetry::log!(onefuzz_telemetry::Verbose, msg);
     }}
 }
 
@@ -400,7 +406,7 @@ macro_rules! verbose {
 macro_rules! info {
     ($($tt: tt)*) => {{
         let msg = format!($($tt)*);
-        $crate::log!(Information, msg);
+        onefuzz_telemetry::log!(onefuzz_telemetry::Information, msg);
     }}
 }
 
@@ -408,7 +414,7 @@ macro_rules! info {
 macro_rules! warn {
     ($($tt: tt)*) => {{
         let msg = format!($($tt)*);
-        $crate::log!(Warning, msg);
+        onefuzz_telemetry::log!(onefuzz_telemetry::Warning, msg);
     }}
 }
 
@@ -416,7 +422,7 @@ macro_rules! warn {
 macro_rules! error {
     ($($tt: tt)*) => {{
         let msg = format!($($tt)*);
-        $crate::log!(Error, msg);
+        onefuzz_telemetry::log!(onefuzz_telemetry::Error, msg);
     }}
 }
 
@@ -424,14 +430,14 @@ macro_rules! error {
 macro_rules! critical {
     ($($tt: tt)*) => {{
         let msg = format!($($tt)*);
-        $crate::log!(Critical, msg);
+        onefuzz_telemetry::log!(onefuzz_telemetry::Critical, msg);
     }}
 }
 
 #[macro_export]
 macro_rules! metric {
     ($name: expr, $value: expr) => {{
-        let client = $crate::telemetry::client($crate::telemetry::ClientType::Instance);
+        let client = onefuzz_telemetry::client(onefuzz_telemetry::ClientType::Instance);
         client.track_metric($name.into(), $value);
     }};
 }

src/agent/onefuzz/Cargo.toml

@@ -38,6 +38,7 @@ strum_macros = "0.19"
 tempfile = "3.1"
 process_control = "2.0"
 reqwest-retry = { path = "../reqwest-retry"}
+onefuzz-telemetry = { path = "../onefuzz-telemetry"}
 
 [target.'cfg(target_os = "windows")'.dependencies]
 winreg = "0.7"

src/agent/onefuzz/src/lib.rs

@@ -11,7 +11,7 @@ extern crate lazy_static;
 extern crate serde;
 
 #[macro_use]
-pub mod telemetry;
+extern crate onefuzz_telemetry;
 
 pub mod asan;
 pub mod az_copy;

src/agent/onefuzz/src/syncdir.rs

@@ -2,15 +2,12 @@
 // Licensed under the MIT License.
 
 use crate::{
-    az_copy,
+    az_copy, blob::BlobContainerUrl, jitter::delay_with_jitter, monitor::DirectoryMonitor,
-    blob::BlobContainerUrl,
-    jitter::delay_with_jitter,
-    monitor::DirectoryMonitor,
-    telemetry::{Event, EventData},
     uploader::BlobUploader,
 };
 use anyhow::{Context, Result};
 use futures::stream::StreamExt;
+use onefuzz_telemetry::{Event, EventData};
 use std::{path::PathBuf, str, time::Duration};
 use tokio::fs;