Add salvo telemetry (#619)

Adds telemetry fields for Salvo.

This PR should not be merged until approved by CELA.

Note: This PR includes #586

docs/telemetry.md

@@ -182,3 +182,69 @@ recorded:
 * Machine ID
 * State - the current state of the task on the node. For a full list, see the
   enum [NodeTaskState](../src/pytypes/onefuzztypes/enums.py).
+
+
+### Data recorded by Salvo
+
+The following information is recorded for Salvo related tasks:
+
+* InputsFuzzed - A u64 representing the count of inputs that were symbolically
+  executed.
+* SatConstraints - A u64 representing the count of satisfiable constraints and
+  hence number new inputs generated.
+* UnsatConstraints - A u64 representing the count of unsatisfiable constraints.
+* AverageVarsPerConstraint - A float64 representing the average count of input
+  bytes used per constraint over all of the inputs fuzzed.
+* MaxConstraintVars - A u64 representing the maximum count of input bytes used
+  for any single constraint.
+* AverageSymexTime - A float64 representing the average time in seconds spent
+  symbolically executing the program under test over all inputs fuzzed.
+* MaxSymexTime - A u64 representing the maximum time in seconds spent
+  symbolically executing the program under test for a single input.
+* AverageSolvingTime - A float64 representing the average time in seconds spent
+  solving constraints over all inputs fuzzed.
+* MaxSolvingTime - A u64 representing the maximum time in seconds spent solving
+  constraints for any single input.
+* UniqueCodeLocationCount - A u64 representing the count of the unique code
+  locations that are of interest to Salvo, e.g. a tainted instruction or branch
+  target.
+* AverageInstructionsExecuted - A float64 representing the average count of
+  instructions that were symbolically executed over all fuzzed inputs.
+* MaxInstructionsExecuted - A u64 representing the maximum count of
+  instructions that were symbolically executed for any single fuzzed input.
+* AverageTaintedInstructions - A float64 representing the count of instructions
+  that make use of tainted input over all inputs fuzzed.
+* MaxTaintedInstructions - A u64 representing the count of instructions that
+  make use of tainted input for any single input.
+* AverageMemoryTaintedInstructions - A float64 representing the count of
+  instructions that make use of tainted input to read or write memory over all
+  inputs fuzzed.
+* MaxMemoryTaintedInstructions - A u64 representing the count of instructions
+  that make use of tainted input to read or write memory for any single input.
+* AveragePathLength - A u64 representing the count of the average constraints
+  tracked for constraint solving while symbolically executing the program.
+* MaxPathLength - A u64 representing the count of the maximum constraints
+  tracked for constraint solving while symbolically executing the program.
+* DivergenceRate - A float64 representing the ratio of inputs that did not
+  branch as expected divided by the number of inputs fuzzed.
+* DivergencePathLength - A u32 that indicates the length of execution path
+  divergence.
+* DivergencePathExpectedIndex - A u32 that indicates the expected index for
+  divergence.
+* DivergencePathActualIndex - A u32 that indicates the actual index for
+  divergence.
+* MissedInstructionCode - The Intel Instruction code for an instruction that
+  was not modelled during symbolic execution but may have been input tainted.
+  Examples include `Cmovs_r16_rm16` and `Movq_mm_rm64`.  For the full list, see
+  [iced_x86::Code](https://docs.rs/iced-x86/1.10.3/iced_x86/enum.Code.html).
+* MissedInstructionMnemonic - The Intel Instruction that was not modelled
+  during symbolic execution but may have been input tainted. Examples include
+  `Cmovs` and `Movq`.  For the full list, see
+  [iced_x86::mnemonic](https://docs.rs/iced-x86/1.10.3/iced_x86/enum.Mnemonic.html).
+* Z3ErrorCode - An error code that corresponds to an error code from Z3 when
+  solving a constraint fails.  Examples include `NoParser` and
+  `InvalidPattern`.  For the full list, see
+  [z3_sys::ErrorCode](https://docs.rs/z3-sys/0.6.3/z3_sys/enum.ErrorCode.html)
+* SymexTimeout - A u64 representing the maximum time in seconds to spend during
+  symbolic execution, reported each time symbolic execution was stopped due to
+  the limit.

src/agent/Cargo.lock

@@ -1648,10 +1648,12 @@ name = "onefuzz-telemetry"
 version = "0.1.0"
 dependencies = [
  "appinsights",
+ "iced-x86",
  "log",
  "serde",
  "tokio",
  "uuid",
+ "z3-sys",
 ]
 
 [[package]]
@@ -3067,3 +3069,9 @@ name = "xml-rs"
 version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b07db065a5cf61a7e4ba64f29e67db906fb1787316516c4e6e5ff0fea1efcd8a"
+
+[[package]]
+name = "z3-sys"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "afa18ba5fbd4933e41ffb440c3fd91f91fe9cdb7310cce3ddfb6648563811de0"

src/agent/onefuzz-telemetry/Cargo.toml

@@ -5,11 +5,19 @@ authors = ["fuzzing@microsoft.com"]
 edition = "2018"
 license = "MIT"
 
+[features]
+default = []
+z3 = ["z3-sys"]
+intel_instructions = ["iced-x86"]
+
 [dependencies]
 appinsights = "0.1"
 log = "0.4"
 uuid = { version = "0.8", features = ["serde", "v4"] }
 serde = { version = "1.0", features = ["derive"] }
+z3-sys = { version = "0.6", optional = true}
+iced-x86 = { version = "1.1", optional = true}
+
 
 [dev-dependencies]
 tokio = { version = "0.2" }

src/agent/onefuzz-telemetry/src/lib.rs

@@ -1,10 +1,14 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
+#[cfg(feature = "intel_instructions")]
+use iced_x86::{Code as IntelInstructionCode, Mnemonic as IntelInstructionMnemonic};
 use serde::{Deserialize, Serialize};
 use std::fmt;
 use std::sync::{LockResult, RwLockReadGuard, RwLockWriteGuard};
 use uuid::Uuid;
+#[cfg(feature = "z3")]
+use z3_sys::ErrorCode as Z3ErrorCode;
 
 pub use appinsights::telemetry::SeverityLevel::{Critical, Error, Information, Verbose, Warning};
 
@@ -31,6 +35,25 @@ impl InstanceTelemetryKey {
     }
 }
 
+#[cfg(feature = "z3")]
+pub fn z3_error_as_str(code: &Z3ErrorCode) -> &'static str {
+    match code {
+        Z3ErrorCode::OK => "OK",
+        Z3ErrorCode::SortError => "SortError",
+        Z3ErrorCode::IOB => "IOB",
+        Z3ErrorCode::InvalidArg => "InvalidArg",
+        Z3ErrorCode::ParserError => "ParserError",
+        Z3ErrorCode::NoParser => "NoParser",
+        Z3ErrorCode::InvalidPattern => "InvalidPattern",
+        Z3ErrorCode::MemoutFail => "MemoutFail",
+        Z3ErrorCode::FileAccessError => "FileAccessError",
+        Z3ErrorCode::InternalFatal => "InternalFatal",
+        Z3ErrorCode::InvalidUsage => "InvalidUsage",
+        Z3ErrorCode::DecRefError => "DecRefError",
+        Z3ErrorCode::Exception => "Exception",
+    }
+}
+
 impl fmt::Display for InstanceTelemetryKey {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         write!(f, "{}", self.0)
@@ -124,6 +147,37 @@ pub enum EventData {
     ToolName(String),
     Region(String),
     Role(Role),
+    InputsFuzzed(u64),
+    SatConstraints(u64),
+    UnsatConstraints(u64),
+    AverageVarsPerConstraint(u64),
+    MaxConstraintVars(u64),
+    AverageSymexTime(f64),
+    MaxSymexTime(u64),
+    AverageSolvingTime(f64),
+    MaxSolvingTime(u64),
+    UniqueCodeLocationCount(u64),
+    AverageInstructionsExecuted(f64),
+    MaxInstructionsExecuted(u64),
+    AverageTaintedInstructions(f64),
+    MaxTaintedInstructions(u64),
+    AverageMemoryTaintedInstructions(f64),
+    MaxMemoryTaintedInstructions(u64),
+    AveragePathLength(f64),
+    MaxPathLength(u64),
+    DivergenceRate(f64),
+    DivergencePathLength(u32),
+    DivergencePathExpectedIndex(u32),
+    DivergencePathActualIndex(u32),
+    #[cfg(feature = "intel_instructions")]
+    MissedInstructionCode(IntelInstructionCode),
+    #[cfg(feature = "intel_instructions")]
+    MissedInstructionMnemonic(IntelInstructionMnemonic),
+    #[cfg(feature = "z3")]
+    Z3ErrorCode(Z3ErrorCode),
+    #[cfg(feature = "z3")]
+    Z3ErrorString(String),
+    SymexTimeout(u64),
 }
 
 impl EventData {
@@ -161,6 +215,47 @@ impl EventData {
             Self::ToolName(x) => ("tool_name", x.to_owned()),
             Self::Region(x) => ("region", x.to_owned()),
             Self::Role(x) => ("role", x.as_str().to_owned()),
+            #[cfg(feature = "intel_instructions")]
+            Self::MissedInstructionCode(x) => ("missed_instruction_code", format!("{:?}", x)),
+            #[cfg(feature = "intel_instructions")]
+            Self::MissedInstructionMnemonic(x) => {
+                ("missed_instruction_mnemonic", format!("{:?}", x))
+            }
+            Self::InputsFuzzed(x) => ("inputs_fuzzed", x.to_string()),
+            Self::SatConstraints(x) => ("sat_constraints", x.to_string()),
+            Self::UnsatConstraints(x) => ("unsat_constraints", x.to_string()),
+            Self::AverageVarsPerConstraint(x) => ("average_vars_per_constraint", x.to_string()),
+            Self::MaxConstraintVars(x) => ("max_constraint_vars", x.to_string()),
+            Self::AverageSymexTime(x) => ("average_symex_time", x.to_string()),
+            Self::MaxSymexTime(x) => ("max_symex_time", x.to_string()),
+            Self::AverageSolvingTime(x) => ("average_solving_time", x.to_string()),
+            Self::MaxSolvingTime(x) => ("max_solving_time", x.to_string()),
+            Self::UniqueCodeLocationCount(x) => ("unique_code_locations_count", x.to_string()),
+            Self::AverageInstructionsExecuted(x) => {
+                ("average_instructions_executed", x.to_string())
+            }
+            Self::MaxInstructionsExecuted(x) => ("max_instructions_executed", x.to_string()),
+            Self::AverageTaintedInstructions(x) => ("average_tainted_instructions", x.to_string()),
+            Self::MaxTaintedInstructions(x) => ("max_tainted_instructions", x.to_string()),
+            Self::AverageMemoryTaintedInstructions(x) => {
+                ("average_memory_tainted_instructions", x.to_string())
+            }
+            Self::MaxMemoryTaintedInstructions(x) => {
+                ("max_memory_tainted_instructions", x.to_string())
+            }
+            Self::AveragePathLength(x) => ("average_path_length", x.to_string()),
+            Self::MaxPathLength(x) => ("max_path_length", x.to_string()),
+            Self::DivergenceRate(x) => ("divergence_rate", x.to_string()),
+            Self::DivergencePathLength(x) => ("divergence_path_length", x.to_string()),
+            Self::DivergencePathExpectedIndex(x) => {
+                ("divergence_path_expected_index", x.to_string())
+            }
+            Self::DivergencePathActualIndex(x) => ("divergence_path_actual_index", x.to_string()),
+            #[cfg(feature = "z3")]
+            Self::Z3ErrorCode(x) => ("z3_error_code", z3_error_as_str(x).to_owned()),
+            #[cfg(feature = "z3")]
+            Self::Z3ErrorString(x) => ("z3_error_string", x.to_owned()),
+            Self::SymexTimeout(x) => ("symex_timeout", x.to_string()),
         }
     }
 
@@ -198,6 +293,37 @@ impl EventData {
             Self::ToolName(_) => true,
             Self::Region(_) => false,
             Self::Role(_) => true,
+            Self::InputsFuzzed(_) => true,
+            Self::SatConstraints(_) => true,
+            Self::UnsatConstraints(_) => true,
+            Self::AverageVarsPerConstraint(_) => true,
+            Self::MaxConstraintVars(_) => true,
+            Self::AverageSymexTime(_) => true,
+            Self::MaxSymexTime(_) => true,
+            Self::AverageSolvingTime(_) => true,
+            Self::MaxSolvingTime(_) => true,
+            Self::UniqueCodeLocationCount(_) => true,
+            Self::AverageInstructionsExecuted(_) => true,
+            Self::MaxInstructionsExecuted(_) => true,
+            Self::AverageTaintedInstructions(_) => true,
+            Self::MaxTaintedInstructions(_) => true,
+            Self::AverageMemoryTaintedInstructions(_) => true,
+            Self::MaxMemoryTaintedInstructions(_) => true,
+            Self::AveragePathLength(_) => true,
+            Self::MaxPathLength(_) => true,
+            Self::DivergenceRate(_) => true,
+            Self::DivergencePathLength(_) => true,
+            Self::DivergencePathExpectedIndex(_) => true,
+            Self::DivergencePathActualIndex(_) => true,
+            #[cfg(feature = "intel_instructions")]
+            Self::MissedInstructionCode(_) => true,
+            #[cfg(feature = "intel_instructions")]
+            Self::MissedInstructionMnemonic(_) => true,
+            #[cfg(feature = "z3")]
+            Self::Z3ErrorCode(_) => true,
+            #[cfg(feature = "z3")]
+            Self::Z3ErrorString(_) => false,
+            Self::SymexTimeout(_) => true,
         }
     }
 }

src/ci/agent.sh

@@ -43,6 +43,9 @@ cargo test --release --workspace
 # TODO: re-enable integration tests.
 # cargo test --release --manifest-path ./onefuzz-agent/Cargo.toml --features integration_test -- --nocapture
 
+# TODO: once Salvo is integrated, this can get deleted
+cargo build --release --manifest-path ./onefuzz-telemetry/Cargo.toml --all-features
+
 cp target/release/onefuzz-agent* ../../artifacts/agent
 cp target/release/onefuzz-supervisor* ../../artifacts/agent