Merge branch 'main' into report-all-loader-errors

CHANGELOG.md

@@ -7,6 +7,35 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## 8.9.0
+
+### Added
+
+* Agent: Added fuzz tests for coverage recording [#3322](https://github.com/microsoft/onefuzz/pull/3322)
+* Agent: Added version checking in local tasks [#3517](https://github.com/microsoft/onefuzz/pull/3517)
+* Agent: Create directories from template specification in local task if they don't exist [#3522](https://github.com/microsoft/onefuzz/pull/3522)
+* CLI: Added a new command for template creation in the local task `onefuzz-task local create-template` [#3531](https://github.com/microsoft/onefuzz/pull/3531)
+* CLI/Deployment/Service: Support for retention policies on containers [#3501](https://github.com/microsoft/onefuzz/pull/3501)
+* Service: Add onefuzz service version to job created events [#3504](https://github.com/microsoft/onefuzz/pull/3504)
+* Service: Added a start time to job and task records [#3440](https://github.com/microsoft/onefuzz/pull/3440)
+
+### Changed
+
+* Agent: Improved handling of unexpected breakpoints [#3493](https://github.com/microsoft/onefuzz/pull/3493)
+* Agent: Updated windows interceptor list [#3528](https://github.com/microsoft/onefuzz/pull/3528), [#3549](https://github.com/microsoft/onefuzz/pull/3549)
+* Agent: Reporting coverage on task start up, ensuring `coverage_data` is emitted at the beginning of every task instead of when `new_coverage` is identified [#3502](https://github.com/microsoft/onefuzz/pull/3502)
+* CLI/Deployment: Updating onefuzz cli requirements.txt to accept `>= onefuzztypes` versions [#3477](https://github.com/microsoft/onefuzz/pull/3477), [#3486](https://github.com/microsoft/onefuzz/pull/3486)
+* Service: Improve area/iteration path validation in notifications [#3489](https://github.com/microsoft/onefuzz/pull/3489)
+* Service: Remove feature flag from heartbeat metrics [#3505](https://github.com/microsoft/onefuzz/pull/3505)
+
+### Fixed
+
+* Agent: Terminate process on timeout in Windows agents for the coverage task [#3529](https://github.com/microsoft/onefuzz/pull/3529)
+* Agent/Service: Bumped several C#, Python, and Rust dependencies [#3425](https://github.com/microsoft/onefuzz/pull/3425), [#3424](https://github.com/microsoft/onefuzz/pull/3424), [#3411](https://github.com/microsoft/onefuzz/pull/3411), [#3437](https://github.com/microsoft/onefuzz/pull/3437), [#3436](https://github.com/microsoft/onefuzz/pull/3436), [#3435](https://github.com/microsoft/onefuzz/pull/3435), [#3478](https://github.com/microsoft/onefuzz/pull/3478), [#3484](https://github.com/microsoft/onefuzz/pull/3484), [#3414](https://github.com/microsoft/onefuzz/pull/3414), [#3474](https://github.com/microsoft/onefuzz/pull/3474), [#3434](https://github.com/microsoft/onefuzz/pull/3434), [#3488](https://github.com/microsoft/onefuzz/pull/3488), [#3503](https://github.com/microsoft/onefuzz/pull/3503), [#3520](https://github.com/microsoft/onefuzz/pull/3520), [#3521](https://github.com/microsoft/onefuzz/pull/3521)
+* Agent: Fixed dependencies in `onefuzz-task` [#3552](https://github.com/microsoft/onefuzz/pull/3552)
+* Service: Removed unnecessary method argument in notifications processing [#3473](https://github.com/microsoft/onefuzz/pull/3473)
+* Service: Ignore regression work item updates when the work item is in some states [#3532](https://github.com/microsoft/onefuzz/pull/3532)
+
 ## 8.8.0
 
 ### Added

CURRENT_VERSION

@@ -1 +1 @@
-8.8.0
+8.9.0

src/ApiService/ApiService/FeatureFlags.cs

@@ -9,4 +9,5 @@ public static class FeatureFlagConstants {
     public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention";
     public const string EnableWorkItemCreation = "EnableWorkItemCreation";
     public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies";
+    public const string EnableSlimEventSerialization = "EnableSlimEventSerialization";
 }

src/ApiService/ApiService/onefuzzlib/Events.cs

@@ -35,7 +35,9 @@ namespace Microsoft.OneFuzz.Service {
         private readonly IContainers _containers;
         private readonly ICreds _creds;
         private readonly JsonSerializerOptions _options;
+        private readonly JsonSerializerOptions _optionsSlim;
         private readonly JsonSerializerOptions _deserializingFromBlobOptions;
+        private readonly IOnefuzzContext _context;
 
         public Events(ILogger<Events> log, IOnefuzzContext context) {
             _queue = context.Queue;
@@ -47,9 +49,12 @@ namespace Microsoft.OneFuzz.Service {
                 DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
             };
             _options.Converters.Add(new RemoveUserInfo());
+            _optionsSlim = new JsonSerializerOptions(_options);
+            _optionsSlim.Converters.Add(new EventExportConverter<DownloadableEventMessage>());
             _deserializingFromBlobOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()) {
                 DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
             };
+            _context = context;
         }
 
         public virtual async Async.Task QueueSignalrEvent(DownloadableEventMessage message) {
@@ -58,7 +63,13 @@ namespace Microsoft.OneFuzz.Service {
                 ("event_id", message.EventId.ToString())
             };
             var ev = new SignalREvent("events", new List<DownloadableEventMessage>() { message });
-            var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: _options);
+
+            var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch {
+                true => _optionsSlim,
+                false => _options
+            };
+
+            var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: opts);
 
             if (!queueResult) {
                 _log.AddTags(tags);
@@ -155,16 +166,4 @@ namespace Microsoft.OneFuzz.Service {
             );
         }
     }
-
-
-    public class RemoveUserInfo : JsonConverter<UserInfo> {
-        public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
-            throw new NotSupportedException("reading UserInfo is not supported");
-        }
-
-        public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) {
-            writer.WriteStartObject();
-            writer.WriteEndObject();
-        }
-    }
 }

src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs

@@ -20,10 +20,15 @@ public interface IWebhookOperations : IOrm<Webhook> {
 public class WebhookOperations : Orm<Webhook>, IWebhookOperations {
 
     private readonly IHttpClientFactory _httpFactory;
+    private readonly JsonSerializerOptions _options;
+    private readonly JsonSerializerOptions _optionsSlim;
 
     public WebhookOperations(IHttpClientFactory httpFactory, ILogger<WebhookOperations> log, IOnefuzzContext context)
         : base(log, context) {
         _httpFactory = httpFactory;
+        _options = EntityConverter.GetJsonSerializerOptions();
+        _optionsSlim = new JsonSerializerOptions(_options);
+        _optionsSlim.Converters.Add(new EventExportConverter<WebhookMessage>());
     }
 
     public async Async.Task SendEvent(DownloadableEventMessage eventMessage) {
@@ -139,11 +144,15 @@ public class WebhookOperations : Orm<Webhook>, IWebhookOperations {
         string data;
         var instanceId = await _context.Containers.GetInstanceId();
         var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName(), CreatedAt: eventData.CreatedAt, SasUrl: eventData.SasUrl);
+        var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch {
+            true => _optionsSlim,
+            false => _options
+        };
         if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) {
             var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookMessage, DataVersion: "2.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) };
-            data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions());
+            data = JsonSerializer.Serialize(eventGridMessage, options: opts);
         } else {
-            data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions());
+            data = JsonSerializer.Serialize(webhookMessage, options: opts);
         }
 
         string? digest = null;

src/ApiService/ApiService/onefuzzlib/events/Converters.cs

@@ -0,0 +1,76 @@
+using System.Collections;
+using System.Reflection;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace Microsoft.OneFuzz.Service {
+    public class RemoveUserInfo : JsonConverter<UserInfo> {
+        public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+            throw new NotSupportedException("reading UserInfo is not supported");
+        }
+
+        public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) {
+            writer.WriteStartObject();
+            writer.WriteEndObject();
+        }
+    }
+
+    /// <summary>
+    /// <b>THIS IS A WRITE ONLY JSON CONVERTER</b>
+    /// <br/>
+    /// It should only be used when serializing events to be sent outside of the service
+    /// </summary>
+    public class EventExportConverter<T> : JsonConverter<T>
+        where T : DownloadableEventMessage {
+        public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
+            throw new NotSupportedException("This converter should only be used when serializing event messages to sent outside of the service");
+        }
+
+        public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) {
+            BoundedSerializer.WriteInternal(writer, value, options);
+        }
+    }
+
+    public class BoundedSerializer {
+        private static HashSet<Type> boundedTypes = new HashSet<Type>{
+            typeof(Guid),
+            typeof(DateTime),
+            typeof(int),
+            typeof(bool),
+            typeof(float),
+            typeof(double),
+            typeof(long),
+            typeof(char),
+            typeof(Uri)
+        };
+
+        public static void WriteInternal(Utf8JsonWriter writer, object type, JsonSerializerOptions options) {
+            writer.WriteStartObject();
+            var properties = type.GetType().GetProperties();
+            foreach (var property in properties) {
+                if (property.GetValue(type, null) == null
+                    || typeof(IEnumerable).IsAssignableFrom(property.PropertyType)
+                    || type.GetType() == property.PropertyType) {
+                    continue;
+                }
+                if (HasBoundedSerialization(property)) {
+                    var serialized = JsonSerializer.Serialize(property.GetValue(type, null), property.PropertyType, options);
+                    if (!string.IsNullOrEmpty(serialized)) {
+                        writer.WritePropertyName(property.Name);
+                        writer.WriteRawValue(serialized);
+                    }
+                } else if (property.PropertyType.IsClass) {
+                    writer.WritePropertyName(property.Name);
+                    WriteInternal(writer, property.GetValue(type, null)!, options);
+                }
+            }
+            writer.WriteEndObject();
+        }
+
+        public static bool HasBoundedSerialization(PropertyInfo propertyInfo) {
+            return propertyInfo.PropertyType.IsEnum ||
+                boundedTypes.Contains(propertyInfo.PropertyType) ||
+                typeof(IValidatedString).IsAssignableFrom(propertyInfo.PropertyType);
+        }
+    }
+}

src/ApiService/Tests/EventExportConverterTests.cs

@@ -0,0 +1,219 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text.Json;
+using FluentAssertions;
+using FsCheck;
+using FsCheck.Xunit;
+using Microsoft.OneFuzz.Service;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
+using Xunit;
+
+
+namespace Tests;
+
+public class EventExportConverterTests {
+    enum Color {
+        Red,
+        Blue
+    }
+
+    [Fact]
+    public void BaseTypesAreBounded() {
+        var a = new {
+            guid = Guid.NewGuid(),
+            date = new DateTime(),
+            en = Color.Red,
+            b = 1,
+            boo = false,
+            flo = float.Pi,
+            doub = double.Tau,
+            lon = long.MinValue,
+            cha = 'a'
+        };
+
+        a.GetType().GetProperties().All(p => BoundedSerializer.HasBoundedSerialization(p)).Should().BeTrue();
+    }
+
+    [Fact]
+    public void StringIsNotBounded() {
+        var a = new {
+            bad = "this is not bounded"
+        };
+
+        BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("bad")!).Should().BeFalse();
+    }
+
+    [Fact]
+    public void ValidatedStringIsBounded() {
+        var a = new {
+            scalesetid = ScalesetId.Parse("abc-123")
+        };
+
+        BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("scalesetid")!).Should().BeTrue();
+    }
+
+    [Fact]
+    public void ComplexObjectsAreSerialized() {
+        var randomGuid = Guid.NewGuid();
+        var a = new DownloadableEventMessage(
+            randomGuid,
+            EventType.CrashReported,
+            new EventCrashReported(
+               new Report(
+                   "https://example.com",
+                   null,
+                   "target.exe",
+                   "crash",
+                   string.Empty,
+                   new List<string> { "this", "is", "a", "stacktrace" },
+                   string.Empty,
+                   string.Empty,
+                   null,
+                   Guid.NewGuid(),
+                   Guid.NewGuid(),
+                   null,
+                   null,
+                   null,
+                   null,
+                   null,
+                   null,
+                   null,
+                   null,
+                   null,
+                   null,
+                   null,
+                   null
+                   ),
+               Container.Parse("this-is-a-container"),
+               "crash-abc123",
+               null
+           ),
+           Guid.NewGuid(),
+           "onefuzz",
+           DateTime.Now,
+           new Uri("https://example.com"),
+           null
+        );
+        var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions());
+        serializerOptions.Converters.Add(new EventExportConverter<DownloadableEventMessage>());
+
+        var serialized = JsonSerializer.Serialize(a, serializerOptions);
+
+        serialized.Should().NotBeNullOrEmpty();
+        serialized.Should().NotContain("stacktrace"); // List<string> is not serialized
+        serialized.Should().NotContain("crash-abc123"); // string is not serialized
+        serialized.Should().Contain("this-is-a-container"); // ValidatedString is serialized
+        serialized.Should().Contain("crash_reported"); // Enum is serialized
+        serialized.Should().Contain(DateTime.Now.Year.ToString()); // DateTime is serialized
+        serialized.Should().Contain(randomGuid.ToString()); // Guid id serialized
+    }
+
+    [Fact]
+    public void TestWebhookMessage() {
+        var a = new WebhookMessageEventGrid(
+            "2.0.0",
+            "eventsubject",
+            EventType.JobCreated,
+            DateTime.Now,
+            Guid.NewGuid(),
+            new WebhookMessage(
+                Guid.NewGuid(),
+                EventType.JobCreated,
+                new EventJobCreated(
+                   Guid.NewGuid(),
+                   new JobConfig("some project", "some name", "some build", 1, "some logs"),
+                   null,
+                   "8.0"),
+                Guid.NewGuid(),
+                "onefuzz",
+                Guid.NewGuid(),
+                DateTime.Now,
+                new Uri("https://example.com")
+            )
+        );
+
+        var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions());
+        serializerOptions.Converters.Add(new EventExportConverter<WebhookMessage>());
+
+        var serialized = JsonSerializer.Serialize(a, serializerOptions);
+
+        serialized.Should().Contain("eventsubject");
+        serialized.Should().NotContain("some project");
+    }
+
+    public class EventExportConverterSerializationTests {
+        private readonly JsonSerializerOptions _opts = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions());
+        public EventExportConverterSerializationTests() {
+            _ = Arb.Register<Arbitraries>();
+            _opts.Converters.Add(new EventExportConverter<DownloadableEventMessage>());
+        }
+
+        void Test<T>(T v) {
+            // TODO: Try cloning/creating a new serializer options from the existing one?
+            var serialized = JsonSerializer.Serialize(v, _opts);
+            var _ = JsonSerializer.Deserialize<dynamic>(serialized);
+        }
+
+        [Property]
+        public void EventNodeHeartbeat(EventNodeHeartbeat e) => Test(e);
+
+
+        [Property]
+        public void EventTaskHeartbeat(EventTaskHeartbeat e) => Test(e);
+
+        [Property]
+        public void EventTaskStopped(EventTaskStopped e) => Test(e);
+
+        [Property]
+        public void EventInstanceConfigUpdated(EventInstanceConfigUpdated e) => Test(e);
+
+        [Property]
+        public void EventProxyCreated(EventProxyCreated e) => Test(e);
+
+        [Property]
+        public void EventProxyDeleted(EventProxyDeleted e) => Test(e);
+
+        [Property]
+        public void EventProxyFailed(EventProxyFailed e) => Test(e);
+
+        [Property]
+        public void EventProxyStateUpdated(EventProxyStateUpdated e) => Test(e);
+
+
+        [Property]
+        public void EventCrashReported(EventCrashReported e) => Test(e);
+
+
+        [Property]
+        public void EventRegressionReported(EventRegressionReported e) => Test(e);
+
+
+        [Property]
+        public void EventFileAdded(EventFileAdded e) => Test(e);
+
+        [Property]
+        public void EventTaskFailed(EventTaskFailed e) => Test(e);
+
+        [Property]
+        public void EventTaskStateUpdated(EventTaskStateUpdated e) => Test(e);
+
+        [Property]
+        public void EventScalesetFailed(EventScalesetFailed e) => Test(e);
+
+        [Property]
+        public void EventScalesetResizeScheduled(EventScalesetResizeScheduled e) => Test(e);
+
+        [Property]
+        public void EventScalesetStateUpdated(EventScalesetStateUpdated e) => Test(e);
+
+        [Property]
+        public void EventNodeDeleted(EventNodeDeleted e) => Test(e);
+
+        [Property]
+        public void EventNodeCreated(EventNodeCreated e) => Test(e);
+
+        [Property]
+        public void EventMessage(DownloadableEventMessage e) => Test(e);
+    }
+}

src/ApiService/Tests/OrmModelsTest.cs

@@ -530,12 +530,13 @@ namespace Tests {
 
         //Sample function on how repro a failing test run, using Replay
         //functionality of FsCheck. Feel free to
-        /*
+
+        [Fact]
         void Replay() {
-            var seed = FsCheck.Random.StdGen.NewStdGen(811038773, 297085737);
-            var p = Prop.ForAll((NotificationTemplate x) => NotificationTemplate(x));
+            var seed = FsCheck.Random.StdGen.NewStdGen(1687595065, 297240661);
+            var p = Prop.ForAll((DownloadableEventMessage x) => EventMessage(x));
             p.Check(new Configuration { Replay = seed });
         }
-        */
+
     }
 }

src/deployment/bicep-templates/feature-flags.bicep

@@ -102,4 +102,17 @@ resource enableContainerRetentionPolicies 'Microsoft.AppConfiguration/configurat
   }
 }
 
+resource enableSlimEventSerialization 'Microsoft.AppConfiguration/configurationStores/keyValues@2021-10-01-preview' = {
+  parent: featureFlags
+  name: '.appconfig.featureflag~2FEnableSlimEventSerialization'
+  properties: {
+    value: string({
+      id: 'EnableSlimEventSerialization'
+      description: 'Enable serializing events as smaller payloads'
+      enabled: false
+    })
+    contentType: 'application/vnd.microsoft.appconfig.ff+json;charset=utf-8'
+  }
+}
+
 output AppConfigEndpoint string = 'https://${appConfigName}.azconfig.io'

src/integration-tests/integration-test.py

@@ -244,7 +244,7 @@ TARGETS: Dict[str, Integration] = {
             "--test:{extra_setup_dir}",
             "--write_test_file={extra_output_dir}/test.txt",
         ],
-        pool=PoolName("mariner")
+        pool=PoolName("mariner"),
     ),
     "windows-libfuzzer": Integration(
         template=TemplateType.libfuzzer,
@@ -401,10 +401,13 @@ class TestOnefuzz:
         self.of.pools.create(name, OS.linux)
         self.logger.info("creating scaleset for pool: %s", name)
         self.of.scalesets.create(
-            name, pool_size, region=region, initial_size=pool_size, image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest"
+            name,
+            pool_size,
+            region=region,
+            initial_size=pool_size,
+            image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest",
         )
 
-
     class UnmanagedPool:
         def __init__(
             self,
@@ -644,7 +647,7 @@ class TestOnefuzz:
                 setup = Directory(os.path.join(setup, config.nested_setup_dir))
 
             job: Optional[Job] = None
-                
+
             job = self.build_job(
                 duration, pool, target, config, setup, target_exe, inputs
             )
@@ -1277,7 +1280,7 @@ class TestOnefuzz:
 
         if seen_errors:
             raise Exception("logs included errors")
-        
+
     def build_pool_name(self, os_type: str) -> PoolName:
         return PoolName(f"testpool-{os_type}-{self.test_id}")
 
@@ -1592,13 +1595,6 @@ class Run(Command):
             result = tester.check_jobs(poll=True, stop_on_complete_check=True)
             if not result:
                 raise Exception("jobs failed")
-            if skip_repro:
-                self.logger.warning("not testing crash repro")
-            else:
-                launch_result, repros = tester.launch_repro()
-                result = tester.check_repro(repros)
-                if not (result and launch_result):
-                    raise Exception("repros failed")
 
             tester.check_logs_for_errors()