diff --git a/CHANGELOG.md b/CHANGELOG.md index f02721fa4..528b406c9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,35 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 8.9.0 + +### Added + +* Agent: Added fuzz tests for coverage recording [#3322](https://github.com/microsoft/onefuzz/pull/3322) +* Agent: Added version checking in local tasks [#3517](https://github.com/microsoft/onefuzz/pull/3517) +* Agent: Create directories from template specification in local task if they don't exist [#3522](https://github.com/microsoft/onefuzz/pull/3522) +* CLI: Added a new command for template creation in the local task `onefuzz-task local create-template` [#3531](https://github.com/microsoft/onefuzz/pull/3531) +* CLI/Deployment/Service: Support for retention policies on containers [#3501](https://github.com/microsoft/onefuzz/pull/3501) +* Service: Add onefuzz service version to job created events [#3504](https://github.com/microsoft/onefuzz/pull/3504) +* Service: Added a start time to job and task records [#3440](https://github.com/microsoft/onefuzz/pull/3440) + +### Changed + +* Agent: Improved handling of unexpected breakpoints [#3493](https://github.com/microsoft/onefuzz/pull/3493) +* Agent: Updated windows interceptor list [#3528](https://github.com/microsoft/onefuzz/pull/3528), [#3549](https://github.com/microsoft/onefuzz/pull/3549) +* Agent: Reporting coverage on task start up, ensuring `coverage_data` is emitted at the beginning of every task instead of when `new_coverage` is identified [#3502](https://github.com/microsoft/onefuzz/pull/3502) +* CLI/Deployment: Updating onefuzz cli requirements.txt to accept `>= onefuzztypes` versions [#3477](https://github.com/microsoft/onefuzz/pull/3477), [#3486](https://github.com/microsoft/onefuzz/pull/3486) +* Service: Improve area/iteration path validation in notifications [#3489](https://github.com/microsoft/onefuzz/pull/3489) +* Service: Remove feature flag from heartbeat metrics [#3505](https://github.com/microsoft/onefuzz/pull/3505) + +### Fixed + +* Agent: Terminate process on timeout in Windows agents for the coverage task [#3529](https://github.com/microsoft/onefuzz/pull/3529) +* Agent/Service: Bumped several C#, Python, and Rust dependencies [#3425](https://github.com/microsoft/onefuzz/pull/3425), [#3424](https://github.com/microsoft/onefuzz/pull/3424), [#3411](https://github.com/microsoft/onefuzz/pull/3411), [#3437](https://github.com/microsoft/onefuzz/pull/3437), [#3436](https://github.com/microsoft/onefuzz/pull/3436), [#3435](https://github.com/microsoft/onefuzz/pull/3435), [#3478](https://github.com/microsoft/onefuzz/pull/3478), [#3484](https://github.com/microsoft/onefuzz/pull/3484), [#3414](https://github.com/microsoft/onefuzz/pull/3414), [#3474](https://github.com/microsoft/onefuzz/pull/3474), [#3434](https://github.com/microsoft/onefuzz/pull/3434), [#3488](https://github.com/microsoft/onefuzz/pull/3488), [#3503](https://github.com/microsoft/onefuzz/pull/3503), [#3520](https://github.com/microsoft/onefuzz/pull/3520), [#3521](https://github.com/microsoft/onefuzz/pull/3521) +* Agent: Fixed dependencies in `onefuzz-task` [#3552](https://github.com/microsoft/onefuzz/pull/3552) +* Service: Removed unnecessary method argument in notifications processing [#3473](https://github.com/microsoft/onefuzz/pull/3473) +* Service: Ignore regression work item updates when the work item is in some states [#3532](https://github.com/microsoft/onefuzz/pull/3532) + ## 8.8.0 ### Added diff --git a/CURRENT_VERSION b/CURRENT_VERSION index cfc27b4fa..3ecb39816 100644 --- a/CURRENT_VERSION +++ b/CURRENT_VERSION @@ -1 +1 @@ -8.8.0 \ No newline at end of file +8.9.0 \ No newline at end of file diff --git a/src/ApiService/ApiService/FeatureFlags.cs b/src/ApiService/ApiService/FeatureFlags.cs index e74396e88..5a88d7232 100644 --- a/src/ApiService/ApiService/FeatureFlags.cs +++ b/src/ApiService/ApiService/FeatureFlags.cs @@ -9,4 +9,5 @@ public static class FeatureFlagConstants { public const string EnableDryRunBlobRetention = "EnableDryRunBlobRetention"; public const string EnableWorkItemCreation = "EnableWorkItemCreation"; public const string EnableContainerRetentionPolicies = "EnableContainerRetentionPolicies"; + public const string EnableSlimEventSerialization = "EnableSlimEventSerialization"; } diff --git a/src/ApiService/ApiService/onefuzzlib/Events.cs b/src/ApiService/ApiService/onefuzzlib/Events.cs index 0b806c580..b6343ac95 100644 --- a/src/ApiService/ApiService/onefuzzlib/Events.cs +++ b/src/ApiService/ApiService/onefuzzlib/Events.cs @@ -35,7 +35,9 @@ namespace Microsoft.OneFuzz.Service { private readonly IContainers _containers; private readonly ICreds _creds; private readonly JsonSerializerOptions _options; + private readonly JsonSerializerOptions _optionsSlim; private readonly JsonSerializerOptions _deserializingFromBlobOptions; + private readonly IOnefuzzContext _context; public Events(ILogger log, IOnefuzzContext context) { _queue = context.Queue; @@ -47,9 +49,12 @@ namespace Microsoft.OneFuzz.Service { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; _options.Converters.Add(new RemoveUserInfo()); + _optionsSlim = new JsonSerializerOptions(_options); + _optionsSlim.Converters.Add(new EventExportConverter()); _deserializingFromBlobOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()) { DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull }; + _context = context; } public virtual async Async.Task QueueSignalrEvent(DownloadableEventMessage message) { @@ -58,7 +63,13 @@ namespace Microsoft.OneFuzz.Service { ("event_id", message.EventId.ToString()) }; var ev = new SignalREvent("events", new List() { message }); - var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: _options); + + var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch { + true => _optionsSlim, + false => _options + }; + + var queueResult = await _queue.QueueObject("signalr-events", ev, StorageType.Config, serializerOptions: opts); if (!queueResult) { _log.AddTags(tags); @@ -155,16 +166,4 @@ namespace Microsoft.OneFuzz.Service { ); } } - - - public class RemoveUserInfo : JsonConverter { - public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - throw new NotSupportedException("reading UserInfo is not supported"); - } - - public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) { - writer.WriteStartObject(); - writer.WriteEndObject(); - } - } } diff --git a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs index cb43a536e..cc5367b21 100644 --- a/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs @@ -20,10 +20,15 @@ public interface IWebhookOperations : IOrm { public class WebhookOperations : Orm, IWebhookOperations { private readonly IHttpClientFactory _httpFactory; + private readonly JsonSerializerOptions _options; + private readonly JsonSerializerOptions _optionsSlim; public WebhookOperations(IHttpClientFactory httpFactory, ILogger log, IOnefuzzContext context) : base(log, context) { _httpFactory = httpFactory; + _options = EntityConverter.GetJsonSerializerOptions(); + _optionsSlim = new JsonSerializerOptions(_options); + _optionsSlim.Converters.Add(new EventExportConverter()); } public async Async.Task SendEvent(DownloadableEventMessage eventMessage) { @@ -139,11 +144,15 @@ public class WebhookOperations : Orm, IWebhookOperations { string data; var instanceId = await _context.Containers.GetInstanceId(); var webhookMessage = new WebhookMessage(WebhookId: webhookId, EventId: eventId, EventType: eventType, Event: webhookEvent, InstanceId: instanceId, InstanceName: _context.Creds.GetInstanceName(), CreatedAt: eventData.CreatedAt, SasUrl: eventData.SasUrl); + var opts = await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableSlimEventSerialization) switch { + true => _optionsSlim, + false => _options + }; if (messageFormat != null && messageFormat == WebhookMessageFormat.EventGrid) { var eventGridMessage = new[] { new WebhookMessageEventGrid(Id: eventId, Data: webhookMessage, DataVersion: "2.0.0", Subject: _context.Creds.GetInstanceName(), EventType: eventType, EventTime: DateTimeOffset.UtcNow) }; - data = JsonSerializer.Serialize(eventGridMessage, options: EntityConverter.GetJsonSerializerOptions()); + data = JsonSerializer.Serialize(eventGridMessage, options: opts); } else { - data = JsonSerializer.Serialize(webhookMessage, options: EntityConverter.GetJsonSerializerOptions()); + data = JsonSerializer.Serialize(webhookMessage, options: opts); } string? digest = null; diff --git a/src/ApiService/ApiService/onefuzzlib/events/Converters.cs b/src/ApiService/ApiService/onefuzzlib/events/Converters.cs new file mode 100644 index 000000000..47050a496 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/events/Converters.cs @@ -0,0 +1,76 @@ +using System.Collections; +using System.Reflection; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.OneFuzz.Service { + public class RemoveUserInfo : JsonConverter { + public override UserInfo? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + throw new NotSupportedException("reading UserInfo is not supported"); + } + + public override void Write(Utf8JsonWriter writer, UserInfo value, JsonSerializerOptions options) { + writer.WriteStartObject(); + writer.WriteEndObject(); + } + } + + /// + /// THIS IS A WRITE ONLY JSON CONVERTER + ///
+ /// It should only be used when serializing events to be sent outside of the service + /// + public class EventExportConverter : JsonConverter + where T : DownloadableEventMessage { + public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + throw new NotSupportedException("This converter should only be used when serializing event messages to sent outside of the service"); + } + + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) { + BoundedSerializer.WriteInternal(writer, value, options); + } + } + + public class BoundedSerializer { + private static HashSet boundedTypes = new HashSet{ + typeof(Guid), + typeof(DateTime), + typeof(int), + typeof(bool), + typeof(float), + typeof(double), + typeof(long), + typeof(char), + typeof(Uri) + }; + + public static void WriteInternal(Utf8JsonWriter writer, object type, JsonSerializerOptions options) { + writer.WriteStartObject(); + var properties = type.GetType().GetProperties(); + foreach (var property in properties) { + if (property.GetValue(type, null) == null + || typeof(IEnumerable).IsAssignableFrom(property.PropertyType) + || type.GetType() == property.PropertyType) { + continue; + } + if (HasBoundedSerialization(property)) { + var serialized = JsonSerializer.Serialize(property.GetValue(type, null), property.PropertyType, options); + if (!string.IsNullOrEmpty(serialized)) { + writer.WritePropertyName(property.Name); + writer.WriteRawValue(serialized); + } + } else if (property.PropertyType.IsClass) { + writer.WritePropertyName(property.Name); + WriteInternal(writer, property.GetValue(type, null)!, options); + } + } + writer.WriteEndObject(); + } + + public static bool HasBoundedSerialization(PropertyInfo propertyInfo) { + return propertyInfo.PropertyType.IsEnum || + boundedTypes.Contains(propertyInfo.PropertyType) || + typeof(IValidatedString).IsAssignableFrom(propertyInfo.PropertyType); + } + } +} diff --git a/src/ApiService/Tests/EventExportConverterTests.cs b/src/ApiService/Tests/EventExportConverterTests.cs new file mode 100644 index 000000000..4d129c18c --- /dev/null +++ b/src/ApiService/Tests/EventExportConverterTests.cs @@ -0,0 +1,219 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using FluentAssertions; +using FsCheck; +using FsCheck.Xunit; +using Microsoft.OneFuzz.Service; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +using Xunit; + + +namespace Tests; + +public class EventExportConverterTests { + enum Color { + Red, + Blue + } + + [Fact] + public void BaseTypesAreBounded() { + var a = new { + guid = Guid.NewGuid(), + date = new DateTime(), + en = Color.Red, + b = 1, + boo = false, + flo = float.Pi, + doub = double.Tau, + lon = long.MinValue, + cha = 'a' + }; + + a.GetType().GetProperties().All(p => BoundedSerializer.HasBoundedSerialization(p)).Should().BeTrue(); + } + + [Fact] + public void StringIsNotBounded() { + var a = new { + bad = "this is not bounded" + }; + + BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("bad")!).Should().BeFalse(); + } + + [Fact] + public void ValidatedStringIsBounded() { + var a = new { + scalesetid = ScalesetId.Parse("abc-123") + }; + + BoundedSerializer.HasBoundedSerialization(a.GetType().GetProperty("scalesetid")!).Should().BeTrue(); + } + + [Fact] + public void ComplexObjectsAreSerialized() { + var randomGuid = Guid.NewGuid(); + var a = new DownloadableEventMessage( + randomGuid, + EventType.CrashReported, + new EventCrashReported( + new Report( + "https://example.com", + null, + "target.exe", + "crash", + string.Empty, + new List { "this", "is", "a", "stacktrace" }, + string.Empty, + string.Empty, + null, + Guid.NewGuid(), + Guid.NewGuid(), + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null, + null + ), + Container.Parse("this-is-a-container"), + "crash-abc123", + null + ), + Guid.NewGuid(), + "onefuzz", + DateTime.Now, + new Uri("https://example.com"), + null + ); + var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + serializerOptions.Converters.Add(new EventExportConverter()); + + var serialized = JsonSerializer.Serialize(a, serializerOptions); + + serialized.Should().NotBeNullOrEmpty(); + serialized.Should().NotContain("stacktrace"); // List is not serialized + serialized.Should().NotContain("crash-abc123"); // string is not serialized + serialized.Should().Contain("this-is-a-container"); // ValidatedString is serialized + serialized.Should().Contain("crash_reported"); // Enum is serialized + serialized.Should().Contain(DateTime.Now.Year.ToString()); // DateTime is serialized + serialized.Should().Contain(randomGuid.ToString()); // Guid id serialized + } + + [Fact] + public void TestWebhookMessage() { + var a = new WebhookMessageEventGrid( + "2.0.0", + "eventsubject", + EventType.JobCreated, + DateTime.Now, + Guid.NewGuid(), + new WebhookMessage( + Guid.NewGuid(), + EventType.JobCreated, + new EventJobCreated( + Guid.NewGuid(), + new JobConfig("some project", "some name", "some build", 1, "some logs"), + null, + "8.0"), + Guid.NewGuid(), + "onefuzz", + Guid.NewGuid(), + DateTime.Now, + new Uri("https://example.com") + ) + ); + + var serializerOptions = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + serializerOptions.Converters.Add(new EventExportConverter()); + + var serialized = JsonSerializer.Serialize(a, serializerOptions); + + serialized.Should().Contain("eventsubject"); + serialized.Should().NotContain("some project"); + } + + public class EventExportConverterSerializationTests { + private readonly JsonSerializerOptions _opts = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()); + public EventExportConverterSerializationTests() { + _ = Arb.Register(); + _opts.Converters.Add(new EventExportConverter()); + } + + void Test(T v) { + // TODO: Try cloning/creating a new serializer options from the existing one? + var serialized = JsonSerializer.Serialize(v, _opts); + var _ = JsonSerializer.Deserialize(serialized); + } + + [Property] + public void EventNodeHeartbeat(EventNodeHeartbeat e) => Test(e); + + + [Property] + public void EventTaskHeartbeat(EventTaskHeartbeat e) => Test(e); + + [Property] + public void EventTaskStopped(EventTaskStopped e) => Test(e); + + [Property] + public void EventInstanceConfigUpdated(EventInstanceConfigUpdated e) => Test(e); + + [Property] + public void EventProxyCreated(EventProxyCreated e) => Test(e); + + [Property] + public void EventProxyDeleted(EventProxyDeleted e) => Test(e); + + [Property] + public void EventProxyFailed(EventProxyFailed e) => Test(e); + + [Property] + public void EventProxyStateUpdated(EventProxyStateUpdated e) => Test(e); + + + [Property] + public void EventCrashReported(EventCrashReported e) => Test(e); + + + [Property] + public void EventRegressionReported(EventRegressionReported e) => Test(e); + + + [Property] + public void EventFileAdded(EventFileAdded e) => Test(e); + + [Property] + public void EventTaskFailed(EventTaskFailed e) => Test(e); + + [Property] + public void EventTaskStateUpdated(EventTaskStateUpdated e) => Test(e); + + [Property] + public void EventScalesetFailed(EventScalesetFailed e) => Test(e); + + [Property] + public void EventScalesetResizeScheduled(EventScalesetResizeScheduled e) => Test(e); + + [Property] + public void EventScalesetStateUpdated(EventScalesetStateUpdated e) => Test(e); + + [Property] + public void EventNodeDeleted(EventNodeDeleted e) => Test(e); + + [Property] + public void EventNodeCreated(EventNodeCreated e) => Test(e); + + [Property] + public void EventMessage(DownloadableEventMessage e) => Test(e); + } +} diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index 956d0c30c..6828d7f2f 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -530,12 +530,13 @@ namespace Tests { //Sample function on how repro a failing test run, using Replay //functionality of FsCheck. Feel free to - /* + + [Fact] void Replay() { - var seed = FsCheck.Random.StdGen.NewStdGen(811038773, 297085737); - var p = Prop.ForAll((NotificationTemplate x) => NotificationTemplate(x)); + var seed = FsCheck.Random.StdGen.NewStdGen(1687595065, 297240661); + var p = Prop.ForAll((DownloadableEventMessage x) => EventMessage(x)); p.Check(new Configuration { Replay = seed }); } - */ + } } diff --git a/src/deployment/bicep-templates/feature-flags.bicep b/src/deployment/bicep-templates/feature-flags.bicep index 46fccb085..cd39edb52 100644 --- a/src/deployment/bicep-templates/feature-flags.bicep +++ b/src/deployment/bicep-templates/feature-flags.bicep @@ -102,4 +102,17 @@ resource enableContainerRetentionPolicies 'Microsoft.AppConfiguration/configurat } } +resource enableSlimEventSerialization 'Microsoft.AppConfiguration/configurationStores/keyValues@2021-10-01-preview' = { + parent: featureFlags + name: '.appconfig.featureflag~2FEnableSlimEventSerialization' + properties: { + value: string({ + id: 'EnableSlimEventSerialization' + description: 'Enable serializing events as smaller payloads' + enabled: false + }) + contentType: 'application/vnd.microsoft.appconfig.ff+json;charset=utf-8' + } +} + output AppConfigEndpoint string = 'https://${appConfigName}.azconfig.io' diff --git a/src/integration-tests/integration-test.py b/src/integration-tests/integration-test.py index 15ffcfb9f..b71b0b737 100755 --- a/src/integration-tests/integration-test.py +++ b/src/integration-tests/integration-test.py @@ -244,7 +244,7 @@ TARGETS: Dict[str, Integration] = { "--test:{extra_setup_dir}", "--write_test_file={extra_output_dir}/test.txt", ], - pool=PoolName("mariner") + pool=PoolName("mariner"), ), "windows-libfuzzer": Integration( template=TemplateType.libfuzzer, @@ -401,10 +401,13 @@ class TestOnefuzz: self.of.pools.create(name, OS.linux) self.logger.info("creating scaleset for pool: %s", name) self.of.scalesets.create( - name, pool_size, region=region, initial_size=pool_size, image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest" + name, + pool_size, + region=region, + initial_size=pool_size, + image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest", ) - class UnmanagedPool: def __init__( self, @@ -644,7 +647,7 @@ class TestOnefuzz: setup = Directory(os.path.join(setup, config.nested_setup_dir)) job: Optional[Job] = None - + job = self.build_job( duration, pool, target, config, setup, target_exe, inputs ) @@ -1277,7 +1280,7 @@ class TestOnefuzz: if seen_errors: raise Exception("logs included errors") - + def build_pool_name(self, os_type: str) -> PoolName: return PoolName(f"testpool-{os_type}-{self.test_id}") @@ -1592,13 +1595,6 @@ class Run(Command): result = tester.check_jobs(poll=True, stop_on_complete_check=True) if not result: raise Exception("jobs failed") - if skip_repro: - self.logger.warning("not testing crash repro") - else: - launch_result, repros = tester.launch_repro() - result = tester.check_repro(repros) - if not (result and launch_result): - raise Exception("repros failed") tester.check_logs_for_errors()