diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index 79817bf31..ea5be2c74 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -53,7 +53,7 @@ public enum TaskState Init, Waiting, Scheduled, - Setting_up, + SettingUp, Running, Stopping, Stopped, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Events.cs b/src/ApiService/ApiService/OneFuzzTypes/Events.cs index f2fb10d59..699177cf1 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Events.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Events.cs @@ -51,6 +51,7 @@ public abstract record BaseEvent() this switch { EventNodeHeartbeat _ => EventType.NodeHeartbeat, + EventTaskHeartbeat _ => EventType.TaskHeartbeat, EventInstanceConfigUpdated _ => EventType.InstanceConfigUpdated, _ => throw new NotImplementedException(), }; @@ -113,7 +114,7 @@ public abstract record BaseEvent() // ) : BaseEvent(); -record EventTaskHeartbeat( +public record EventTaskHeartbeat( Guid JobId, Guid TaskId, TaskConfig Config @@ -252,7 +253,7 @@ public record EventNodeHeartbeat( // ) : BaseEvent(); -record EventInstanceConfigUpdated( +public record EventInstanceConfigUpdated( InstanceConfig Config ) : BaseEvent(); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 41cdb4dcb..b64647f41 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -144,7 +144,7 @@ public partial record Proxy string? Ip, Error? Error, string Version, - ProxyHeartbeat? heartbeat + ProxyHeartbeat? Heartbeat ) : EntityBase(); public record Error(ErrorCode Code, string[]? Errors = null); @@ -334,6 +334,8 @@ public record InstanceConfig null) { } + public InstanceConfig() : this(String.Empty) { } + public List? CheckAdmins(List? value) { if (value is not null && value.Count == 0) diff --git a/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs b/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs index e1275e8d3..e1f82889d 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs @@ -29,7 +29,8 @@ public record WebhookMessageEventGrid( BaseEvent data); - +// TODO: This should inherit from Entity Base ? no, since there is +// a table WebhookMessaageLog public record WebhookMessageLog( [RowKey] Guid EventId, EventType EventType, @@ -49,7 +50,7 @@ public record WebhookMessageLog( public record Webhook( [PartitionKey] Guid WebhookId, [RowKey] string Name, - Uri? url, + Uri? Url, List EventTypes, string SecretToken, // SecretString?? WebhookMessageFormat? MessageFormat diff --git a/src/ApiService/ApiService/QueueProxyHeartbeat.cs b/src/ApiService/ApiService/QueueProxyHeartbeat.cs index fb053f695..198badddc 100644 --- a/src/ApiService/ApiService/QueueProxyHeartbeat.cs +++ b/src/ApiService/ApiService/QueueProxyHeartbeat.cs @@ -34,7 +34,7 @@ public class QueueProxyHearbeat log.Warning($"invalid proxy id: {newHb.ProxyId}"); return; } - var newProxy = proxy with { heartbeat = newHb }; + var newProxy = proxy with { Heartbeat = newHb }; var r = await _proxy.Replace(newProxy); if (!r.IsOk) diff --git a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs index 6691d28af..8c8ea50e7 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs @@ -47,6 +47,7 @@ public class EntityConverter private readonly ConcurrentDictionary _cache; + private readonly ETag _emptyETag = new ETag(); public EntityConverter() { @@ -285,7 +286,11 @@ public class EntityConverter ).ToArray(); var entityRecord = (T)entityInfo.constructor.Invoke(parameters); - entityRecord.ETag = entity.ETag; + + if (entity.ETag != _emptyETag) + { + entityRecord.ETag = entity.ETag; + } entityRecord.TimeStamp = entity.Timestamp; return entityRecord; diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs new file mode 100644 index 000000000..b59ddbf29 --- /dev/null +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -0,0 +1,750 @@ +using FsCheck; +using FsCheck.Xunit; +using Xunit.Abstractions; +using Microsoft.OneFuzz.Service; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +using System.Collections.Generic; +using System; +using System.Linq; +using System.Security; +using System.Text.Json; + +namespace Tests +{ + + public class OrmGenerators + { + + public static Gen BaseEvent() + { + return Gen.OneOf(new[] { + Arb.Generate().Select(e => e as BaseEvent), + Arb.Generate().Select(e => e as BaseEvent), + Arb.Generate().Select(e => e as BaseEvent) + }); + } + + public static Gen EventType() + { + return Gen.OneOf(new[] { + Gen.Constant(Microsoft.OneFuzz.Service.EventType.NodeHeartbeat), + Gen.Constant(Microsoft.OneFuzz.Service.EventType.TaskHeartbeat), + Gen.Constant(Microsoft.OneFuzz.Service.EventType.InstanceConfigUpdated) + }); + + } + + + public static Gen Uri() + { + return Arb.Generate().Select( + arg => new Uri($"https://{arg.Item.ToString()}:8080") + ); + } + + public static Gen WebhookMessageLog() + { + return Arb.Generate, Tuple>>().Select( + arg => new WebhookMessageLog( + EventId: arg.Item1.Item1, + EventType: arg.Item1.Item2, + Event: arg.Item1.Item3, + InstanceId: arg.Item1.Item4, + InstanceName: arg.Item1.Item5, + WebhookId: arg.Item1.Item6, + State: arg.Item2.Item1, + TryCount: arg.Item2.Item2 + )); + } + + public static Gen Node() + { + return Arb.Generate, Tuple>>().Select( + arg => new Node( + InitializedAt: arg.Item1.Item1, + PoolName: arg.Item1.Item2, + PoolId: arg.Item1.Item3, + MachineId: arg.Item1.Item4, + State: arg.Item1.Item5, + ScalesetId: arg.Item2.Item1, + Heartbeat: arg.Item2.Item2, + Version: arg.Item2.Item3, + ReimageRequested: arg.Item2.Item4, + DeleteRequested: arg.Item2.Item5, + DebugKeepNode: arg.Item2.Item6)); + } + + public static Gen ProxyForward() + { + return Arb.Generate>().Select( + arg => + new ProxyForward( + Region: arg.Item1, + DstPort: arg.Item2, + SrcPort: arg.Item3, + DstIp: arg.Item4.Item.ToString() + ) + ); + } + + public static Gen Proxy() + { + return Arb.Generate, Tuple>>().Select( + arg => + new Proxy( + Region: arg.Item1.Item1, + ProxyId: arg.Item1.Item2, + CreatedTimestamp: arg.Item1.Item3, + State: arg.Item1.Item4, + Auth: arg.Item1.Item5, + Ip: arg.Item1.Item6, + Error: arg.Item1.Item7, + Version: arg.Item2.Item1, + Heartbeat: arg.Item2.Item2 + ) + ); + } + + public static Gen EventMessage() + { + return Arb.Generate>().Select( + arg => + new EventMessage( + EventId: arg.Item1, + EventType: arg.Item2.GetEventType(), + Event: arg.Item2, + InstanceId: arg.Item3, + InstanceName: arg.Item4 + ) + ); + } + + public static Gen NetworkConfig() + { + return Arb.Generate>().Select( + arg => + new NetworkConfig( + AddressSpace: arg.Item1.Item.ToString(), + Subnet: arg.Item2.Item.ToString() + ) + ); + } + + public static Gen NetworkSecurityGroupConfig() + { + return Arb.Generate>().Select( + arg => + new NetworkSecurityGroupConfig( + AllowedServiceTags: arg.Item1, + AllowedIps: (from ip in arg.Item2 select ip.Item.ToString()).ToArray() + ) + ); + } + + public static Gen InstanceConfig() + { + return Arb.Generate, + Tuple?, IDictionary?, IDictionary?, IDictionary?>>>().Select( + arg => + new InstanceConfig( + InstanceName: arg.Item1.Item1, + Admins: arg.Item1.Item2, + AllowPoolManagement: arg.Item1.Item3, + AllowedAadTenants: arg.Item1.Item4, + NetworkConfig: arg.Item1.Item5, + ProxyNsgConfig: arg.Item1.Item6, + Extensions: arg.Item1.Item7, + + ProxyVmSku: arg.Item2.Item1, + ApiAccessRules: arg.Item2.Item2, + GroupMembership: arg.Item2.Item3, + VmTags: arg.Item2.Item4, + VmssTags: arg.Item2.Item5 + ) + ); + } + + public static Gen Task() + { + return Arb.Generate, + Tuple>>().Select( + arg => + new Task( + JobId: arg.Item1.Item1, + TaskId: arg.Item1.Item2, + State: arg.Item1.Item3, + Os: arg.Item1.Item4, + Config: arg.Item1.Item5, + Error: arg.Item1.Item6, + Auth: arg.Item1.Item7, + + Heartbeat: arg.Item2.Item1, + EndTime: arg.Item2.Item2, + UserInfo: arg.Item2.Item3 + ) + ); + } + + + public static Gen Scaleset() + { + return Arb.Generate, + Tuple, Guid?, Guid?>, + Tuple>>>().Select( + arg => + new Scaleset( + PoolName: arg.Item1.Item1, + ScalesetId: arg.Item1.Item2, + State: arg.Item1.Item3, + Auth: arg.Item1.Item4, + VmSku: arg.Item1.Item5, + Image: arg.Item1.Item6, + Region: arg.Item1.Item7, + + Size: arg.Item2.Item1, + SpotInstance: arg.Item2.Item2, + EphemeralOsDisks: arg.Item2.Item3, + NeedsConfigUpdate: arg.Item2.Item4, + Nodes: arg.Item2.Item5, + ClientId: arg.Item2.Item6, + ClientObjectId: arg.Item2.Item7, + + Tags: arg.Item3.Item1 + ) + ); + } + + + public static Gen Webhook() + { + return Arb.Generate, string, WebhookMessageFormat>>().Select( + arg => + new Webhook( + WebhookId: arg.Item1, + Name: arg.Item2, + Url: arg.Item3, + EventTypes: arg.Item4, + SecretToken: arg.Item5, + MessageFormat: arg.Item6 + ) + ); + + } + + public static Gen WebhookMessage() + { + return Arb.Generate>().Select( + arg => + new WebhookMessage( + EventId: arg.Item1, + EventType: arg.Item2, + Event: arg.Item3, + InstanceId: arg.Item4, + InstanceName: arg.Item5, + WebhookId: arg.Item6 + ) + ); + + } + } + + public class OrmArb + { + public static Arbitrary Uri() + { + return Arb.From(OrmGenerators.Uri()); + } + + public static Arbitrary BaseEvent() + { + return Arb.From(OrmGenerators.BaseEvent()); + } + + public static Arbitrary EventType() + { + return Arb.From(OrmGenerators.EventType()); + } + + public static Arbitrary Node() + { + return Arb.From(OrmGenerators.Node()); + } + + public static Arbitrary ProxyForward() + { + return Arb.From(OrmGenerators.ProxyForward()); + } + + public static Arbitrary Proxy() + { + return Arb.From(OrmGenerators.Proxy()); + } + + public static Arbitrary EventMessage() + { + return Arb.From(OrmGenerators.EventMessage()); + } + + public static Arbitrary NetworkConfig() + { + return Arb.From(OrmGenerators.NetworkConfig()); + } + + public static Arbitrary NetworkSecurityConfig() + { + return Arb.From(OrmGenerators.NetworkSecurityGroupConfig()); + } + + public static Arbitrary InstanceConfig() + { + return Arb.From(OrmGenerators.InstanceConfig()); + } + + public static Arbitrary WebhookMessageLog() + { + return Arb.From(OrmGenerators.WebhookMessageLog()); + } + + public static Arbitrary Task() + { + return Arb.From(OrmGenerators.Task()); + } + + public static Arbitrary Scaleset() + { + return Arb.From(OrmGenerators.Scaleset()); + } + + public static Arbitrary Webhook() + { + return Arb.From(OrmGenerators.Webhook()); + } + + public static Arbitrary WebhookMessage() + { + return Arb.From(OrmGenerators.WebhookMessage()); + } + } + + + public static class EqualityComparison + { + private static HashSet _baseTypes = new HashSet( + new[]{ + typeof(byte), + typeof(char), + typeof(bool), + typeof(int), + typeof(long), + typeof(float), + typeof(double), + typeof(string), + typeof(Guid), + typeof(Uri), + typeof(DateTime), + typeof(DateTime?), + typeof(DateTimeOffset), + typeof(DateTimeOffset?), + typeof(SecureString) + }); + static bool IEnumerableEqual(IEnumerable? a, IEnumerable? b) + { + if (a is null && b is null) + { + return true; + } + if (a!.Count() != b!.Count()) + { + return false; + } + + if (a!.Count() == 0 && b!.Count() == 0) + { + return true; + } + + foreach (var v in a!.Zip(b!)) + { + if (!AreEqual(v.First, v.Second)) + { + return false; + } + } + + return true; + } + + static bool IDictionaryEqual(IDictionary? a, IDictionary? b, Func cmp) + { + if (a is null && b is null) + return true; + + if (a!.Count == 0 && b!.Count == 0) + return true; + + if (a!.Count != b!.Count) + return false; + + return a!.Any(v => cmp(v.Value, b[v.Key])); + } + + static bool IDictionaryEqual(IDictionary? a, IDictionary? b) + { + if (a is null && b is null) + return true; + + if (a!.Count == 0 && b!.Count == 0) + return true; + + if (a!.Count != b!.Count) + return false; + + return a!.Any(v => AreEqual(v.Value, b[v.Key])); + } + + + public static bool AreEqual(T r1, T r2) + { + var t = typeof(T); + + if (r1 is null && r2 is null) + return true; + + if (_baseTypes.Contains(t)) + return r1!.Equals(r2); + + foreach (var p in t.GetProperties()) + { + var v1 = p.GetValue(r1); + var v2 = p.GetValue(r2); + var tt = p.PropertyType; + + if (v1 is null && v2 is null) + continue; + + if (v1 is null || v2 is null) + return false; + + if (_baseTypes.Contains(tt) && !v1!.Equals(v2)) + return false; + + if (tt.GetInterface("IEnumerable") is not null) + { + if (!IEnumerableEqual(v1 as IEnumerable, v2 as IEnumerable)) + return false; + } + + if (tt.GetInterface("IDictionary") is not null) + { + if (!IDictionaryEqual(v1 as IDictionary, v2 as IDictionary)) + return false; + } + } + return true; + } + } + + public class OrmModelsTest + { + EntityConverter _converter = new EntityConverter(); + ITestOutputHelper _output; + + public OrmModelsTest(ITestOutputHelper output) + { + Arb.Register(); + _output = output; + } + + bool Test(T e) where T : EntityBase + { + var v = _converter.ToTableEntity(e); + var r = _converter.ToRecord(v); + return EqualityComparison.AreEqual(e, r); + + } + + [Property] + public bool Node(Node node) + { + return Test(node); + } + + [Property] + public bool ProxyForward(ProxyForward proxyForward) + { + return Test(proxyForward); + } + + [Property] + public bool Proxy(Proxy proxy) + { + return Test(proxy); + } + + [Property] + public bool Task(Task task) + { + return Test(task); + } + + + [Property] + public bool InstanceConfig(InstanceConfig cfg) + { + return Test(cfg); + } + + [Property] + public bool Scaleset(Scaleset ss) + { + return Test(ss); + } + + /* @Cheick + [Property] + public bool WebhookMessageLog(WebhookMessageLog log) + { + return Test(log); + } + */ + + + [Property] + public bool Webhook(Webhook wh) + { + return Test(wh); + } + + + + //Sample function on how repro a failing test run, using Replay + //functionality of FsCheck. Feel free to + /* + [Property] + void Replay() + { + var seed = FsCheck.Random.StdGen.NewStdGen(1384212554,297026222); + var p = Prop.ForAll((Task x) => Task(x) ); + p.Check(new Configuration { Replay = seed }); + } + */ + } + + + public class OrmJsonSerialization + { + + JsonSerializerOptions _opts = EntityConverter.GetJsonSerializerOptions(); + ITestOutputHelper _output; + + public OrmJsonSerialization(ITestOutputHelper output) + { + Arb.Register(); + _output = output; + } + + + string serialize(T x) + { + return JsonSerializer.Serialize(x, _opts); + } + + T? deserialize(string json) + { + return JsonSerializer.Deserialize(json, _opts); + } + + + bool Test(T v) + { + var j = serialize(v); + var r = deserialize(j); + return EqualityComparison.AreEqual(v, r); + } + + [Property] + public bool Node(Node node) + { + return Test(node); + } + + [Property] + public bool ProxyForward(ProxyForward proxyForward) + { + return Test(proxyForward); + } + + [Property] + public bool Proxy(Proxy proxy) + { + return Test(proxy); + } + + + [Property] + public bool Task(Task task) + { + return Test(task); + } + + + [Property] + public bool InstanceConfig(InstanceConfig cfg) + { + return Test(cfg); + } + + + [Property] + public bool Scaleset(Scaleset ss) + { + return Test(ss); + } + + /* @Cheick + [Property] + public bool WebhookMessageLog(WebhookMessageLog log) + { + return Test(log); + } + */ + + [Property] + public bool Webhook(Webhook wh) + { + return Test(wh); + } + + /* @Cheick + [Property] + public bool WebhookMessageEventGrid(WebhookMessageEventGrid evt) + { + return Teste(evt); + } + */ + + /* @Cheick + [Property] + public bool WebhookMessage(WebhookMessage msg) + { + return Test(msg); + } + */ + + + [Property] + public bool TaskHeartbeatEntry(TaskHeartbeatEntry e) + { + return Test(e); + } + + [Property] + public bool NodeCommand(NodeCommand e) + { + return Test(e); + } + + [Property] + public bool NodeTasks(NodeTasks e) + { + return Test(e); + } + + [Property] + public bool ProxyHeartbeat(ProxyHeartbeat e) + { + return Test(e); + } + + [Property] + public bool ProxyConfig(ProxyConfig e) + { + return Test(e); + } + + [Property] + public bool TaskDetails(TaskDetails e) + { + return Test(e); + } + + [Property] + public bool TaskVm(TaskVm e) + { + return Test(e); + } + + [Property] + public bool TaskPool(TaskPool e) + { + return Test(e); + } + + [Property] + public bool TaskContainers(TaskContainers e) + { + return Test(e); + } + + [Property] + public bool TaskConfig(TaskConfig e) + { + return Test(e); + } + + [Property] + public bool TaskEventSummary(TaskEventSummary e) + { + return Test(e); + } + + [Property] + public bool NodeAssignment(NodeAssignment e) + { + return Test(e); + } + + [Property] + public bool KeyvaultExtensionConfig(KeyvaultExtensionConfig e) + { + return Test(e); + } + + [Property] + public bool AzureMonitorExtensionConfig(AzureMonitorExtensionConfig e) + { + return Test(e); + } + + [Property] + public bool AzureVmExtensionConfig(AzureVmExtensionConfig e) + { + return Test(e); + } + + [Property] + public bool NetworkConfig(NetworkConfig e) + { + return Test(e); + } + + [Property] + public bool NetworkSecurityGroupConfig(NetworkSecurityGroupConfig e) + { + return Test(e); + } + + [Property] + public bool Report(Report e) + { + return Test(e); + } + } + +} + + + diff --git a/src/ApiService/Tests/Tests.csproj b/src/ApiService/Tests/Tests.csproj index 18726c808..851460033 100644 --- a/src/ApiService/Tests/Tests.csproj +++ b/src/ApiService/Tests/Tests.csproj @@ -7,13 +7,15 @@ - + + + runtime; build; native; contentfiles; analyzers; buildtransitive all - + runtime; build; native; contentfiles; analyzers; buildtransitive all diff --git a/src/ApiService/Tests/UnitTest1.cs b/src/ApiService/Tests/UnitTest1.cs deleted file mode 100644 index 0fa554f60..000000000 --- a/src/ApiService/Tests/UnitTest1.cs +++ /dev/null @@ -1,13 +0,0 @@ -using Xunit; - -namespace Tests -{ - public class UnitTest1 - { - [Fact] - public void Test1() - { - - } - } -} \ No newline at end of file