Limited support for polymorphic deserialization (#1814)

src/ApiService/ApiService/OneFuzzTypes/Events.cs

@@ -1,7 +1,5 @@
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using PoolName = System.String;
@@ -44,7 +42,6 @@ public enum EventType
 
 public abstract record BaseEvent()
 {
-
     public EventType GetEventType()
     {
         return
@@ -57,8 +54,28 @@ public abstract record BaseEvent()
             };
 
     }
+
+    public static Type GetTypeInfo(EventType eventType)
+    {
+        return (eventType) switch
+        {
+            EventType.NodeHeartbeat => typeof(EventNodeHeartbeat),
+            EventType.InstanceConfigUpdated => typeof(EventInstanceConfigUpdated),
+            EventType.TaskHeartbeat => typeof(EventTaskHeartbeat),
+            _ => throw new ArgumentException($"invalid input {eventType}"),
+
+        };
+    }
 };
 
+public class EventTypeProvider : ITypeProvider
+{
+    public Type GetTypeInfo(object input)
+    {
+        return BaseEvent.GetTypeInfo((input as EventType?) ?? throw new ArgumentException($"input is expected to be an EventType {input}"));
+    }
+}
+
 //public record EventTaskStopped(
 //    Guid JobId,
 //    Guid TaskId,
@@ -257,92 +274,26 @@ public record EventInstanceConfigUpdated(
     InstanceConfig Config
 ) : BaseEvent();
 
-
-[JsonConverter(typeof(EventConverter))]
 public record EventMessage(
     Guid EventId,
     EventType EventType,
+    [property: TypeDiscrimnatorAttribute("EventType", typeof(EventTypeProvider))]
+    [property: JsonConverter(typeof(BaseEventConverter))]
     BaseEvent Event,
     Guid InstanceId,
     String InstanceName
 ) : EntityBase();
 
-public class EventConverter : JsonConverter<EventMessage>
+public class BaseEventConverter : JsonConverter<BaseEvent>
 {
-
+    public override BaseEvent? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
-    private readonly Dictionary<string, Type> _allBaseEvents;
-
-    public EventConverter()
     {
-        _allBaseEvents = typeof(BaseEvent).Assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(BaseEvent))).ToDictionary(x => x.Name);
+        return null;
     }
 
-
+    public override void Write(Utf8JsonWriter writer, BaseEvent value, JsonSerializerOptions options)
-    public override EventMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
     {
-
+        var eventType = value.GetType();
-
+        JsonSerializer.Serialize(writer, value, eventType, options);
-        if (reader.TokenType != JsonTokenType.StartObject)
-        {
-            throw new JsonException();
-        }
-
-        using (var jsonDocument = JsonDocument.ParseValue(ref reader))
-        {
-            if (!jsonDocument.RootElement.TryGetProperty("event_type", out var eventType))
-            {
-                throw new JsonException();
-            }
-            if (!jsonDocument.RootElement.TryGetProperty("event", out var eventData))
-            {
-                throw new JsonException();
-            }
-            if (!jsonDocument.RootElement.TryGetProperty("event_id", out var eventId))
-            {
-                throw new JsonException();
-            }
-            if (!jsonDocument.RootElement.TryGetProperty("instance_id", out var instanceId))
-            {
-                throw new JsonException();
-            }
-            if (!jsonDocument.RootElement.TryGetProperty("instance_name", out var instanceName))
-            {
-                throw new JsonException();
-            }
-
-
-            var eventTypeName = eventType.GetString() ?? throw new JsonException();
-
-            if (!_allBaseEvents.TryGetValue($"Event{CaseConverter.SnakeToPascal(eventTypeName)}", out var eType))
-            {
-                throw new JsonException($"Unknown eventType {eventTypeName}");
-            }
-
-            var rawTest = eventData.GetRawText();
-            var eventClass = (BaseEvent)(JsonSerializer.Deserialize(eventData.GetRawText(), eType, options: options) ?? throw new JsonException());
-            var eventTypeEnum = Enum.Parse<EventType>(CaseConverter.SnakeToPascal(eventTypeName));
-
-
-            return new EventMessage(
-                eventId.GetGuid(),
-                eventTypeEnum,
-                eventClass,
-                instanceId.GetGuid(),
-                instanceName.GetString() ?? throw new JsonException()
-            );
-        }
     }
-
+}
-    public override void Write(Utf8JsonWriter writer, EventMessage value, JsonSerializerOptions options)
-    {
-        writer.WriteStartObject();
-        writer.WriteString("event_id", value.EventId.ToString());
-        writer.WriteString("event_type", CaseConverter.PascalToSnake(value.EventType.ToString()));
-        writer.WritePropertyName("event");
-        var eventstr = JsonSerializer.Serialize(value.Event, value.Event.GetType(), options);
-        writer.WriteRawValue(eventstr);
-        writer.WriteString("instance_id", value.InstanceId.ToString());
-        writer.WriteString("instance_name", value.InstanceName);
-        writer.WriteEndObject();
-    }
-}

src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs

@@ -26,6 +26,8 @@ public record WebhookMessageEventGrid(
     [property: JsonPropertyName("EventType")] EventType EventType,
     [property: JsonPropertyName("eventTime")] DateTimeOffset eventTime,
     Guid Id,
+    [property: TypeDiscrimnatorAttribute("EventType", typeof(EventTypeProvider))]
+    [property: JsonConverter(typeof(BaseEventConverter))]
     BaseEvent data);
 
 

src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs

@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
+using System.Runtime.CompilerServices;
 using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Text.Json.Serialization;
@@ -165,3 +166,141 @@ public sealed class CustomEnumConverter<T> : JsonConverter<T> where T : Enum
     }
 }
 
+
+public sealed class PolymorphicConverterFactory : JsonConverterFactory
+{
+    public override bool CanConvert(Type typeToConvert)
+    {
+        var converter = typeToConvert.GetCustomAttribute<JsonConverterAttribute>();
+        if (converter != null)
+        {
+            return false;
+        }
+
+        var propertyAndAttributes =
+            typeToConvert.GetProperties()
+                .Select(p => new { property = p, attribute = p.GetCustomAttribute<TypeDiscrimnatorAttribute>() })
+                .Where(p => p.attribute != null)
+                .ToList();
+
+        if (propertyAndAttributes.Count == 0)
+        {
+            return false;
+        }
+
+        if (propertyAndAttributes.Count == 1)
+        {
+            return true;
+        }
+
+        else
+        {
+            throw new InvalidOperationException("the attribute TypeDiscrimnatorAttribute can only be aplied once");
+        }
+    }
+
+    public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
+    {
+        var (field, attribute) = typeToConvert.GetProperties()
+                .Select(p => (p.Name, p.GetCustomAttribute<TypeDiscrimnatorAttribute>()))
+                .Where(p => p.Item2 != null)
+                .First();
+
+
+        return (JsonConverter)Activator.CreateInstance(
+            typeof(PolymorphicConverter<>).MakeGenericType(typeToConvert),
+            BindingFlags.Instance | BindingFlags.Public,
+            binder: null,
+            args: new object?[] { attribute, field },
+            culture: null)!;
+    }
+}
+
+public sealed class PolymorphicConverter<T> : JsonConverter<T>
+{
+    private readonly ITypeProvider _typeProvider;
+    private readonly string _discriminatorField;
+    private readonly string _discriminatedField;
+    private readonly ConstructorInfo _constructorInfo;
+    private readonly Dictionary<string, ParameterInfo> _parameters;
+    private readonly Func<object?[], object> _constructor;
+    private readonly Type _discriminatorType;
+    private readonly ConditionalWeakTable<JsonSerializerOptions, JsonSerializerOptions> _options;
+    private readonly Dictionary<string, string> _renamedViaJsonPropery;
+
+
+
+    public PolymorphicConverter(TypeDiscrimnatorAttribute typeDiscriminator, string discriminatedField) : base()
+    {
+        _discriminatorField = typeDiscriminator.FieldName;
+        _typeProvider = (ITypeProvider)(typeDiscriminator.ConverterType.GetConstructor(new Type[] { })?.Invoke(null) ?? throw new JsonException());
+        _discriminatedField = discriminatedField;
+        _constructorInfo = typeof(T).GetConstructors().FirstOrDefault() ?? throw new JsonException("No Constructor found");
+        _parameters = _constructorInfo.GetParameters()?.ToDictionary(x => x.Name ?? "") ?? throw new JsonException();
+        _constructor = EntityConverter.BuildConstructerFrom(_constructorInfo);
+        _discriminatorType = _parameters[_discriminatorField].ParameterType;
+        _options = new ConditionalWeakTable<JsonSerializerOptions, JsonSerializerOptions>();
+
+        _renamedViaJsonPropery =
+            typeof(T).GetProperties()
+                .Select(x => (x.Name, x.GetCustomAttribute<JsonPropertyNameAttribute>()?.Name))
+                .Where(x => x.Item2 != null)
+                .ToDictionary(x => x.Item1, x => x.Item2!) ?? new Dictionary<string, string>();
+    }
+
+    private string ConvertName(string name, JsonSerializerOptions options)
+    {
+
+        if (_renamedViaJsonPropery.TryGetValue(name, out var renamed))
+        {
+            return renamed;
+        }
+
+        return options.PropertyNamingPolicy?.ConvertName(name) ?? name;
+    }
+
+    public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        if (reader.TokenType != JsonTokenType.StartObject)
+        {
+            throw new JsonException();
+        }
+
+        using (var jsonDocument = JsonDocument.ParseValue(ref reader))
+        {
+            var discriminatorName = ConvertName(_discriminatorField, options);
+            var discriminatorValue = jsonDocument.RootElement.GetProperty(discriminatorName).GetRawText();
+            var discriminatorTypedValue = JsonSerializer.Deserialize(discriminatorValue, _discriminatorType, options) ?? throw new JsonException("unable to read deserialize discriminator value");
+            var discriminatedType = _typeProvider.GetTypeInfo(discriminatorTypedValue);
+            var constructorParams =
+                _constructorInfo.GetParameters().Select(p =>
+                {
+                    var parameterName = p.Name ?? throw new JsonException();
+                    var parameterType = parameterName == _discriminatedField ? discriminatedType : p.ParameterType;
+                    var fName = ConvertName(parameterName, options);
+                    var prop = jsonDocument.RootElement.GetProperty(fName);
+                    return JsonSerializer.Deserialize(prop.GetRawText(), parameterType, options);
+
+                }).ToArray();
+            return (T?)_constructor(constructorParams);
+        }
+    }
+
+    public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options)
+    {
+        var newOptions =
+            _options.GetValue(options, k =>
+            {
+                var newOptions = new JsonSerializerOptions(k);
+                var thisConverter = newOptions.Converters.FirstOrDefault(c => c.GetType() == typeof(PolymorphicConverterFactory));
+                if (thisConverter != null)
+                {
+                    newOptions.Converters.Remove(thisConverter);
+                }
+
+                return newOptions;
+            });
+
+        JsonSerializer.Serialize(writer, value, newOptions);
+    }
+}

src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs

@@ -7,6 +7,7 @@ using System.Text.Json;
 using System.Text.Json.Serialization;
 using System.Collections.Concurrent;
 using Azure;
+using System.Collections.Generic;
 
 namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 
@@ -22,17 +23,39 @@ public abstract record EntityBase
 /// Indicates that the enum cases should no be renamed
 [AttributeUsage(AttributeTargets.Enum)]
 public class SkipRename : Attribute { }
-
 public class RowKeyAttribute : Attribute { }
 public class PartitionKeyAttribute : Attribute { }
+public class TypeDiscrimnatorAttribute : Attribute
+{
+    public string FieldName { get; }
+    // the type of a function that takes the value of fieldName as an input and return the type
+    public Type ConverterType { get; }
+
+    public TypeDiscrimnatorAttribute(string fieldName, Type converterType)
+    {
+        if (!converterType.IsAssignableTo(typeof(ITypeProvider)))
+        {
+            throw new ArgumentException($"the provided type needs to implement ITypeProvider");
+        }
+
+        FieldName = fieldName;
+        ConverterType = converterType;
+    }
+}
+
+public interface ITypeProvider
+{
+    Type GetTypeInfo(object input);
+}
+
 public enum EntityPropertyKind
 {
     PartitionKey,
     RowKey,
     Column
 }
-public record EntityProperty(string name, string columnName, Type type, EntityPropertyKind kind);
+public record EntityProperty(string name, string columnName, Type type, EntityPropertyKind kind, (TypeDiscrimnatorAttribute, ITypeProvider)? discriminator);
-public record EntityInfo(Type type, EntityProperty[] properties, Func<object?[], object> constructor);
+public record EntityInfo(Type type, Dictionary<string, EntityProperty> properties, Func<object?[], object> constructor);
 
 class OnefuzzNamingPolicy : JsonNamingPolicy
 {
@@ -63,10 +86,11 @@ public class EntityConverter
             PropertyNamingPolicy = new OnefuzzNamingPolicy(),
         };
         options.Converters.Add(new CustomEnumConverterFactory());
+        options.Converters.Add(new PolymorphicConverterFactory());
         return options;
     }
 
-    internal Func<object?[], object> BuildConstructerFrom(ConstructorInfo constructorInfo)
+    internal static Func<object?[], object> BuildConstructerFrom(ConstructorInfo constructorInfo)
     {
         var constructorParameters = Expression.Parameter(typeof(object?[]));
 
@@ -112,11 +136,18 @@ public class EntityConverter
                                 ?? CaseConverter.PascalToSnake(name),
                             EntityPropertyKind.Column
                         );
+                var discriminatorAttribute = type.GetProperty(name)?.GetCustomAttribute<TypeDiscrimnatorAttribute>();
 
-                return new EntityProperty(name, columnName, parameterType, kind);
+                (TypeDiscrimnatorAttribute, ITypeProvider)? discriminator = null;
+                if (discriminatorAttribute != null)
+                {
+                    var t = (ITypeProvider)(discriminatorAttribute.ConverterType.GetConstructor(new Type[] { })?.Invoke(null) ?? throw new Exception("unable to retrive the type provider"));
+                    discriminator = (discriminatorAttribute, t);
+                }
+                return new EntityProperty(name, columnName, parameterType, kind, discriminator);
             }).ToArray();
 
-            return new EntityInfo(typeof(T), parameters, BuildConstructerFrom(constructor));
+            return new EntityInfo(typeof(T), parameters.ToDictionary(x => x.name), BuildConstructerFrom(constructor));
         });
     }
 
@@ -139,9 +170,9 @@ public class EntityConverter
         }
         var tableEntity = new TableEntity();
         var entityInfo = GetEntityInfo<T>();
-        foreach (var prop in entityInfo.properties)
+        foreach (var kvp in entityInfo.properties)
         {
-
+            var prop = kvp.Value;
             var value = entityInfo.type.GetProperty(prop.name)?.GetValue(typedEntity);
             if (prop.kind == EntityPropertyKind.PartitionKey || prop.kind == EntityPropertyKind.RowKey)
             {
@@ -194,96 +225,108 @@ public class EntityConverter
     }
 
 
+    private object? GetFieldValue(EntityInfo info, string name, TableEntity entity)
+    {
+        var ef = info.properties[name];
+        if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey)
+        {
+            if (ef.type == typeof(string))
+                return entity.GetString(ef.kind.ToString());
+            else if (ef.type == typeof(Guid))
+                return Guid.Parse(entity.GetString(ef.kind.ToString()));
+            else if (ef.type == typeof(int))
+                return int.Parse(entity.GetString(ef.kind.ToString()));
+            else
+            {
+                throw new Exception("invalid ");
+            }
+        }
+
+        var fieldName = ef.columnName;
+        var obj = entity[fieldName];
+        if (obj == null)
+        {
+            return null;
+        }
+        var objType = obj.GetType();
+
+        if (ef.type == typeof(string))
+        {
+            return entity.GetString(fieldName);
+        }
+        else if (ef.type == typeof(bool))
+        {
+            return entity.GetBoolean(fieldName);
+        }
+        else if (ef.type == typeof(DateTimeOffset) || ef.type == typeof(DateTimeOffset?))
+        {
+            return entity.GetDateTimeOffset(fieldName);
+        }
+        else if (ef.type == typeof(DateTime))
+        {
+            return entity.GetDateTime(fieldName);
+        }
+        else if (ef.type == typeof(double))
+        {
+            return entity.GetDouble(fieldName);
+        }
+        else if (ef.type == typeof(Guid) || ef.type == typeof(Guid?))
+        {
+            return (object?)Guid.Parse(entity.GetString(fieldName));
+        }
+        else if (ef.type == typeof(int))
+        {
+            return entity.GetInt32(fieldName);
+        }
+        else if (ef.type == typeof(Int64))
+        {
+            return entity.GetInt64(fieldName);
+        }
+        else if (ef.type.IsEnum)
+        {
+            var stringValues =
+                entity.GetString(fieldName).Split(",", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
+                .Select(CaseConverter.SnakeToPascal);
+
+            return Enum.Parse(ef.type, string.Join(",", stringValues));
+        }
+        else
+        {
+            var outputType = ef.type;
+            if (ef.discriminator != null)
+            {
+                var (attr, typeProvider) = ef.discriminator.Value;
+                var v = GetFieldValue(info, attr.FieldName, entity) ?? throw new Exception($"No value for {attr.FieldName}");
+                outputType = typeProvider.GetTypeInfo(v);
+            }
+
+
+            if (objType == typeof(string))
+            {
+                var value = entity.GetString(fieldName);
+                if (value.StartsWith('[') || value.StartsWith('{') || value == "null")
+                {
+                    return JsonSerializer.Deserialize(value, outputType, options: _options);
+                }
+                else
+                {
+                    return JsonSerializer.Deserialize($"\"{value}\"", outputType, options: _options);
+                }
+            }
+            else
+            {
+                var value = entity.GetString(fieldName);
+                return JsonSerializer.Deserialize(value, outputType, options: _options);
+            }
+        }
+    }
+
+
     public T ToRecord<T>(TableEntity entity) where T : EntityBase
     {
         var entityInfo = GetEntityInfo<T>();
         var parameters =
-            entityInfo.properties.Select(ef =>
+            entityInfo.properties.Keys.Select(k => GetFieldValue(entityInfo, k, entity)).ToArray();
-                {
-                    if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey)
-                    {
-                        if (ef.type == typeof(string))
-                            return entity.GetString(ef.kind.ToString());
-                        else if (ef.type == typeof(Guid))
-                            return Guid.Parse(entity.GetString(ef.kind.ToString()));
-                        else if (ef.type == typeof(int))
-                            return int.Parse(entity.GetString(ef.kind.ToString()));
-                        else
-                        {
-                            throw new Exception("invalid ");
-                        }
-                    }
-
-                    var fieldName = ef.columnName;
-                    var obj = entity[fieldName];
-                    if (obj == null)
-                    {
-                        return null;
-                    }
-                    var objType = obj.GetType();
-
-                    if (ef.type == typeof(string))
-                    {
-                        return entity.GetString(fieldName);
-                    }
-                    else if (ef.type == typeof(bool))
-                    {
-                        return entity.GetBoolean(fieldName);
-                    }
-                    else if (ef.type == typeof(DateTimeOffset) || ef.type == typeof(DateTimeOffset?))
-                    {
-                        return entity.GetDateTimeOffset(fieldName);
-                    }
-                    else if (ef.type == typeof(DateTime))
-                    {
-                        return entity.GetDateTime(fieldName);
-                    }
-                    else if (ef.type == typeof(double))
-                    {
-                        return entity.GetDouble(fieldName);
-                    }
-                    else if (ef.type == typeof(Guid) || ef.type == typeof(Guid?))
-                    {
-                        return (object?)Guid.Parse(entity.GetString(fieldName));
-                    }
-                    else if (ef.type == typeof(int))
-                    {
-                        return entity.GetInt32(fieldName);
-                    }
-                    else if (ef.type == typeof(Int64))
-                    {
-                        return entity.GetInt64(fieldName);
-                    }
-                    else if (ef.type.IsEnum)
-                    {
-                        var stringValues =
-                            entity.GetString(fieldName).Split(",", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
-                            .Select(CaseConverter.SnakeToPascal);
-
-                        return Enum.Parse(ef.type, string.Join(",", stringValues));
-                    }
-                    else
-                    {
-                        if (objType == typeof(string))
-                        {
-                            var value = entity.GetString(fieldName);
-                            if (value.StartsWith('[') || value.StartsWith('{') || value == "null")
-                            {
-                                return JsonSerializer.Deserialize(value, ef.type, options: _options);
-                            }
-                            else
-                            {
-                                return JsonSerializer.Deserialize($"\"{value}\"", ef.type, options: _options);
-                            }
-                        }
-                        else
-                        {
-                            var value = entity.GetString(fieldName);
-                            return JsonSerializer.Deserialize(value, ef.type, options: _options);
-                        }
-                    }
-                }
-            ).ToArray();
 
         var entityRecord = (T)entityInfo.constructor.Invoke(parameters);
 

src/ApiService/Tests/OrmModelsTest.cs

@@ -503,13 +503,12 @@ namespace Tests
             return Test(ss);
         }
 
-        /* @Cheick
         [Property]
         public bool WebhookMessageLog(WebhookMessageLog log)
         {
             return Test(log);
         }
-        */
+
 
 
         [Property]
@@ -604,13 +603,11 @@ namespace Tests
             return Test(ss);
         }
 
-        /* @Cheick
         [Property]
         public bool WebhookMessageLog(WebhookMessageLog log)
         {
             return Test(log);
         }
-        */
 
         [Property]
         public bool Webhook(Webhook wh)
@@ -618,21 +615,18 @@ namespace Tests
             return Test(wh);
         }
 
-        /* @Cheick
         [Property]
         public bool WebhookMessageEventGrid(WebhookMessageEventGrid evt)
-        { 
+        {
-            return Teste(evt);
+            return Test(evt);
         }
-        */
 
-        /* @Cheick
+
         [Property]
         public bool WebhookMessage(WebhookMessage msg)
         {
             return Test(msg);
         }
-        */
 
 
         [Property]

src/ApiService/Tests/OrmTest.cs

@@ -261,5 +261,20 @@ namespace Tests
             Assert.Equal(expected.Id, actual.Id);
             Assert.Equal(expected.TheName, actual.TheName);
         }
+
+
+        [Fact]
+        public void TestEventSerialization2()
+        {
+
+            var converter = new EntityConverter();
+            var expectedEvent = new EventMessage(Guid.NewGuid(), EventType.NodeHeartbeat, new EventNodeHeartbeat(Guid.NewGuid(), Guid.NewGuid(), "test Poool"), Guid.NewGuid(), "test")
+            {
+                ETag = new Azure.ETag("33a64df551425fcc55e4d42a148795d9f25f89d4")
+            };
+            var te = converter.ToTableEntity(expectedEvent);
+            var actualEvent = converter.ToRecord<EventMessage>(te);
+            Assert.Equal(expectedEvent, actualEvent);
+        }
     }
 }