ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-15 03:18:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

Limited support for polymorphic deserialization (#1814)

Browse Source
This commit is contained in:
Cheick Keita
2022-04-19 18:36:50 -07:00
committed by GitHub
parent 4be286ada5
commit e32f42cc1b
6 changed files with 326 additions and 182 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
src/ApiService/ApiService/OneFuzzTypes/Events.cs
View File

@ -1,7 +1,5 @@
using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Text.Json.Serialization;
using PoolName = System.String;
@ -44,7 +42,6 @@ public enum EventType
public abstract record BaseEvent()
{
public EventType GetEventType()
{
return
@ -57,8 +54,28 @@ public abstract record BaseEvent()
};
}
public static Type GetTypeInfo(EventType eventType)
{
return (eventType) switch
{
EventType.NodeHeartbeat => typeof(EventNodeHeartbeat),
EventType.InstanceConfigUpdated => typeof(EventInstanceConfigUpdated),
EventType.TaskHeartbeat => typeof(EventTaskHeartbeat),
_ => throw new ArgumentException($"invalid input {eventType}"),
};
}
};
public class EventTypeProvider : ITypeProvider
{
public Type GetTypeInfo(object input)
{
return BaseEvent.GetTypeInfo((input as EventType?) ?? throw new ArgumentException($"input is expected to be an EventType {input}"));
}
}
//public record EventTaskStopped(
// Guid JobId,
// Guid TaskId,
@ -257,92 +274,26 @@ public record EventInstanceConfigUpdated(
InstanceConfig Config
) : BaseEvent();
[JsonConverter(typeof(EventConverter))]
public record EventMessage(
Guid EventId,
EventType EventType,
[property: TypeDiscrimnatorAttribute("EventType", typeof(EventTypeProvider))]
[property: JsonConverter(typeof(BaseEventConverter))]
BaseEvent Event,
Guid InstanceId,
String InstanceName
) : EntityBase();
public class EventConverter : JsonConverter<EventMessage>
public class BaseEventConverter : JsonConverter<BaseEvent>
{
private readonly Dictionary<string, Type> _allBaseEvents;
public EventConverter()
public override BaseEvent? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
_allBaseEvents = typeof(BaseEvent).Assembly.GetTypes().Where(t => t.IsSubclassOf(typeof(BaseEvent))).ToDictionary(x => x.Name);
return null;
}
public override EventMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
public override void Write(Utf8JsonWriter writer, BaseEvent value, JsonSerializerOptions options)
{
if (reader.TokenType != JsonTokenType.StartObject)
{
throw new JsonException();
}
using (var jsonDocument = JsonDocument.ParseValue(ref reader))
{
if (!jsonDocument.RootElement.TryGetProperty("event_type", out var eventType))
{
throw new JsonException();
}
if (!jsonDocument.RootElement.TryGetProperty("event", out var eventData))
{
throw new JsonException();
}
if (!jsonDocument.RootElement.TryGetProperty("event_id", out var eventId))
{
throw new JsonException();
}
if (!jsonDocument.RootElement.TryGetProperty("instance_id", out var instanceId))
{
throw new JsonException();
}
if (!jsonDocument.RootElement.TryGetProperty("instance_name", out var instanceName))
{
throw new JsonException();
}
var eventTypeName = eventType.GetString() ?? throw new JsonException();
if (!_allBaseEvents.TryGetValue($"Event{CaseConverter.SnakeToPascal(eventTypeName)}", out var eType))
{
throw new JsonException($"Unknown eventType {eventTypeName}");
}
var rawTest = eventData.GetRawText();
var eventClass = (BaseEvent)(JsonSerializer.Deserialize(eventData.GetRawText(), eType, options: options) ?? throw new JsonException());
var eventTypeEnum = Enum.Parse<EventType>(CaseConverter.SnakeToPascal(eventTypeName));
return new EventMessage(
eventId.GetGuid(),
eventTypeEnum,
eventClass,
instanceId.GetGuid(),
instanceName.GetString() ?? throw new JsonException()
);
}
}
public override void Write(Utf8JsonWriter writer, EventMessage value, JsonSerializerOptions options)
{
writer.WriteStartObject();
writer.WriteString("event_id", value.EventId.ToString());
writer.WriteString("event_type", CaseConverter.PascalToSnake(value.EventType.ToString()));
writer.WritePropertyName("event");
var eventstr = JsonSerializer.Serialize(value.Event, value.Event.GetType(), options);
writer.WriteRawValue(eventstr);
writer.WriteString("instance_id", value.InstanceId.ToString());
writer.WriteString("instance_name", value.InstanceName);
writer.WriteEndObject();
var eventType = value.GetType();
JsonSerializer.Serialize(writer, value, eventType, options);
}
}

2
src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs
View File

@ -26,6 +26,8 @@ public record WebhookMessageEventGrid(
[property: JsonPropertyName("EventType")] EventType EventType,
[property: JsonPropertyName("eventTime")] DateTimeOffset eventTime,
Guid Id,
[property: TypeDiscrimnatorAttribute("EventType", typeof(EventTypeProvider))]
[property: JsonConverter(typeof(BaseEventConverter))]
BaseEvent data);

139
src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs
View File

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Runtime.CompilerServices;
using System.Text.Encodings.Web;
using System.Text.Json;
using System.Text.Json.Serialization;
@ -165,3 +166,141 @@ public sealed class CustomEnumConverter<T> : JsonConverter<T> where T : Enum
}
}
public sealed class PolymorphicConverterFactory : JsonConverterFactory
{
public override bool CanConvert(Type typeToConvert)
{
var converter = typeToConvert.GetCustomAttribute<JsonConverterAttribute>();
if (converter != null)
{
return false;
}
var propertyAndAttributes =
typeToConvert.GetProperties()
.Select(p => new { property = p, attribute = p.GetCustomAttribute<TypeDiscrimnatorAttribute>() })
.Where(p => p.attribute != null)
.ToList();
if (propertyAndAttributes.Count == 0)
{
return false;
}
if (propertyAndAttributes.Count == 1)
{
return true;
}
else
{
throw new InvalidOperationException("the attribute TypeDiscrimnatorAttribute can only be aplied once");
}
}
public override JsonConverter? CreateConverter(Type typeToConvert, JsonSerializerOptions options)
{
var (field, attribute) = typeToConvert.GetProperties()
.Select(p => (p.Name, p.GetCustomAttribute<TypeDiscrimnatorAttribute>()))
.Where(p => p.Item2 != null)
.First();
return (JsonConverter)Activator.CreateInstance(
typeof(PolymorphicConverter<>).MakeGenericType(typeToConvert),
BindingFlags.Instance | BindingFlags.Public,
binder: null,
args: new object?[] { attribute, field },
culture: null)!;
}
}
public sealed class PolymorphicConverter<T> : JsonConverter<T>
{
private readonly ITypeProvider _typeProvider;
private readonly string _discriminatorField;
private readonly string _discriminatedField;
private readonly ConstructorInfo _constructorInfo;
private readonly Dictionary<string, ParameterInfo> _parameters;
private readonly Func<object?[], object> _constructor;
private readonly Type _discriminatorType;
private readonly ConditionalWeakTable<JsonSerializerOptions, JsonSerializerOptions> _options;
private readonly Dictionary<string, string> _renamedViaJsonPropery;
public PolymorphicConverter(TypeDiscrimnatorAttribute typeDiscriminator, string discriminatedField) : base()
{
_discriminatorField = typeDiscriminator.FieldName;
_typeProvider = (ITypeProvider)(typeDiscriminator.ConverterType.GetConstructor(new Type[] { })?.Invoke(null) ?? throw new JsonException());
_discriminatedField = discriminatedField;
_constructorInfo = typeof(T).GetConstructors().FirstOrDefault() ?? throw new JsonException("No Constructor found");
_parameters = _constructorInfo.GetParameters()?.ToDictionary(x => x.Name ?? "") ?? throw new JsonException();
_constructor = EntityConverter.BuildConstructerFrom(_constructorInfo);
_discriminatorType = _parameters[_discriminatorField].ParameterType;
_options = new ConditionalWeakTable<JsonSerializerOptions, JsonSerializerOptions>();
_renamedViaJsonPropery =
typeof(T).GetProperties()
.Select(x => (x.Name, x.GetCustomAttribute<JsonPropertyNameAttribute>()?.Name))
.Where(x => x.Item2 != null)
.ToDictionary(x => x.Item1, x => x.Item2!) ?? new Dictionary<string, string>();
}
private string ConvertName(string name, JsonSerializerOptions options)
{
if (_renamedViaJsonPropery.TryGetValue(name, out var renamed))
{
return renamed;
}
return options.PropertyNamingPolicy?.ConvertName(name) ?? name;
}
public override T? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
{
if (reader.TokenType != JsonTokenType.StartObject)
{
throw new JsonException();
}
using (var jsonDocument = JsonDocument.ParseValue(ref reader))
{
var discriminatorName = ConvertName(_discriminatorField, options);
var discriminatorValue = jsonDocument.RootElement.GetProperty(discriminatorName).GetRawText();
var discriminatorTypedValue = JsonSerializer.Deserialize(discriminatorValue, _discriminatorType, options) ?? throw new JsonException("unable to read deserialize discriminator value");
var discriminatedType = _typeProvider.GetTypeInfo(discriminatorTypedValue);
var constructorParams =
_constructorInfo.GetParameters().Select(p =>
{
var parameterName = p.Name ?? throw new JsonException();
var parameterType = parameterName == _discriminatedField ? discriminatedType : p.ParameterType;
var fName = ConvertName(parameterName, options);
var prop = jsonDocument.RootElement.GetProperty(fName);
return JsonSerializer.Deserialize(prop.GetRawText(), parameterType, options);
}).ToArray();
return (T?)_constructor(constructorParams);
}
}
public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options)
{
var newOptions =
_options.GetValue(options, k =>
{
var newOptions = new JsonSerializerOptions(k);
var thisConverter = newOptions.Converters.FirstOrDefault(c => c.GetType() == typeof(PolymorphicConverterFactory));
if (thisConverter != null)
{
newOptions.Converters.Remove(thisConverter);
}
return newOptions;
});
JsonSerializer.Serialize(writer, value, newOptions);
}
}

77
src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs
View File

@ -7,6 +7,7 @@ using System.Text.Json;
using System.Text.Json.Serialization;
using System.Collections.Concurrent;
using Azure;
using System.Collections.Generic;
namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
@ -22,17 +23,39 @@ public abstract record EntityBase
/// Indicates that the enum cases should no be renamed
[AttributeUsage(AttributeTargets.Enum)]
public class SkipRename : Attribute { }
public class RowKeyAttribute : Attribute { }
public class PartitionKeyAttribute : Attribute { }
public class TypeDiscrimnatorAttribute : Attribute
{
public string FieldName { get; }
// the type of a function that takes the value of fieldName as an input and return the type
public Type ConverterType { get; }
public TypeDiscrimnatorAttribute(string fieldName, Type converterType)
{
if (!converterType.IsAssignableTo(typeof(ITypeProvider)))
{
throw new ArgumentException($"the provided type needs to implement ITypeProvider");
}
FieldName = fieldName;
ConverterType = converterType;
}
}
public interface ITypeProvider
{
Type GetTypeInfo(object input);
}
public enum EntityPropertyKind
{
PartitionKey,
RowKey,
Column
}
public record EntityProperty(string name, string columnName, Type type, EntityPropertyKind kind);
public record EntityInfo(Type type, EntityProperty[] properties, Func<object?[], object> constructor);
public record EntityProperty(string name, string columnName, Type type, EntityPropertyKind kind, (TypeDiscrimnatorAttribute, ITypeProvider)? discriminator);
public record EntityInfo(Type type, Dictionary<string, EntityProperty> properties, Func<object?[], object> constructor);
class OnefuzzNamingPolicy : JsonNamingPolicy
{
@ -63,10 +86,11 @@ public class EntityConverter
PropertyNamingPolicy = new OnefuzzNamingPolicy(),
};
options.Converters.Add(new CustomEnumConverterFactory());
options.Converters.Add(new PolymorphicConverterFactory());
return options;
}
internal Func<object?[], object> BuildConstructerFrom(ConstructorInfo constructorInfo)
internal static Func<object?[], object> BuildConstructerFrom(ConstructorInfo constructorInfo)
{
var constructorParameters = Expression.Parameter(typeof(object?[]));
@ -112,11 +136,18 @@ public class EntityConverter
?? CaseConverter.PascalToSnake(name),
EntityPropertyKind.Column
);
var discriminatorAttribute = type.GetProperty(name)?.GetCustomAttribute<TypeDiscrimnatorAttribute>();
return new EntityProperty(name, columnName, parameterType, kind);
(TypeDiscrimnatorAttribute, ITypeProvider)? discriminator = null;
if (discriminatorAttribute != null)
{
var t = (ITypeProvider)(discriminatorAttribute.ConverterType.GetConstructor(new Type[] { })?.Invoke(null) ?? throw new Exception("unable to retrive the type provider"));
discriminator = (discriminatorAttribute, t);
}
return new EntityProperty(name, columnName, parameterType, kind, discriminator);
}).ToArray();
return new EntityInfo(typeof(T), parameters, BuildConstructerFrom(constructor));
return new EntityInfo(typeof(T), parameters.ToDictionary(x => x.name), BuildConstructerFrom(constructor));
});
}
@ -139,9 +170,9 @@ public class EntityConverter
}
var tableEntity = new TableEntity();
var entityInfo = GetEntityInfo<T>();
foreach (var prop in entityInfo.properties)
foreach (var kvp in entityInfo.properties)
{
var prop = kvp.Value;
var value = entityInfo.type.GetProperty(prop.name)?.GetValue(typedEntity);
if (prop.kind == EntityPropertyKind.PartitionKey || prop.kind == EntityPropertyKind.RowKey)
{
@ -194,12 +225,9 @@ public class EntityConverter
}
public T ToRecord<T>(TableEntity entity) where T : EntityBase
{
var entityInfo = GetEntityInfo<T>();
var parameters =
entityInfo.properties.Select(ef =>
private object? GetFieldValue(EntityInfo info, string name, TableEntity entity)
{
var ef = info.properties[name];
if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey)
{
if (ef.type == typeof(string))
@ -264,26 +292,41 @@ public class EntityConverter
}
else
{
var outputType = ef.type;
if (ef.discriminator != null)
{
var (attr, typeProvider) = ef.discriminator.Value;
var v = GetFieldValue(info, attr.FieldName, entity) ?? throw new Exception($"No value for {attr.FieldName}");
outputType = typeProvider.GetTypeInfo(v);
}
if (objType == typeof(string))
{
var value = entity.GetString(fieldName);
if (value.StartsWith('[') || value.StartsWith('{') || value == "null")
{
return JsonSerializer.Deserialize(value, ef.type, options: _options);
return JsonSerializer.Deserialize(value, outputType, options: _options);
}
else
{
return JsonSerializer.Deserialize($"\"{value}\"", ef.type, options: _options);
return JsonSerializer.Deserialize($"\"{value}\"", outputType, options: _options);
}
}
else
{
var value = entity.GetString(fieldName);
return JsonSerializer.Deserialize(value, ef.type, options: _options);
return JsonSerializer.Deserialize(value, outputType, options: _options);
}
}
}
).ToArray();
public T ToRecord<T>(TableEntity entity) where T : EntityBase
{
var entityInfo = GetEntityInfo<T>();
var parameters =
entityInfo.properties.Keys.Select(k => GetFieldValue(entityInfo, k, entity)).ToArray();
var entityRecord = (T)entityInfo.constructor.Invoke(parameters);

12
src/ApiService/Tests/OrmModelsTest.cs
View File

@ -503,13 +503,12 @@ namespace Tests
return Test(ss);
}
/* @Cheick
[Property]
public bool WebhookMessageLog(WebhookMessageLog log)
{
return Test(log);
}
*/
[Property]
@ -604,13 +603,11 @@ namespace Tests
return Test(ss);
}
/* @Cheick
[Property]
public bool WebhookMessageLog(WebhookMessageLog log)
{
return Test(log);
}
*/
[Property]
public bool Webhook(Webhook wh)
@ -618,21 +615,18 @@ namespace Tests
return Test(wh);
}
/* @Cheick
[Property]
public bool WebhookMessageEventGrid(WebhookMessageEventGrid evt)
{
return Teste(evt);
return Test(evt);
}
*/
/* @Cheick
[Property]
public bool WebhookMessage(WebhookMessage msg)
{
return Test(msg);
}
*/
[Property]

15
src/ApiService/Tests/OrmTest.cs
View File

@ -261,5 +261,20 @@ namespace Tests
Assert.Equal(expected.Id, actual.Id);
Assert.Equal(expected.TheName, actual.TheName);
}
[Fact]
public void TestEventSerialization2()
{
var converter = new EntityConverter();
var expectedEvent = new EventMessage(Guid.NewGuid(), EventType.NodeHeartbeat, new EventNodeHeartbeat(Guid.NewGuid(), Guid.NewGuid(), "test Poool"), Guid.NewGuid(), "test")
{
ETag = new Azure.ETag("33a64df551425fcc55e4d42a148795d9f25f89d4")
};
var te = converter.ToTableEntity(expectedEvent);
var actualEvent = converter.ToRecord<EventMessage>(te);
Assert.Equal(expectedEvent, actualEvent);
}
}
}