From ae6df1e22f78097131f554c8b6fb4580b72b9dc5 Mon Sep 17 00:00:00 2001 From: George Pollard Date: Mon, 29 Aug 2022 15:12:19 +1200 Subject: [PATCH] Create tables on startup (#2309) --- src/ApiService/ApiService/Program.cs | 32 +++++++- .../ApiService/onefuzzlib/orm/Orm.cs | 5 +- .../IntegrationTests/_FunctionTestBase.cs | 78 +++++++++++-------- 3 files changed, 76 insertions(+), 39 deletions(-) diff --git a/src/ApiService/ApiService/Program.cs b/src/ApiService/ApiService/Program.cs index 7975911a0..e7c8fa7a2 100644 --- a/src/ApiService/ApiService/Program.cs +++ b/src/ApiService/ApiService/Program.cs @@ -7,6 +7,7 @@ using System.Linq; global using Async = System.Threading.Tasks; using System.Text.Json; +using ApiService.OneFuzzLib.Orm; using Azure.Core.Serialization; using Microsoft.Azure.Functions.Worker; using Microsoft.Azure.Functions.Worker.Middleware; @@ -109,12 +110,39 @@ public class Program { .AddSingleton() .AddHttpClient() .AddMemoryCache(); - } - ) + }) .Build(); + await SetupStorage( + host.Services.GetRequiredService(), + host.Services.GetRequiredService()); + await host.RunAsync(); } + public static async Async.Task SetupStorage(IStorage storage, IServiceConfig serviceConfig) { + // Creates the tables for each implementor of IOrm + // locate all IOrm instances and collect the Ts + var toCreate = new List(); + var types = typeof(Program).Assembly.GetTypes(); + foreach (var type in types) { + if (type.IsAbstract) { + continue; + } + + foreach (var iface in type.GetInterfaces()) { + if (iface.IsGenericType && iface.GetGenericTypeDefinition() == typeof(IOrm<>)) { + toCreate.Add(iface.GenericTypeArguments.Single()); + break; + } + } + } + + var storageAccount = serviceConfig.OneFuzzFuncStorage; + if (storageAccount is not null) { + var tableClient = await storage.GetTableServiceClientForAccount(storageAccount); + await Async.Task.WhenAll(toCreate.Select(t => tableClient.CreateTableIfNotExistsAsync(serviceConfig.OneFuzzStoragePrefix + t.Name))); + } + } } diff --git a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs index 42adc6608..9acdab898 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs @@ -27,7 +27,7 @@ namespace ApiService.OneFuzzLib.Orm { } - public class Orm : IOrm where T : EntityBase { + public abstract class Orm : IOrm where T : EntityBase { #pragma warning disable CA1051 // permit visible instance fields protected readonly EntityConverter _entityConverter; protected readonly IOnefuzzContext _context; @@ -109,7 +109,6 @@ namespace ApiService.OneFuzzLib.Orm { var account = accountId ?? _context.ServiceConfiguration.OneFuzzFuncStorage ?? throw new ArgumentNullException(nameof(accountId)); var tableClient = await _context.Storage.GetTableServiceClientForAccount(account); - await tableClient.CreateTableIfNotExistsAsync(tableName); return tableClient.GetTableClient(tableName); } @@ -146,7 +145,7 @@ namespace ApiService.OneFuzzLib.Orm { } - public class StatefulOrm : Orm, IStatefulOrm where T : StatefulEntityBase where TState : Enum { + public abstract class StatefulOrm : Orm, IStatefulOrm where T : StatefulEntityBase where TState : Enum { static Lazy>? _partitionKeyGetter; static Lazy>? _rowKeyGetter; static ConcurrentDictionary>?> _stateFuncs = new ConcurrentDictionary>?>(); diff --git a/src/ApiService/IntegrationTests/_FunctionTestBase.cs b/src/ApiService/IntegrationTests/_FunctionTestBase.cs index c43f5895a..2586ad4ef 100644 --- a/src/ApiService/IntegrationTests/_FunctionTestBase.cs +++ b/src/ApiService/IntegrationTests/_FunctionTestBase.cs @@ -1,6 +1,7 @@ using System; using System.IO; using System.Linq; +using System.Threading.Tasks; using ApiService.OneFuzzLib.Orm; using Azure.Data.Tables; using Azure.Storage.Blobs; @@ -8,8 +9,11 @@ using IntegrationTests.Fakes; using Microsoft.Azure.Functions.Worker.Http; using Microsoft.OneFuzz.Service; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +using Xunit; using Xunit.Abstractions; +using Task = System.Threading.Tasks.Task; + namespace IntegrationTests; // FunctionTestBase contains shared implementations for running @@ -22,7 +26,7 @@ namespace IntegrationTests; // - one for Azure Storage (marked with [Trait("Category", "Live")]) // // See AgentEventsTests for an example. -public abstract class FunctionTestBase : IDisposable { +public abstract class FunctionTestBase : IAsyncLifetime { private readonly IStorage _storage; // each test will use a different prefix for storage (tables, blobs) so they don't interfere @@ -52,6 +56,18 @@ public abstract class FunctionTestBase : IDisposable { _blobClient = _storage.GetBlobServiceClientForAccount("").Result; // for test implementations this is always sync } + public async Task InitializeAsync() { + await Program.SetupStorage(Context.Storage, Context.ServiceConfiguration); + } + + public async Task DisposeAsync() { + // clean up any tables & blobs that this test created + // these Get methods are always sync for test impls + await ( + CleanupTables(_storage.GetTableServiceClientForAccount("").Result), + CleanupBlobs(_storage.GetBlobServiceClientForAccount("").Result)); + } + protected static string BodyAsString(HttpResponseData data) { data.Body.Seek(0, SeekOrigin.Begin); using var sr = new StreamReader(data.Body); @@ -61,38 +77,32 @@ public abstract class FunctionTestBase : IDisposable { protected static T BodyAs(HttpResponseData data) => EntityConverter.FromJsonString(BodyAsString(data)) ?? throw new Exception($"unable to deserialize body as {typeof(T)}"); - public void Dispose() { - GC.SuppressFinalize(this); + private async Task CleanupBlobs(BlobServiceClient blobClient) + => await Task.WhenAll( + await blobClient + .GetBlobContainersAsync(prefix: _storagePrefix) + .Where(c => c.IsDeleted != true) + .Select(async container => { + try { + await blobClient.DeleteBlobContainerAsync(container.Name); + Logger.Info($"cleaned up container {container.Name}"); + } catch (Exception ex) { + // swallow any exceptions: this is a best-effort attempt to cleanup + Logger.Exception(ex, "error deleting container at end of test"); + } + }).ToListAsync()); - // clean up any tables & blobs that this test created - // these Get methods are always sync for test impls - CleanupTables(_storage.GetTableServiceClientForAccount("").Result); - CleanupBlobs(_storage.GetBlobServiceClientForAccount("").Result); - } - - private void CleanupBlobs(BlobServiceClient blobClient) { - var containersToDelete = blobClient.GetBlobContainers(prefix: _storagePrefix); - foreach (var container in containersToDelete.Where(c => c.IsDeleted != true)) { - try { - blobClient.DeleteBlobContainer(container.Name); - Logger.Info($"cleaned up container {container.Name}"); - } catch (Exception ex) { - // swallow any exceptions: this is a best-effort attempt to cleanup - Logger.Exception(ex, "error deleting container at end of test"); - } - } - } - - private void CleanupTables(TableServiceClient tableClient) { - var tablesToDelete = tableClient.Query(filter: Query.StartsWith("TableName", _storagePrefix)); - foreach (var table in tablesToDelete) { - try { - tableClient.DeleteTable(table.Name); - Logger.Info($"cleaned up table {table.Name}"); - } catch (Exception ex) { - // swallow any exceptions: this is a best-effort attempt to cleanup - Logger.Exception(ex, "error deleting table at end of test"); - } - } - } + private async Task CleanupTables(TableServiceClient tableClient) + => await Task.WhenAll( + await tableClient + .QueryAsync(filter: Query.StartsWith("TableName", _storagePrefix)) + .Select(async table => { + try { + await tableClient.DeleteTableAsync(table.Name); + Logger.Info($"cleaned up table {table.Name}"); + } catch (Exception ex) { + // swallow any exceptions: this is a best-effort attempt to cleanup + Logger.Exception(ex, "error deleting table at end of test"); + } + }).ToListAsync()); }