Port autoscaling to C# (#2296)

Co-authored-by: stas <statis@microsoft.com>

src/ApiService/ApiService.sln

@@ -7,7 +7,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApiService", "ApiService\Ap
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Tests", "Tests\Tests.csproj", "{06C9FE9B-6DDD-45EC-B716-CB074512DAA9}"
 EndProject
-Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "IntegrationTests", "IntegrationTests\IntegrationTests.csproj", "{AAB88572-BA8E-4661-913E-61FC9827005D}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "IntegrationTests", "IntegrationTests\IntegrationTests.csproj", "{AAB88572-BA8E-4661-913E-61FC9827005D}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FunctionalTests", "FunctionalTests\FunctionalTests.csproj", "{915DAFA4-595D-4A89-BDB9-C0DB96676148}"
 EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
@@ -27,6 +29,10 @@ Global
 		{AAB88572-BA8E-4661-913E-61FC9827005D}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{AAB88572-BA8E-4661-913E-61FC9827005D}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{AAB88572-BA8E-4661-913E-61FC9827005D}.Release|Any CPU.Build.0 = Release|Any CPU
+		{915DAFA4-595D-4A89-BDB9-C0DB96676148}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{915DAFA4-595D-4A89-BDB9-C0DB96676148}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{915DAFA4-595D-4A89-BDB9-C0DB96676148}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{915DAFA4-595D-4A89-BDB9-C0DB96676148}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE

src/ApiService/ApiService/Functions/Pool.cs

@@ -66,18 +66,7 @@ public class Pool {
                     Errors: new string[] { "pool with that name already exists" }),
                 "PoolCreate");
         }
-
+        var newPool = await _context.PoolOperations.Create(name: create.Name, os: create.Os, architecture: create.Arch, managed: create.Managed, clientId: create.ClientId);
-        // logging.Info(request)
-
-        var newPool = new Service.Pool(
-            PoolId: Guid.NewGuid(),
-            State: PoolState.Init,
-            Name: create.Name,
-            Os: create.Os,
-            Managed: create.Managed,
-            Arch: create.Arch);
-
-        await _context.PoolOperations.Insert(newPool);
         return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(newPool), true));
     }
 

src/ApiService/ApiService/Functions/Scaleset.cs

@@ -102,7 +102,7 @@ public class Scaleset {
                 context: "ScalesetCreate");
         }
 
-        var tags = create.Tags;
+        var tags = create.Tags ?? new Dictionary<string, string>();
         var configTags = (await _context.ConfigOperations.Fetch()).VmssTags;
         if (configTags is not null) {
             foreach (var (key, value) in configTags) {

src/ApiService/ApiService/HttpClient.cs

@@ -37,11 +37,23 @@ public class Request {
         return await _httpClient.SendAsync(request, HttpCompletionOption.ResponseHeadersRead);
     }
 
-    public async Task<HttpResponseMessage> Get(Uri url) {
+    public async Task<HttpResponseMessage> Get(Uri url, string? json = null) {
-        return await Send(method: HttpMethod.Get, url: url);
+        if (json is not null) {
+            using var b = new StringContent(json);
+            b.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            return await Send(method: HttpMethod.Get, url: url, content: b);
+        } else {
+            return await Send(method: HttpMethod.Get, url: url);
+        }
     }
-    public async Task<HttpResponseMessage> Delete(Uri url) {
+    public async Task<HttpResponseMessage> Delete(Uri url, string? json = null) {
-        return await Send(method: HttpMethod.Delete, url: url);
+        if (json is not null) {
+            using var b = new StringContent(json);
+            b.Headers.ContentType = MediaTypeHeaderValue.Parse("application/json");
+            return await Send(method: HttpMethod.Delete, url: url, content: b);
+        } else {
+            return await Send(method: HttpMethod.Delete, url: url);
+        }
     }
 
     public async Task<HttpResponseMessage> Post(Uri url, String json, IDictionary<string, string>? headers = null) {

src/ApiService/ApiService/onefuzzlib/AutoScale.cs

@@ -68,8 +68,13 @@ public class AutoScaleOperations : Orm<AutoScale>, IAutoScaleOperations {
     }
 
     public async Async.Task<AutoScale?> GetSettingsForScaleset(Guid scalesetId) {
-        var autoscale = await GetEntityAsync(scalesetId.ToString(), scalesetId.ToString());
+        try {
-        return autoscale;
+            var autoscale = await GetEntityAsync(scalesetId.ToString(), scalesetId.ToString());
+            return autoscale;
+        } catch (Exception ex) {
+            _logTracer.Exception(ex, "Failed to get auto-scale entity");
+            return null;
+        }
     }
 
 
@@ -83,15 +88,17 @@ public class AutoScaleOperations : Orm<AutoScale>, IAutoScaleOperations {
         }
 
         if (existingAutoScaleResource.OkV != null) {
-            _logTracer.Warning($"Scaleset {vmss} already has auto scale resource");
+            return OneFuzzResultVoid.Ok;
         }
 
         var autoScaleResource = await CreateAutoScaleResourceFor(vmss, await _context.Creds.GetBaseRegion(), autoScaleProfile);
         if (!autoScaleResource.IsOk) {
             return OneFuzzResultVoid.Error(autoScaleResource.ErrorV);
         }
+        var workspaceId = _context.LogAnalytics.GetWorkspaceId().ToString();
+        _logTracer.Info($"Setting up diagnostics for id: {autoScaleResource.OkV.Id!} with name: {autoScaleResource.OkV.Data.Name} and workspace id: {workspaceId}");
 
-        var diagnosticsResource = await SetupAutoScaleDiagnostics(autoScaleResource.OkV.Id!, autoScaleResource.OkV.Data.Name, _context.LogAnalytics.GetWorkspaceId().ToString());
+        var diagnosticsResource = await SetupAutoScaleDiagnostics(autoScaleResource.OkV, workspaceId);
         if (!diagnosticsResource.IsOk) {
             return OneFuzzResultVoid.Error(diagnosticsResource.ErrorV);
         }
@@ -111,11 +118,11 @@ public class AutoScaleOperations : Orm<AutoScale>, IAutoScaleOperations {
         };
 
         try {
-            var autoScaleResource = await _context.Creds.GetResourceGroupResource().GetAutoscaleSettings()
+            var autoScaleSettings = _context.Creds.GetResourceGroupResource().GetAutoscaleSettings();
-                .CreateOrUpdateAsync(WaitUntil.Completed, Guid.NewGuid().ToString(), parameters);
+            var autoScaleResource = await autoScaleSettings.CreateOrUpdateAsync(WaitUntil.Started, Guid.NewGuid().ToString(), parameters);
 
             if (autoScaleResource != null && autoScaleResource.HasValue) {
-                _logTracer.Info($"Successfully created auto scale resource {autoScaleResource.Id} for {resourceId}");
+                _logTracer.Info($"Successfully created auto scale resource {autoScaleResource.Value.Id} for {resourceId}");
                 return OneFuzzResult<AutoscaleSettingResource>.Ok(autoScaleResource.Value);
             }
 
@@ -123,6 +130,15 @@ public class AutoScaleOperations : Orm<AutoScale>, IAutoScaleOperations {
                 ErrorCode.UNABLE_TO_CREATE,
                 $"Could not get auto scale resource value after creating for {resourceId}"
             );
+        } catch (RequestFailedException ex) when (ex.Status == 409 && ex.Message.Contains("\"code\":\"SettingAlreadyExists\"")) {
+            var existingAutoScaleResource = GetAutoscaleSettings(resourceId);
+            if (existingAutoScaleResource.IsOk) {
+                _logTracer.Info($"Successfully created auto scale resource {existingAutoScaleResource.OkV!.Data.Id} for {resourceId}");
+                return OneFuzzResult<AutoscaleSettingResource>.Ok(existingAutoScaleResource.OkV!);
+            } else {
+                return existingAutoScaleResource.ErrorV;
+            }
+
         } catch (Exception ex) {
             _logTracer.Exception(ex);
             return OneFuzzResult<AutoscaleSettingResource>.Error(
@@ -203,27 +219,34 @@ public class AutoScaleOperations : Orm<AutoScale>, IAutoScaleOperations {
     }
 
 
-    private async Async.Task<OneFuzzResult<DiagnosticSettingsResource>> SetupAutoScaleDiagnostics(string autoScaleResourceUri, string autoScaleResourceName, string logAnalyticsWorkspaceId) {
+    private async Async.Task<OneFuzzResult<DiagnosticSettingsResource>> SetupAutoScaleDiagnostics(AutoscaleSettingResource autoscaleSettingResource, string logAnalyticsWorkspaceId) {
-        var logSettings = new LogSettings(true) { Category = "allLogs", RetentionPolicy = new RetentionPolicy(true, 30) };
-
         try {
+            // TODO: we are missing CategoryGroup = "allLogs", we cannot set it since current released dotnet SDK is missing the field
+            // The field is there in github though, so need to update this code once that code is released:
+            // https://github.com/Azure/azure-sdk-for-net/blob/main/sdk/monitor/Azure.ResourceManager.Monitor/src/Generated/Models/LogSettings.cs
+            // But setting logs one by one works the same as "allLogs" being set...
+            var logSettings1 = new LogSettings(true) { RetentionPolicy = new RetentionPolicy(true, 30), Category = "AutoscaleEvaluations" };
+            var logSettings2 = new LogSettings(true) { RetentionPolicy = new RetentionPolicy(true, 30), Category = "AutoscaleScaleActions" };
+
             var parameters = new DiagnosticSettingsData {
                 WorkspaceId = logAnalyticsWorkspaceId
             };
-            parameters.Logs.Add(logSettings);
+            parameters.Logs.Add(logSettings1);
-            var diagnostics = await _context.Creds.GetResourceGroupResource().GetDiagnosticSettings().CreateOrUpdateAsync(WaitUntil.Completed, $"{autoScaleResourceName}-diagnostics", parameters);
+            parameters.Logs.Add(logSettings2);
+
+            var diagnostics = await autoscaleSettingResource.GetDiagnosticSettings().CreateOrUpdateAsync(WaitUntil.Started, $"{autoscaleSettingResource.Data.Name}-diagnostics", parameters);
             if (diagnostics != null && diagnostics.HasValue) {
                 return OneFuzzResult.Ok(diagnostics.Value);
             }
             return OneFuzzResult<DiagnosticSettingsResource>.Error(
                 ErrorCode.UNABLE_TO_CREATE,
-                $"The resulting diagnostics settings resource was null when attempting to create for {autoScaleResourceUri}"
+                $"The resulting diagnostics settings resource was null when attempting to create for {autoscaleSettingResource.Id}"
             );
         } catch (Exception ex) {
             _logTracer.Exception(ex);
             return OneFuzzResult<DiagnosticSettingsResource>.Error(
                     ErrorCode.UNABLE_TO_CREATE,
-                    $"unable to setup diagnostics for auto-scale resource: {autoScaleResourceUri}"
+                    $"unable to setup diagnostics for auto-scale resource: {autoscaleSettingResource.Id} and name: {autoscaleSettingResource.Data.Name}"
                 );
         }
     }

src/ApiService/ApiService/onefuzzlib/NodeOperations.cs

@@ -496,7 +496,7 @@ public class NodeOperations : StatefulOrm<Node, NodeState, NodeOperations>, INod
     }
 
     public IAsyncEnumerable<Node> SearchByPoolName(PoolName poolName) {
-        return QueryAsync(TableClient.CreateQueryFilter($"(pool_name eq {poolName})"));
+        return QueryAsync($"(pool_name eq '{poolName}')");
     }
 
 

src/ApiService/ApiService/onefuzzlib/NsgOperations.cs

@@ -140,7 +140,7 @@ namespace Microsoft.OneFuzz.Service {
             _logTracer.Info($"deleting nsg: {name}");
             try {
                 var nsg = await _context.Creds.GetResourceGroupResource().GetNetworkSecurityGroupAsync(name);
-                await nsg.Value.DeleteAsync(WaitUntil.Completed);
+                await nsg.Value.DeleteAsync(WaitUntil.Started);
                 return true;
             } catch (RequestFailedException ex) {
                 if (ex.ErrorCode == "ResourceNotFound") {

src/ApiService/ApiService/onefuzzlib/PoolOperations.cs

@@ -1,7 +1,6 @@
 using System.Threading.Tasks;
 using ApiService.OneFuzzLib.Orm;
 using Azure.Data.Tables;
-
 namespace Microsoft.OneFuzz.Service;
 
 public interface IPoolOperations : IStatefulOrm<Pool, PoolState> {
@@ -16,6 +15,9 @@ public interface IPoolOperations : IStatefulOrm<Pool, PoolState> {
     Async.Task<Pool> SetShutdown(Pool pool, bool Now);
 
     Async.Task<Pool> Init(Pool pool);
+
+    Async.Task<Pool> Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? clientId = null);
+    new Async.Task Delete(Pool pool);
 }
 
 public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoolOperations {
@@ -25,6 +27,23 @@ public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoo
 
     }
 
+    public async Async.Task<Pool> Create(PoolName name, Os os, Architecture architecture, bool managed, Guid? clientId = null) {
+        var newPool = new Service.Pool(
+        PoolId: Guid.NewGuid(),
+        State: PoolState.Init,
+        Name: name,
+        Os: os,
+        Managed: managed,
+        Arch: architecture,
+        ClientId: clientId);
+
+        var r = await Insert(newPool);
+        if (!r.IsOk) {
+            _logTracer.Error($"Failed to save new pool. Pool name: {newPool.Name}, PoolId: {newPool.PoolId} due to {r.ErrorV}");
+        }
+        await _context.Events.SendEvent(new EventPoolCreated(PoolName: newPool.Name, Os: newPool.Os, Arch: newPool.Arch, Managed: newPool.Managed));
+        return newPool;
+    }
     public async Async.Task<OneFuzzResult<Pool>> GetByName(PoolName poolName) {
         var pools = QueryAsync(Query.PartitionKey(poolName.String));
 
@@ -124,7 +143,7 @@ public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoo
         }
 
         pool = pool with { State = state };
-        await Update(pool);
+        await Replace(pool);
         return pool;
     }
 
@@ -136,6 +155,14 @@ public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoo
         return pool;
     }
 
+    new public async Async.Task Delete(Pool pool) {
+        var r = await base.Delete(pool);
+        if (!r.IsOk) {
+            _logTracer.Error($"Failed to delete pool: {pool.Name} due to {r.ErrorV}");
+        }
+        await _context.Events.SendEvent(new EventPoolDeleted(PoolName: pool.Name));
+    }
+
     public async Async.Task<Pool> Shutdown(Pool pool) {
         var scalesets = _context.ScalesetOperations.SearchByPool(pool.Name);
         var nodes = _context.NodeOperations.SearchByPoolName(pool.Name);
@@ -180,10 +207,7 @@ public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoo
             var shrinkQueue = new ShrinkQueue(pool.PoolId, _context.Queue, _logTracer);
             await shrinkQueue.Delete();
             _logTracer.Info($"pool stopped, deleting: {pool.Name}");
-            var r = await Delete(pool);
+            await Delete(pool);
-            if (!r.IsOk) {
-                _logTracer.Error($"Failed to delete pool: {pool.Name} due to {r.ErrorV}");
-            }
         }
 
         if (scalesets is not null) {

src/ApiService/ApiService/onefuzzlib/Queue.cs

@@ -92,7 +92,7 @@ public class Queue : IQueue {
     public async Async.Task DeleteQueue(string name, StorageType storageType) {
         var client = await GetQueueClient(name, storageType);
         var resp = await client.DeleteIfExistsAsync();
-        if (resp.GetRawResponse().IsError) {
+        if (resp.GetRawResponse() is not null && resp.GetRawResponse().IsError) {
             _log.Error($"failed to delete queue {name} due to {resp.GetRawResponse().ReasonPhrase}");
         }
     }
@@ -100,7 +100,7 @@ public class Queue : IQueue {
     public async Async.Task ClearQueue(string name, StorageType storageType) {
         var client = await GetQueueClient(name, storageType);
         var resp = await client.ClearMessagesAsync();
-        if (resp.IsError) {
+        if (resp is not null && resp.IsError) {
             _log.Error($"failed to clear the queue {name} due to {resp.ReasonPhrase}");
         }
     }

src/ApiService/ApiService/onefuzzlib/Request.cs

@@ -45,6 +45,11 @@ public class RequestHandling : IRequestHandling {
                         Code: ErrorCode.INVALID_REQUEST,
                         Errors: validationResults.Select(vr => vr.ToString()).ToArray());
                 }
+            } else {
+                return OneFuzzResult<T>.Error(
+                    ErrorCode.INVALID_REQUEST,
+                    $"Failed to deserialize message into type: {typeof(T)} - null"
+                );
             }
         } catch (Exception e) {
             exception = e;

src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs

@@ -124,7 +124,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
         }
 
         var updatedScaleSet = scaleset with { State = state };
-        var r = await Update(updatedScaleSet);
+        var r = await Replace(updatedScaleSet);
         if (!r.IsOk) {
             var msg = "Failed to update scaleset {scaleSet.ScalesetId} when updating state from {scaleSet.State} to {state}";
             _log.Error(msg);
@@ -279,7 +279,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
             }
         }
 
-        var rr = await Update(scaleset);
+        var rr = await Replace(scaleset);
         if (!rr.IsOk) {
             _logTracer.Error($"Failed to save scale data for scale set: {scaleset.ScalesetId}");
         }

src/ApiService/ApiService/onefuzzlib/VmssOperations.cs

@@ -127,7 +127,10 @@ public class VmssOperations : IVmssOperations {
         if (canUpdate.IsOk) {
             _log.Info($"updating VM extensions: {name}");
             var res = GetVmssResource(name);
-            var patch = new VirtualMachineScaleSetPatch();
+            var patch = new VirtualMachineScaleSetPatch() {
+                VirtualMachineProfile =
+                    new VirtualMachineScaleSetUpdateVmProfile() { ExtensionProfile = new VirtualMachineScaleSetExtensionProfile() }
+            };
 
             foreach (var ext in extensions) {
                 patch.VirtualMachineProfile.ExtensionProfile.Extensions.Add(ext);

src/ApiService/FunctionalTests/Auth.cs

@@ -0,0 +1,63 @@
+using Microsoft.Identity.Client;
+
+namespace Microsoft.Morse;
+
+public record AuthenticationConfig(string ClientId, string TenantId, string Secret, string[] Scopes);
+
+interface IServiceAuth {
+    Task<AuthenticationResult> Auth(CancellationToken cancelationToken);
+}
+
+public class ServiceAuth : IServiceAuth, IDisposable {
+    private SemaphoreSlim _lockObj = new SemaphoreSlim(1);
+    private AuthenticationResult? _token;
+    private IConfidentialClientApplication _app;
+    private AuthenticationConfig _authConfig;
+
+    public ServiceAuth(AuthenticationConfig authConfig) {
+        _authConfig = authConfig;
+
+        _app = ConfidentialClientApplicationBuilder
+                .Create(authConfig.ClientId)
+                .WithClientSecret(authConfig.Secret)
+                .WithTenantId(authConfig.TenantId)
+                .WithLegacyCacheCompatibility(false)
+                .Build();
+    }
+
+    public async Task<AuthenticationResult> Auth(CancellationToken cancelationToken) {
+        await _lockObj.WaitAsync(cancelationToken);
+        if (cancelationToken.IsCancellationRequested)
+            throw new System.Exception("Canellation requested, aborting Auth");
+
+        try {
+            if (_token is null) {
+                throw new MsalUiRequiredException(MsalError.ActivityRequired, "Authenticating for the first time");
+            } else {
+                var now = System.DateTimeOffset.UtcNow;
+                if (_token.ExpiresOn < now) {
+                    //_log.LogInformation("Cached token expired on : {token}. DateTime Offset Now: {now}", _token.ExpiresOn, now);
+                    throw new MsalUiRequiredException(MsalError.ActivityRequired, "Cached token expired");
+                } else {
+                    return _token;
+                }
+            }
+        } catch (MsalUiRequiredException) {
+            //_log.LogInformation("Getting new token due to {msg}", ex.Message);
+            _token = await _app.AcquireTokenForClient(_authConfig.Scopes).ExecuteAsync(cancelationToken);
+            return _token;
+        } finally {
+            _lockObj.Release();
+        }
+    }
+
+    public void Dispose() {
+        ((IDisposable)_lockObj).Dispose();
+        GC.SuppressFinalize(this);
+    }
+
+    public async Task<(string, string)> Token(CancellationToken cancellationToken) {
+        var t = await Auth(cancellationToken);
+        return (t.TokenType, t.AccessToken);
+    }
+}

src/ApiService/FunctionalTests/FunctionalTests.csproj

@@ -0,0 +1,29 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Microsoft.Identity.Client" Version="4.46.1" />
+    <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.1.0" />
+    <PackageReference Include="xunit" Version="2.4.1" />
+    <PackageReference Include="xunit.runner.visualstudio" Version="2.4.3">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+    <PackageReference Include="coverlet.collector" Version="3.1.2">
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+      <PrivateAssets>all</PrivateAssets>
+    </PackageReference>
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\ApiService\ApiService.csproj" />
+  </ItemGroup>
+
+</Project>

src/ApiService/FunctionalTests/Scalesets.cs

@@ -0,0 +1,233 @@
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using Microsoft.OneFuzz.Service;
+using Xunit.Abstractions;
+
+namespace FunctionalTests {
+
+    [Trait("Category", "Live")]
+
+    public class Scalesets {
+        static Uri endpoint = new Uri("http://localhost:7071");
+
+        static Microsoft.Morse.AuthenticationConfig authConfig =
+                new Microsoft.Morse.AuthenticationConfig(
+                    ClientId: System.Environment.GetEnvironmentVariable("ONEFUZZ_CLIENT_ID")!,
+                    TenantId: System.Environment.GetEnvironmentVariable("ONEFUZZ_TENANT_ID")!,
+                    Scopes: new[] { System.Environment.GetEnvironmentVariable("ONEFUZZ_SCOPES")! },
+                    Secret: System.Environment.GetEnvironmentVariable("ONEFUZZ_SECRET")!);
+
+        static Microsoft.Morse.ServiceAuth auth = new Microsoft.Morse.ServiceAuth(authConfig);
+        static Microsoft.OneFuzz.Service.Request request = new Microsoft.OneFuzz.Service.Request(new HttpClient(), () => auth.Token(new CancellationToken()));
+
+        JsonSerializerOptions _opts = Microsoft.OneFuzz.Service.OneFuzzLib.Orm.EntityConverter.GetJsonSerializerOptions();
+
+        Uri poolEndpoint = new Uri(endpoint, "/api/Pool");
+        Uri scalesetEndpoint = new Uri(endpoint, "/api/Scaleset");
+
+
+        string serialize<T>(T x) {
+            return JsonSerializer.Serialize(x, _opts);
+        }
+
+        T? deserialize<T>(string json) {
+            return JsonSerializer.Deserialize<T>(json, _opts);
+        }
+
+        private readonly ITestOutputHelper output;
+        public Scalesets(ITestOutputHelper output) {
+            this.output = output;
+        }
+
+        public async Task<HttpResponseMessage> DeletePool(PoolName name) {
+            var root = new JsonObject();
+            root.Add("name", JsonValue.Create(name));
+            root.Add("now", JsonValue.Create(true));
+            var body = root.ToJsonString();
+            var r = await request.Delete(poolEndpoint, body); ;
+            return r;
+        }
+
+        async Task<Pool> GetPool(string poolName) {
+            var root = new JsonObject();
+            root.Add("pool_id", null);
+            root.Add("name", poolName);
+            root.Add("state", null);
+
+            var body = root.ToJsonString();
+            var r2 = await request.Get(poolEndpoint, body);
+            var resPoolSearch = await r2.Content.ReadAsStringAsync();
+            var doc = await System.Text.Json.JsonDocument.ParseAsync(r2.Content.ReadAsStream());
+            return deserialize<Pool>(resPoolSearch)!;
+        }
+
+
+        async Task<Pool[]> GetAllPools() {
+            var root = new JsonObject();
+            root.Add("pool_id", null);
+            root.Add("name", null);
+            root.Add("state", null);
+
+            var body = root.ToJsonString();
+            var r2 = await request.Get(poolEndpoint, body);
+            var resPoolSearch = await r2.Content.ReadAsStringAsync();
+            var doc = await System.Text.Json.JsonDocument.ParseAsync(r2.Content.ReadAsStream());
+            return deserialize<Pool[]>(resPoolSearch)!;
+        }
+
+        async Task<Scaleset[]> GetAllScalesets() {
+            var root = new JsonObject();
+            root.Add("scaleset_id", null);
+            root.Add("state", null);
+            root.Add("include_auth", false);
+
+            var scalesetSearchBody = root.ToJsonString();
+            var r = await request.Get(scalesetEndpoint, scalesetSearchBody);
+            var scalesets = deserialize<Scaleset[]>(await r.Content.ReadAsStringAsync());
+            return scalesets!;
+        }
+
+        async Task<Scaleset?> GetScaleset(Guid id) {
+            var root = new JsonObject();
+            root.Add("scaleset_id", id.ToString());
+            root.Add("state", null);
+            root.Add("include_auth", false);
+
+            var scalesetSearchBody = root.ToJsonString();
+            var r = await request.Get(scalesetEndpoint, scalesetSearchBody);
+            var s = await r.Content.ReadAsStringAsync();
+            var scaleset = deserialize<Scaleset>(s);
+            return scaleset;
+        }
+
+        async System.Threading.Tasks.Task DeleteAllTestPools() {
+            var pools = await GetAllPools();
+            foreach (var p in pools) {
+                if (p.Name.String.StartsWith("FT-DELETE-")) {
+                    output.WriteLine($"Deleting {p.Name}");
+                    await DeletePool(p.Name);
+                }
+            }
+        }
+
+        [Fact]
+        public async System.Threading.Tasks.Task DeleteFunctionalTestPools() {
+            await DeleteAllTestPools();
+        }
+
+        [Fact]
+        public async System.Threading.Tasks.Task GetPoolsAndScalesets() {
+            var scalesets = await GetAllScalesets();
+            if (scalesets is null) {
+                output.WriteLine("Got null when getting scalesets");
+            } else if (scalesets.Length == 0) {
+                output.WriteLine("Got empty scalesets");
+            } else {
+                foreach (var sc in scalesets!) {
+                    output.WriteLine($"Pool: {sc.PoolName} Scaleset: {sc.ScalesetId}");
+                }
+            }
+
+            var pools = await GetAllPools();
+
+            if (pools is null) {
+                output.WriteLine("Got null when getting pools");
+            } else if (pools.Length == 0) {
+                output.WriteLine("Got empty pools");
+            } else {
+                foreach (var p in pools) {
+                    output.WriteLine($"Pool: {p.Name}, PoolId : {p.PoolId}, OS: {p.Os}");
+                }
+            }
+        }
+
+        [Fact]
+        public async System.Threading.Tasks.Task CreateAndDelete() {
+            var scalesets = await GetAllScalesets();
+            var pools = await GetAllPools();
+
+            var newPoolId = System.Guid.NewGuid().ToString();
+            var newPoolName = "FT-DELETE-" + newPoolId;
+
+            try {
+                Pool? newPool;
+                {
+                    var rootPoolCreate = new JsonObject();
+                    rootPoolCreate.Add("name", newPoolName);
+                    rootPoolCreate.Add("os", "linux");
+                    rootPoolCreate.Add("architecture", "x86_64");
+                    rootPoolCreate.Add("managed", true);
+
+                    var newPoolCreate = rootPoolCreate.ToJsonString();
+
+                    var r = await request.Post(poolEndpoint, newPoolCreate);
+                    var s = await r.Content.ReadAsStringAsync();
+                    newPool = deserialize<Pool>(s);
+                }
+
+                Scaleset? newScaleset;
+                {
+                    var rootScalesetCreate = new JsonObject();
+                    rootScalesetCreate.Add("pool_name", newPool!.Name.String);
+                    rootScalesetCreate.Add("vm_sku", "Standard_D2s_v3");
+                    rootScalesetCreate.Add("image", "Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest");
+                    rootScalesetCreate.Add("size", 2);
+                    rootScalesetCreate.Add("spot_instance", false);
+                    var tags = new JsonObject();
+                    tags.Add("Purpose", "Functional-Test");
+                    rootScalesetCreate.Add("tags", tags);
+
+                    var newScalesetCreate = rootScalesetCreate.ToJsonString();
+
+                    var r = await request.Post(scalesetEndpoint, newScalesetCreate);
+                    var s = await r.Content.ReadAsStringAsync();
+                    newScaleset = deserialize<Scaleset>(s);
+                }
+
+                output.WriteLine($"New scale set info id: {newScaleset!.ScalesetId}, pool: {newScaleset!.PoolName}, state: {newScaleset.State}, error: {newScaleset.Error}");
+
+                var scalesetsCreated = await GetAllScalesets();
+                var poolsCreated = await GetAllPools();
+
+                var newPools = poolsCreated.Where(p => p.Name.String == newPoolName);
+                var newScalesets = scalesetsCreated.Where(sc => sc.ScalesetId == newScaleset.ScalesetId);
+
+                Assert.True(newPools.Count() == 1);
+                Assert.True(newScalesets.Count() == 1);
+
+
+                var currentState = ScalesetState.Init;
+                System.Console.WriteLine($"Waiting for scaleset to move out from Init State");
+                while (newScaleset.State == ScalesetState.Init || newScaleset.State == ScalesetState.Setup) {
+                    System.Threading.Thread.Sleep(TimeSpan.FromSeconds(10.0));
+                    newScaleset = await GetScaleset(id: newScaleset.ScalesetId);
+                    if (currentState != newScaleset!.State) {
+                        output.WriteLine($"Scaleset is in {newScaleset.State}");
+                        currentState = newScaleset!.State;
+                    }
+                }
+                output.WriteLine($"Scaleset is in {newScaleset.State}");
+
+                if (currentState == ScalesetState.CreationFailed) {
+                    throw new Exception($"Scaleset creation failed due {newScaleset.Error}");
+                } else if (currentState != ScalesetState.Running) {
+                    throw new Exception($"Expected scaleset to be in Running state, instead got {currentState}");
+                }
+            } finally {
+                var preDelete = (await GetAllScalesets()).Where(sc => sc.PoolName.String == newPoolName);
+                Assert.True(preDelete.Count() == 1);
+
+                await DeletePool(new PoolName(newPoolName));
+
+                Pool deletedPool;
+                do {
+                    System.Threading.Thread.Sleep(TimeSpan.FromSeconds(10.0));
+                    deletedPool = await GetPool(newPoolName);
+                } while (deletedPool != null);
+                var postDelete = (await GetAllScalesets()).Where(sc => sc.PoolName.String == newPoolName);
+                Assert.True(postDelete.Any() == false);
+            }
+            return;
+        }
+    }
+}

src/ApiService/FunctionalTests/Usings.cs

@@ -0,0 +1 @@
+global using Xunit;