ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-15 11:28:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Convert pool function to C# (#2178)

Browse Source 
#2177
This commit is contained in:
George Pollard
2022-08-11 09:29:13 +12:00
committed by GitHub
parent 20faf88902
commit e3e001a172
23 changed files with 670 additions and 48 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
src/ApiService/ApiService/Functions/AgentRegistration.cs
View File

@ -65,13 +65,13 @@ public class AgentRegistration {
return await RequestHandling.Ok(req, await CreateRegistrationResponse(pool.OkV));
}
private async Async.Task<AgentRegistrationResponse> CreateRegistrationResponse(Pool pool) {
private async Async.Task<AgentRegistrationResponse> CreateRegistrationResponse(Service.Pool pool) {
var baseAddress = _context.Creds.GetInstanceUrl();
var eventsUrl = new Uri(baseAddress, "/api/agents/events");
var commandsUrl = new Uri(baseAddress, "/api/agents/commands");
var workQueue = await _context.Queue.GetQueueSas(
_context.PoolOperations.GetPoolQueue(pool),
_context.PoolOperations.GetPoolQueue(pool.PoolId),
StorageType.Corpus,
QueueSasPermissions.Read | QueueSasPermissions.Update | QueueSasPermissions.Process,
TimeSpan.FromHours(24));

2
src/ApiService/ApiService/Functions/Node.cs
View File

@ -29,7 +29,7 @@ public class Node {
private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
var request = await RequestHandling.ParseRequest<NodeSearch>(req);
if (!request.IsOk) {
return await _context.RequestHandling.NotOk(req, request.ErrorV, "pool get");
return await _context.RequestHandling.NotOk(req, request.ErrorV, "node get");
}
var search = request.OkV;

149
src/ApiService/ApiService/Functions/Pool.cs Normal file
View File

@ -0,0 +1,149 @@
using System.Threading.Tasks;
using Azure.Storage.Sas;
using Microsoft.Azure.Functions.Worker;
using Microsoft.Azure.Functions.Worker.Http;
namespace Microsoft.OneFuzz.Service.Functions;
public class Pool {
private readonly ILogTracer _log;
private readonly IEndpointAuthorization _auth;
private readonly IOnefuzzContext _context;
public Pool(ILogTracer log, IEndpointAuthorization auth, IOnefuzzContext context) {
_log = log;
_auth = auth;
_context = context;
}
[Function("Pool")]
public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", "POST", "DELETE")] HttpRequestData req)
=> _auth.CallIfUser(req, r => r.Method switch {
"GET" => Get(r),
"POST" => Post(r),
"DELETE" => Delete(r),
var m => throw new InvalidOperationException("Unsupported HTTP method {m}"),
});
private async Task<HttpResponseData> Delete(HttpRequestData r) {
var request = await RequestHandling.ParseRequest<PoolStop>(r);
if (!request.IsOk) {
return await _context.RequestHandling.NotOk(r, request.ErrorV, "PoolDelete");
}
var answer = await _auth.CheckRequireAdmins(r);
if (!answer.IsOk) {
return await _context.RequestHandling.NotOk(r, answer.ErrorV, "PoolDelete");
}
var poolResult = await _context.PoolOperations.GetByName(request.OkV.Name);
if (!poolResult.IsOk) {
return await _context.RequestHandling.NotOk(r, poolResult.ErrorV, "pool stop");
}
await _context.PoolOperations.SetShutdown(poolResult.OkV, Now: request.OkV.Now);
return await RequestHandling.Ok(r, true);
}
private async Task<HttpResponseData> Post(HttpRequestData req) {
var request = await RequestHandling.ParseRequest<PoolCreate>(req);
if (!request.IsOk) {
return await _context.RequestHandling.NotOk(req, request.ErrorV, "PoolCreate");
}
var answer = await _auth.CheckRequireAdmins(req);
if (!answer.IsOk) {
return await _context.RequestHandling.NotOk(req, answer.ErrorV, "PoolCreate");
}
var create = request.OkV;
var pool = await _context.PoolOperations.GetByName(create.Name);
if (pool.IsOk) {
return await _context.RequestHandling.NotOk(
req,
new Error(
Code: ErrorCode.INVALID_REQUEST,
Errors: new string[] { "pool with that name already exists" }),
"PoolCreate");
}
// logging.Info(request)
var newPool = new Service.Pool(
PoolId: Guid.NewGuid(),
State: PoolState.Init,
Name: create.Name,
Os: create.Os,
Managed: create.Managed,
Arch: create.Arch);
await _context.PoolOperations.Insert(newPool);
return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(newPool), true));
}
private async Task<HttpResponseData> Get(HttpRequestData req) {
var request = await RequestHandling.ParseRequest<PoolSearch>(req);
if (!request.IsOk) {
return await _context.RequestHandling.NotOk(req, request.ErrorV, "pool get");
}
var search = request.OkV;
if (search.Name is not null) {
var poolResult = await _context.PoolOperations.GetByName(search.Name);
if (!poolResult.IsOk) {
return await _context.RequestHandling.NotOk(req, poolResult.ErrorV, context: search.Name.ToString());
}
return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(poolResult.OkV)));
}
if (search.PoolId is Guid poolId) {
var poolResult = await _context.PoolOperations.GetById(poolId);
if (!poolResult.IsOk) {
return await _context.RequestHandling.NotOk(req, poolResult.ErrorV, context: poolId.ToString());
}
return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(poolResult.OkV)));
}
var pools = await _context.PoolOperations.SearchStates(search.State ?? Enumerable.Empty<PoolState>()).ToListAsync();
return await RequestHandling.Ok(req, pools.Select(PoolToPoolResponse));
}
private static PoolGetResult PoolToPoolResponse(Service.Pool p)
=> new(
Name: p.Name,
PoolId: p.PoolId,
Os: p.Os,
State: p.State,
ClientId: p.ClientId,
Managed: p.Managed,
Arch: p.Arch,
Nodes: p.Nodes,
Config: p.Config,
WorkQueue: null,
ScalesetSummary: null);
private async Task<PoolGetResult> Populate(PoolGetResult p, bool skipSummaries = false) {
var (queueSas, instanceId, workQueue, scalesetSummary) = await (
_context.Queue.GetQueueSas("node-heartbeat", StorageType.Config, QueueSasPermissions.Add),
_context.Containers.GetInstanceId(),
skipSummaries ? Async.Task.FromResult(new List<WorkSetSummary>()) : _context.PoolOperations.GetWorkQueue(p.PoolId, p.State),
skipSummaries ? Async.Task.FromResult(new List<ScalesetSummary>()) : _context.PoolOperations.GetScalesetSummary(p.Name));
return p with {
WorkQueue = workQueue,
ScalesetSummary = scalesetSummary,
Config =
new AgentConfig(
PoolName: p.Name,
OneFuzzUrl: _context.Creds.GetInstanceUrl(),
InstanceTelemetryKey: _context.ServiceConfiguration.ApplicationInsightsInstrumentationKey,
MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry,
HeartbeatQueue: queueSas,
InstanceId: instanceId,
ClientCredentials: null,
MultiTenantDomain: _context.ServiceConfiguration.MultiTenantDomain)
};
}
}

40
src/ApiService/ApiService/OneFuzzTypes/Model.cs
View File

@ -594,14 +594,26 @@ public record Pool(
bool Managed,
Architecture Arch,
PoolState State,
Guid? ClientId
Guid? ClientId = null
) : StatefulEntityBase<PoolState>(State) {
public List<Node>? Nodes { get; set; }
public AgentConfig? Config { get; set; }
public List<WorkSetSummary>? WorkQueue { get; set; }
public List<ScalesetSummary>? ScalesetSummary { get; set; }
}
public record WorkUnitSummary(
Guid JobId,
Guid TaskId,
TaskType TaskType
);
public record WorkSetSummary(
List<WorkUnitSummary> WorkUnits
);
public record ScalesetSummary(
Guid ScalesetId,
ScalesetState State
);
public record ClientCredentials
(
@ -623,8 +635,6 @@ public record AgentConfig(
public record WorkSetSummary();
public record ScalesetSummary();
public record Vm(
string Name,
@ -788,7 +798,25 @@ public record MultipleContainer(List<SyncedDir> SyncedDirs) : IContainerDef;
public class ContainerDefConverter : JsonConverter<IContainerDef> {
public override IContainerDef? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
throw new NotSupportedException("reading IContainerDef is not supported");
if (reader.TokenType == JsonTokenType.StartObject) {
var result = (SyncedDir?)JsonSerializer.Deserialize(ref reader, typeof(SyncedDir), options);
if (result is null) {
return null;
}
return new SingleContainer(result);
}
if (reader.TokenType == JsonTokenType.StartArray) {
var result = (List<SyncedDir>?)JsonSerializer.Deserialize(ref reader, typeof(List<SyncedDir>), options);
if (result is null) {
return null;
}
return new MultipleContainer(result);
}
throw new JsonException("expecting array or object");
}
public override void Write(Utf8JsonWriter writer, IContainerDef value, JsonSerializerOptions options) {

19
src/ApiService/ApiService/OneFuzzTypes/Requests.cs
View File

@ -163,3 +163,22 @@ public record ProxyDelete(
public record ProxyReset(
string Region
);
public record PoolSearch(
Guid? PoolId = null,
PoolName? Name = null,
List<PoolState>? State = null
);
public record PoolStop(
PoolName Name,
bool Now
);
public record PoolCreate(
PoolName Name,
Os Os,
Architecture Arch,
bool Managed,
Guid? ClientId = null
);

14
src/ApiService/ApiService/OneFuzzTypes/Responses.cs
View File

@ -91,6 +91,20 @@ public record JobResponse(
);
}
public record PoolGetResult(
PoolName Name,
Guid PoolId,
Os Os,
bool Managed,
Architecture Arch,
PoolState State,
Guid? ClientId,
List<Node>? Nodes,
AgentConfig? Config,
List<WorkSetSummary>? WorkQueue,
List<ScalesetSummary>? ScalesetSummary
) : BaseResponse();
public class BaseResponseConverter : JsonConverter<BaseResponse> {
public override BaseResponse? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) {
return null;

3
src/ApiService/ApiService/Program.cs
View File

@ -118,7 +118,8 @@ public class Program {
.AddSingleton<ICreds, Creds>()
.AddSingleton<IServiceConfig, ServiceConfiguration>()
.AddSingleton<IStorage, Storage>()
.AddHttpClient();
.AddHttpClient()
.AddMemoryCache();
}
)
.Build();

2
src/ApiService/ApiService/onefuzzlib/Containers.cs
View File

@ -181,7 +181,7 @@ public class Containers : IContainers {
await client.GetBlobClient(name).UploadAsync(new BinaryData(data), overwrite: true);
}
public Async.Task<Guid> GetInstanceId() => _getInstanceId.Value;
public virtual Async.Task<Guid> GetInstanceId() => _getInstanceId.Value;
private readonly Lazy<Async.Task<Guid>> _getInstanceId;
public static Uri? GetContainerSasUrlService(

20
src/ApiService/ApiService/onefuzzlib/Creds.cs
View File

@ -5,6 +5,7 @@ using Azure.Core;
using Azure.Identity;
using Azure.ResourceManager;
using Azure.ResourceManager.Resources;
using Microsoft.Extensions.Caching.Memory;
namespace Microsoft.OneFuzz.Service;
@ -32,6 +33,7 @@ public interface ICreds {
public GenericResource ParseResourceId(string resourceId);
public Async.Task<GenericResource> GetData(GenericResource resource);
Async.Task<IReadOnlyList<string>> GetRegions();
}
public class Creds : ICreds {
@ -39,12 +41,14 @@ public class Creds : ICreds {
private readonly DefaultAzureCredential _azureCredential;
private readonly IServiceConfig _config;
private readonly IHttpClientFactory _httpClientFactory;
private readonly IMemoryCache _cache;
public ArmClient ArmClient => _armClient;
public Creds(IServiceConfig config, IHttpClientFactory httpClientFactory) {
public Creds(IServiceConfig config, IHttpClientFactory httpClientFactory, IMemoryCache cache) {
_config = config;
_httpClientFactory = httpClientFactory;
_cache = cache;
_azureCredential = new DefaultAzureCredential();
_armClient = new ArmClient(this.GetIdentity(), this.GetSubscription());
}
@ -161,8 +165,22 @@ public class Creds : ICreds {
}
return resource;
}
public Task<IReadOnlyList<string>> GetRegions()
=> _cache.GetOrCreateAsync<IReadOnlyList<string>>(
nameof(Creds) + "." + nameof(GetRegions),
async entry => {
// cache for one day
entry.AbsoluteExpirationRelativeToNow = TimeSpan.FromDays(1);
var subscriptionId = SubscriptionResource.CreateResourceIdentifier(GetSubscription());
return await ArmClient.GetSubscriptionResource(subscriptionId)
.GetLocationsAsync()
.Select(x => x.Name)
.ToListAsync();
});
}
class GraphQueryException : Exception {
public GraphQueryException(string? message) : base(message) {
}

98
src/ApiService/ApiService/onefuzzlib/PoolOperations.cs
View File

@ -1,13 +1,19 @@
using System.Threading.Tasks;
using ApiService.OneFuzzLib.Orm;
using Azure.Data.Tables;
namespace Microsoft.OneFuzz.Service;
public interface IPoolOperations : IOrm<Pool> {
public Async.Task<OneFuzzResult<Pool>> GetByName(PoolName poolName);
public Async.Task<OneFuzzResult<Pool>> GetById(Guid poolId);
Task<bool> ScheduleWorkset(Pool pool, WorkSet workSet);
IAsyncEnumerable<Pool> GetByClientId(Guid clientId);
string GetPoolQueue(Pool pool);
string GetPoolQueue(Guid poolId);
public Async.Task<List<ScalesetSummary>> GetScalesetSummary(PoolName name);
public Async.Task<List<WorkSetSummary>> GetWorkQueue(Guid poolId, PoolState state);
IAsyncEnumerable<Pool> SearchStates(IEnumerable<PoolState> states);
Async.Task<Pool> SetShutdown(Pool pool, bool Now);
}
public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoolOperations {
@ -18,17 +24,33 @@ public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoo
}
public async Async.Task<OneFuzzResult<Pool>> GetByName(PoolName poolName) {
var pools = QueryAsync(filter: $"PartitionKey eq '{poolName.String}'");
var pools = QueryAsync(Query.PartitionKey(poolName.String));
if (pools == null || await pools.CountAsync() == 0) {
var result = await pools.ToListAsync();
if (result.Count == 0) {
return OneFuzzResult<Pool>.Error(ErrorCode.INVALID_REQUEST, "unable to find pool");
}
if (await pools.CountAsync() != 1) {
if (result.Count != 1) {
return OneFuzzResult<Pool>.Error(ErrorCode.INVALID_REQUEST, "error identifying pool");
}
return OneFuzzResult<Pool>.Ok(await pools.SingleAsync());
return OneFuzzResult<Pool>.Ok(result.Single());
}
public async Async.Task<OneFuzzResult<Pool>> GetById(Guid poolId) {
var pools = QueryAsync(Query.RowKey(poolId.ToString()));
var result = await pools.ToListAsync();
if (result.Count == 0) {
return OneFuzzResult<Pool>.Error(ErrorCode.INVALID_REQUEST, "unable to find pool");
}
if (result.Count != 1) {
return OneFuzzResult<Pool>.Error(ErrorCode.INVALID_REQUEST, "error identifying pool");
}
return OneFuzzResult<Pool>.Ok(result.Single());
}
public async Task<bool> ScheduleWorkset(Pool pool, WorkSet workSet) {
@ -36,13 +58,71 @@ public class PoolOperations : StatefulOrm<Pool, PoolState, PoolOperations>, IPoo
return false;
}
return await _context.Queue.QueueObject(GetPoolQueue(pool), workSet, StorageType.Corpus);
return await _context.Queue.QueueObject(GetPoolQueue(pool.PoolId), workSet, StorageType.Corpus);
}
public IAsyncEnumerable<Pool> GetByClientId(Guid clientId) {
return QueryAsync(filter: $"client_id eq '{clientId.ToString()}'");
return QueryAsync(filter: TableClient.CreateQueryFilter($"client_id eq {clientId}"));
}
public string GetPoolQueue(Pool pool)
=> $"pool-{pool.PoolId:N}";
public string GetPoolQueue(Guid poolId)
=> $"pool-{poolId:N}";
public async Async.Task<List<ScalesetSummary>> GetScalesetSummary(PoolName name)
=> await _context.ScalesetOperations.SearchByPool(name)
.Select(x => new ScalesetSummary(ScalesetId: x.ScalesetId, State: x.State))
.ToListAsync();
public async Async.Task<List<WorkSetSummary>> GetWorkQueue(Guid poolId, PoolState state) {
var result = new List<WorkSetSummary>();
// Only populate the work queue summaries if the pool is initialized. We
// can then be sure that the queue is available in the operations below.
if (state == PoolState.Init) {
return result;
}
var workSets = await PeekWorkQueue(poolId);
foreach (var workSet in workSets) {
if (!workSet.WorkUnits.Any()) {
continue;
}
var workUnits = workSet.WorkUnits
.Select(x => new WorkUnitSummary(
JobId: x.JobId,
TaskId: x.TaskId,
TaskType: x.TaskType))
.ToList();
result.Add(new WorkSetSummary(workUnits));
}
return result;
}
private Async.Task<IList<WorkSet>> PeekWorkQueue(Guid poolId)
=> _context.Queue.PeekQueue<WorkSet>(GetPoolQueue(poolId), StorageType.Corpus);
public IAsyncEnumerable<Pool> SearchStates(IEnumerable<PoolState> states)
=> QueryAsync(Query.EqualAnyEnum("state", states));
public Async.Task<Pool> SetShutdown(Pool pool, bool Now)
=> SetState(pool, Now ? PoolState.Halt : PoolState.Shutdown);
public async Async.Task<Pool> SetState(Pool pool, PoolState state) {
if (pool.State == state) {
return pool;
}
// scalesets should never leave the `halt` state
// it is terminal
if (pool.State == PoolState.Halt) {
return pool;
}
pool = pool with { State = state };
await Update(pool);
return pool;
}
}

16
src/ApiService/ApiService/onefuzzlib/Request.cs
View File

@ -90,21 +90,19 @@ public class RequestHandling : IRequestHandling {
return resp;
}
public async static Async.Task<HttpResponseData> Ok(HttpRequestData req, IEnumerable<BaseResponse> response) {
public static async Async.ValueTask<HttpResponseData> Ok(HttpRequestData req, IEnumerable<BaseResponse> response) {
// TODO: ModelMixin stuff
var resp = req.CreateResponse();
resp.StatusCode = HttpStatusCode.OK;
if (response.Count() > 1) {
await resp.WriteAsJsonAsync(response);
return resp;
} else if (response.Any()) {
await resp.WriteAsJsonAsync(response.Single());
}
// TODO: ModelMixin stuff
public static async Async.ValueTask<HttpResponseData> Ok(HttpRequestData req, BaseResponse response) {
// TODO: ModelMixin stuff
var resp = req.CreateResponse();
resp.StatusCode = HttpStatusCode.OK;
await resp.WriteAsJsonAsync(response);
return resp;
}
public async static Async.Task<HttpResponseData> Ok(HttpRequestData req, BaseResponse response) {
return await Ok(req, new BaseResponse[] { response });
}
}

8
src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs
View File

@ -6,7 +6,7 @@ namespace Microsoft.OneFuzz.Service;
public interface IScalesetOperations : IOrm<Scaleset> {
IAsyncEnumerable<Scaleset> Search();
public IAsyncEnumerable<Scaleset?> SearchByPool(PoolName poolName);
public IAsyncEnumerable<Scaleset> SearchByPool(PoolName poolName);
public Async.Task UpdateConfigs(Scaleset scaleSet);
@ -29,10 +29,8 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
return QueryAsync();
}
public IAsyncEnumerable<Scaleset> SearchByPool(PoolName poolName) {
return QueryAsync(filter: $"PartitionKey eq '{poolName}'");
}
public IAsyncEnumerable<Scaleset> SearchByPool(PoolName poolName)
=> QueryAsync(Query.PartitionKey(poolName.String));
async Async.Task SetState(Scaleset scaleSet, ScalesetState state) {
if (scaleSet.State == state)

4
src/ApiService/ApiService/onefuzzlib/orm/Orm.cs
View File

@ -44,6 +44,10 @@ namespace ApiService.OneFuzzLib.Orm {
public async IAsyncEnumerable<T> QueryAsync(string? filter = null) {
var tableClient = await GetTableClient(typeof(T).Name);
if (filter == "") {
filter = null;
}
await foreach (var x in tableClient.QueryAsync<TableEntity>(filter).Select(x => _entityConverter.ToRecord<T>(x))) {
yield return x;
}

2
src/ApiService/IntegrationTests/AgentRegistrationTests.cs
View File

@ -6,8 +6,10 @@ using Microsoft.OneFuzz.Service;
using Microsoft.OneFuzz.Service.Functions;
using Xunit;
using Xunit.Abstractions;
using Async = System.Threading.Tasks;
using Node = Microsoft.OneFuzz.Service.Node;
using Pool = Microsoft.OneFuzz.Service.Pool;
namespace IntegrationTests;

14
src/ApiService/IntegrationTests/Fakes/TestContainers.cs Normal file
View File

@ -0,0 +1,14 @@
using System;
using System.Threading.Tasks;
using Microsoft.OneFuzz.Service;
// TestContainers class allows use of InstanceID without having to set it up in blob storage
class TestContainers : Containers {
public TestContainers(ILogTracer log, IStorage storage, ICreds creds, IServiceConfig config)
: base(log, storage, creds, config) { }
public Guid InstanceId { get; } = Guid.NewGuid();
public override Task<Guid> GetInstanceId()
=> System.Threading.Tasks.Task.FromResult(InstanceId);
}

6
src/ApiService/IntegrationTests/Fakes/TestContext.cs
View File

@ -27,6 +27,7 @@ public sealed class TestContext : IOnefuzzContext {
NodeMessageOperations = new NodeMessageOperations(logTracer, this);
ConfigOperations = new ConfigOperations(logTracer, this);
PoolOperations = new PoolOperations(logTracer, this);
ScalesetOperations = new ScalesetOperations(logTracer, this);
UserCredentials = new UserCredentials(logTracer, ConfigOperations);
}
@ -54,7 +55,7 @@ public sealed class TestContext : IOnefuzzContext {
public IStorage Storage { get; }
public ICreds Creds { get; }
public IContainers Containers { get; }
public IContainers Containers { get; set; }
public IQueue Queue { get; }
public IUserCredentials UserCredentials { get; set; }
@ -68,6 +69,7 @@ public sealed class TestContext : IOnefuzzContext {
public INodeMessageOperations NodeMessageOperations { get; }
public IConfigOperations ConfigOperations { get; }
public IPoolOperations PoolOperations { get; }
public IScalesetOperations ScalesetOperations { get; }
// -- Remainder not implemented --
@ -92,8 +94,6 @@ public sealed class TestContext : IOnefuzzContext {
public IReproOperations ReproOperations => throw new System.NotImplementedException();
public IScalesetOperations ScalesetOperations => throw new System.NotImplementedException();
public IScheduler Scheduler => throw new System.NotImplementedException();
public ISecretsOperations SecretsOperations => throw new System.NotImplementedException();

2
src/ApiService/IntegrationTests/Fakes/TestCreds.cs
View File

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Net.Http;
using System.Threading.Tasks;
using Azure.Core;
@ -26,6 +27,7 @@ class TestCreds : ICreds {
// we have to return something in some test cases, even if it isn’t used
public Task<string> GetBaseRegion() => Task.FromResult(_region);
public Task<IReadOnlyList<string>> GetRegions() => Task.FromResult<IReadOnlyList<string>>(new[] { _region });
public string GetBaseResourceGroup() => _resourceGroup;

4
src/ApiService/IntegrationTests/Fakes/TestEndpointAuthorization.cs
View File

@ -5,7 +5,9 @@ using System.Threading.Tasks;
using Microsoft.Azure.Functions.Worker.Http;
using Microsoft.OneFuzz.Service;
enum RequestType {
namespace IntegrationTests.Fakes;
public enum RequestType {
NoAuthorization,
User,
Agent,

8
src/ApiService/IntegrationTests/Fakes/TestServiceConfiguration.cs
View File

@ -19,6 +19,10 @@ public sealed class TestServiceConfiguration : IServiceConfig {
public string? ApplicationInsightsInstrumentationKey { get; set; } = "TestAppInsightsInstrumentationKey";
public string? OneFuzzTelemetry => "TestOneFuzzTelemetry";
public string? MultiTenantDomain => null;
public string? OneFuzzInstanceName => "UnitTestInstance";
// -- Remainder not implemented --
@ -40,8 +44,6 @@ public sealed class TestServiceConfiguration : IServiceConfig {
public string? DiagnosticsAzureBlobRetentionDays => throw new System.NotImplementedException();
public string? MultiTenantDomain => throw new System.NotImplementedException();
public string? OneFuzzInstance => throw new System.NotImplementedException();
public string? OneFuzzKeyvault => throw new System.NotImplementedException();
@ -52,8 +54,6 @@ public sealed class TestServiceConfiguration : IServiceConfig {
public string OneFuzzNodeDisposalStrategy => throw new System.NotImplementedException();
public string? OneFuzzTelemetry => throw new System.NotImplementedException();
public string? OneFuzzDataStorage => throw new NotImplementedException();
public string? OneFuzzResourceGroup => throw new NotImplementedException();

4
src/ApiService/IntegrationTests/JobsTests.cs
View File

@ -126,8 +126,8 @@ public abstract class JobsTestBase : FunctionTestBase {
var result = await func.Run(TestHttpRequestData.FromJson("GET", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
var response = BodyAs<JobResponse>(result);
Assert.Equal(JobState.Enabled, response.State);
var response = BodyAs<JobResponse[]>(result);
Assert.Equal(JobState.Enabled, response.Single().State);
}
[Fact]

2
src/ApiService/IntegrationTests/NodeTests.cs
View File

@ -67,7 +67,7 @@ public abstract class NodeTestBase : FunctionTestBase {
var func = new NodeFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.FromJson("GET", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
Assert.Equal(0, result.Body.Length);
Assert.Equal("[]", BodyAsString(result));
}
[Fact]

260
src/ApiService/IntegrationTests/PoolTests.cs Normal file
View File

@ -0,0 +1,260 @@

using System;
using System.Collections.Generic;
using System.Linq;
using System.Net;
using IntegrationTests.Fakes;
using Microsoft.OneFuzz.Service;
using Xunit;
using Xunit.Abstractions;
using Async = System.Threading.Tasks;
using PoolFunction = Microsoft.OneFuzz.Service.Functions.Pool;
namespace IntegrationTests.Functions;
[Trait("Category", "Live")]
public class AzureStoragePoolTest : PoolTestBase {
public AzureStoragePoolTest(ITestOutputHelper output)
: base(output, Integration.AzureStorage.FromEnvironment()) { }
}
public class AzuritePoolTest : PoolTestBase {
public AzuritePoolTest(ITestOutputHelper output)
: base(output, new Integration.AzuriteStorage()) { }
}
public abstract class PoolTestBase : FunctionTestBase {
public PoolTestBase(ITestOutputHelper output, IStorage storage)
: base(output, storage) { }
private readonly Guid _userObjectId = Guid.NewGuid();
private readonly Guid _poolId = Guid.NewGuid();
private readonly PoolName _poolName = PoolName.Parse("pool-" + Guid.NewGuid());
[Theory]
[InlineData("POST", RequestType.Agent)]
[InlineData("POST", RequestType.NoAuthorization)]
[InlineData("GET", RequestType.Agent)]
[InlineData("GET", RequestType.NoAuthorization)]
[InlineData("DELETE", RequestType.Agent)]
[InlineData("DELETE", RequestType.NoAuthorization)]
public async Async.Task UserAuthorization_IsRequired(string method, RequestType authType) {
var auth = new TestEndpointAuthorization(authType, Logger, Context);
var func = new PoolFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.Empty(method));
Assert.Equal(HttpStatusCode.Unauthorized, result.StatusCode);
}
[Fact]
public async Async.Task Search_SpecificPool_ById_NotFound_ReturnsBadRequest() {
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var req = new PoolSearch(PoolId: _poolId);
var func = new PoolFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.FromJson("GET", req));
Assert.Equal(HttpStatusCode.BadRequest, result.StatusCode);
}
[Fact]
public async Async.Task Search_SpecificPool_ById_CanFind() {
await Context.InsertAll(
new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null));
// queue must exist
await Context.Queue.CreateQueue(Context.PoolOperations.GetPoolQueue(_poolId), StorageType.Corpus);
// use test class to override instance ID
Context.Containers = new TestContainers(Logger, Context.Storage, Context.Creds, Context.ServiceConfiguration);
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var req = new PoolSearch(PoolId: _poolId);
var func = new PoolFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.FromJson("GET", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
var pool = BodyAs<PoolGetResult>(result);
Assert.Equal(_poolId, pool.PoolId);
}
[Fact]
public async Async.Task Search_SpecificPool_ByName_NotFound_ReturnsBadRequest() {
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var req = new PoolSearch(Name: _poolName);
var func = new PoolFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.FromJson("GET", req));
Assert.Equal(HttpStatusCode.BadRequest, result.StatusCode);
}
[Fact]
public async Async.Task Search_SpecificPool_ByName_CanFind() {
await Context.InsertAll(
new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null));
// queue must exist
await Context.Queue.CreateQueue(Context.PoolOperations.GetPoolQueue(_poolId), StorageType.Corpus);
// use test class to override instance ID
Context.Containers = new TestContainers(Logger, Context.Storage, Context.Creds, Context.ServiceConfiguration);
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var req = new PoolSearch(Name: _poolName);
var func = new PoolFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.FromJson("GET", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
var pool = BodyAs<PoolGetResult>(result);
Assert.Equal(_poolName, pool.Name);
}
[Fact]
public async Async.Task Search_SpecificPool_ByState_NotFound_ReturnsEmptyResult() {
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var req = new PoolSearch(State: new List<PoolState> { PoolState.Init });
var func = new PoolFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.FromJson("GET", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
Assert.Equal("[]", BodyAsString(result));
}
[Fact]
public async Async.Task Search_SpecificPool_NoQuery_ReturnsAllPools() {
await Context.InsertAll(
new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }, // needed for admin check
new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null));
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var func = new PoolFunction(Logger, auth, Context);
var result = await func.Run(TestHttpRequestData.FromJson("GET", new PoolSearch()));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
var pool = BodyAs<PoolGetResult[]>(result);
Assert.Equal(_poolName, pool.Single().Name);
}
[Fact]
public async Async.Task Delete_NotNow_PoolEntersShutdownState() {
await Context.InsertAll(
new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }, // needed for admin check
new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null));
// override the found user credentials - need these to check for admin
var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn");
Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult<UserInfo>.Ok(userInfo));
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var func = new PoolFunction(Logger, auth, Context);
var req = new PoolStop(Name: _poolName, Now: false);
var result = await func.Run(TestHttpRequestData.FromJson("DELETE", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
var pool = await Context.PoolOperations.GetByName(_poolName);
Assert.True(pool.IsOk);
Assert.Equal(PoolState.Shutdown, pool.OkV!.State);
}
[Fact]
public async Async.Task Delete_NotNow_PoolStaysInHaltedState_IfAlreadyHalted() {
await Context.InsertAll(
new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }, // needed for admin check
new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Halt, null));
// override the found user credentials - need these to check for admin
var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn");
Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult<UserInfo>.Ok(userInfo));
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var func = new PoolFunction(Logger, auth, Context);
var req = new PoolStop(Name: _poolName, Now: false);
var result = await func.Run(TestHttpRequestData.FromJson("DELETE", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
var pool = await Context.PoolOperations.GetByName(_poolName);
Assert.True(pool.IsOk);
Assert.Equal(PoolState.Halt, pool.OkV!.State);
}
[Fact]
public async Async.Task Delete_Now_PoolEntersHaltState() {
await Context.InsertAll(
new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }, // needed for admin check
new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null));
// override the found user credentials - need these to check for admin
var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn");
Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult<UserInfo>.Ok(userInfo));
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var func = new PoolFunction(Logger, auth, Context);
var req = new PoolStop(Name: _poolName, Now: true);
var result = await func.Run(TestHttpRequestData.FromJson("DELETE", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
var pool = await Context.PoolOperations.GetByName(_poolName);
Assert.True(pool.IsOk);
Assert.Equal(PoolState.Halt, pool.OkV!.State);
}
[Fact]
public async Async.Task Post_CreatesNewPool() {
await Context.InsertAll(
new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }); // needed for admin check
// override the found user credentials - need these to check for admin
var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn");
Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult<UserInfo>.Ok(userInfo));
// need to override instance id
Context.Containers = new TestContainers(Logger, Context.Storage, Context.Creds, Context.ServiceConfiguration);
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var func = new PoolFunction(Logger, auth, Context);
var req = new PoolCreate(Name: _poolName, Os.Linux, Architecture.x86_64, true);
var result = await func.Run(TestHttpRequestData.FromJson("POST", req));
Assert.Equal(HttpStatusCode.OK, result.StatusCode);
// should get a pool back
var returnedPool = BodyAs<PoolGetResult>(result);
Assert.Equal(_poolName, returnedPool.Name);
var poolId = returnedPool.PoolId;
// should exist in storage
var pool = await Context.PoolOperations.GetByName(_poolName);
Assert.True(pool.IsOk);
Assert.Equal(poolId, pool.OkV!.PoolId);
}
[Fact]
public async Async.Task Post_DoesNotCreatePool_IfOneWithTheSameNameAlreadyExists() {
await Context.InsertAll(
new InstanceConfig(Context.ServiceConfiguration.OneFuzzInstanceName!) { Admins = new[] { _userObjectId } }, // needed for admin check
new Pool(_poolName, _poolId, Os.Linux, true, Architecture.x86_64, PoolState.Running, null));
// override the found user credentials - need these to check for admin
var userInfo = new UserInfo(ApplicationId: Guid.NewGuid(), ObjectId: _userObjectId, "upn");
Context.UserCredentials = new TestUserCredentials(Logger, Context.ConfigOperations, OneFuzzResult<UserInfo>.Ok(userInfo));
var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context);
var func = new PoolFunction(Logger, auth, Context);
var req = new PoolCreate(Name: _poolName, Os.Linux, Architecture.x86_64, true);
var result = await func.Run(TestHttpRequestData.FromJson("POST", req));
Assert.Equal(HttpStatusCode.BadRequest, result.StatusCode);
// should get an error back
var returnedPool = BodyAs<Error>(result);
Assert.Contains(returnedPool.Errors, c => c == "pool with that name already exists");
}
}

33
src/ApiService/Tests/JsonTests.cs Normal file
View File

@ -0,0 +1,33 @@
using System;
using System.Collections.Generic;
using System.Text.Json;
using Microsoft.OneFuzz.Service;
using Xunit;
namespace Tests;
public class JsonTests {
private static IContainerDef? Roundtrip(IContainerDef def)
=> JsonSerializer.Deserialize<IContainerDef>(JsonSerializer.Serialize(def));
[Fact]
public void CanRoundtripMultipleContainer() {
var it = new MultipleContainer(new List<SyncedDir>{
new SyncedDir("path", new Uri("https://example.com/1")),
new SyncedDir("path2", new Uri("https://example.com/2")),
});
var result = Roundtrip(it);
var multiple = Assert.IsType<MultipleContainer>(result);
Assert.Equal(it.SyncedDirs, multiple.SyncedDirs);
}
[Fact]
public void CanRoundtripSingleContainer() {
var it = new SingleContainer(new SyncedDir("path", new Uri("https://example.com")));
var result = Roundtrip(it);
var single = Assert.IsType<SingleContainer>(result);
Assert.Equal(it.SyncedDir, single.SyncedDir);
}
}