diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 020846aae..e5b3d5fd5 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -1329,7 +1329,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -1369,7 +1368,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -1429,7 +1427,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -1487,7 +1484,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, @@ -2597,7 +2593,7 @@ If webhook is set to have Event Grid message format then the payload will look a "image": "Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", "pool_name": "example", "region": "eastus", - "scaleset_id": "00000000-0000-0000-0000-000000000000", + "scaleset_id": "example-000", "size": 10, "vm_sku": "Standard_D2s_v3" } @@ -2621,7 +2617,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, @@ -2654,7 +2649,7 @@ If webhook is set to have Event Grid message format then the payload will look a ```json { "pool_name": "example", - "scaleset_id": "00000000-0000-0000-0000-000000000000" + "scaleset_id": "example-000" } ``` @@ -2668,7 +2663,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -2695,7 +2689,7 @@ If webhook is set to have Event Grid message format then the payload will look a ] }, "pool_name": "example", - "scaleset_id": "00000000-0000-0000-0000-000000000000" + "scaleset_id": "example-000" } ``` @@ -2764,7 +2758,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -2786,7 +2779,7 @@ If webhook is set to have Event Grid message format then the payload will look a ```json { "pool_name": "example", - "scaleset_id": "00000000-0000-0000-0000-000000000000", + "scaleset_id": "example-000", "size": 0 } ``` @@ -2801,7 +2794,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, @@ -2827,7 +2819,7 @@ If webhook is set to have Event Grid message format then the payload will look a ```json { "pool_name": "example", - "scaleset_id": "00000000-0000-0000-0000-000000000000", + "scaleset_id": "example-000", "state": "init" } ``` @@ -2857,7 +2849,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, @@ -5658,7 +5649,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -5682,7 +5672,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -5709,7 +5698,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -5733,7 +5721,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, @@ -5926,7 +5913,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, @@ -5957,7 +5943,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -5979,7 +5964,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" } @@ -5999,7 +5983,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, @@ -6023,7 +6006,6 @@ If webhook is set to have Event Grid message format then the payload will look a "type": "string" }, "scaleset_id": { - "format": "uuid", "title": "Scaleset Id", "type": "string" }, diff --git a/src/ApiService/ApiService/Functions/Proxy.cs b/src/ApiService/ApiService/Functions/Proxy.cs index 17ab4d6ed..ff56dd5fa 100644 --- a/src/ApiService/ApiService/Functions/Proxy.cs +++ b/src/ApiService/ApiService/Functions/Proxy.cs @@ -52,7 +52,7 @@ public class Proxy { var proxyGet = request.OkV; switch ((proxyGet.ScalesetId, proxyGet.MachineId, proxyGet.DstPort)) { - case (Guid scalesetId, Guid machineId, int dstPort): + case (ScalesetId scalesetId, Guid machineId, int dstPort): var scaleset = await _context.ScalesetOperations.GetById(scalesetId); if (!scaleset.IsOk) { return await _context.RequestHandling.NotOk(req, scaleset.ErrorV, "ProxyGet"); diff --git a/src/ApiService/ApiService/Functions/Scaleset.cs b/src/ApiService/ApiService/Functions/Scaleset.cs index 64f9ac52b..45c3bd858 100644 --- a/src/ApiService/ApiService/Functions/Scaleset.cs +++ b/src/ApiService/ApiService/Functions/Scaleset.cs @@ -118,7 +118,7 @@ public class Scaleset { } var scaleset = new Service.Scaleset( - ScalesetId: Guid.NewGuid(), + ScalesetId: Service.Scaleset.GenerateNewScalesetId(create.PoolName), State: ScalesetState.Init, NeedsConfigUpdate: false, Auth: await Auth.BuildAuth(_log), @@ -206,7 +206,7 @@ public class Scaleset { } var search = request.OkV; - if (search.ScalesetId is Guid id) { + if (search.ScalesetId is ScalesetId id) { var scalesetResult = await _context.ScalesetOperations.GetById(id); if (!scalesetResult.IsOk) { return await _context.RequestHandling.NotOk(req, scalesetResult.ErrorV, "ScalesetSearch"); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Events.cs b/src/ApiService/ApiService/OneFuzzTypes/Events.cs index 265035e68..02ef9e431 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Events.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Events.cs @@ -185,7 +185,7 @@ public record EventPing( [EventType(EventType.ScalesetCreated)] public record EventScalesetCreated( - Guid ScalesetId, + ScalesetId ScalesetId, PoolName PoolName, string VmSku, string Image, @@ -195,7 +195,7 @@ public record EventScalesetCreated( [EventType(EventType.ScalesetFailed)] public sealed record EventScalesetFailed( - Guid ScalesetId, + ScalesetId ScalesetId, PoolName PoolName, Error Error ) : BaseEvent(); @@ -203,7 +203,7 @@ public sealed record EventScalesetFailed( [EventType(EventType.ScalesetDeleted)] public record EventScalesetDeleted( - Guid ScalesetId, + ScalesetId ScalesetId, PoolName PoolName ) : BaseEvent(); @@ -211,7 +211,7 @@ public record EventScalesetDeleted( [EventType(EventType.ScalesetResizeScheduled)] public record EventScalesetResizeScheduled( - Guid ScalesetId, + ScalesetId ScalesetId, PoolName PoolName, long size ) : BaseEvent(); @@ -267,14 +267,14 @@ public record EventProxyStateUpdated( [EventType(EventType.NodeCreated)] public record EventNodeCreated( Guid MachineId, - Guid? ScalesetId, + ScalesetId? ScalesetId, PoolName PoolName ) : BaseEvent(); [EventType(EventType.NodeHeartbeat)] public record EventNodeHeartbeat( Guid MachineId, - Guid? ScalesetId, + ScalesetId? ScalesetId, PoolName PoolName, NodeState state ) : BaseEvent(); @@ -283,7 +283,7 @@ public record EventNodeHeartbeat( [EventType(EventType.NodeDeleted)] public record EventNodeDeleted( Guid MachineId, - Guid? ScalesetId, + ScalesetId? ScalesetId, PoolName PoolName, NodeState? MachineState ) : BaseEvent(); @@ -291,7 +291,7 @@ public record EventNodeDeleted( [EventType(EventType.ScalesetStateUpdated)] public record EventScalesetStateUpdated( - Guid ScalesetId, + ScalesetId ScalesetId, PoolName PoolName, ScalesetState State ) : BaseEvent(); @@ -299,7 +299,7 @@ public record EventScalesetStateUpdated( [EventType(EventType.NodeStateUpdated)] public record EventNodeStateUpdated( Guid MachineId, - Guid? ScalesetId, + ScalesetId? ScalesetId, PoolName PoolName, NodeState state ) : BaseEvent(); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index abb096ce6..f59b83358 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -1,6 +1,7 @@ using System.Reflection; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.RegularExpressions; using System.Threading.Tasks; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; using Endpoint = System.String; @@ -107,7 +108,7 @@ public record Node // a string internally. string? InstanceId = null, - Guid? ScalesetId = null, + ScalesetId? ScalesetId = null, bool ReimageRequested = false, bool DeleteRequested = false, @@ -132,7 +133,7 @@ public record ProxyForward ( [PartitionKey] Region Region, [RowKey] long Port, - Guid ScalesetId, + ScalesetId ScalesetId, Guid MachineId, Guid? ProxyId, long DstPort, @@ -263,7 +264,7 @@ public record TaskEventSummary( public record NodeAssignment( Guid NodeId, - Guid? ScalesetId, + ScalesetId? ScalesetId, NodeTaskState State ); @@ -392,7 +393,7 @@ public record InstanceConfig } public record AutoScale( - [PartitionKey, RowKey] Guid ScalesetId, + [PartitionKey, RowKey] ScalesetId ScalesetId, long Min, long Max, long Default, @@ -402,15 +403,10 @@ public record AutoScale( long ScaleInCooldown ) : EntityBase; -public record ScalesetNodeState( - Guid MachineId, - string InstanceId, - NodeState? State -); -public record Scaleset( +public partial record Scaleset( [PartitionKey] PoolName PoolName, - [RowKey] Guid ScalesetId, + [RowKey] ScalesetId ScalesetId, ScalesetState State, string VmSku, ImageReference Image, @@ -425,7 +421,31 @@ public record Scaleset( Guid? ClientId = null, Guid? ClientObjectId = null // 'Nodes' removed when porting from Python: only used in search response -) : StatefulEntityBase(State); +) : StatefulEntityBase(State) { + + [GeneratedRegex(@"[^a-zA-Z0-9\-]+")] + private static partial Regex InvalidCharacterRegex(); + + public static ScalesetId GenerateNewScalesetId(PoolName poolName) + => GenerateNewScalesetIdUsingGuid(poolName, Guid.NewGuid()); + + public static ScalesetId GenerateNewScalesetIdUsingGuid(PoolName poolName, Guid guid) { + // poolnames permit underscores but not scaleset names; use hyphen instead: + var name = poolName.ToString().Replace("_", "-"); + + // since poolnames are not actually validated, take only the valid characters: + name = InvalidCharacterRegex().Replace(name, ""); + + // trim off any starting and ending dashes: + name = name.Trim('-'); + + // this should now be a valid name; generate a unique suffix: + // max length is 64; length of Guid in "N" format is 32, -1 for the hyphen + name = name[..Math.Min(64 - 32 - 1, name.Length)] + "-" + guid.ToString("N"); + + return ScalesetId.Parse(name); + } +} public record Notification( [PartitionKey] Guid NotificationId, @@ -733,7 +753,7 @@ public record WorkSetSummary( ); public record ScalesetSummary( - Guid ScalesetId, + ScalesetId ScalesetId, ScalesetState State ); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index b0ffcc500..8f3d16aa6 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -35,7 +35,7 @@ public record NodeUpdate( public record NodeSearch( Guid? MachineId = null, List? State = null, - Guid? ScalesetId = null, + ScalesetId? ScalesetId = null, PoolName? PoolName = null ) : BaseRequest; @@ -172,20 +172,20 @@ public record ReproCreate( ) : BaseRequest; public record ProxyGet( - Guid? ScalesetId, + ScalesetId? ScalesetId, Guid? MachineId, int? DstPort ) : BaseRequest; public record ProxyCreate( - [property: Required] Guid ScalesetId, + [property: Required] ScalesetId ScalesetId, [property: Required] Guid MachineId, [property: Required] int DstPort, [property: Required] int Duration ) : BaseRequest; public record ProxyDelete( - [property: Required] Guid ScalesetId, + [property: Required] ScalesetId ScalesetId, [property: Required] Guid MachineId, int? DstPort ) : BaseRequest; @@ -217,18 +217,18 @@ public record AutoScaleOptions( ); public record ScalesetSearch( - Guid? ScalesetId = null, + ScalesetId? ScalesetId = null, List? State = null, bool IncludeAuth = false ) : BaseRequest; public record ScalesetStop( - [property: Required] Guid ScalesetId, + [property: Required] ScalesetId ScalesetId, [property: Required] bool Now ) : BaseRequest; public record ScalesetUpdate( - [property: Required] Guid ScalesetId, + [property: Required] ScalesetId ScalesetId, [property: Range(1, long.MaxValue)] long? Size ) : BaseRequest; @@ -313,7 +313,7 @@ public record AgentRegistrationGet( public record AgentRegistrationPost( [property: Required] PoolName PoolName, - Guid? ScalesetId, + ScalesetId? ScalesetId, [property: Required] Guid MachineId, Os? Os, string? MachineName, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs index 41d34a4f6..dad2bb53e 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs @@ -30,7 +30,7 @@ public record NodeSearchResult( DateTimeOffset? Heartbeat, DateTimeOffset? InitializedAt, NodeState State, - Guid? ScalesetId, + ScalesetId? ScalesetId, bool ReimageRequested, bool DeleteRequested, bool DebugKeepNode @@ -123,7 +123,7 @@ public record PoolGetResult( public record ScalesetResponse( PoolName PoolName, - Guid ScalesetId, + ScalesetId ScalesetId, ScalesetState State, Authentication? Auth, string VmSku, @@ -159,6 +159,12 @@ public record ScalesetResponse( Nodes: null); } +public record ScalesetNodeState( + Guid MachineId, + string? InstanceId, + NodeState? State +); + public record ConfigResponse( string? Authority, string? ClientId, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Validated.cs b/src/ApiService/ApiService/OneFuzzTypes/Validated.cs index 5db96fe8a..41fa254c4 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Validated.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Validated.cs @@ -16,9 +16,15 @@ static partial class Check { public static bool IsAlnumDash(string input) => IsAlnumDashRegex().IsMatch(input); // Permits 1-64 characters: alphanumeric, underscore, period, or dash. - [GeneratedRegex("\\A[._a-zA-Z0-9\\-]{1,64}\\z")] - private static partial Regex IsNameLikeRegex(); - public static bool IsNameLike(string input) => IsNameLikeRegex().IsMatch(input); + // Cannot start with underscore (or dash) or end with period or dash. + [GeneratedRegex(@"\A(?![_\-])[._a-zA-Z0-9\-]{1,64}(? ResourceNameRegex().IsMatch(input); + + // The same as ResourceNameRegex but underscore and period are not permitted. + [GeneratedRegex(@"\A(?!-)[a-zA-Z0-9\-]{1,64}(? VmssNameRegex().IsMatch(input); // This regex is based upon DNS labels but more restricted. // It is used for many different Storage resources. @@ -31,13 +37,16 @@ static partial class Check { public static bool IsStorageDnsLabel(string input) => StorageDnsLabelRegex().IsMatch(input); } -public interface IValidatedString where T : IValidatedString { - public static abstract T Parse(string input); +public interface IValidatedString { public static abstract bool IsValid(string input); public static abstract string Requirements { get; } public string String { get; } } +public interface IValidatedString : IValidatedString where T : IValidatedString { + public static abstract T Parse(string input); +} + public abstract record ValidatedStringBase where T : IValidatedString { protected ValidatedStringBase(string value) { if (!T.IsValid(value)) { @@ -109,3 +118,11 @@ public sealed record Container : ValidatedStringBase, IValidatedStrin public static bool IsValid(string input) => Check.IsStorageDnsLabel(input); public static string Requirements => "Container name must be 3-63 lowercase letters, numbers, or non-consecutive hyphens (see: https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage)"; } + +[JsonConverter(typeof(ValidatedStringConverter))] +public sealed record ScalesetId : ValidatedStringBase, IValidatedString { + private ScalesetId(string value) : base(value) { } + public static ScalesetId Parse(string input) => new(input); + public static bool IsValid(string input) => Check.IsVmssName(input); + public static string Requirements => "Virtual machine scaleset names must be 1-64 numbers, letters, or dashes (not at start or end)."; +} diff --git a/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs b/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs index 2aa8b773c..11ad95891 100644 --- a/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs +++ b/src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs @@ -159,7 +159,10 @@ namespace ApiService.TestHooks { var query = UriExtension.GetQueryComponents(req.Url); Guid? poolId = UriExtension.GetGuid("poolId", query); - Guid? scaleSetId = UriExtension.GetGuid("scaleSetId", query); + var scaleSetId = UriExtension.GetString("scaleSetId", query) + is string scalesetId + ? ScalesetId.Parse(scalesetId) + : null; List? states = default; if (query.TryGetValue("states", out var value)) { @@ -196,7 +199,7 @@ namespace ApiService.TestHooks { _log.Info($"reimage long lived nodes"); var query = UriExtension.GetQueryComponents(req.Url); - var r = _nodeOps.ReimageLongLivedNodes(Guid.Parse(query["scaleSetId"])); + var r = _nodeOps.ReimageLongLivedNodes(ScalesetId.Parse(query["scaleSetId"])); var resp = req.CreateResponse(HttpStatusCode.OK); await resp.WriteAsJsonAsync(r); return resp; @@ -213,9 +216,9 @@ namespace ApiService.TestHooks { var poolName = PoolName.Parse(query["poolName"]); Guid machineId = Guid.Parse(query["machineId"]); - Guid? scaleSetId = default; + ScalesetId? scaleSetId = null; if (query.TryGetValue("scaleSetId", out var value)) { - scaleSetId = Guid.Parse(value); + scaleSetId = ScalesetId.Parse(value); } string version = query["version"]; @@ -236,10 +239,10 @@ namespace ApiService.TestHooks { var query = UriExtension.GetQueryComponents(req.Url); - Guid scaleSetId = Guid.Parse(query["scaleSetId"]); + var scaleSetId = ScalesetId.Parse(query["scaleSetId"]); TimeSpan timeSpan = TimeSpan.Parse(query["timeSpan"]); - var nodes = await (_nodeOps.GetDeadNodes(scaleSetId, timeSpan).ToListAsync()); + var nodes = await _nodeOps.GetDeadNodes(scaleSetId, timeSpan).ToListAsync(); var json = JsonSerializer.Serialize(nodes, EntityConverter.GetJsonSerializerOptions()); var resp = req.CreateResponse(HttpStatusCode.OK); await resp.WriteStringAsync(json); diff --git a/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs b/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs index 12a5d2ed9..3e83b8c82 100644 --- a/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs +++ b/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs @@ -27,7 +27,7 @@ namespace ApiService.TestHooks { var query = UriExtension.GetQueryComponents(req.Url); var poolRes = _proxyForward.SearchForward( - UriExtension.GetGuid("scaleSetId", query), + UriExtension.GetString("scaleSetId", query) is string scalesetId ? ScalesetId.Parse(scalesetId) : null, UriExtension.GetString("region", query) is string region ? Region.Parse(region) : null, UriExtension.GetGuid("machineId", query), UriExtension.GetGuid("proxyId", query), diff --git a/src/ApiService/ApiService/TestHooks/VmssTestHooks.cs b/src/ApiService/ApiService/TestHooks/VmssTestHooks.cs index d919e22df..73bf64878 100644 --- a/src/ApiService/ApiService/TestHooks/VmssTestHooks.cs +++ b/src/ApiService/ApiService/TestHooks/VmssTestHooks.cs @@ -27,8 +27,8 @@ namespace ApiService.TestHooks { public async Task ListInstanceIds([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/vmssOperations/listInstanceIds")] HttpRequestData req) { _log.Info($"list instance ids"); var query = UriExtension.GetQueryComponents(req.Url); - var name = UriExtension.GetGuid("name", query) ?? throw new Exception("name must be set"); - var ids = await _vmssOps.ListInstanceIds(name); + var name = UriExtension.GetString("name", query) ?? throw new Exception("name must be set"); + var ids = await _vmssOps.ListInstanceIds(ScalesetId.Parse(name)); var json = JsonSerializer.Serialize(ids, EntityConverter.GetJsonSerializerOptions()); var resp = req.CreateResponse(HttpStatusCode.OK); @@ -40,9 +40,9 @@ namespace ApiService.TestHooks { public async Task GetInstanceId([HttpTrigger(AuthorizationLevel.Anonymous, "get", Route = "testhooks/vmssOperations/getInstanceId")] HttpRequestData req) { _log.Info($"list instance ids"); var query = UriExtension.GetQueryComponents(req.Url); - var name = UriExtension.GetGuid("name", query) ?? throw new Exception("name must be set"); + var name = UriExtension.GetString("name", query) ?? throw new Exception("name must be set"); var vmId = UriExtension.GetGuid("vmId", query) ?? throw new Exception("vmId must be set"); - var id = await _vmssOps.GetInstanceId(name, vmId); + var id = await _vmssOps.GetInstanceId(ScalesetId.Parse(name), vmId); var json = JsonSerializer.Serialize(id, EntityConverter.GetJsonSerializerOptions()); var resp = req.CreateResponse(HttpStatusCode.OK); @@ -54,9 +54,9 @@ namespace ApiService.TestHooks { public async Task UpdateScaleInProtection([HttpTrigger(AuthorizationLevel.Anonymous, "put", Route = "testhooks/vmssOperations/updateScaleInProtection")] HttpRequestData req) { _log.Info($"list instance ids"); var query = UriExtension.GetQueryComponents(req.Url); - var name = UriExtension.GetGuid("name", query) ?? throw new Exception("name must be set"); + var name = UriExtension.GetString("name", query) ?? throw new Exception("name must be set"); var instanceId = UriExtension.GetString("instanceId", query) ?? throw new Exception("instanceId must be set"); - var scalesetResult = await _scalesetOperations.GetById(name); + var scalesetResult = await _scalesetOperations.GetById(ScalesetId.Parse(name)); if (!scalesetResult.IsOk) { throw new Exception("invalid scaleset name"); } diff --git a/src/ApiService/ApiService/onefuzzlib/AutoScale.cs b/src/ApiService/ApiService/onefuzzlib/AutoScale.cs index 999eb1f3f..9d35085ab 100644 --- a/src/ApiService/ApiService/onefuzzlib/AutoScale.cs +++ b/src/ApiService/ApiService/onefuzzlib/AutoScale.cs @@ -13,7 +13,7 @@ public interface IAutoScaleOperations { public Async.Task> Insert(AutoScale autoScale); - public Async.Task GetSettingsForScaleset(Guid scalesetId); + public Async.Task GetSettingsForScaleset(ScalesetId scalesetId); AutoscaleProfile CreateAutoScaleProfile( string queueUri, @@ -26,16 +26,16 @@ public interface IAutoScaleOperations { double scaleInCooldownMinutes); AutoscaleProfile DefaultAutoScaleProfile(string queueUri, long scaleSetSize); - Async.Task AddAutoScaleToVmss(Guid vmss, AutoscaleProfile autoScaleProfile); + Async.Task AddAutoScaleToVmss(ScalesetId vmss, AutoscaleProfile autoScaleProfile); - OneFuzzResult GetAutoscaleSettings(Guid vmss); + OneFuzzResult GetAutoscaleSettings(ScalesetId vmss); Async.Task UpdateAutoscale(AutoscaleSettingData autoscale); - Async.Task> GetAutoScaleProfile(Guid scalesetId); + Async.Task> GetAutoScaleProfile(ScalesetId scalesetId); Async.Task Update( - Guid scalesetId, + ScalesetId scalesetId, long minAmount, long maxAmount, long defaultAmount, @@ -54,7 +54,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { } public async Async.Task Create( - Guid scalesetId, + ScalesetId scalesetId, long minAmount, long maxAmount, long defaultAmount, @@ -81,7 +81,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { return entry; } - public async Async.Task GetSettingsForScaleset(Guid scalesetId) { + public async Async.Task GetSettingsForScaleset(ScalesetId scalesetId) { try { var autoscale = await GetEntityAsync(scalesetId.ToString(), scalesetId.ToString()); return autoscale; @@ -91,7 +91,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { } } - public async Async.Task> GetAutoScaleProfile(Guid scalesetId) { + public async Async.Task> GetAutoScaleProfile(ScalesetId scalesetId) { _logTracer.Info($"getting scaleset for existing auto-scale resources {scalesetId:Tag:ScalesetId}"); var settings = _context.Creds.GetResourceGroupResource().GetAutoscaleSettings(); if (settings is null) { @@ -114,7 +114,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { return OneFuzzResult.Error(ErrorCode.INVALID_CONFIGURATION, $"could not find auto-scale settings for scaleset {scalesetId}"); } - public async Async.Task AddAutoScaleToVmss(Guid vmss, AutoscaleProfile autoScaleProfile) { + public async Async.Task AddAutoScaleToVmss(ScalesetId vmss, AutoscaleProfile autoScaleProfile) { _logTracer.Info($"Checking scaleset {vmss:Tag:ScalesetId} for existing auto scale resource"); var existingAutoScaleResource = GetAutoscaleSettings(vmss); @@ -141,7 +141,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { return OneFuzzResultVoid.Ok; } - private async Async.Task> CreateAutoScaleResourceFor(Guid resourceId, Region location, AutoscaleProfile profile) { + private async Async.Task> CreateAutoScaleResourceFor(ScalesetId resourceId, Region location, AutoscaleProfile profile) { _logTracer.Info($"Creating auto-scale resource for: {resourceId:Tag:AutoscaleResourceId}"); var resourceGroup = _context.Creds.GetBaseResourceGroup(); @@ -287,7 +287,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { } } - public OneFuzzResult GetAutoscaleSettings(Guid vmss) { + public OneFuzzResult GetAutoscaleSettings(ScalesetId vmss) { _logTracer.Info($"Checking scaleset {vmss:Tag:ScalesetId} for existing auto scale resource"); try { var autoscale = _context.Creds.GetResourceGroupResource().GetAutoscaleSettings() @@ -354,7 +354,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { } public async Async.Task Update( - Guid scalesetId, + ScalesetId scalesetId, long minAmount, long maxAmount, long defaultAmount, diff --git a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs index 4158a4da1..e4e00febe 100644 --- a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs @@ -24,16 +24,15 @@ public interface IIpOperations { public Async.Task DeleteIp(string resourceGroup, string name); - public Async.Task GetScalesetInstanceIp(Guid scalesetId, Guid machineId); + public Async.Task GetScalesetInstanceIp(ScalesetId scalesetId, Guid machineId); public Async.Task CreateIp(string resourceGroup, string name, Region region); } public class IpOperations : IIpOperations { - private ILogTracer _logTracer; - - private IOnefuzzContext _context; + private readonly ILogTracer _logTracer; + private readonly IOnefuzzContext _context; private readonly NetworkInterfaceQuery _networkInterfaceQuery; public IpOperations(ILogTracer log, IOnefuzzContext context) { @@ -86,7 +85,7 @@ public class IpOperations : IIpOperations { } } - public async Task GetScalesetInstanceIp(Guid scalesetId, Guid machineId) { + public async Task GetScalesetInstanceIp(ScalesetId scalesetId, Guid machineId) { var instance = await _context.VmssOperations.GetInstanceId(scalesetId, machineId); if (!instance.IsOk) { _logTracer.Verbose($"failed to get vmss {scalesetId:Tag:ScalesetId} for instance id {machineId:Tag:MachineId} due to {instance.ErrorV:Tag:Error}"); @@ -253,7 +252,7 @@ public class IpOperations : IIpOperations { } - public async Task> ListInstancePrivateIps(Guid scalesetId, string instanceId) { + public async Task> ListInstancePrivateIps(ScalesetId scalesetId, string instanceId) { var token = _context.Creds.GetIdentity().GetToken( new TokenRequestContext( new[] { $"https://management.azure.com" })); diff --git a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs index 8415af3df..fea5e72f6 100644 --- a/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NodeOperations.cs @@ -27,7 +27,7 @@ public interface INodeOperations : IStatefulOrm { Async.Task ToReimage(Node node, bool done = false); Async.Task SendStopIfFree(Node node); IAsyncEnumerable SearchStates(Guid? poolId = default, - Guid? scalesetId = default, + ScalesetId? scalesetId = default, IEnumerable? states = default, PoolName? poolName = default, bool excludeUpdateScheduled = false, @@ -35,18 +35,18 @@ public interface INodeOperations : IStatefulOrm { Async.Task Delete(Node node, string reason); - Async.Task ReimageLongLivedNodes(Guid scaleSetId); + Async.Task ReimageLongLivedNodes(ScalesetId scaleSetId); Async.Task Create( Guid poolId, PoolName poolName, Guid machineId, string? instanceId, - Guid? scaleSetId, + ScalesetId? scaleSetId, string version, bool isNew = false); - IAsyncEnumerable GetDeadNodes(Guid scaleSetId, TimeSpan expirationPeriod); + IAsyncEnumerable GetDeadNodes(ScalesetId scaleSetId, TimeSpan expirationPeriod); Async.Task MarkTasksStoppedEarly(Node node, Error? error); static readonly TimeSpan NODE_EXPIRATION_TIME = TimeSpan.FromHours(1.0); @@ -88,7 +88,7 @@ public class NodeOperations : StatefulOrm, INod } public async Task> AcquireScaleInProtection(Node node) { - if (node.ScalesetId is Guid scalesetId && + if (node.ScalesetId is ScalesetId scalesetId && await TryGetNodeInfo(node) is NodeInfo nodeInfo) { _logTracer.Info($"Setting scale-in protection on node {node.MachineId:Tag:MachineId}"); @@ -118,7 +118,7 @@ public class NodeOperations : StatefulOrm, INod public async Task ReleaseScaleInProtection(Node node) { if (!node.DebugKeepNode && - node.ScalesetId is Guid scalesetId && + node.ScalesetId is ScalesetId scalesetId && await TryGetNodeInfo(node) is NodeInfo nodeInfo) { _logTracer.Info($"Removing scale-in protection on node {node.MachineId:Tag:MachineId}"); @@ -150,7 +150,7 @@ public class NodeOperations : StatefulOrm, INod return null; } - var scalesetResult = await _context.ScalesetOperations.GetById(scalesetId.Value); + var scalesetResult = await _context.ScalesetOperations.GetById(scalesetId); if (!scalesetResult.IsOk || scalesetResult.OkV == null) { return null; } @@ -205,8 +205,8 @@ public class NodeOperations : StatefulOrm, INod return CanProcessNewWorkResponse.NotAllowed("node is scheduled to shrink"); } - if (node.ScalesetId != null) { - var scalesetResult = await _context.ScalesetOperations.GetById(node.ScalesetId.Value); + if (node.ScalesetId is not null) { + var scalesetResult = await _context.ScalesetOperations.GetById(node.ScalesetId); if (!scalesetResult.IsOk) { return CanProcessNewWorkResponse.NotAllowed("invalid scaleset"); } @@ -235,7 +235,7 @@ public class NodeOperations : StatefulOrm, INod /// This helps keep nodes on scalesets that use `latest` OS image SKUs /// reasonably up-to-date with OS patches without disrupting running /// fuzzing tasks with patch reboot cycles. - public async Async.Task ReimageLongLivedNodes(Guid scaleSetId) { + public async Async.Task ReimageLongLivedNodes(ScalesetId scaleSetId) { var timeFilter = Query.OlderThan("initialized_at", DateTimeOffset.UtcNow - INodeOperations.NODE_REIMAGE_TIME); await foreach (var node in QueryAsync(Query.And(Query.CreateQueryFilter($"scaleset_id eq {scaleSetId}"), timeFilter))) { @@ -270,7 +270,7 @@ public class NodeOperations : StatefulOrm, INod public static string SearchOutdatedQuery( string oneFuzzVersion, Guid? poolId = null, - Guid? scalesetId = null, + ScalesetId? scalesetId = null, IEnumerable? states = null, PoolName? poolName = null, bool excludeUpdateScheduled = false, @@ -283,7 +283,7 @@ public class NodeOperations : StatefulOrm, INod } if (poolName is not null) { - queryParts.Add(Query.CreateQueryFilter($"(pool_name eq {poolName.String})")); + queryParts.Add(Query.CreateQueryFilter($"(pool_name eq {poolName})")); } if (scalesetId is not null) { @@ -310,7 +310,7 @@ public class NodeOperations : StatefulOrm, INod IAsyncEnumerable SearchOutdated( Guid? poolId = null, - Guid? scalesetId = null, + ScalesetId? scalesetId = null, IEnumerable? states = null, PoolName? poolName = null, bool excludeUpdateScheduled = false, @@ -366,11 +366,11 @@ public class NodeOperations : StatefulOrm, INod return updatedNode; } - public IAsyncEnumerable GetDeadNodes(Guid scaleSetId, TimeSpan expirationPeriod) { + public IAsyncEnumerable GetDeadNodes(ScalesetId scaleSetId, TimeSpan expirationPeriod) { var minDate = DateTimeOffset.UtcNow - expirationPeriod; var filter = $"heartbeat lt datetime'{minDate.ToString("o")}' or Timestamp lt datetime'{minDate.ToString("o")}'"; - var query = Query.And(filter, $"scaleset_id eq '{scaleSetId}'"); + var query = Query.And(filter, Query.CreateQueryFilter($"scaleset_id eq {scaleSetId}")); return QueryAsync(query); } @@ -380,7 +380,7 @@ public class NodeOperations : StatefulOrm, INod PoolName poolName, Guid machineId, string? instanceId, - Guid? scaleSetId, + ScalesetId? scaleSetId, string version, bool isNew = false) { @@ -495,7 +495,7 @@ public class NodeOperations : StatefulOrm, INod } public async Task CouldShrinkScaleset(Node node) { - if (node.ScalesetId is Guid scalesetId) { + if (node.ScalesetId is ScalesetId scalesetId) { var queue = new ShrinkQueue(scalesetId, _context.Queue, _logTracer); if (await queue.ShouldShrink()) { return true; @@ -536,7 +536,7 @@ public class NodeOperations : StatefulOrm, INod public static string SearchStatesQuery( Guid? poolId = default, - Guid? scaleSetId = default, + ScalesetId? scaleSetId = default, IEnumerable? states = default, PoolName? poolName = default, int? numResults = default) { @@ -544,15 +544,15 @@ public class NodeOperations : StatefulOrm, INod List queryParts = new(); if (poolId is not null) { - queryParts.Add($"(pool_id eq '{poolId}')"); + queryParts.Add(Query.CreateQueryFilter($"(pool_id eq {poolId})")); } if (poolName is not null) { - queryParts.Add($"(PartitionKey eq '{poolName.String}')"); + queryParts.Add(Query.CreateQueryFilter($"(PartitionKey eq {poolName})")); } if (scaleSetId is not null) { - queryParts.Add($"(scaleset_id eq '{scaleSetId}')"); + queryParts.Add(Query.CreateQueryFilter($"(scaleset_id eq {scaleSetId})")); } if (states is not null) { @@ -566,7 +566,7 @@ public class NodeOperations : StatefulOrm, INod public IAsyncEnumerable SearchStates( Guid? poolId = default, - Guid? scalesetId = default, + ScalesetId? scalesetId = default, IEnumerable? states = default, PoolName? poolName = default, bool excludeUpdateScheduled = false, diff --git a/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs b/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs index f9f45df52..2a8dd3184 100644 --- a/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs @@ -5,10 +5,10 @@ namespace Microsoft.OneFuzz.Service; public interface IProxyForwardOperations : IOrm { - IAsyncEnumerable SearchForward(Guid? scalesetId = null, Region? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null); + IAsyncEnumerable SearchForward(ScalesetId? scalesetId = null, Region? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null); Forward ToForward(ProxyForward proxyForward); - Task> UpdateOrCreate(Region region, Guid scalesetId, Guid machineId, int dstPort, int duration); - Task> RemoveForward(Guid scalesetId, Guid? machineId = null, int? dstPort = null, Guid? proxyId = null); + Task> UpdateOrCreate(Region region, ScalesetId scalesetId, Guid machineId, int dstPort, int duration); + Task> RemoveForward(ScalesetId scalesetId, Guid? machineId = null, int? dstPort = null, Guid? proxyId = null); } @@ -20,12 +20,12 @@ public class ProxyForwardOperations : Orm, IProxyForwardOperations } - public IAsyncEnumerable SearchForward(Guid? scalesetId = null, Region? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null) { + public IAsyncEnumerable SearchForward(ScalesetId? scalesetId = null, Region? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null) { var conditions = new[] { scalesetId is not null ? Query.CreateQueryFilter($"scaleset_id eq {scalesetId}") : null, - region is not null ? Query.CreateQueryFilter($"PartitionKey eq {region.String}") : null , + region is not null ? Query.CreateQueryFilter($"PartitionKey eq {region}") : null , machineId is not null ? Query.CreateQueryFilter($"machine_id eq {machineId}") : null , proxyId is not null ? Query.CreateQueryFilter($"proxy_id eq {proxyId}") : null , dstPort is not null ? Query.CreateQueryFilter($"dst_port eq {dstPort}") : null , @@ -39,7 +39,7 @@ public class ProxyForwardOperations : Orm, IProxyForwardOperations return new Forward(proxyForward.Port, proxyForward.DstPort, proxyForward.DstIp); } - public async Task> UpdateOrCreate(Region region, Guid scalesetId, Guid machineId, int dstPort, int duration) { + public async Task> UpdateOrCreate(Region region, ScalesetId scalesetId, Guid machineId, int dstPort, int duration) { var privateIp = await _context.IpOperations.GetScalesetInstanceIp(scalesetId, machineId); if (privateIp == null) { @@ -87,7 +87,7 @@ public class ProxyForwardOperations : Orm, IProxyForwardOperations } - public async Task> RemoveForward(Guid scalesetId, Guid? machineId, int? dstPort, Guid? proxyId) { + public async Task> RemoveForward(ScalesetId scalesetId, Guid? machineId, int? dstPort, Guid? proxyId) { var entries = await SearchForward(scalesetId: scalesetId, machineId: machineId, proxyId: proxyId, dstPort: dstPort).ToListAsync(); var regions = new HashSet(); diff --git a/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs b/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs index 47a46fd55..37b1d2993 100644 --- a/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs @@ -15,7 +15,7 @@ public interface IScalesetOperations : IStatefulOrm { Async.Task UpdateConfigs(Scaleset scaleSet); - Async.Task> GetById(Guid scalesetId); + Async.Task> GetById(ScalesetId scalesetId); IAsyncEnumerable GetByObjectId(Guid objectId); Async.Task<(bool, Scaleset)> CleanupNodes(Scaleset scaleSet); @@ -584,7 +584,7 @@ public class ScalesetOperations : StatefulOrm> GetById(Guid scalesetId) { + public async Task> GetById(ScalesetId scalesetId) { var data = QueryAsync(filter: Query.RowKey(scalesetId.ToString())); var scaleSets = data is not null ? (await data.ToListAsync()) : null; @@ -800,17 +800,11 @@ public class ScalesetOperations : StatefulOrm(); } - var (nodes, azureNodes) = await ( - _context.NodeOperations.SearchStates(scaleset.ScalesetId).ToListAsync().AsTask(), - _context.VmssOperations.ListInstanceIds(scaleset.ScalesetId)); + var nodes = _context.NodeOperations.SearchStates(scalesetId: scaleset.ScalesetId); var result = new List(); - foreach (var (machineId, instanceId) in azureNodes) { - var node = nodes.FirstOrDefault(n => n.MachineId == machineId); - result.Add(new ScalesetNodeState( - MachineId: machineId, - InstanceId: instanceId, - node?.State)); + await foreach (var node in nodes) { + result.Add(new ScalesetNodeState(node.MachineId, node.InstanceId, node.State)); } return result; diff --git a/src/ApiService/ApiService/onefuzzlib/ShrinkQueue.cs b/src/ApiService/ApiService/onefuzzlib/ShrinkQueue.cs index d58199b23..4fb480cdb 100644 --- a/src/ApiService/ApiService/onefuzzlib/ShrinkQueue.cs +++ b/src/ApiService/ApiService/onefuzzlib/ShrinkQueue.cs @@ -2,25 +2,40 @@ public record ShrinkEntry(Guid ShrinkId); - -public class ShrinkQueue { - readonly Guid _baseId; +public sealed class ShrinkQueue { readonly IQueue _queueOps; readonly ILogTracer _log; - public ShrinkQueue(Guid baseId, IQueue queueOps, ILogTracer log) { - _baseId = baseId; + public ShrinkQueue(ScalesetId baseId, IQueue queueOps, ILogTracer log) + // backwards compat + // scaleset ID used to be a GUID and then this class would format it with "N" format + // to retain the same behaviour remove any dashes in the name + : this(baseId.ToString().Replace("-", ""), queueOps, log) { } + + public ShrinkQueue(Guid poolId, IQueue queueOps, ILogTracer log) + : this(poolId.ToString("N"), queueOps, log) { } + + private ShrinkQueue(string baseId, IQueue queueOps, ILogTracer log) { + var name = ShrinkQueueNamePrefix + baseId.ToLowerInvariant(); + + // queue names can be no longer than 63 characters + // if we exceed that, trim off the end. we will still have + // sufficient random chracters to stop collisions from happening + if (name.Length > 63) { + name = name[..63]; + } + + QueueName = name; _queueOps = queueOps; _log = log; } public static string ShrinkQueueNamePrefix => "to-shrink-"; - public override string ToString() { - return $"{ShrinkQueueNamePrefix}{_baseId:N}"; - } + public override string ToString() + => QueueName; - public string QueueName => ToString(); + public string QueueName { get; } public async Async.Task Clear() { await _queueOps.ClearQueue(QueueName, StorageType.Config); diff --git a/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs b/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs index 77efd5eec..feb0bca51 100644 --- a/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs @@ -13,23 +13,23 @@ namespace Microsoft.OneFuzz.Service; public interface IVmssOperations { Async.Task UpdateScaleInProtection(Scaleset scaleset, string instanceId, bool protectFromScaleIn); - Async.Task> GetInstanceId(Guid name, Guid vmId); - Async.Task UpdateExtensions(Guid name, IList extensions); - Async.Task GetVmss(Guid name); + Async.Task> GetInstanceId(ScalesetId name, Guid vmId); + Async.Task UpdateExtensions(ScalesetId name, IList extensions); + Async.Task GetVmss(ScalesetId name); Async.Task> ListAvailableSkus(Region region); - Async.Task DeleteVmss(Guid name, bool? forceDeletion = null); + Async.Task DeleteVmss(ScalesetId name, bool? forceDeletion = null); - Async.Task> ListInstanceIds(Guid name); + Async.Task> ListInstanceIds(ScalesetId name); - Async.Task GetVmssSize(Guid name); + Async.Task GetVmssSize(ScalesetId name); - Async.Task ResizeVmss(Guid name, long capacity); + Async.Task ResizeVmss(ScalesetId name, long capacity); Async.Task CreateVmss( Region location, - Guid name, + ScalesetId name, string vmSku, long vmCount, ImageReference image, @@ -41,9 +41,9 @@ public interface IVmssOperations { string sshPublicKey, IDictionary tags); - IAsyncEnumerable ListVmss(Guid name); - Async.Task ReimageNodes(Guid scalesetId, IEnumerable nodes); - Async.Task DeleteNodes(Guid scalesetId, IEnumerable nodes); + IAsyncEnumerable ListVmss(ScalesetId name); + Async.Task ReimageNodes(ScalesetId scalesetId, IEnumerable nodes); + Async.Task DeleteNodes(ScalesetId scalesetId, IEnumerable nodes); } public class VmssOperations : IVmssOperations { @@ -60,7 +60,7 @@ public class VmssOperations : IVmssOperations { _cache = cache; } - public async Async.Task DeleteVmss(Guid name, bool? forceDeletion = null) { + public async Async.Task DeleteVmss(ScalesetId name, bool? forceDeletion = null) { var r = GetVmssResource(name); var result = await r.DeleteAsync(WaitUntil.Started, forceDeletion: forceDeletion); var raw = result.GetRawResponse(); @@ -72,7 +72,7 @@ public class VmssOperations : IVmssOperations { } } - public async Async.Task GetVmssSize(Guid name) { + public async Async.Task GetVmssSize(ScalesetId name) { var vmss = await GetVmss(name); if (vmss == null) { return null; @@ -80,7 +80,7 @@ public class VmssOperations : IVmssOperations { return vmss.Sku.Capacity; } - public async Async.Task ResizeVmss(Guid name, long capacity) { + public async Async.Task ResizeVmss(ScalesetId name, long capacity) { var canUpdate = await CheckCanUpdate(name); if (canUpdate.IsOk) { var scalesetResource = GetVmssResource(name); @@ -100,7 +100,7 @@ public class VmssOperations : IVmssOperations { } - private VirtualMachineScaleSetResource GetVmssResource(Guid name) { + private VirtualMachineScaleSetResource GetVmssResource(ScalesetId name) { var id = VirtualMachineScaleSetResource.CreateResourceIdentifier( _creds.GetSubscription(), _creds.GetBaseResourceGroup(), @@ -108,7 +108,7 @@ public class VmssOperations : IVmssOperations { return _creds.ArmClient.GetVirtualMachineScaleSetResource(id); } - private VirtualMachineScaleSetVmResource GetVmssVmResource(Guid name, string instanceId) { + private VirtualMachineScaleSetVmResource GetVmssVmResource(ScalesetId name, string instanceId) { var id = VirtualMachineScaleSetVmResource.CreateResourceIdentifier( _creds.GetSubscription(), _creds.GetBaseResourceGroup(), @@ -117,7 +117,7 @@ public class VmssOperations : IVmssOperations { return _creds.ArmClient.GetVirtualMachineScaleSetVmResource(id); } - public async Async.Task GetVmss(Guid name) { + public async Async.Task GetVmss(ScalesetId name) { try { var res = await GetVmssResource(name).GetAsync(); _log.Verbose($"getting vmss: {name:Tag:VmssName}"); @@ -127,7 +127,7 @@ public class VmssOperations : IVmssOperations { } } - public async Async.Task> CheckCanUpdate(Guid name) { + public async Async.Task> CheckCanUpdate(ScalesetId name) { var vmss = await GetVmss(name); if (vmss is null) { return OneFuzzResult.Error(ErrorCode.UNABLE_TO_UPDATE, $"vmss not found: {name}"); @@ -139,7 +139,7 @@ public class VmssOperations : IVmssOperations { } - public async Async.Task UpdateExtensions(Guid name, IList extensions) { + public async Async.Task UpdateExtensions(ScalesetId name, IList extensions) { var canUpdate = await CheckCanUpdate(name); if (canUpdate.IsOk) { var res = GetVmssResource(name); @@ -165,7 +165,7 @@ public class VmssOperations : IVmssOperations { } } - public async Async.Task> ListInstanceIds(Guid name) { + public async Async.Task> ListInstanceIds(ScalesetId name) { _log.Verbose($"get instance IDs for scaleset {name:Tag:VmssName}"); try { var results = new Dictionary(); @@ -185,8 +185,8 @@ public class VmssOperations : IVmssOperations { } } - private sealed record InstanceIdKey(Guid Scaleset, Guid VmId); - private Task GetInstanceIdForVmId(Guid scaleset, Guid vmId) + private sealed record InstanceIdKey(ScalesetId Scaleset, Guid VmId); + private Task GetInstanceIdForVmId(ScalesetId scaleset, Guid vmId) => _cache.GetOrCreateAsync(new InstanceIdKey(scaleset, vmId), async entry => { var scalesetResource = GetVmssResource(scaleset); var vmIdString = vmId.ToString(); @@ -214,7 +214,7 @@ public class VmssOperations : IVmssOperations { } })!; // NULLABLE: only this method inserts InstanceIdKey so it cannot be null - public async Async.Task> GetInstanceVm(Guid name, Guid vmId) { + public async Async.Task> GetInstanceVm(ScalesetId name, Guid vmId) { _log.Info($"get instance ID for scaleset node: {name:Tag:VmssName}:{vmId:Tag:VmId}"); var instanceId = await GetInstanceId(name, vmId); if (!instanceId.IsOk) { @@ -231,7 +231,7 @@ public class VmssOperations : IVmssOperations { } } - public async Async.Task> GetInstanceId(Guid name, Guid vmId) { + public async Async.Task> GetInstanceId(ScalesetId name, Guid vmId) { try { return OneFuzzResult.Ok(await GetInstanceIdForVmId(name, vmId)); } catch { @@ -265,7 +265,7 @@ public class VmssOperations : IVmssOperations { public async Async.Task CreateVmss( Region location, - Guid name, + ScalesetId name, string vmSku, long vmCount, ImageReference image, @@ -397,7 +397,7 @@ public class VmssOperations : IVmssOperations { } } - public IAsyncEnumerable ListVmss(Guid name) + public IAsyncEnumerable ListVmss(ScalesetId name) => GetVmssResource(name) .GetVirtualMachineScaleSetVms() .SelectAwait(async vm => vm.HasData ? vm : await vm.GetAsync()); @@ -431,7 +431,7 @@ public class VmssOperations : IVmssOperations { return skuNames; })!; // NULLABLE: only this method inserts AvailableSkusKey so it cannot be null - private async Async.Task> ResolveInstanceIds(Guid scalesetId, IEnumerable nodes) { + private async Async.Task> ResolveInstanceIds(ScalesetId scalesetId, IEnumerable nodes) { // only initialize this if we find a missing InstanceId var machineToInstanceLazy = new Lazy>>(async () => { @@ -461,7 +461,7 @@ public class VmssOperations : IVmssOperations { return instanceIds; } - public async Async.Task ReimageNodes(Guid scalesetId, IEnumerable nodes) { + public async Async.Task ReimageNodes(ScalesetId scalesetId, IEnumerable nodes) { var result = await CheckCanUpdate(scalesetId); if (!result.IsOk) { return OneFuzzResultVoid.Error(result.ErrorV); @@ -514,7 +514,7 @@ public class VmssOperations : IVmssOperations { return OneFuzzResultVoid.Ok; } - public async Async.Task DeleteNodes(Guid scalesetId, IEnumerable nodes) { + public async Async.Task DeleteNodes(ScalesetId scalesetId, IEnumerable nodes) { var result = await CheckCanUpdate(scalesetId); if (!result.IsOk) { _log.Warning($"cannot delete nodes from scaleset {scalesetId} : {result.ErrorV}"); diff --git a/src/ApiService/ApiService/onefuzzlib/orm/Queries.cs b/src/ApiService/ApiService/onefuzzlib/orm/Queries.cs index 48852a1a0..6bb0c9b19 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/Queries.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/Queries.cs @@ -1,6 +1,7 @@ using System.Runtime.CompilerServices; using System.Text.Json; using Azure.Data.Tables; +using Microsoft.OneFuzz.Service; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; namespace ApiService.OneFuzzLib.Orm { @@ -15,6 +16,8 @@ namespace ApiService.OneFuzzLib.Orm { for (int i = 0; i < args.Length; i++) { if (args[i] is Guid g) { args[i] = g.ToString(); + } else if (args[i] is IValidatedString s) { + args[i] = s.String; } } diff --git a/src/ApiService/FunctionalTests/1f-api/Node.cs b/src/ApiService/FunctionalTests/1f-api/Node.cs index c36ff514f..4a1dd7dd3 100644 --- a/src/ApiService/FunctionalTests/1f-api/Node.cs +++ b/src/ApiService/FunctionalTests/1f-api/Node.cs @@ -24,7 +24,7 @@ public class Node : IFromJsonElement { public string State => _e.GetStringProperty("state"); - public Guid? ScalesetId => _e.GetNullableGuidProperty("scaleset_id"); + public string? ScalesetId => _e.GetNullableStringProperty("scaleset_id"); public bool ReimageRequested => _e.GetBoolProperty("reimage_requested"); public bool DeleteRequested => _e.GetBoolProperty("delete_requested"); public bool DebugKeepNode => _e.GetBoolProperty("debug_keep_node"); @@ -45,7 +45,7 @@ public class NodeApi : ApiBase { .AddV("machine_id", machineId); return Return(await Post(j)); } - public async Task, Error>> Get(Guid? machineId = null, IEnumerable? state = null, Guid? scalesetId = null, string? poolName = null) { + public async Task, Error>> Get(Guid? machineId = null, IEnumerable? state = null, string? scalesetId = null, string? poolName = null) { var j = new JsonObject() .AddIfNotNullV("machine_id", machineId) .AddIfNotNullEnumerableV("state", state) diff --git a/src/ApiService/FunctionalTests/1f-api/Proxy.cs b/src/ApiService/FunctionalTests/1f-api/Proxy.cs index a93b984e0..645e90559 100644 --- a/src/ApiService/FunctionalTests/1f-api/Proxy.cs +++ b/src/ApiService/FunctionalTests/1f-api/Proxy.cs @@ -91,7 +91,7 @@ public class ProxyApi : ApiBase { base(endpoint, "/api/proxy", request, output) { } - public async Task, Error>> Get(Guid? scalesetId = null, Guid? machineId = null, int? dstPort = null) { + public async Task, Error>> Get(string? scalesetId = null, Guid? machineId = null, int? dstPort = null) { var root = new JsonObject() .AddIfNotNullV("scaleset_id", scalesetId) .AddIfNotNullV("machine_id", machineId) @@ -101,7 +101,7 @@ public class ProxyApi : ApiBase { return IEnumerableResult(r.GetProperty("proxies")); } - public async Task Delete(Guid scalesetId, Guid machineId, int? dstPort = null) { + public async Task Delete(string scalesetId, Guid machineId, int? dstPort = null) { var root = new JsonObject() .AddV("scaleset_id", scalesetId) .AddV("machine_id", machineId) @@ -115,10 +115,10 @@ public class ProxyApi : ApiBase { return Return(r); } - public async Task> Create(Guid scalesetId, Guid machineId, int dstPort, int duration) { + public async Task> Create(string scalesetId, Guid machineId, int dstPort, int duration) { var root = new JsonObject() .AddV("scaleset_id", scalesetId) - .AddV("machin_id", machineId) + .AddV("machine_id", machineId) .AddV("dst_port", dstPort) .AddV("duration", duration); diff --git a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs index ceeaf3186..6d8bc6923 100644 --- a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs +++ b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs @@ -23,7 +23,7 @@ public class Scaleset : IFromJsonElement { public Scaleset(JsonElement e) => _e = e; public static Scaleset Convert(JsonElement e) => new(e); - public Guid ScalesetId => _e.GetGuidProperty("scaleset_id"); + public string ScalesetId => _e.GetStringProperty("scaleset_id"); public string PoolName => _e.GetStringProperty("pool_name"); public string State => _e.GetStringProperty("state"); @@ -59,7 +59,7 @@ public class ScalesetApi : ApiBase { base(endpoint, "/api/Scaleset", request, output) { } - public async Task, Error>> Get(Guid? id = null, string? state = null, bool? includeAuth = false) { + public async Task, Error>> Get(string? id = null, string? state = null, bool? includeAuth = false) { var j = new JsonObject() .AddIfNotNullV("scaleset_id", id) .AddIfNotNullV("state", state) @@ -84,14 +84,14 @@ public class ScalesetApi : ApiBase { return Result(await Post(rootScalesetCreate)); } - public async Task> Patch(Guid id, int size) { + public async Task> Patch(string id, int size) { var scalesetPatch = new JsonObject() .AddV("scaleset_id", id) .AddV("size", size); return Result(await Patch(scalesetPatch)); } - public async Task Delete(Guid id, bool now) { + public async Task Delete(string id, bool now) { var scalesetDelete = new JsonObject() .AddV("scaleset_id", id) .AddV("now", now); @@ -99,7 +99,7 @@ public class ScalesetApi : ApiBase { } - public async Task WaitWhile(Guid id, Func wait) { + public async Task WaitWhile(string id, Func wait) { var currentState = ""; Scaleset newScaleset; do { diff --git a/src/ApiService/IntegrationTests/AgentRegistrationTests.cs b/src/ApiService/IntegrationTests/AgentRegistrationTests.cs index 0ca8fd881..618616bbd 100644 --- a/src/ApiService/IntegrationTests/AgentRegistrationTests.cs +++ b/src/ApiService/IntegrationTests/AgentRegistrationTests.cs @@ -30,7 +30,7 @@ public abstract class AgentRegistrationTestsBase : FunctionTestBase { private readonly Guid _machineId = Guid.NewGuid(); private readonly Guid _poolId = Guid.NewGuid(); - private readonly Guid _scalesetId = Guid.NewGuid(); + private readonly ScalesetId _scalesetId = ScalesetId.Parse($"scaleset-{Guid.NewGuid()}"); private readonly PoolName _poolName = PoolName.Parse($"pool-{Guid.NewGuid()}"); [Fact] diff --git a/src/ApiService/IntegrationTests/Fakes/TestContext.cs b/src/ApiService/IntegrationTests/Fakes/TestContext.cs index 127f9a04e..2f5e30bdb 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestContext.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestContext.cs @@ -58,6 +58,7 @@ public sealed class TestContext : IOnefuzzContext { Pool p => PoolOperations.Insert(p), Job j => JobOperations.Insert(j), Repro r => ReproOperations.Insert(r), + Scaleset ss => ScalesetOperations.Insert(ss), NodeTasks nt => NodeTasksOperations.Insert(nt), InstanceConfig ic => ConfigOperations.Insert(ic), Notification n => NotificationOperations.Insert(n), diff --git a/src/ApiService/IntegrationTests/Fakes/TestVmssOperations.cs b/src/ApiService/IntegrationTests/Fakes/TestVmssOperations.cs index e4d0537fc..40b7c3342 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestVmssOperations.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestVmssOperations.cs @@ -20,40 +20,40 @@ sealed class TestVmssOperations : IVmssOperations { /* below not implemented */ - public Task CreateVmss(Region location, Guid name, string vmSku, long vmCount, ImageReference image, string networkId, bool? spotInstance, bool ephemeralOsDisks, IList? extensions, string password, string sshPublicKey, IDictionary tags) { + public Task CreateVmss(Region location, ScalesetId name, string vmSku, long vmCount, ImageReference image, string networkId, bool? spotInstance, bool ephemeralOsDisks, IList? extensions, string password, string sshPublicKey, IDictionary tags) { throw new NotImplementedException(); } - public Task DeleteVmss(Guid name, bool? forceDeletion = null) { + public Task DeleteVmss(ScalesetId name, bool? forceDeletion = null) { throw new NotImplementedException(); } - public Task> GetInstanceId(Guid name, Guid vmId) { + public Task> GetInstanceId(ScalesetId name, Guid vmId) { throw new NotImplementedException(); } - public Task GetVmss(Guid name) { + public Task GetVmss(ScalesetId name) { throw new NotImplementedException(); } - public Task GetVmssSize(Guid name) { + public Task GetVmssSize(ScalesetId name) { throw new NotImplementedException(); } - public Task> ListInstanceIds(Guid name) { + public Task> ListInstanceIds(ScalesetId name) { throw new NotImplementedException(); } - public IAsyncEnumerable ListVmss(Guid name) { + public IAsyncEnumerable ListVmss(ScalesetId name) { throw new NotImplementedException(); } - public Task ResizeVmss(Guid name, long capacity) { + public Task ResizeVmss(ScalesetId name, long capacity) { throw new NotImplementedException(); } - public Task UpdateExtensions(Guid name, IList extensions) { + public Task UpdateExtensions(ScalesetId name, IList extensions) { throw new NotImplementedException(); } @@ -61,11 +61,11 @@ sealed class TestVmssOperations : IVmssOperations { throw new NotImplementedException(); } - public Task ReimageNodes(Guid scalesetId, IEnumerable nodes) { + public Task ReimageNodes(ScalesetId scalesetId, IEnumerable nodes) { throw new NotImplementedException(); } - public Async.Task DeleteNodes(Guid scalesetId, IEnumerable nodes) { + public Async.Task DeleteNodes(ScalesetId scalesetId, IEnumerable nodes) { throw new NotImplementedException(); } } diff --git a/src/ApiService/IntegrationTests/NodeTests.cs b/src/ApiService/IntegrationTests/NodeTests.cs index e3f63cb86..8902a01ab 100644 --- a/src/ApiService/IntegrationTests/NodeTests.cs +++ b/src/ApiService/IntegrationTests/NodeTests.cs @@ -24,10 +24,12 @@ public class AzuriteNodeTest : NodeTestBase { public abstract class NodeTestBase : FunctionTestBase { public NodeTestBase(ITestOutputHelper output, IStorage storage) - : base(output, storage) { } + : base(output, storage) { + _scalesetId = Scaleset.GenerateNewScalesetId(_poolName); + } private readonly Guid _machineId = Guid.NewGuid(); - private readonly Guid _scalesetId = Guid.NewGuid(); + private readonly ScalesetId _scalesetId; private readonly PoolName _poolName = PoolName.Parse($"pool-{Guid.NewGuid()}"); private readonly string _version = Guid.NewGuid().ToString(); diff --git a/src/ApiService/IntegrationTests/ScalesetTests.cs b/src/ApiService/IntegrationTests/ScalesetTests.cs index 8e7f76061..6ee51271a 100644 --- a/src/ApiService/IntegrationTests/ScalesetTests.cs +++ b/src/ApiService/IntegrationTests/ScalesetTests.cs @@ -45,7 +45,7 @@ public abstract class ScalesetTestBase : FunctionTestBase { public async Async.Task Search_SpecificScaleset_ReturnsErrorIfNoneFound() { var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); - var req = new ScalesetSearch(ScalesetId: Guid.NewGuid()); + var req = new ScalesetSearch(ScalesetId: ScalesetId.Parse(Guid.NewGuid().ToString())); var func = new ScalesetFunction(Logger, auth, Context); var result = await func.Run(TestHttpRequestData.FromJson("GET", req)); @@ -66,6 +66,33 @@ public abstract class ScalesetTestBase : FunctionTestBase { Assert.Equal("[]", BodyAsString(result)); } + [Fact] + public async Async.Task Search_CanFindScaleset_AndReturnsNodes() { + var scalesetId = ScalesetId.Parse(Guid.NewGuid().ToString()); + var poolName = PoolName.Parse($"pool-${Guid.NewGuid()}"); + var poolId = Guid.NewGuid(); + + await Context.InsertAll( + new Pool(poolName, poolId, Os.Linux, Managed: true, Architecture.x86_64, PoolState.Running), + // scaleset to be found must exist + new Scaleset(poolName, scalesetId, ScalesetState.Running, "", ImageReference.MustParse("x:y:z:v"), Region.Parse("region"), 1, null, false, false, new Dictionary()), + // some nodes + new Node(poolName, Guid.NewGuid(), poolId, "version", ScalesetId: scalesetId), + new Node(poolName, Guid.NewGuid(), poolId, "version", ScalesetId: scalesetId) + ); + + var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); + + var req = new ScalesetSearch(ScalesetId: scalesetId); + var func = new ScalesetFunction(Logger, auth, Context); + var result = await func.Run(TestHttpRequestData.FromJson("GET", req)); + + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + var resp = BodyAs(result); + Assert.Equal(scalesetId, resp.ScalesetId); + Assert.Equal(2, resp.Nodes?.Count); + } + [Fact] public async Async.Task Create_Scaleset() { var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index 72711682d..58ebd5b56 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -91,26 +91,32 @@ namespace Tests { where PoolName.IsValid(name.Get) select PoolName.Parse(name.Get); + public static Gen ScalesetIdGen { get; } + = from name in Arb.Generate() + where ScalesetId.IsValid(name.Get) + select ScalesetId.Parse(name.Get); + public static Gen RegionGen { get; } = from name in Arb.Generate() where Region.IsValid(name.Get) select Region.Parse(name.Get); public static Gen Node { get; } - = from arg in Arb.Generate, Tuple>>() + = from arg in Arb.Generate, Tuple>>() from poolName in PoolNameGen + from scalesetId in Arb.Generate() select new Node( InitializedAt: arg.Item1.Item1, PoolName: poolName, PoolId: arg.Item1.Item3, MachineId: arg.Item1.Item3, State: arg.Item1.Item4, - ScalesetId: arg.Item2.Item1, - Heartbeat: arg.Item2.Item2, - Version: arg.Item2.Item3, - ReimageRequested: arg.Item2.Item4, - DeleteRequested: arg.Item2.Item5, - DebugKeepNode: arg.Item2.Item6); + ScalesetId: ScalesetId.Parse(scalesetId.ToString()), + Heartbeat: arg.Item2.Item1, + Version: arg.Item2.Item2, + ReimageRequested: arg.Item2.Item3, + DeleteRequested: arg.Item2.Item4, + DebugKeepNode: arg.Item2.Item5); public static Gen ProxyForward { get; } = from region in RegionGen @@ -124,7 +130,7 @@ namespace Tests { select new ProxyForward( Region: region, Port: port, - ScalesetId: scalesetId, + ScalesetId: ScalesetId.Parse(scalesetId.ToString()), MachineId: machineId, ProxyId: proxyId, DstPort: dstPort, @@ -241,18 +247,19 @@ namespace Tests { public static Gen Scaleset { get; } = from arg in Arb.Generate, + Tuple, Tuple, Tuple>>>() + from scalesetId in Arb.Generate() from poolName in PoolNameGen from region in RegionGen from image in ImageReferenceGen select new Scaleset( PoolName: poolName, - ScalesetId: arg.Item1.Item1, - State: arg.Item1.Item2, - Auth: arg.Item1.Item3, - VmSku: arg.Item1.Item4, + ScalesetId: ScalesetId.Parse(scalesetId.ToString()), + State: arg.Item1.Item1, + Auth: arg.Item1.Item2, + VmSku: arg.Item1.Item3, Image: image, Region: region, @@ -511,6 +518,7 @@ namespace Tests { public class OrmArb { public static Arbitrary PoolName { get; } = OrmGenerators.PoolNameGen.ToArbitrary(); + public static Arbitrary ScalesetId { get; } = OrmGenerators.ScalesetIdGen.ToArbitrary(); public static Arbitrary> ReadOnlyList() => Arb.Default.List().Convert(x => (IReadOnlyList)x, x => (List)x); diff --git a/src/ApiService/Tests/OrmTest.cs b/src/ApiService/Tests/OrmTest.cs index 656acf409..21aa65570 100644 --- a/src/ApiService/Tests/OrmTest.cs +++ b/src/ApiService/Tests/OrmTest.cs @@ -236,7 +236,9 @@ namespace Tests { [Fact] public void TestEventSerialization() { - var expectedEvent = new EventMessage(Guid.NewGuid(), EventType.NodeHeartbeat, new EventNodeHeartbeat(Guid.NewGuid(), Guid.NewGuid(), PoolName.Parse("test-Poool"), NodeState.Busy), Guid.NewGuid(), "test", DateTime.UtcNow); + var scalesetId = ScalesetId.Parse(Guid.NewGuid().ToString()); + var hb = new EventNodeHeartbeat(Guid.NewGuid(), scalesetId, PoolName.Parse("test-Poool"), NodeState.Busy); + var expectedEvent = new EventMessage(Guid.NewGuid(), EventType.NodeHeartbeat, hb, Guid.NewGuid(), "test", DateTime.UtcNow); var serialized = JsonSerializer.Serialize(expectedEvent, EntityConverter.GetJsonSerializerOptions()); var actualEvent = JsonSerializer.Deserialize((string)serialized, EntityConverter.GetJsonSerializerOptions()); Assert.Equal(expectedEvent, actualEvent); diff --git a/src/ApiService/Tests/QueryTest.cs b/src/ApiService/Tests/QueryTest.cs index 23443be38..bc32a59cc 100644 --- a/src/ApiService/Tests/QueryTest.cs +++ b/src/ApiService/Tests/QueryTest.cs @@ -17,7 +17,7 @@ namespace Tests { var query2 = NodeOperations.SearchStatesQuery(poolId: Guid.Parse("3b0426d3-9bde-4ae8-89ac-4edf0d3b3618")); Assert.Equal("((pool_id eq '3b0426d3-9bde-4ae8-89ac-4edf0d3b3618'))", query2); - var query3 = NodeOperations.SearchStatesQuery(scaleSetId: Guid.Parse("4c96dd6b-9bdb-4758-9720-1010c244fa4b")); + var query3 = NodeOperations.SearchStatesQuery(scaleSetId: ScalesetId.Parse("4c96dd6b-9bdb-4758-9720-1010c244fa4b")); Assert.Equal("((scaleset_id eq '4c96dd6b-9bdb-4758-9720-1010c244fa4b'))", query3); var query4 = NodeOperations.SearchStatesQuery(states: new[] { NodeState.Free, NodeState.Done, NodeState.Ready }); @@ -25,7 +25,7 @@ namespace Tests { var query7 = NodeOperations.SearchStatesQuery( poolId: Guid.Parse("3b0426d3-9bde-4ae8-89ac-4edf0d3b3618"), - scaleSetId: Guid.Parse("4c96dd6b-9bdb-4758-9720-1010c244fa4b"), + scaleSetId: ScalesetId.Parse("4c96dd6b-9bdb-4758-9720-1010c244fa4b"), states: new[] { NodeState.Free, NodeState.Done, NodeState.Ready }); Assert.Equal("((pool_id eq '3b0426d3-9bde-4ae8-89ac-4edf0d3b3618')) and ((scaleset_id eq '4c96dd6b-9bdb-4758-9720-1010c244fa4b')) and (((state eq 'free') or (state eq 'done') or (state eq 'ready')))", query7); } @@ -33,7 +33,7 @@ namespace Tests { [Fact] public void QueryFilterTest() { - var scalesetId = Guid.Parse("3b0426d3-9bde-4ae8-89ac-4edf0d3b3618"); + var scalesetId = ScalesetId.Parse(Guid.Parse("3b0426d3-9bde-4ae8-89ac-4edf0d3b3618").ToString()); var proxyId = Guid.Parse("4c96dd6b-9bdb-4758-9720-1010c244fa4b"); var region = "westus2"; var outdated = false; diff --git a/src/ApiService/Tests/ValidatedStringTests.cs b/src/ApiService/Tests/ValidatedStringTests.cs index c839b1226..fcc0d0a0f 100644 --- a/src/ApiService/Tests/ValidatedStringTests.cs +++ b/src/ApiService/Tests/ValidatedStringTests.cs @@ -1,4 +1,5 @@ -using System.Text.Json; +using System; +using System.Text.Json; using Microsoft.OneFuzz.Service; using Xunit; @@ -42,4 +43,40 @@ public class ValidatedStringTests { public void PoolNames(string name, bool valid) { Assert.Equal(valid, PoolName.IsValid(name)); } + + [Theory] + [InlineData("", false)] + [InlineData("abc", true)] + [InlineData("a-bc", true)] + [InlineData("-abc", false)] + [InlineData("abc-", false)] + [InlineData("ef052a0d-f235-4115-bd47-b359bcc5078b", true)] + public void ScalesetIds(string name, bool valid) { + Assert.Equal(valid, ScalesetId.IsValid(name)); + } + + private static readonly Guid _fixedGuid = Guid.Parse("3b24ba21-1cad-4b07-8655-914754485838"); + + [Fact] + public void ScalesetId_FromBasicPool() { + var pool = PoolName.Parse("pool"); + var id = Scaleset.GenerateNewScalesetIdUsingGuid(pool, _fixedGuid).ToString(); + Assert.Equal("pool-3b24ba211cad4b078655914754485838", id); + } + + [Fact] + public void ScalesetId_FromReallyLongPool() { + var pool = PoolName.Parse(new string('x', 100)); + var id = Scaleset.GenerateNewScalesetIdUsingGuid(pool, _fixedGuid).ToString(); + Assert.Equal(64, id.Length); + Assert.Equal($"xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx-3b24ba211cad4b078655914754485838", id); + } + + [Fact] + public void ScalesetId_FromPoolWithBadCharacters() { + var pool = PoolName.Parse("_.-po-!?(*!&@#$)o_.l-._"); + var id = Scaleset.GenerateNewScalesetIdUsingGuid(pool, _fixedGuid).ToString(); + // hyphens preserved except at start and end, and underscores turned into hyphens + Assert.Equal($"po-o-l-3b24ba211cad4b078655914754485838", id); + } } diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 713d1c14f..0e0a32a91 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1451,11 +1451,10 @@ class Node(Endpoint): self, *, state: Optional[List[enums.NodeState]] = None, - scaleset_id: Optional[UUID_EXPANSION] = None, + scaleset_id: Optional[str] = None, pool_name: Optional[primitives.PoolName] = None, ) -> List[models.Node]: self.logger.debug("list nodes") - scaleset_id_expanded: Optional[UUID] = None if pool_name is not None: pool_name = primitives.PoolName( @@ -1467,18 +1466,11 @@ class Node(Endpoint): ) ) - if scaleset_id is not None: - scaleset_id_expanded = self._disambiguate_uuid( - "scaleset_id", - scaleset_id, - lambda: [str(x.scaleset_id) for x in self.onefuzz.scalesets.list()], - ) - return self._req_model_list( "GET", models.Node, data=requests.NodeSearch( - scaleset_id=scaleset_id_expanded, state=state, pool_name=pool_name + scaleset_id=scaleset_id, state=state, pool_name=pool_name ), ) @@ -1510,7 +1502,7 @@ class Scaleset(Endpoint): def _expand_scaleset_machine( self, - scaleset_id: UUID_EXPANSION, + scaleset_id: str, machine_id: UUID_EXPANSION, *, include_auth: bool = False, @@ -1577,54 +1569,32 @@ class Scaleset(Endpoint): ), ) - def shutdown( - self, scaleset_id: UUID_EXPANSION, *, now: bool = False - ) -> responses.BoolResult: - scaleset_id_expanded = self._disambiguate_uuid( - "scaleset_id", - scaleset_id, - lambda: [str(x.scaleset_id) for x in self.list()], - ) - - self.logger.debug("shutdown scaleset: %s (now: %s)", scaleset_id_expanded, now) + def shutdown(self, scaleset_id: str, *, now: bool = False) -> responses.BoolResult: + self.logger.debug("shutdown scaleset: %s (now: %s)", scaleset_id, now) return self._req_model( "DELETE", responses.BoolResult, - data=requests.ScalesetStop(scaleset_id=scaleset_id_expanded, now=now), + data=requests.ScalesetStop(scaleset_id=scaleset_id, now=now), ) - def get( - self, scaleset_id: UUID_EXPANSION, *, include_auth: bool = False - ) -> models.Scaleset: + def get(self, scaleset_id: str, *, include_auth: bool = False) -> models.Scaleset: self.logger.debug("get scaleset: %s", scaleset_id) - scaleset_id_expanded = self._disambiguate_uuid( - "scaleset_id", - scaleset_id, - lambda: [str(x.scaleset_id) for x in self.list()], - ) - return self._req_model( "GET", models.Scaleset, data=requests.ScalesetSearch( - scaleset_id=scaleset_id_expanded, include_auth=include_auth + scaleset_id=scaleset_id, include_auth=include_auth ), ) def update( - self, scaleset_id: UUID_EXPANSION, *, size: Optional[int] = None + self, scaleset_id: str, *, size: Optional[int] = None ) -> models.Scaleset: self.logger.debug("update scaleset: %s", scaleset_id) - scaleset_id_expanded = self._disambiguate_uuid( - "scaleset_id", - scaleset_id, - lambda: [str(x.scaleset_id) for x in self.list()], - ) - return self._req_model( "PATCH", models.Scaleset, - data=requests.ScalesetUpdate(scaleset_id=scaleset_id_expanded, size=size), + data=requests.ScalesetUpdate(scaleset_id=scaleset_id, size=size), ) def list( @@ -1645,7 +1615,7 @@ class ScalesetProxy(Endpoint): def delete( self, - scaleset_id: UUID_EXPANSION, + scaleset_id: str, machine_id: UUID_EXPANSION, *, dst_port: Optional[int] = None, @@ -1681,7 +1651,7 @@ class ScalesetProxy(Endpoint): ) def get( - self, scaleset_id: UUID_EXPANSION, machine_id: UUID_EXPANSION, dst_port: int + self, scaleset_id: str, machine_id: UUID_EXPANSION, dst_port: int ) -> responses.ProxyGetResult: """Get information about a specific job""" ( @@ -1705,7 +1675,7 @@ class ScalesetProxy(Endpoint): def create( self, - scaleset_id: UUID_EXPANSION, + scaleset_id: str, machine_id: UUID_EXPANSION, dst_port: int, *, diff --git a/src/cli/onefuzz/debug.py b/src/cli/onefuzz/debug.py index 4562d2d2e..534cbbd42 100644 --- a/src/cli/onefuzz/debug.py +++ b/src/cli/onefuzz/debug.py @@ -103,7 +103,7 @@ class DebugScaleset(Command): """Debug tasks""" def _get_proxy_setup( - self, scaleset_id: UUID, machine_id: UUID, port: int, duration: Optional[int] + self, scaleset_id: str, machine_id: UUID, port: int, duration: Optional[int] ) -> Tuple[bool, str, Optional[Tuple[str, int]]]: proxy = self.onefuzz.scaleset_proxy.create( scaleset_id, machine_id, port, duration=duration @@ -115,7 +115,7 @@ class DebugScaleset(Command): def rdp( self, - scaleset_id: UUID_EXPANSION, + scaleset_id: str, machine_id: UUID_EXPANSION, duration: Optional[int] = 1, ) -> None: @@ -144,7 +144,7 @@ class DebugScaleset(Command): def ssh( self, - scaleset_id: UUID_EXPANSION, + scaleset_id: str, machine_id: UUID_EXPANSION, duration: Optional[int] = 1, command: Optional[str] = None, @@ -185,7 +185,7 @@ class DebugTask(Command): def _get_node( self, task_id: UUID_EXPANSION, node_id: Optional[UUID] - ) -> Tuple[UUID, UUID]: + ) -> Tuple[str, UUID]: nodes = self.list_nodes(task_id) if not nodes: raise Exception("task is not currently executing on nodes") diff --git a/src/pytypes/extra/generate-docs.py b/src/pytypes/extra/generate-docs.py index 34c908a1b..799cd042d 100755 --- a/src/pytypes/extra/generate-docs.py +++ b/src/pytypes/extra/generate-docs.py @@ -186,7 +186,7 @@ def main() -> None: ), EventPoolDeleted(pool_name=PoolName("example")), EventScalesetCreated( - scaleset_id=UUID(int=0), + scaleset_id="example-000", pool_name=PoolName("example"), vm_sku="Standard_D2s_v3", image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", @@ -194,20 +194,20 @@ def main() -> None: size=10, ), EventScalesetFailed( - scaleset_id=UUID(int=0), + scaleset_id="example-000", pool_name=PoolName("example"), error=Error( code=ErrorCode.UNABLE_TO_RESIZE, errors=["example error message"] ), ), - EventScalesetDeleted(scaleset_id=UUID(int=0), pool_name=PoolName("example")), + EventScalesetDeleted(scaleset_id="example-000", pool_name=PoolName("example")), EventScalesetStateUpdated( - scaleset_id=UUID(int=0), + scaleset_id="example-000", pool_name=PoolName("example"), state=ScalesetState.init, ), EventScalesetResizeScheduled( - scaleset_id=UUID(int=0), pool_name=PoolName("example"), size=0 + scaleset_id="example-000", pool_name=PoolName("example"), size=0 ), EventJobCreated( job_id=UUID(int=0), diff --git a/src/pytypes/onefuzztypes/events.py b/src/pytypes/onefuzztypes/events.py index bee02ca90..6efdb21d8 100644 --- a/src/pytypes/onefuzztypes/events.py +++ b/src/pytypes/onefuzztypes/events.py @@ -98,7 +98,7 @@ class EventPing(BaseEvent, BaseResponse): class EventScalesetCreated(BaseEvent): - scaleset_id: UUID + scaleset_id: str pool_name: PoolName vm_sku: str image: str @@ -107,18 +107,18 @@ class EventScalesetCreated(BaseEvent): class EventScalesetFailed(BaseEvent): - scaleset_id: UUID + scaleset_id: str pool_name: PoolName error: Error class EventScalesetDeleted(BaseEvent): - scaleset_id: UUID + scaleset_id: str pool_name: PoolName class EventScalesetResizeScheduled(BaseEvent): - scaleset_id: UUID + scaleset_id: str pool_name: PoolName size: int @@ -159,32 +159,32 @@ class EventProxyStateUpdated(BaseEvent): class EventNodeCreated(BaseEvent): machine_id: UUID - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] pool_name: PoolName class EventNodeHeartbeat(BaseEvent): machine_id: UUID - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] pool_name: PoolName machine_state: Optional[NodeState] class EventNodeDeleted(BaseEvent): machine_id: UUID - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] pool_name: PoolName class EventScalesetStateUpdated(BaseEvent): - scaleset_id: UUID + scaleset_id: str pool_name: PoolName state: ScalesetState class EventNodeStateUpdated(BaseEvent): machine_id: UUID - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] pool_name: PoolName state: NodeState diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 9f9cb9fbe..cc246d42a 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -604,7 +604,7 @@ class Node(BaseModel): pool_id: Optional[UUID] machine_id: UUID state: NodeState = Field(default=NodeState.init) - scaleset_id: Optional[UUID] = None + scaleset_id: Optional[str] = None tasks: Optional[List[NodeTasks]] = None messages: Optional[List[NodeCommand]] = None heartbeat: Optional[datetime] @@ -615,7 +615,7 @@ class Node(BaseModel): class ScalesetSummary(BaseModel): - scaleset_id: UUID + scaleset_id: str state: ScalesetState @@ -668,7 +668,7 @@ class ScalesetNodeState(BaseModel): class Scaleset(BaseModel): timestamp: Optional[datetime] = Field(alias="Timestamp") pool_name: PoolName - scaleset_id: UUID = Field(default_factory=uuid4) + scaleset_id: str state: ScalesetState = Field(default=ScalesetState.init) auth: Optional[Authentication] vm_sku: str @@ -686,7 +686,7 @@ class Scaleset(BaseModel): class AutoScale(BaseModel): - scaleset_id: UUID + scaleset_id: str min: int = Field(ge=0) max: int = Field(ge=1) default: int = Field(ge=0) @@ -812,7 +812,7 @@ class TaskEventSummary(BaseModel): class NodeAssignment(BaseModel): node_id: UUID - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] state: NodeTaskState diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index 7ce73b27f..f72991feb 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -91,7 +91,7 @@ class AgentRegistrationGet(BaseRequest): class AgentRegistrationPost(BaseRequest): pool_name: PoolName - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] machine_id: UUID version: str = Field(default="1.0.0") @@ -122,7 +122,7 @@ class PoolStop(BaseRequest): class ProxyGet(BaseRequest): - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] machine_id: Optional[UUID] dst_port: Optional[int] @@ -139,14 +139,14 @@ class ProxyGet(BaseRequest): class ProxyCreate(BaseRequest): - scaleset_id: UUID + scaleset_id: str machine_id: UUID dst_port: int duration: int = Field(ge=ONE_HOUR, le=SEVEN_DAYS) class ProxyDelete(BaseRequest): - scaleset_id: UUID + scaleset_id: str machine_id: UUID dst_port: Optional[int] @@ -154,7 +154,7 @@ class ProxyDelete(BaseRequest): class NodeSearch(BaseRequest): machine_id: Optional[UUID] state: Optional[List[NodeState]] - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] pool_name: Optional[PoolName] @@ -168,18 +168,18 @@ class NodeUpdate(BaseRequest): class ScalesetSearch(BaseRequest): - scaleset_id: Optional[UUID] + scaleset_id: Optional[str] state: Optional[List[ScalesetState]] include_auth: bool = Field(default=False) class ScalesetStop(BaseRequest): - scaleset_id: UUID + scaleset_id: str now: bool class ScalesetUpdate(BaseRequest): - scaleset_id: UUID + scaleset_id: str size: Optional[int] = Field(ge=1) diff --git a/src/pytypes/tests/test_models.py b/src/pytypes/tests/test_models.py index 2b6eaa734..04902f680 100755 --- a/src/pytypes/tests/test_models.py +++ b/src/pytypes/tests/test_models.py @@ -54,6 +54,7 @@ class TestScaleset(unittest.TestCase): def test_scaleset_size(self) -> None: with self.assertRaises(ValueError): Scaleset( + scaleset_id="test-pool-000", pool_name=PoolName("test-pool"), vm_sku="Standard_D2ds_v4", image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", @@ -63,6 +64,7 @@ class TestScaleset(unittest.TestCase): ) scaleset = Scaleset( + scaleset_id="test-pool-000", pool_name=PoolName("test-pool"), vm_sku="Standard_D2ds_v4", image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest", @@ -73,6 +75,7 @@ class TestScaleset(unittest.TestCase): self.assertEqual(scaleset.size, 0) scaleset = Scaleset( + scaleset_id="test-pool-000", pool_name=PoolName("test-pool"), vm_sku="Standard_D2ds_v4", image="Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest",