diff --git a/src/ApiService/ApiService/Functions/Containers.cs b/src/ApiService/ApiService/Functions/Containers.cs index dd04b4106..fcb1627c7 100644 --- a/src/ApiService/ApiService/Functions/Containers.cs +++ b/src/ApiService/ApiService/Functions/Containers.cs @@ -42,7 +42,7 @@ public class ContainersFunction { new Error( Code: ErrorCode.INVALID_REQUEST, Errors: new[] { "invalid container" }), - context: get.Name.ContainerName); + context: get.Name.String); } var metadata = (await container.GetPropertiesAsync()).Value.Metadata; @@ -63,7 +63,7 @@ public class ContainersFunction { // otherwise list all containers var containers = await _context.Containers.GetContainers(StorageType.Corpus); - var result = containers.Select(c => new ContainerInfoBase(new Container(c.Key), c.Value)); + var result = containers.Select(c => new ContainerInfoBase(c.Key, c.Value)); return await RequestHandling.Ok(req, result); } @@ -104,7 +104,7 @@ public class ContainersFunction { new Error( Code: ErrorCode.INVALID_REQUEST, Errors: new[] { "invalid container" }), - context: post.Name.ContainerName); + context: post.Name.String); } return await RequestHandling.Ok( diff --git a/src/ApiService/ApiService/Functions/Download.cs b/src/ApiService/ApiService/Functions/Download.cs index a61635555..65b3fbcf7 100644 --- a/src/ApiService/ApiService/Functions/Download.cs +++ b/src/ApiService/ApiService/Functions/Download.cs @@ -21,13 +21,13 @@ public class Download { private async Async.Task Get(HttpRequestData req) { var query = HttpUtility.ParseQueryString(req.Url.Query); - var container = query["container"]; - if (container is null) { + var queryContainer = query["container"]; + if (queryContainer is null || !Container.TryParse(queryContainer, out var container)) { return await _context.RequestHandling.NotOk( req, new Error( ErrorCode.INVALID_REQUEST, - new string[] { "'container' query parameter must be provided" }), + new string[] { "'container' query parameter must be provided and valid" }), "download"); } @@ -42,7 +42,7 @@ public class Download { } var sasUri = await _context.Containers.GetFileSasUrl( - new Container(container), + container, filename, StorageType.Corpus, BlobSasPermissions.Read, diff --git a/src/ApiService/ApiService/Functions/Jobs.cs b/src/ApiService/ApiService/Functions/Jobs.cs index dd8e61e2b..5ede97f73 100644 --- a/src/ApiService/ApiService/Functions/Jobs.cs +++ b/src/ApiService/ApiService/Functions/Jobs.cs @@ -52,7 +52,7 @@ public class Jobs { var metadata = new Dictionary{ { "container_type", "logs" }, // TODO: use ContainerType.Logs enum somehow; needs snake case name }; - var containerName = new Container($"logs-{job.JobId}"); + var containerName = Container.Parse($"logs-{job.JobId}"); var containerSas = await _context.Containers.CreateContainer(containerName, StorageType.Corpus, metadata); if (containerSas is null) { return await _context.RequestHandling.NotOk( diff --git a/src/ApiService/ApiService/Functions/Notifications.cs b/src/ApiService/ApiService/Functions/Notifications.cs index 2cb80f675..61c547936 100644 --- a/src/ApiService/ApiService/Functions/Notifications.cs +++ b/src/ApiService/ApiService/Functions/Notifications.cs @@ -22,8 +22,9 @@ public class Notifications { return await _context.RequestHandling.NotOk(req, request.ErrorV, "notification search"); } - var entries = request.OkV switch { { Container: null } => _context.NotificationOperations.SearchAll(), { Container: var c } => _context.NotificationOperations.SearchByRowKeys(c.Select(x => x.ContainerName)) + var entries = request.OkV switch { { Container: null } => _context.NotificationOperations.SearchAll(), { Container: var c } => _context.NotificationOperations.SearchByRowKeys(c.Select(x => x.String)) }; + var response = req.CreateResponse(HttpStatusCode.OK); await response.WriteAsJsonAsync(entries); return response; diff --git a/src/ApiService/ApiService/Functions/QueueFileChanges.cs b/src/ApiService/ApiService/Functions/QueueFileChanges.cs index 88a5129ca..22832d3a9 100644 --- a/src/ApiService/ApiService/Functions/QueueFileChanges.cs +++ b/src/ApiService/ApiService/Functions/QueueFileChanges.cs @@ -56,6 +56,6 @@ public class QueueFileChanges { var path = string.Join('/', parts.Skip(1)); log.Info($"file added container: {container} - path: {path}"); - await _notificationOperations.NewFiles(new Container(container), path, failTaskOnTransientError); + await _notificationOperations.NewFiles(Container.Parse(container), path, failTaskOnTransientError); } } diff --git a/src/ApiService/ApiService/Functions/Scaleset.cs b/src/ApiService/ApiService/Functions/Scaleset.cs index 1055f30ae..5a9571879 100644 --- a/src/ApiService/ApiService/Functions/Scaleset.cs +++ b/src/ApiService/ApiService/Functions/Scaleset.cs @@ -75,7 +75,7 @@ public class Scaleset { context: "ScalesetCreate"); } - string region; + Region region; if (create.Region is null) { region = await _context.Creds.GetBaseRegion(); } else { diff --git a/src/ApiService/ApiService/Functions/TimerProxy.cs b/src/ApiService/ApiService/Functions/TimerProxy.cs index e5e89ba2f..82eaaecd4 100644 --- a/src/ApiService/ApiService/Functions/TimerProxy.cs +++ b/src/ApiService/ApiService/Functions/TimerProxy.cs @@ -62,15 +62,16 @@ public class TimerProxy { // nsg enabled OneFuzz this will overwrite existing NSG // assignment though. This behavior is acceptable at this point // since we do not support bring your own NSG + var nsgName = Nsg.NameFromRegion(region); - if (await nsgOpertions.GetNsg(region) != null) { + if (await nsgOpertions.GetNsg(nsgName) != null) { var network = await Network.Init(region, _context); var subnet = await network.GetSubnet(); if (subnet != null) { var vnet = await network.GetVnet(); if (vnet != null) { - var result = await nsgOpertions.AssociateSubnet(region, vnet, subnet); + var result = await nsgOpertions.AssociateSubnet(nsgName, vnet, subnet); if (!result.OkV) { _logger.Error($"Failed to associate NSG and subnet due to {result.ErrorV} in region {region}"); } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Events.cs b/src/ApiService/ApiService/OneFuzzTypes/Events.cs index 6afd76537..4257a7baa 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Events.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Events.cs @@ -2,7 +2,6 @@ using System.Text.Json; using System.Text.Json.Serialization; using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; -using Region = System.String; namespace Microsoft.OneFuzz.Service; diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 850aa1ddd..fae2e51d6 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -5,7 +5,6 @@ using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; using Endpoint = System.String; using GroupId = System.Guid; using PrincipalId = System.Guid; -using Region = System.String; namespace Microsoft.OneFuzz.Service; @@ -406,25 +405,6 @@ public record Scaleset( // 'Nodes' removed when porting from Python: only used in search response ) : StatefulEntityBase(State); -[JsonConverter(typeof(ContainerConverter))] -public record Container(string ContainerName) { - public string ContainerName { get; } = ContainerName.All(c => char.IsLetterOrDigit(c) || c == '-') ? ContainerName : throw new ArgumentException("Container name must have only numbers, letters or dashes"); - public override string ToString() { - return ContainerName; - } -} - -public class ContainerConverter : JsonConverter { - public override Container? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { - var containerName = reader.GetString(); - return containerName == null ? null : new Container(containerName); - } - - public override void Write(Utf8JsonWriter writer, Container value, JsonSerializerOptions options) { - writer.WriteStringValue(value.ContainerName); - } -} - public record Notification( [PartitionKey] Guid NotificationId, [RowKey] Container Container, @@ -732,7 +712,14 @@ public record Job( public UserInfo? UserInfo { get; set; } } -public record Nsg(string Name, Region Region); +public record Nsg(string Name, Region Region) { + public static Nsg ForRegion(Region region) + => new(NameFromRegion(region), region); + + // Currently, the name of a NSG is the same as the region it is in. + public static string NameFromRegion(Region region) + => region.String; +}; public record WorkUnit( Guid JobId, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index d37c5e1de..adccbfa9a 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -191,7 +191,7 @@ public record ScalesetCreate( [property: Required] PoolName PoolName, [property: Required] string VmSku, [property: Required] string Image, - string? Region, + Region? Region, [property: Range(1, long.MaxValue), Required] long Size, [property: Required] bool SpotInstances, [property: Required] Dictionary Tags, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs index 82eb087f0..f9a6dfb5d 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Responses.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Responses.cs @@ -56,7 +56,7 @@ public record BoolResult( public record InfoResponse( string ResourceGroup, - string Region, + Region Region, string Subscription, IReadOnlyDictionary Versions, Guid? InstanceId, @@ -127,7 +127,7 @@ public record ScalesetResponse( Authentication? Auth, string VmSku, string Image, - string Region, + Region Region, long Size, bool? SpotInstances, bool EmphemeralOsDisks, @@ -175,7 +175,7 @@ public record ProxyGetResult( ); public record ProxyInfo( - string Region, + Region Region, Guid ProxyId, VmState State ); diff --git a/src/ApiService/ApiService/OneFuzzTypes/Validated.cs b/src/ApiService/ApiService/OneFuzzTypes/Validated.cs index 61bf2a82c..ddab03bb5 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Validated.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Validated.cs @@ -1,7 +1,9 @@ -using System.Diagnostics.CodeAnalysis; +using System.Diagnostics; +using System.Diagnostics.CodeAnalysis; using System.Text.Json; using System.Text.Json.Serialization; using System.Text.RegularExpressions; +using Azure.Core; namespace Microsoft.OneFuzz.Service; @@ -11,6 +13,10 @@ static class Check { private static readonly Regex _isAlnumDash = new(@"\A[a-zA-Z0-9\-]+\z", RegexOptions.Compiled); public static bool IsAlnumDash(string input) => _isAlnumDash.IsMatch(input); + + // Permits 1-64 characters: alphanumeric, underscore, or dash. + private static readonly Regex _isNameLike = new(@"\A[_a-zA-Z0-9\-]{1,64}\z", RegexOptions.Compiled); + public static bool IsNameLike(string input) => _isNameLike.IsMatch(input); } // Base class for types that are wrappers around a validated string. @@ -47,9 +53,11 @@ public abstract class ValidatedStringConverter : JsonConverter where T : V } [JsonConverter(typeof(Converter))] -public record PoolName : ValidatedString { - public PoolName(string value) : base(value) { - // Debug.Assert(Check.IsAlnumDash(value)); +public sealed record PoolName : ValidatedString { + private static bool IsValid(string input) => Check.IsNameLike(input); + + private PoolName(string value) : base(value) { + Debug.Assert(IsValid(value)); } public static PoolName Parse(string input) { @@ -61,14 +69,10 @@ public record PoolName : ValidatedString { } public static bool TryParse(string input, [NotNullWhen(returnValue: true)] out PoolName? result) { - - // bypassing the validation because this code has a stricter validation than the python equivalent - // see (issue #2080) - - // if (!Check.IsAlnumDash(input)) { - // result = default; - // return false; - // } + if (!IsValid(input)) { + result = default; + return false; + } result = new PoolName(input); return true; @@ -80,12 +84,12 @@ public record PoolName : ValidatedString { } } -/* TODO: to be enabled in a separate PR - [JsonConverter(typeof(Converter))] public record Region : ValidatedString { - private Region(string value) : base(value) { - Debug.Assert(Check.IsAlnum(value)); + private static bool IsValid(string input) => Check.IsAlnum(input); + + private Region(string value) : base(value.ToLowerInvariant()) { + Debug.Assert(IsValid(value)); } public static Region Parse(string input) { @@ -93,11 +97,11 @@ public record Region : ValidatedString { return result; } - throw new ArgumentException("Region name must have only numbers, letters or dashes"); + throw new ArgumentException("Region name must have only numbers or letters"); } public static bool TryParse(string input, [NotNullWhen(returnValue: true)] out Region? result) { - if (!Check.IsAlnum(input)) { + if (!IsValid(input)) { result = default; return false; } @@ -106,6 +110,9 @@ public record Region : ValidatedString { return true; } + public static implicit operator AzureLocation(Region me) => new(me.String); + public static implicit operator Region(AzureLocation it) => new(it.Name); + public sealed class Converter : ValidatedStringConverter { protected override bool TryParse(string input, out Region? output) => Region.TryParse(input, out output); @@ -114,8 +121,16 @@ public record Region : ValidatedString { [JsonConverter(typeof(Converter))] public record Container : ValidatedString { + // See: https://docs.microsoft.com/en-us/azure/azure-resource-manager/management/resource-name-rules#microsoftstorage + // - 3-63 + // - Lowercase letters, numbers, and hyphens. + // - Start with lowercase letter or number. Can't use consecutive hyphens. + private static readonly Regex _containerRegex = new(@"\A(?!-)(?!.*--)[a-z0-9\-]{3,63}\z", RegexOptions.Compiled); + + private static bool IsValid(string input) => _containerRegex.IsMatch(input); + private Container(string value) : base(value) { - Debug.Assert(Check.IsAlnumDash(value)); + Debug.Assert(IsValid(value)); } public static Container Parse(string input) { @@ -127,7 +142,7 @@ public record Container : ValidatedString { } public static bool TryParse(string input, [NotNullWhen(returnValue: true)] out Container? result) { - if (!Check.IsAlnumDash(input)) { + if (!IsValid(input)) { result = default; return false; } @@ -141,4 +156,3 @@ public record Container : ValidatedString { => Container.TryParse(input, out output); } } -*/ diff --git a/src/ApiService/ApiService/TestHooks/CredsTestHookks.cs b/src/ApiService/ApiService/TestHooks/CredsTestHookks.cs index a0cc8c860..4a5888ebb 100644 --- a/src/ApiService/ApiService/TestHooks/CredsTestHookks.cs +++ b/src/ApiService/ApiService/TestHooks/CredsTestHookks.cs @@ -49,7 +49,7 @@ namespace ApiService.TestHooks { _log.Info("Get base region"); var resp = req.CreateResponse(HttpStatusCode.OK); var region = await _creds.GetBaseRegion(); - await resp.WriteStringAsync(region); + await resp.WriteStringAsync(region.String); return resp; } diff --git a/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs b/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs index 37bc9251a..2a8275258 100644 --- a/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs +++ b/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs @@ -30,7 +30,7 @@ namespace ApiService.TestHooks { var fileName = query["fileName"]; var failTaskOnTransientError = UriExtension.GetBool("failTaskOnTransientError", query, true); - await _notificationOps.NewFiles(new Container(container), fileName, failTaskOnTransientError); + await _notificationOps.NewFiles(Container.Parse(container), fileName, failTaskOnTransientError); var resp = req.CreateResponse(HttpStatusCode.OK); return resp; } @@ -43,7 +43,7 @@ namespace ApiService.TestHooks { var query = UriExtension.GetQueryComponents(req.Url); var container = query["container"]; - var notifications = _notificationOps.GetNotifications(new Container(container)); + var notifications = _notificationOps.GetNotifications(Container.Parse(container)); var json = JsonSerializer.Serialize(await notifications.ToListAsync(), EntityConverter.GetJsonSerializerOptions()); var resp = req.CreateResponse(HttpStatusCode.OK); diff --git a/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs b/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs index 8a6752eec..366bccec1 100644 --- a/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs +++ b/src/ApiService/ApiService/TestHooks/ProxyForwardTestHooks.cs @@ -28,7 +28,7 @@ namespace ApiService.TestHooks { var poolRes = _proxyForward.SearchForward( UriExtension.GetGuid("scaleSetId", query), - UriExtension.GetString("region", query), + UriExtension.GetString("region", query) is string region ? Region.Parse(region) : null, UriExtension.GetGuid("machineId", query), UriExtension.GetGuid("proxyId", query), UriExtension.GetInt("dstPort", query)); diff --git a/src/ApiService/ApiService/onefuzzlib/AutoScale.cs b/src/ApiService/ApiService/onefuzzlib/AutoScale.cs index c5f5f956c..e8af3d08f 100644 --- a/src/ApiService/ApiService/onefuzzlib/AutoScale.cs +++ b/src/ApiService/ApiService/onefuzzlib/AutoScale.cs @@ -105,7 +105,7 @@ public class AutoScaleOperations : Orm, IAutoScaleOperations { return OneFuzzResultVoid.Ok; } - private async Async.Task> CreateAutoScaleResourceFor(Guid resourceId, string location, AutoscaleProfile profile) { + private async Async.Task> CreateAutoScaleResourceFor(Guid resourceId, Region location, AutoscaleProfile profile) { _logTracer.Info($"Creating auto-scale resource for: {resourceId}"); var resourceGroup = _context.Creds.GetBaseResourceGroup(); diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs index 11d8bf830..d376ae3fd 100644 --- a/src/ApiService/ApiService/onefuzzlib/Config.cs +++ b/src/ApiService/ApiService/onefuzzlib/Config.cs @@ -458,21 +458,24 @@ public class Config : IConfig { return ResultVoid.Ok(); } - var exist = new HashSet(); + var exist = new HashSet(); var containers = new Dictionary>(); foreach (var container in config.Containers) { - if (exist.Contains(container.Name.ContainerName)) { + if (exist.Contains(container.Name)) { continue; } + if (await _containers.FindContainer(container.Name, StorageType.Corpus) == null) { return ResultVoid.Error(new TaskConfigError($"missing container: {container.Name}")); } - exist.Add(container.Name.ContainerName); + + exist.Add(container.Name); if (!containers.ContainsKey(container.Type)) { containers.Add(container.Type, new List()); } + containers[container.Type].Add(container.Name); } diff --git a/src/ApiService/ApiService/onefuzzlib/Containers.cs b/src/ApiService/ApiService/onefuzzlib/Containers.cs index e731f50dc..d2a99689a 100644 --- a/src/ApiService/ApiService/onefuzzlib/Containers.cs +++ b/src/ApiService/ApiService/onefuzzlib/Containers.cs @@ -28,7 +28,7 @@ public interface IContainers { public Async.Task BlobExists(Container container, string name, StorageType storageType); public Async.Task AddContainerSasUrl(Uri uri, TimeSpan? duration = null); - public Async.Task>> GetContainers(StorageType corpus); + public Async.Task>> GetContainers(StorageType corpus); } public class Containers : IContainers { @@ -36,7 +36,7 @@ public class Containers : IContainers { private readonly IStorage _storage; private readonly IServiceConfig _config; - static TimeSpan CONTAINER_SAS_DEFAULT_DURATION = TimeSpan.FromDays(30); + static readonly TimeSpan CONTAINER_SAS_DEFAULT_DURATION = TimeSpan.FromDays(30); public Containers(ILogTracer log, IStorage storage, IServiceConfig config) { _log = log; @@ -44,7 +44,7 @@ public class Containers : IContainers { _config = config; _getInstanceId = new Lazy>(async () => { - var blob = await GetBlob(new Container("base-config"), "instance_id", StorageType.Config); + var blob = await GetBlob(WellKnownContainers.BaseConfig, "instance_id", StorageType.Config); if (blob == null) { throw new Exception("Blob Not Found"); } @@ -99,14 +99,14 @@ public class Containers : IContainers { var account = _storage.ChooseAccount(storageType); var client = await _storage.GetBlobServiceClientForAccount(account); - var containerName = _config.OneFuzzStoragePrefix + container.ContainerName; + var containerName = _config.OneFuzzStoragePrefix + container; var cc = client.GetBlobContainerClient(containerName); try { await cc.CreateAsync(metadata: metadata); } catch (RequestFailedException ex) when (ex.ErrorCode == "ContainerAlreadyExists") { // note: resource exists error happens during creation if the container // is being deleted - _log.Error($"unable to create container. account: {account} container: {container.ContainerName} metadata: {metadata} - {ex.Message}"); + _log.Error($"unable to create container. account: {account} container: {container} metadata: {metadata} - {ex.Message}"); return null; } @@ -123,7 +123,7 @@ public class Containers : IContainers { // # Secondary accounts, if they exist, are preferred for containers and have // # increased IOP rates, this should be a slight optimization - var containerName = _config.OneFuzzStoragePrefix + container.ContainerName; + var containerName = _config.OneFuzzStoragePrefix + container; foreach (var account in _storage.GetAccounts(storageType).Reverse()) { var accountClient = await _storage.GetBlobServiceClientForAccount(account); @@ -137,7 +137,7 @@ public class Containers : IContainers { } public async Async.Task GetFileSasUrl(Container container, string name, StorageType storageType, BlobSasPermissions permissions, TimeSpan? duration = null) { - var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}"); + var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container} - {storageType}"); var blobClient = client.GetBlobClient(name); var timeWindow = SasTimeWindow(duration ?? TimeSpan.FromDays(30)); return _storage.GenerateBlobSasUri(permissions, blobClient, timeWindow); @@ -160,7 +160,7 @@ public class Containers : IContainers { } public async Async.Task SaveBlob(Container container, string name, string data, StorageType storageType) { - var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}"); + var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container} - {storageType}"); await client.GetBlobClient(name).UploadAsync(new BinaryData(data), overwrite: true); } @@ -192,23 +192,25 @@ public class Containers : IContainers { } public async Task GetContainerSasUrl(Container container, StorageType storageType, BlobContainerSasPermissions permissions, TimeSpan? duration = null) { - var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}"); + var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container} - {storageType}"); var timeWindow = SasTimeWindow(duration ?? CONTAINER_SAS_DEFAULT_DURATION); return _storage.GenerateBlobContainerSasUri(permissions, client, timeWindow); } public async Async.Task BlobExists(Container container, string name, StorageType storageType) { - var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container.ContainerName} - {storageType}"); + var client = await FindContainer(container, storageType) ?? throw new Exception($"unable to find container: {container} - {storageType}"); return await client.GetBlobClient(name).ExistsAsync(); } - public async Task>> GetContainers(StorageType corpus) { + public async Task>> GetContainers(StorageType corpus) { var accounts = _storage.GetAccounts(corpus); - IEnumerable>>> data = + IEnumerable>>> data = await Async.Task.WhenAll(accounts.Select(async acc => { var service = await _storage.GetBlobServiceClientForAccount(acc); - return await service.GetBlobContainersAsync(BlobContainerTraits.Metadata).Select(container => - KeyValuePair.Create(container.Name, container.Properties.Metadata)).ToListAsync(); + return await service + .GetBlobContainersAsync(BlobContainerTraits.Metadata) + .Select(container => KeyValuePair.Create(Container.Parse(container.Name), container.Properties.Metadata)) + .ToListAsync(); })); return new(data.SelectMany(x => x)); diff --git a/src/ApiService/ApiService/onefuzzlib/Creds.cs b/src/ApiService/ApiService/onefuzzlib/Creds.cs index 44fbd81be..69d82d546 100644 --- a/src/ApiService/ApiService/onefuzzlib/Creds.cs +++ b/src/ApiService/ApiService/onefuzzlib/Creds.cs @@ -25,14 +25,14 @@ public interface ICreds { public SubscriptionResource GetSubscriptionResource(); - public Async.Task GetBaseRegion(); + public Async.Task GetBaseRegion(); + public Async.Task> GetRegions(); public Uri GetInstanceUrl(); public Async.Task GetScalesetPrincipalId(); public GenericResource ParseResourceId(string resourceId); public GenericResource ParseResourceId(ResourceIdentifier resourceId); public Async.Task GetData(GenericResource resource); - Async.Task> GetRegions(); public ResourceIdentifier GetScalesetIdentityResourcePath(); } @@ -95,13 +95,13 @@ public sealed class Creds : ICreds { return ArmClient.GetSubscriptionResource(id); } - public Async.Task GetBaseRegion() { + public Async.Task GetBaseRegion() { return _cache.GetOrCreateAsync(nameof(GetBaseRegion), async _ => { var rg = await ArmClient.GetResourceGroupResource(GetResourceGroupResourceIdentifier()).GetAsync(); if (rg.GetRawResponse().IsError) { throw new Exception($"Failed to get base region due to [{rg.GetRawResponse().Status}] {rg.GetRawResponse().ReasonPhrase}"); } - return rg.Value.Data.Location.Name; + return Region.Parse(rg.Value.Data.Location.Name); }); } @@ -144,8 +144,8 @@ public sealed class Creds : ICreds { return resource; } - public Task> GetRegions() - => _cache.GetOrCreateAsync>( + public Task> GetRegions() + => _cache.GetOrCreateAsync>( nameof(Creds) + "." + nameof(GetRegions), async entry => { // cache for one day @@ -153,7 +153,7 @@ public sealed class Creds : ICreds { var subscriptionId = SubscriptionResource.CreateResourceIdentifier(GetSubscription()); return await ArmClient.GetSubscriptionResource(subscriptionId) .GetLocationsAsync() - .Select(x => x.Name) + .Select(x => Region.Parse(x.Name)) .ToListAsync(); }); diff --git a/src/ApiService/ApiService/onefuzzlib/Extension.cs b/src/ApiService/ApiService/onefuzzlib/Extension.cs index 66836c026..7351094ce 100644 --- a/src/ApiService/ApiService/onefuzzlib/Extension.cs +++ b/src/ApiService/ApiService/onefuzzlib/Extension.cs @@ -12,15 +12,13 @@ public interface IExtensions { Async.Task> FuzzExtensions(Pool pool, Scaleset scaleset); Async.Task> ReproExtensions(AzureLocation region, Os reproOs, Guid reproId, ReproConfig reproConfig, Container? setupContainer); - Task> ProxyManagerExtensions(string region, Guid proxyId); + Task> ProxyManagerExtensions(Region region, Guid proxyId); } public class Extensions : IExtensions { - IOnefuzzContext _context; + private readonly IOnefuzzContext _context; - private static readonly JsonSerializerOptions _extensionSerializerOptions = new JsonSerializerOptions { - PropertyNamingPolicy = JsonNamingPolicy.CamelCase - }; + private static readonly JsonSerializerOptions _extensionSerializerOptions = new() { PropertyNamingPolicy = JsonNamingPolicy.CamelCase }; public Extensions(IOnefuzzContext context) { _context = context; @@ -227,11 +225,12 @@ public class Extensions : IExtensions { var fileName = $"{pool.Name}/config.json"; var configJson = JsonSerializer.Serialize(config, EntityConverter.GetJsonSerializerOptions()); - await _context.Containers.SaveBlob(new Container("vm-scripts"), fileName, configJson, StorageType.Config); - return await ConfigUrl(new Container("vm-scripts"), fileName, false); + await _context.Containers.SaveBlob(WellKnownContainers.VmScripts, fileName, configJson, StorageType.Config); + return await ConfigUrl(WellKnownContainers.VmScripts, fileName, false); } + public async Async.Task BuildScaleSetScript(Pool pool, Scaleset scaleSet) { List commands = new(); var extension = pool.Os == Os.Windows ? "ps1" : "sh"; @@ -244,21 +243,23 @@ public class Extensions : IExtensions { commands.Add($"Set-Content -Path {sshPath} -Value \"{sshKey}\""); } - await _context.Containers.SaveBlob(new Container("vm-scripts"), fileName, string.Join(sep, commands) + sep, StorageType.Config); - return await _context.Containers.GetFileUrl(new Container("vm-scripts"), fileName, StorageType.Config); + await _context.Containers.SaveBlob(WellKnownContainers.VmScripts, fileName, string.Join(sep, commands) + sep, StorageType.Config); + return await _context.Containers.GetFileUrl(WellKnownContainers.VmScripts, fileName, StorageType.Config); } + public async Async.Task UpdateManagedScripts() { - var instanceSpecificSetupSas = await _context.Containers.GetContainerSasUrl(new Container("instance-specific-setup"), StorageType.Config, BlobContainerSasPermissions.List | BlobContainerSasPermissions.Read); - var toolsSas = await _context.Containers.GetContainerSasUrl(new Container("tools"), StorageType.Config, BlobContainerSasPermissions.List | BlobContainerSasPermissions.Read); + var listAndRead = BlobContainerSasPermissions.List | BlobContainerSasPermissions.Read; + var instanceSpecificSetupSas = await _context.Containers.GetContainerSasUrl(WellKnownContainers.InstanceSpecificSetup, StorageType.Config, listAndRead); + var toolsSas = await _context.Containers.GetContainerSasUrl(WellKnownContainers.Tools, StorageType.Config, listAndRead); string[] commands = { $"azcopy sync '{instanceSpecificSetupSas}' instance-specific-setup", $"azcopy sync '{toolsSas}' tools" }; - await _context.Containers.SaveBlob(new Container("vm-scripts"), "managed.ps1", string.Join("\r\n", commands) + "\r\n", StorageType.Config); - await _context.Containers.SaveBlob(new Container("vm-scripts"), "managed.sh", string.Join("\n", commands) + "\n", StorageType.Config); + await _context.Containers.SaveBlob(WellKnownContainers.VmScripts, "managed.ps1", string.Join("\r\n", commands) + "\r\n", StorageType.Config); + await _context.Containers.SaveBlob(WellKnownContainers.VmScripts, "managed.sh", string.Join("\n", commands) + "\n", StorageType.Config); } public async Async.Task AgentConfig(AzureLocation region, Os vmOs, AgentMode mode, List? urls = null, bool withSas = false) { @@ -267,10 +268,10 @@ public class Extensions : IExtensions { var managedIdentity = JsonSerializer.Serialize(new { ManagedIdentity = new Dictionary() }, _extensionSerializerOptions); if (vmOs == Os.Windows) { - var vmScripts = await ConfigUrl(new Container("vm-scripts"), "managed.ps1", withSas) ?? throw new Exception("failed to get VmScripts config url"); - var toolsAzCopy = await ConfigUrl(new Container("tools"), "win64/azcopy.exe", withSas) ?? throw new Exception("failed to get toolsAzCopy config url"); - var toolsSetup = await ConfigUrl(new Container("tools"), "win64/setup.ps1", withSas) ?? throw new Exception("failed to get toolsSetup config url"); - var toolsOneFuzz = await ConfigUrl(new Container("tools"), "win64/onefuzz.ps1", withSas) ?? throw new Exception("failed to get toolsOneFuzz config url"); + var vmScripts = await ConfigUrl(WellKnownContainers.VmScripts, "managed.ps1", withSas) ?? throw new Exception("failed to get VmScripts config url"); + var toolsAzCopy = await ConfigUrl(WellKnownContainers.Tools, "win64/azcopy.exe", withSas) ?? throw new Exception("failed to get toolsAzCopy config url"); + var toolsSetup = await ConfigUrl(WellKnownContainers.Tools, "win64/setup.ps1", withSas) ?? throw new Exception("failed to get toolsSetup config url"); + var toolsOneFuzz = await ConfigUrl(WellKnownContainers.Tools, "win64/onefuzz.ps1", withSas) ?? throw new Exception("failed to get toolsOneFuzz config url"); urlsUpdated.Add(vmScripts); urlsUpdated.Add(toolsAzCopy); @@ -293,9 +294,9 @@ public class Extensions : IExtensions { return extension; } else if (vmOs == Os.Linux) { - var vmScripts = await ConfigUrl(new Container("vm-scripts"), "managed.sh", withSas) ?? throw new Exception("failed to get VmScripts config url"); - var toolsAzCopy = await ConfigUrl(new Container("tools"), "linux/azcopy", withSas) ?? throw new Exception("failed to get toolsAzCopy config url"); - var toolsSetup = await ConfigUrl(new Container("tools"), "linux/setup.sh", withSas) ?? throw new Exception("failed to get toolsSetup config url"); + var vmScripts = await ConfigUrl(WellKnownContainers.VmScripts, "managed.sh", withSas) ?? throw new Exception("failed to get VmScripts config url"); + var toolsAzCopy = await ConfigUrl(WellKnownContainers.Tools, "linux/azcopy", withSas) ?? throw new Exception("failed to get toolsAzCopy config url"); + var toolsSetup = await ConfigUrl(WellKnownContainers.Tools, "linux/setup.sh", withSas) ?? throw new Exception("failed to get toolsSetup config url"); urlsUpdated.Add(vmScripts); urlsUpdated.Add(toolsAzCopy); @@ -423,7 +424,7 @@ public class Extensions : IExtensions { } await _context.Containers.SaveBlob( - new Container("task-configs"), + WellKnownContainers.TaskConfigs, $"{reproId}/{scriptName}", taskScript, StorageType.Config @@ -433,13 +434,13 @@ public class Extensions : IExtensions { urls.AddRange(new List() { await _context.Containers.GetFileSasUrl( - new Container("repro-scripts"), + WellKnownContainers.ReproScripts, reproFile, StorageType.Config, BlobSasPermissions.Read ), await _context.Containers.GetFileSasUrl( - new Container("task-configs"), + WellKnownContainers.TaskConfigs, $"{reproId}/{scriptName}", StorageType.Config, BlobSasPermissions.Read @@ -460,13 +461,18 @@ public class Extensions : IExtensions { return extensionsDict; } - public async Task> ProxyManagerExtensions(string region, Guid proxyId) { - var config = await _context.Containers.GetFileSasUrl(new Container("proxy-configs"), - $"{region}/{proxyId}/config.json", StorageType.Config, BlobSasPermissions.Read); - - var proxyManager = await _context.Containers.GetFileSasUrl(new Container("tools"), - $"linux/onefuzz-proxy-manager", StorageType.Config, BlobSasPermissions.Read); + public async Task> ProxyManagerExtensions(Region region, Guid proxyId) { + var config = await _context.Containers.GetFileSasUrl( + WellKnownContainers.ProxyConfigs, + $"{region}/{proxyId}/config.json", + StorageType.Config, + BlobSasPermissions.Read); + var proxyManager = await _context.Containers.GetFileSasUrl( + WellKnownContainers.Tools, + $"linux/onefuzz-proxy-manager", + StorageType.Config, + BlobSasPermissions.Read); var baseExtension = await AgentConfig(region, Os.Linux, AgentMode.Proxy, new List { config, proxyManager }, true); diff --git a/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs b/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs index 31ec6a13f..688d1bdf6 100644 --- a/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ImageOperations.cs @@ -7,7 +7,7 @@ namespace Microsoft.OneFuzz.Service; public record ImageInfo(string Publisher, string Offer, string Sku, string Version); public interface IImageOperations { - public Async.Task> GetOs(string region, string image); + public Async.Task> GetOs(Region region, string image); public static ImageInfo GetImageInfo(string image) { var imageParts = image.Split(":"); @@ -32,7 +32,7 @@ public class ImageOperations : IImageOperations { _context = context; } - public async Task> GetOs(string region, string image) { + public async Task> GetOs(Region region, string image) { string? name = null; try { var parsed = _context.Creds.ParseResourceId(image); @@ -86,7 +86,7 @@ public class ImageOperations : IImageOperations { if (string.Equals(imageInfo.Version, "latest", StringComparison.Ordinal)) { version = (await subscription.GetVirtualMachineImagesAsync( - region, + region.String, imageInfo.Publisher, imageInfo.Offer, imageInfo.Sku, @@ -97,7 +97,7 @@ public class ImageOperations : IImageOperations { } name = (await subscription.GetVirtualMachineImageAsync( - region, + region.String, imageInfo.Publisher, imageInfo.Offer, imageInfo.Sku diff --git a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs index 25d08ae26..1e342f2c6 100644 --- a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs @@ -12,7 +12,7 @@ namespace Microsoft.OneFuzz.Service; public interface IIpOperations { public Async.Task GetPublicNic(string resourceGroup, string name); - public Async.Task CreatePublicNic(string resourceGroup, string name, string region, Nsg? nsg); + public Async.Task CreatePublicNic(string resourceGroup, string name, Region region, Nsg? nsg); public Async.Task GetPublicIp(ResourceIdentifier resourceId); @@ -26,7 +26,7 @@ public interface IIpOperations { public Async.Task GetScalesetInstanceIp(Guid scalesetId, Guid machineId); - public Async.Task CreateIp(string resourceGroup, string name, string region); + public Async.Task CreateIp(string resourceGroup, string name, Region region); } @@ -120,7 +120,7 @@ public class IpOperations : IIpOperations { } } - public async Task CreatePublicNic(string resourceGroup, string name, string region, Nsg? nsg) { + public async Task CreatePublicNic(string resourceGroup, string name, Region region, Nsg? nsg) { _logTracer.Info($"creating nic for {resourceGroup}:{name} in {region}"); var network = await Network.Init(region, _context); @@ -190,7 +190,7 @@ public class IpOperations : IIpOperations { return OneFuzzResultVoid.Ok; } - public async Async.Task CreateIp(string resourceGroup, string name, string region) { + public async Async.Task CreateIp(string resourceGroup, string name, Region region) { var ipParams = new PublicIPAddressData() { Location = region, PublicIPAllocationMethod = NetworkIPAllocationMethod.Dynamic @@ -259,5 +259,3 @@ public class IpOperations : IIpOperations { } } } - - diff --git a/src/ApiService/ApiService/onefuzzlib/Network.cs b/src/ApiService/ApiService/onefuzzlib/Network.cs index f170fc38b..a3a162f22 100644 --- a/src/ApiService/ApiService/onefuzzlib/Network.cs +++ b/src/ApiService/ApiService/onefuzzlib/Network.cs @@ -6,7 +6,7 @@ namespace Microsoft.OneFuzz.Service; public class Network { private readonly string _name; private readonly string _group; - private readonly string _region; + private readonly Region _region; private readonly IOnefuzzContext _context; private readonly NetworkConfig _networkConfig; @@ -14,7 +14,7 @@ public class Network { // This was generated randomly and should be preserved moving forwards static Guid NETWORK_GUID_NAMESPACE = Guid.Parse("372977ad-b533-416a-b1b4-f770898e0b11"); - public Network(string region, string group, string name, IOnefuzzContext context, NetworkConfig networkConfig) { + public Network(Region region, string group, string name, IOnefuzzContext context, NetworkConfig networkConfig) { _region = region; _group = group; _name = name; @@ -22,7 +22,7 @@ public class Network { _networkConfig = networkConfig; } - public static async Async.Task Init(string region, IOnefuzzContext context) { + public static async Async.Task Init(Region region, IOnefuzzContext context) { var group = context.Creds.GetBaseResourceGroup(); var instanceConfig = await context.ConfigOperations.Fetch(); var networkConfig = instanceConfig.NetworkConfig; @@ -33,16 +33,14 @@ public class Network { // configs. string name; - if (networkConfig.AddressSpace == NetworkConfig.Default.AddressSpace && networkConfig.Subnet == NetworkConfig.Default.Subnet) { - name = region; + name = region.String; } else { //TODO: Remove dependency on "Faithlife" var networkId = Faithlife.Utility.GuidUtility.Create(NETWORK_GUID_NAMESPACE, string.Join("|", networkConfig.AddressSpace, networkConfig.Subnet), 5); name = $"{region}-{networkId}"; } - return new Network(region, group, name, context, networkConfig); } diff --git a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs index 5c5573c62..9c403faae 100644 --- a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs @@ -7,7 +7,7 @@ namespace Microsoft.OneFuzz.Service; public interface INotificationOperations : IOrm { Async.Task NewFiles(Container container, string filename, bool failTaskOnTransientError); IAsyncEnumerable GetNotifications(Container container); - IAsyncEnumerable<(Task, IEnumerable)> GetQueueTasks(); + IAsyncEnumerable<(Task, IEnumerable)> GetQueueTasks(); Async.Task> Create(Container container, NotificationTemplate config, bool replaceExisting); } @@ -52,8 +52,8 @@ public class NotificationOperations : Orm, INotificationOperations } await foreach (var (task, containers) in GetQueueTasks()) { - if (containers.Contains(container.ContainerName)) { - _logTracer.Info($"queuing input {container.ContainerName} {filename} {task.TaskId}"); + if (containers.Contains(container)) { + _logTracer.Info($"queuing input {container} {filename} {task.TaskId}"); var url = _context.Containers.GetFileSasUrl(container, filename, StorageType.Corpus, BlobSasPermissions.Read | BlobSasPermissions.Delete); await _context.Queue.SendMessage(task.TaskId.ToString(), url?.ToString() ?? "", StorageType.Corpus); } @@ -77,10 +77,10 @@ public class NotificationOperations : Orm, INotificationOperations } public IAsyncEnumerable GetNotifications(Container container) { - return QueryAsync(filter: $"container eq '{container.ContainerName}'"); + return QueryAsync(filter: $"container eq '{container}'"); } - public IAsyncEnumerable<(Task, IEnumerable)> GetQueueTasks() { + public IAsyncEnumerable<(Task, IEnumerable)> GetQueueTasks() { // Nullability mismatch: We filter tuples where the containers are null return _context.TaskOperations.SearchStates(states: TaskStateHelper.AvailableStates) .Select(task => (task, _context.TaskOperations.GetInputContainerQueues(task.Config))) @@ -93,7 +93,7 @@ public class NotificationOperations : Orm, INotificationOperations } if (replaceExisting) { - var existing = this.SearchByRowKeys(new[] { container.ContainerName }); + var existing = this.SearchByRowKeys(new[] { container.String }); await foreach (var existingEntry in existing) { _logTracer.Info($"replacing existing notification: {existingEntry.NotificationId} - {container}"); await this.Delete(existingEntry); diff --git a/src/ApiService/ApiService/onefuzzlib/NsgOperations.cs b/src/ApiService/ApiService/onefuzzlib/NsgOperations.cs index 9734e51d5..98cf92fa6 100644 --- a/src/ApiService/ApiService/onefuzzlib/NsgOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NsgOperations.cs @@ -8,7 +8,7 @@ namespace Microsoft.OneFuzz.Service { Async.Task GetNsg(string name); public Async.Task> AssociateSubnet(string name, VirtualNetworkResource vnet, SubnetResource subnet); IAsyncEnumerable ListNsgs(); - bool OkToDelete(HashSet active_regions, string nsg_region, string nsg_name); + bool OkToDelete(IReadOnlySet active_regions, Region nsg_region, string nsg_name); Async.Task StartDeleteNsg(string name); Async.Task DissociateNic(Nsg nsg, NetworkInterfaceResource nic); @@ -128,8 +128,8 @@ namespace Microsoft.OneFuzz.Service { return _context.Creds.GetResourceGroupResource().GetNetworkSecurityGroups().GetAllAsync(); } - public bool OkToDelete(HashSet active_regions, string nsg_region, string nsg_name) { - return !active_regions.Contains(nsg_region) && nsg_region == nsg_name; + public bool OkToDelete(IReadOnlySet active_regions, Region nsg_region, string nsg_name) { + return !active_regions.Contains(nsg_region) && Nsg.NameFromRegion(nsg_region) == nsg_name; } /// @@ -164,7 +164,7 @@ namespace Microsoft.OneFuzz.Service { return await CreateNsg(nsg.Name, nsg.Region); } - private async Task CreateNsg(string name, string location) { + private async Task CreateNsg(string name, Region location) { var resourceGroup = _context.Creds.GetBaseResourceGroup(); _logTracer.Info($"creating nsg {resourceGroup}:{location}:{name}"); diff --git a/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs b/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs index 6909c511e..589f90c10 100644 --- a/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ProxyForwardOperations.cs @@ -5,10 +5,10 @@ namespace Microsoft.OneFuzz.Service; public interface IProxyForwardOperations : IOrm { - IAsyncEnumerable SearchForward(Guid? scalesetId = null, string? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null); + IAsyncEnumerable SearchForward(Guid? scalesetId = null, Region? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null); Forward ToForward(ProxyForward proxyForward); - Task> UpdateOrCreate(string region, Guid scalesetId, Guid machineId, int dstPort, int duration); - Task> RemoveForward(Guid scalesetId, Guid? machineId = null, int? dstPort = null, Guid? proxyId = null); + Task> UpdateOrCreate(Region region, Guid scalesetId, Guid machineId, int dstPort, int duration); + Task> RemoveForward(Guid scalesetId, Guid? machineId = null, int? dstPort = null, Guid? proxyId = null); } @@ -20,7 +20,7 @@ public class ProxyForwardOperations : Orm, IProxyForwardOperations } - public IAsyncEnumerable SearchForward(Guid? scalesetId = null, string? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null) { + public IAsyncEnumerable SearchForward(Guid? scalesetId = null, Region? region = null, Guid? machineId = null, Guid? proxyId = null, int? dstPort = null) { var conditions = new[] { @@ -40,7 +40,7 @@ public class ProxyForwardOperations : Orm, IProxyForwardOperations return new Forward(proxyForward.Port, proxyForward.DstPort, proxyForward.DstIp); } - public async Task> UpdateOrCreate(string region, Guid scalesetId, Guid machineId, int dstPort, int duration) { + public async Task> UpdateOrCreate(Region region, Guid scalesetId, Guid machineId, int dstPort, int duration) { var privateIp = await _context.IpOperations.GetScalesetInstanceIp(scalesetId, machineId); if (privateIp == null) { @@ -88,10 +88,10 @@ public class ProxyForwardOperations : Orm, IProxyForwardOperations } - public async Task> RemoveForward(Guid scalesetId, Guid? machineId, int? dstPort, Guid? proxyId) { + public async Task> RemoveForward(Guid scalesetId, Guid? machineId, int? dstPort, Guid? proxyId) { var entries = await SearchForward(scalesetId: scalesetId, machineId: machineId, proxyId: proxyId, dstPort: dstPort).ToListAsync(); - var regions = new HashSet(); + var regions = new HashSet(); foreach (var entry in entries) { regions.Add(entry.Region); await Delete(entry); diff --git a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs index 996341db0..8a9b15ad3 100644 --- a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs @@ -14,7 +14,7 @@ public interface IProxyOperations : IStatefulOrm { bool IsAlive(Proxy proxy); Async.Task SaveProxyConfig(Proxy proxy); bool IsOutdated(Proxy proxy); - Async.Task GetOrCreate(string region); + Async.Task GetOrCreate(Region region); Task IsUsed(Proxy proxy); // state transitions: @@ -27,13 +27,10 @@ public interface IProxyOperations : IStatefulOrm { Async.Task Stopped(Proxy proxy); } public class ProxyOperations : StatefulOrm, IProxyOperations { - - - static TimeSpan PROXY_LIFESPAN = TimeSpan.FromDays(7); + static readonly TimeSpan PROXY_LIFESPAN = TimeSpan.FromDays(7); public ProxyOperations(ILogTracer log, IOnefuzzContext context) : base(log.WithTag("Component", "scaleset-proxy"), context) { - } @@ -44,7 +41,7 @@ public class ProxyOperations : StatefulOrm, IPr return await data.FirstOrDefaultAsync(); } - public async Async.Task GetOrCreate(string region) { + public async Async.Task GetOrCreate(Region region) { var proxyList = QueryAsync(filter: $"region eq '{region}' and outdated eq false"); await foreach (var proxy in proxyList) { @@ -117,7 +114,7 @@ public class ProxyOperations : StatefulOrm, IPr public async Async.Task SaveProxyConfig(Proxy proxy) { var forwards = await GetForwards(proxy); - var url = (await _context.Containers.GetFileSasUrl(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", StorageType.Config, BlobSasPermissions.Read)).EnsureNotNull("Can't generate file sas"); + var url = (await _context.Containers.GetFileSasUrl(WellKnownContainers.ProxyConfigs, $"{proxy.Region}/{proxy.ProxyId}/config.json", StorageType.Config, BlobSasPermissions.Read)).EnsureNotNull("Can't generate file sas"); var queueSas = await _context.Queue.GetQueueSas("proxy", StorageType.Config, QueueSasPermissions.Add).EnsureNotNull("can't generate queue sas") ?? throw new Exception("Queue sas is null"); var proxyConfig = new ProxyConfig( @@ -130,7 +127,7 @@ public class ProxyOperations : StatefulOrm, IPr MicrosoftTelemetryKey: _context.ServiceConfiguration.OneFuzzTelemetry.EnsureNotNull("missing Telemetry"), InstanceId: await _context.Containers.GetInstanceId()); - await _context.Containers.SaveBlob(new Container("proxy-configs"), $"{proxy.Region}/{proxy.ProxyId}/config.json", EntityConverter.ToJsonString(proxyConfig), StorageType.Config); + await _context.Containers.SaveBlob(WellKnownContainers.ProxyConfigs, $"{proxy.Region}/{proxy.ProxyId}/config.json", EntityConverter.ToJsonString(proxyConfig), StorageType.Config); } @@ -175,7 +172,7 @@ public class ProxyOperations : StatefulOrm, IPr return await SetState(proxy, VmState.ExtensionsLaunch); } } else { - var nsg = new Nsg(proxy.Region, proxy.Region); + var nsg = Nsg.ForRegion(proxy.Region); var result = await _context.NsgOperations.Create(nsg); if (!result.IsOk) { return await SetFailed(proxy, result.ErrorV); diff --git a/src/ApiService/ApiService/onefuzzlib/Reports.cs b/src/ApiService/ApiService/onefuzzlib/Reports.cs index d31f75f7e..333b63726 100644 --- a/src/ApiService/ApiService/onefuzzlib/Reports.cs +++ b/src/ApiService/ApiService/onefuzzlib/Reports.cs @@ -27,7 +27,7 @@ public class Reports : IReports { } public async Async.Task GetReportOrRegression(Container container, string fileName, bool expectReports = false, params string[] args) { - var filePath = String.Join("/", new[] { container.ContainerName, fileName }); + var filePath = string.Join("/", new[] { container.String, fileName }); if (!fileName.EndsWith(".json", StringComparison.Ordinal)) { if (expectReports) { _log.Error($"get_report invalid extension: {filePath}"); diff --git a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs index 3365294f6..05c734848 100644 --- a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs @@ -131,7 +131,7 @@ public class ReproOperations : StatefulOrm, IRe repro = repro with { State = VmState.ExtensionsLaunch }; } } else { - var nsg = new Nsg(vm.Region, vm.Region); + var nsg = Nsg.ForRegion(vm.Region); var result = await _context.NsgOperations.Create(nsg); if (!result.IsOk) { return await _context.ReproOperations.SetError(repro, result.ErrorV); @@ -260,7 +260,7 @@ public class ReproOperations : StatefulOrm, IRe foreach (var (fileName, fileContents) in files) { await _context.Containers.SaveBlob( - new Container("repro-scripts"), + WellKnownContainers.ReproScripts, $"{repro.VmId}/{fileName}", fileContents, StorageType.Config diff --git a/src/ApiService/ApiService/onefuzzlib/Scheduler.cs b/src/ApiService/ApiService/onefuzzlib/Scheduler.cs index fe651a9c9..72a2cfb25 100644 --- a/src/ApiService/ApiService/onefuzzlib/Scheduler.cs +++ b/src/ApiService/ApiService/onefuzzlib/Scheduler.cs @@ -184,7 +184,7 @@ public class Scheduler : IScheduler { return (bucketConfig, workUnit); } - public record struct BucketId(Os os, Guid jobId, (string, string)? vm, PoolName? pool, string setupContainer, bool? reboot, Guid? unique); + public record struct BucketId(Os os, Guid jobId, (string, string)? vm, PoolName? pool, Container setupContainer, bool? reboot, Guid? unique); public static ILookup BucketTasks(IEnumerable tasks) { @@ -221,11 +221,11 @@ public class Scheduler : IScheduler { }); } - static string GetSetupContainer(TaskConfig config) { + static Container GetSetupContainer(TaskConfig config) { foreach (var container in config.Containers ?? throw new Exception("Missing containers")) { if (container.Type == ContainerType.Setup) { - return container.Name.ContainerName; + return container.Name; } } diff --git a/src/ApiService/ApiService/onefuzzlib/Subnet.cs b/src/ApiService/ApiService/onefuzzlib/Subnet.cs index affe69a56..d66e52d58 100644 --- a/src/ApiService/ApiService/onefuzzlib/Subnet.cs +++ b/src/ApiService/ApiService/onefuzzlib/Subnet.cs @@ -11,7 +11,7 @@ public interface ISubnet { Async.Task GetSubnet(string vnetName, string subnetName); - Async.Task CreateVirtualNetwork(string resourceGroup, string name, string region, NetworkConfig networkConfig); + Async.Task CreateVirtualNetwork(string resourceGroup, string name, Region region, NetworkConfig networkConfig); Async.Task GetSubnetId(string name, string subnetName); } @@ -29,7 +29,7 @@ public class Subnet : ISubnet { _context = context; } - public async Task CreateVirtualNetwork(string resourceGroup, string name, string region, NetworkConfig networkConfig) { + public async Task CreateVirtualNetwork(string resourceGroup, string name, Region region, NetworkConfig networkConfig) { _logTracer.Info($"creating subnet - resource group:{resourceGroup} name:{name} region: {region}"); var virtualNetParam = new VirtualNetworkData { diff --git a/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs b/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs index 81344aaa1..0b084fe9e 100644 --- a/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs @@ -15,7 +15,7 @@ public interface ITaskOperations : IStatefulOrm { IAsyncEnumerable SearchStates(Guid? jobId = null, IEnumerable? states = null); - IEnumerable? GetInputContainerQueues(TaskConfig config); + IEnumerable? GetInputContainerQueues(TaskConfig config); IAsyncEnumerable SearchExpired(); Async.Task MarkStopping(Task task); @@ -75,7 +75,7 @@ public class TaskOperations : StatefulOrm, ITas return QueryAsync(filter: queryString); } - public IEnumerable? GetInputContainerQueues(TaskConfig config) { + public IEnumerable? GetInputContainerQueues(TaskConfig config) { throw new NotImplementedException(); } diff --git a/src/ApiService/ApiService/onefuzzlib/VmOperations.cs b/src/ApiService/ApiService/onefuzzlib/VmOperations.cs index 09db62a3e..df0b7ec79 100644 --- a/src/ApiService/ApiService/onefuzzlib/VmOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/VmOperations.cs @@ -226,7 +226,7 @@ public class VmOperations : IVmOperations { async Task CreateVm( string name, - string location, + Region location, string vmSku, string image, string password, diff --git a/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs b/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs index ea2f7881e..c26ecda53 100644 --- a/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/VmssOperations.cs @@ -16,7 +16,7 @@ public interface IVmssOperations { Async.Task UpdateExtensions(Guid name, IList extensions); Async.Task GetVmss(Guid name); - Async.Task> ListAvailableSkus(string region); + Async.Task> ListAvailableSkus(Region region); Async.Task DeleteVmss(Guid name, bool? forceDeletion = null); @@ -27,7 +27,7 @@ public interface IVmssOperations { Async.Task ResizeVmss(Guid name, long capacity); Async.Task CreateVmss( - string location, + Region location, Guid name, string vmSku, long vmCount, @@ -236,7 +236,7 @@ public class VmssOperations : IVmssOperations { } public async Async.Task CreateVmss( - string location, + Region location, Guid name, string vmSku, long vmCount, @@ -394,7 +394,7 @@ public class VmssOperations : IVmssOperations { return null; } - public Async.Task> ListAvailableSkus(string region) + public Async.Task> ListAvailableSkus(Region region) => _cache.GetOrCreateAsync>($"compute-skus-{region}", async entry => { entry.SetAbsoluteExpiration(TimeSpan.FromMinutes(10)); @@ -407,7 +407,7 @@ public class VmssOperations : IVmssOperations { if (sku.Restrictions is not null) { foreach (var restriction in sku.Restrictions) { if (restriction.RestrictionsType == ResourceSkuRestrictionsType.Location && - restriction.Values.Contains(region, StringComparer.OrdinalIgnoreCase)) { + restriction.Values.Contains(region.String, StringComparer.OrdinalIgnoreCase)) { available = false; break; } diff --git a/src/ApiService/ApiService/onefuzzlib/WellKnownContainers.cs b/src/ApiService/ApiService/onefuzzlib/WellKnownContainers.cs new file mode 100644 index 000000000..fc7b44a26 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/WellKnownContainers.cs @@ -0,0 +1,12 @@ + +namespace Microsoft.OneFuzz.Service; + +public static class WellKnownContainers { + public static readonly Container BaseConfig = Container.Parse("base-config"); + public static readonly Container VmScripts = Container.Parse("vm-scripts"); + public static readonly Container InstanceSpecificSetup = Container.Parse("instance-specific-setup"); + public static readonly Container Tools = Container.Parse("tools"); + public static readonly Container ReproScripts = Container.Parse("repro-scripts"); + public static readonly Container TaskConfigs = Container.Parse("task-configs"); + public static readonly Container ProxyConfigs = Container.Parse("proxy-configs"); +} diff --git a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs index 9a0aee462..3591605d3 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs @@ -225,17 +225,23 @@ public class EntityConverter { private object? GetFieldValue(EntityInfo info, string name, TableEntity entity) { var ef = info.properties[name].First(); if (ef.kind == EntityPropertyKind.PartitionKey || ef.kind == EntityPropertyKind.RowKey) { - if (ef.type == typeof(string)) - return entity.GetString(ef.kind.ToString()); - else if (ef.type == typeof(Guid)) - return Guid.Parse(entity.GetString(ef.kind.ToString())); - else if (ef.type == typeof(int)) - return int.Parse(entity.GetString(ef.kind.ToString())); - else if (ef.type == typeof(long)) - return long.Parse(entity.GetString(ef.kind.ToString())); - else if (ef.type.IsClass) - return ef.type.GetConstructor(new[] { typeof(string) })!.Invoke(new[] { entity.GetString(ef.kind.ToString()) }); - else { + // partition & row keys must always be strings + var stringValue = entity.GetString(ef.kind.ToString()); + if (ef.type == typeof(string)) { + return stringValue; + } else if (ef.type == typeof(Guid)) { + return Guid.Parse(stringValue); + } else if (ef.type == typeof(int)) { + return int.Parse(stringValue); + } else if (ef.type == typeof(long)) { + return long.Parse(stringValue); + } else if (ef.type.IsClass) { + if (ef.type.IsAssignableTo(typeof(ValidatedString))) { + return ef.type.GetMethod("Parse")!.Invoke(null, new[] { stringValue }); + } + + return Activator.CreateInstance(ef.type, new[] { stringValue }); + } else { throw new Exception($"invalid partition or row key type of {info.type} property {name}: {ef.type}"); } } diff --git a/src/ApiService/IntegrationTests/ContainersTests.cs b/src/ApiService/IntegrationTests/ContainersTests.cs index 328ab1d81..2a6c1dfdf 100644 --- a/src/ApiService/IntegrationTests/ContainersTests.cs +++ b/src/ApiService/IntegrationTests/ContainersTests.cs @@ -48,11 +48,11 @@ public abstract class ContainersTestBase : FunctionTestBase { [Fact] public async Async.Task CanDelete() { - var containerName = "test"; + var containerName = Container.Parse("test"); var client = GetContainerClient(containerName); await client.CreateIfNotExistsAsync(); - var msg = TestHttpRequestData.FromJson("DELETE", new ContainerDelete(new Container(containerName))); + var msg = TestHttpRequestData.FromJson("DELETE", new ContainerDelete(containerName)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); var func = new ContainersFunction(Logger, auth, Context); @@ -67,8 +67,8 @@ public abstract class ContainersTestBase : FunctionTestBase { [Fact] public async Async.Task CanPost_New() { var meta = new Dictionary { { "some", "value" } }; - var containerName = "test"; - var msg = TestHttpRequestData.FromJson("POST", new ContainerCreate(new Container(containerName), meta)); + var containerName = Container.Parse("test"); + var msg = TestHttpRequestData.FromJson("POST", new ContainerCreate(containerName, meta)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); var func = new ContainersFunction(Logger, auth, Context); @@ -87,12 +87,12 @@ public abstract class ContainersTestBase : FunctionTestBase { [Fact] public async Async.Task CanPost_Existing() { - var containerName = "test"; + var containerName = Container.Parse("test"); var client = GetContainerClient(containerName); await client.CreateIfNotExistsAsync(); var metadata = new Dictionary { { "some", "value" } }; - var msg = TestHttpRequestData.FromJson("POST", new ContainerCreate(new Container(containerName), metadata)); + var msg = TestHttpRequestData.FromJson("POST", new ContainerCreate(containerName, metadata)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); var func = new ContainersFunction(Logger, auth, Context); @@ -110,13 +110,13 @@ public abstract class ContainersTestBase : FunctionTestBase { [Fact] public async Async.Task Get_Existing() { - var containerName = "test"; + var containerName = Container.Parse("test"); { var client = GetContainerClient(containerName); await client.CreateIfNotExistsAsync(); } - var msg = TestHttpRequestData.FromJson("GET", new ContainerGet(new Container(containerName))); + var msg = TestHttpRequestData.FromJson("GET", new ContainerGet(containerName)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); var func = new ContainersFunction(Logger, auth, Context); @@ -130,7 +130,8 @@ public abstract class ContainersTestBase : FunctionTestBase { [Fact] public async Async.Task Get_Missing_Fails() { - var msg = TestHttpRequestData.FromJson("GET", new ContainerGet(new Container("container"))); + var container = Container.Parse("container"); + var msg = TestHttpRequestData.FromJson("GET", new ContainerGet(container)); var auth = new TestEndpointAuthorization(RequestType.User, Logger, Context); var func = new ContainersFunction(Logger, auth, Context); @@ -142,8 +143,8 @@ public abstract class ContainersTestBase : FunctionTestBase { public async Async.Task List_Existing() { var meta1 = new Dictionary { { "key1", "value1" } }; var meta2 = new Dictionary { { "key2", "value2" } }; - await GetContainerClient("one").CreateIfNotExistsAsync(metadata: meta1); - await GetContainerClient("two").CreateIfNotExistsAsync(metadata: meta2); + await GetContainerClient(Container.Parse("one")).CreateIfNotExistsAsync(metadata: meta1); + await GetContainerClient(Container.Parse("two")).CreateIfNotExistsAsync(metadata: meta2); var msg = TestHttpRequestData.Empty("GET"); // this means list all @@ -154,7 +155,7 @@ public abstract class ContainersTestBase : FunctionTestBase { var list = BodyAs(result); // other tests can run in parallel, so filter to just our containers: - var cs = list.Where(ci => ci.Name.ContainerName.StartsWith(Context.ServiceConfiguration.OneFuzzStoragePrefix)).ToList(); + var cs = list.Where(ci => ci.Name.String.StartsWith(Context.ServiceConfiguration.OneFuzzStoragePrefix)).ToList(); Assert.Equal(2, cs.Count); // ensure correct metadata was returned. diff --git a/src/ApiService/IntegrationTests/DownloadTests.cs b/src/ApiService/IntegrationTests/DownloadTests.cs index 3d9fe3636..a3ee87876 100644 --- a/src/ApiService/IntegrationTests/DownloadTests.cs +++ b/src/ApiService/IntegrationTests/DownloadTests.cs @@ -72,7 +72,8 @@ public abstract class DownloadTestBase : FunctionTestBase { [Fact] public async Async.Task Download_RedirectsToResult_WithLocationHeader() { // set up a file to download - var container = GetContainerClient("xxx"); + var containerName = Container.Parse("xxx"); + var container = GetContainerClient(containerName); await container.CreateAsync(); await container.UploadBlobAsync("yyy", new BinaryData("content")); diff --git a/src/ApiService/IntegrationTests/Fakes/TestCreds.cs b/src/ApiService/IntegrationTests/Fakes/TestCreds.cs index 100937858..889dabece 100644 --- a/src/ApiService/IntegrationTests/Fakes/TestCreds.cs +++ b/src/ApiService/IntegrationTests/Fakes/TestCreds.cs @@ -15,9 +15,9 @@ class TestCreds : ICreds { private readonly Guid _subscriptionId; private readonly string _resourceGroup; - private readonly string _region; + private readonly Region _region; - public TestCreds(Guid subscriptionId, string resourceGroup, string region) { + public TestCreds(Guid subscriptionId, string resourceGroup, Region region) { _subscriptionId = subscriptionId; _resourceGroup = resourceGroup; _region = region; @@ -26,8 +26,8 @@ class TestCreds : ICreds { public ArmClient ArmClient => null!; // we have to return something in some test cases, even if it isn’t used - public Task GetBaseRegion() => Task.FromResult(_region); - public Task> GetRegions() => Task.FromResult>(new[] { _region }); + public Task GetBaseRegion() => Task.FromResult(_region); + public Task> GetRegions() => Task.FromResult>(new[] { _region }); public string GetBaseResourceGroup() => _resourceGroup; diff --git a/src/ApiService/IntegrationTests/InfoTests.cs b/src/ApiService/IntegrationTests/InfoTests.cs index 70849a830..37e5c036e 100644 --- a/src/ApiService/IntegrationTests/InfoTests.cs +++ b/src/ApiService/IntegrationTests/InfoTests.cs @@ -48,7 +48,8 @@ public abstract class InfoTestBase : FunctionTestBase { // store the instance ID in the expected location: // for production this is done by the deploy script var instanceId = Guid.NewGuid().ToString(); - var containerClient = GetContainerClient("base-config"); + var baseConfigContainer = WellKnownContainers.BaseConfig; + var containerClient = GetContainerClient(baseConfigContainer); await containerClient.CreateAsync(); await containerClient.GetBlobClient("instance_id").UploadAsync(new BinaryData(instanceId)); diff --git a/src/ApiService/IntegrationTests/JobsTests.cs b/src/ApiService/IntegrationTests/JobsTests.cs index b33469b94..c9b591a03 100644 --- a/src/ApiService/IntegrationTests/JobsTests.cs +++ b/src/ApiService/IntegrationTests/JobsTests.cs @@ -169,7 +169,7 @@ public abstract class JobsTestBase : FunctionTestBase { Assert.NotNull(job.Config.Logs); Assert.Empty(new Uri(job.Config.Logs!).Query); - var container = Assert.Single(await Context.Containers.GetContainers(StorageType.Corpus), c => c.Key.Contains(job.JobId.ToString())); + var container = Assert.Single(await Context.Containers.GetContainers(StorageType.Corpus), c => c.Key.String.Contains(job.JobId.ToString())); var metadata = Assert.Single(container.Value); Assert.Equal(new KeyValuePair("container_type", "logs"), metadata); } diff --git a/src/ApiService/IntegrationTests/_FunctionTestBase.cs b/src/ApiService/IntegrationTests/_FunctionTestBase.cs index 1b9c09d97..d94903360 100644 --- a/src/ApiService/IntegrationTests/_FunctionTestBase.cs +++ b/src/ApiService/IntegrationTests/_FunctionTestBase.cs @@ -35,15 +35,15 @@ public abstract class FunctionTestBase : IAsyncLifetime { private readonly Guid _subscriptionId = Guid.NewGuid(); private readonly string _resourceGroup = "FakeResourceGroup"; - private readonly string _region = "fakeregion"; + private readonly Region _region = Region.Parse("fakeregion"); protected ILogTracer Logger { get; } protected TestContext Context { get; } private readonly BlobServiceClient _blobClient; - protected BlobContainerClient GetContainerClient(string container) - => _blobClient.GetBlobContainerClient(_storagePrefix + container); + protected BlobContainerClient GetContainerClient(Container container) + => _blobClient.GetBlobContainerClient(_storagePrefix + container.String); public FunctionTestBase(ITestOutputHelper output, IStorage storage) { Logger = new TestLogTracer(output); diff --git a/src/ApiService/Tests/OrmModelsTest.cs b/src/ApiService/Tests/OrmModelsTest.cs index f732f020b..d86072bc0 100644 --- a/src/ApiService/Tests/OrmModelsTest.cs +++ b/src/ApiService/Tests/OrmModelsTest.cs @@ -91,6 +91,11 @@ namespace Tests { where PoolName.TryParse(name.Get, out _) select PoolName.Parse(name.Get); + public static Gen RegionGen { get; } + = from name in Arb.Generate() + where Region.TryParse(name.Get, out _) + select Region.Parse(name.Get); + public static Gen Node { get; } = from arg in Arb.Generate, Tuple>>() from poolName in PoolNameGen @@ -107,39 +112,47 @@ namespace Tests { DeleteRequested: arg.Item2.Item5, DebugKeepNode: arg.Item2.Item6); - public static Gen ProxyForward() { - return Arb.Generate, Tuple>>().Select( - arg => - new ProxyForward( - Region: arg.Item1.Item1, - Port: arg.Item1.Item2, - ScalesetId: arg.Item1.Item3, - MachineId: arg.Item1.Item4, - ProxyId: arg.Item1.Item5, - DstPort: arg.Item1.Item6, - DstIp: arg.Item2.Item1.ToString(), - EndTime: arg.Item2.Item2 - ) - ); - } + public static Gen ProxyForward { get; } = + from region in RegionGen + from port in Gen.Choose(0, ushort.MaxValue) + from scalesetId in Arb.Generate() + from machineId in Arb.Generate() + from proxyId in Arb.Generate() + from dstPort in Gen.Choose(0, ushort.MaxValue) + from dstIp in Arb.Generate() + from endTime in Arb.Generate() + select new ProxyForward( + Region: region, + Port: port, + ScalesetId: scalesetId, + MachineId: machineId, + ProxyId: proxyId, + DstPort: dstPort, + DstIp: dstIp.ToString(), + EndTime: endTime); - public static Gen Proxy() { - return Arb.Generate, Tuple>>().Select( - arg => - new Proxy( - Region: arg.Item1.Item1, - ProxyId: arg.Item1.Item2, - CreatedTimestamp: arg.Item1.Item3, - State: arg.Item1.Item4, - Auth: arg.Item1.Item5, - Ip: arg.Item1.Item6, - Error: arg.Item1.Item7, - Version: arg.Item2.Item1, - Heartbeat: arg.Item2.Item2, - Outdated: arg.Item2.Item3 - ) - ); - } + public static Gen Proxy { get; } = + from region in RegionGen + from proxyId in Arb.Generate() + from createdTimestamp in Arb.Generate() + from state in Arb.Generate() + from auth in Arb.Generate() + from ip in Arb.Generate() + from error in Arb.Generate() + from version in Arb.Generate() + from heartbeat in Arb.Generate() + from outdated in Arb.Generate() + select new Proxy( + Region: region, + ProxyId: proxyId, + CreatedTimestamp: createdTimestamp, + State: state, + Auth: auth, + Ip: ip, + Error: error, + Version: version, + Heartbeat: heartbeat, + Outdated: outdated); public static Gen EventMessage() { return Arb.Generate>().Select( @@ -219,10 +232,11 @@ namespace Tests { } public static Gen Scaleset { get; } = from arg in Arb.Generate, + Tuple, Tuple, Tuple>>>() from poolName in PoolNameGen + from region in RegionGen select new Scaleset( PoolName: poolName, ScalesetId: arg.Item1.Item1, @@ -230,7 +244,7 @@ namespace Tests { Auth: arg.Item1.Item3, VmSku: arg.Item1.Item4, Image: arg.Item1.Item5, - Region: arg.Item1.Item6, + Region: region, Size: arg.Item2.Item1, SpotInstances: arg.Item2.Item2, @@ -346,11 +360,12 @@ namespace Tests { ); } - public static Gen Container() { - return Arb.Generate>>().Select( - arg => new Container(string.Join("", arg.Item1.Get.Where(c => char.IsLetterOrDigit(c) || c == '-'))!) - ); - } + public static Gen ContainerGen { get; } = + from len in Gen.Choose(3, 63) + from name in Gen.ArrayOf(len, Gen.Elements("abcdefghijklmnopqrstuvwxyz0123456789-")) + let nameString = new string(name) + where Container.TryParse(nameString, out var _) + select Container.Parse(nameString); public static Gen NotificationTemplate() { return Gen.OneOf(new[] { @@ -411,11 +426,11 @@ namespace Tests { } public static Arbitrary ProxyForward() { - return Arb.From(OrmGenerators.ProxyForward()); + return Arb.From(OrmGenerators.ProxyForward); } public static Arbitrary Proxy() { - return Arb.From(OrmGenerators.Proxy()); + return Arb.From(OrmGenerators.Proxy); } public static Arbitrary EventMessage() { @@ -458,9 +473,12 @@ namespace Tests { } public static Arbitrary Container() { - return Arb.From(OrmGenerators.Container()); + return Arb.From(OrmGenerators.ContainerGen); } + public static Arbitrary Region() { + return Arb.From(OrmGenerators.RegionGen); + } public static Arbitrary NotificationTemplate() { return Arb.From(OrmGenerators.NotificationTemplate()); diff --git a/src/ApiService/Tests/OrmTest.cs b/src/ApiService/Tests/OrmTest.cs index 089d64af3..8603877e5 100644 --- a/src/ApiService/Tests/OrmTest.cs +++ b/src/ApiService/Tests/OrmTest.cs @@ -265,15 +265,15 @@ namespace Tests { [Fact] public void TestContainerSerialization() { - var container = new Container("abc-123"); + var container = Container.Parse("abc-123"); var expected = new Entity3(123, "abc", container); var converter = new EntityConverter(); var tableEntity = converter.ToTableEntity(expected); var actual = converter.ToRecord(tableEntity); - Assert.Equal(expected.Container.ContainerName, actual.Container.ContainerName); - Assert.Equal(expected.Container.ContainerName, tableEntity.GetString("container")); + Assert.Equal(expected.Container, actual.Container); + Assert.Equal(expected.Container.String, tableEntity.GetString("container")); } [Fact] @@ -288,7 +288,7 @@ namespace Tests { Assert.Equal(123, entity?.Id); Assert.Equal("abc", entity?.TheName); - Assert.Equal("abc-123", entity?.Container.ContainerName); + Assert.Equal("abc-123", entity?.Container.String); } @@ -300,7 +300,7 @@ namespace Tests { [Fact] public void TestPartitionKeyIsRowKey() { - var container = new Container("abc-123"); + var container = Container.Parse("abc-123"); var expected = new Entity4(123, "abc", container); var converter = new EntityConverter(); @@ -310,8 +310,8 @@ namespace Tests { var actual = converter.ToRecord(tableEntity); - Assert.Equal(expected.Container.ContainerName, actual.Container.ContainerName); - Assert.Equal(expected.Container.ContainerName, tableEntity.GetString("container")); + Assert.Equal(expected.Container, actual.Container); + Assert.Equal(expected.Container.String, tableEntity.GetString("container")); } diff --git a/src/ApiService/Tests/SchedulerTests.cs b/src/ApiService/Tests/SchedulerTests.cs index 78aba6e8e..de8058d8c 100644 --- a/src/ApiService/Tests/SchedulerTests.cs +++ b/src/ApiService/Tests/SchedulerTests.cs @@ -25,7 +25,9 @@ public class SchedulerTests { TargetEnv: new Dictionary(), TargetOptions: new List()), Pool: new TaskPool(1, PoolName.Parse("pool")), - Containers: new List { new TaskContainers(ContainerType.Setup, new Container("setup")) }, + Containers: new List { + new TaskContainers(ContainerType.Setup, Container.Parse("setup")) + }, Colocate: true ), @@ -105,7 +107,7 @@ public class SchedulerTests { var tasks = BuildTasks(100).Select((task, i) => { var containers = new List(task.Config.Containers!); if (i % 4 == 0) { - containers[0] = containers[0] with { Name = new Container("setup2") }; + containers[0] = containers[0] with { Name = Container.Parse("setup2") }; } return task with { JobId = i % 2 == 0 ? jobId : task.JobId, diff --git a/src/ApiService/Tests/TimerReproTests.cs b/src/ApiService/Tests/TimerReproTests.cs index 3a85d4982..38c7ef68c 100644 --- a/src/ApiService/Tests/TimerReproTests.cs +++ b/src/ApiService/Tests/TimerReproTests.cs @@ -101,10 +101,9 @@ public class TimerReproTests { Guid.NewGuid(), Guid.Empty, new ReproConfig( - new Container(String.Empty), - String.Empty, - 0 - ), + Container.Parse("container"), + "", + 0), null, Os.Linux, VmState.Init, diff --git a/src/ApiService/Tests/ValidatedStringTests.cs b/src/ApiService/Tests/ValidatedStringTests.cs index 26eebd35e..0ae6ad468 100644 --- a/src/ApiService/Tests/ValidatedStringTests.cs +++ b/src/ApiService/Tests/ValidatedStringTests.cs @@ -19,4 +19,18 @@ public class ValidatedStringTests { var result = JsonSerializer.Serialize(new ThingContainingPoolName(PoolName.Parse("is-a-pool"))); Assert.Equal("{\"PoolName\":\"is-a-pool\"}", result); } + + [Theory] + [InlineData("x", false)] // too short + [InlineData("xy", false)] // too short + [InlineData("xyz", true)] + [InlineData("-container", false)] // can't start with hyphen + [InlineData("container-", true)] // can end with hyphen + [InlineData("container-name", true)] // can have middle hyphen + [InlineData("container--name", false)] // can't have two consecutive hyphens + [InlineData("container-Name", false)] // can't have capitals + [InlineData("container-name-09", true)] // can have numbers + public void ContainerNames(string name, bool valid) { + Assert.Equal(valid, Container.TryParse(name, out var _)); + } }