diff --git a/docs/webhook_events.md b/docs/webhook_events.md index 34dcd8f47..07fc1a0d9 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -682,6 +682,8 @@ If webhook is set to have Event Grid message format then the payload will look a "allowed_aad_tenants": [ "00000000-0000-0000-0000-000000000000" ], + "default_linux_vm_image": "Canonical:UbuntuServer:18.04-LTS:latest", + "default_windows_vm_image": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", "network_config": { "address_space": "10.0.0.0/8", "subnet": "10.0.0.0/16" @@ -822,6 +824,16 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Api Access Rules", "type": "object" }, + "default_linux_vm_image": { + "default": "Canonical:UbuntuServer:18.04-LTS:latest", + "title": "Default Linux Vm Image", + "type": "string" + }, + "default_windows_vm_image": { + "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + "title": "Default Windows Vm Image", + "type": "string" + }, "extensions": { "$ref": "#/definitions/AzureVmExtensionConfig" }, @@ -6046,6 +6058,16 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Api Access Rules", "type": "object" }, + "default_linux_vm_image": { + "default": "Canonical:UbuntuServer:18.04-LTS:latest", + "title": "Default Linux Vm Image", + "type": "string" + }, + "default_windows_vm_image": { + "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + "title": "Default Windows Vm Image", + "type": "string" + }, "extensions": { "$ref": "#/definitions/AzureVmExtensionConfig" }, diff --git a/src/ApiService/ApiService/Functions/Scaleset.cs b/src/ApiService/ApiService/Functions/Scaleset.cs index 67f65316f..1d01657b3 100644 --- a/src/ApiService/ApiService/Functions/Scaleset.cs +++ b/src/ApiService/ApiService/Functions/Scaleset.cs @@ -75,6 +75,18 @@ public class Scaleset { context: "ScalesetCreate"); } + string image; + if (create.Image is null) { + var config = await _context.ConfigOperations.Fetch(); + if (pool.Os == Os.Windows) { + image = config.DefaultWindowsVmImage; + } else { + image = config.DefaultLinuxVmImage; + } + } else { + image = create.Image; + } + Region region; if (create.Region is null) { region = await _context.Creds.GetBaseRegion(); @@ -117,7 +129,7 @@ public class Scaleset { Auth: await Auth.BuildAuth(_log), PoolName: create.PoolName, VmSku: create.VmSku, - Image: create.Image, + Image: image, Region: region, Size: create.Size, SpotInstances: create.SpotInstances, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index f39d4f88f..a1104d678 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -329,6 +329,8 @@ public record InstanceConfig [DefaultValue(InitMethod.DefaultConstructor)] NetworkConfig NetworkConfig, [DefaultValue(InitMethod.DefaultConstructor)] NetworkSecurityGroupConfig ProxyNsgConfig, AzureVmExtensionConfig? Extensions, + string DefaultWindowsVmImage = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + string DefaultLinuxVmImage = "Canonical:UbuntuServer:18.04-LTS:latest", string ProxyVmSku = "Standard_B2s", bool RequireAdminPrivileges = false, IDictionary? ApiAccessRules = null, @@ -343,6 +345,8 @@ public record InstanceConfig new NetworkConfig(), new NetworkSecurityGroupConfig(), null, + "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest", + "Canonical:UbuntuServer:18.04-LTS:latest", "Standard_B2s", false ) { } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs index 320470be7..63488a492 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Requests.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Requests.cs @@ -190,7 +190,7 @@ public record ProxyReset( public record ScalesetCreate( [property: Required] PoolName PoolName, [property: Required] string VmSku, - [property: Required] string Image, + string? Image, Region? Region, [property: Range(1, long.MaxValue), Required] long Size, [property: Required] bool SpotInstances, diff --git a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs index 6c2310d35..59b846181 100644 --- a/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs @@ -236,7 +236,7 @@ public class ProxyOperations : StatefulOrm, IPr public static Vm GetVm(Proxy proxy, InstanceConfig config) { var tags = config.VmssTags; string proxyVmSku; - const string PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"; + string proxyImage = config.DefaultLinuxVmImage; if (config.ProxyVmSku is null) { proxyVmSku = "Standard_B2s"; } else { @@ -247,7 +247,7 @@ public class ProxyOperations : StatefulOrm, IPr Name: $"proxy-{proxy.ProxyId:N}", Region: proxy.Region, Sku: proxyVmSku, - Image: PROXY_IMAGE, + Image: proxyImage, Auth: proxy.Auth, Tags: tags, Nsg: null diff --git a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs index 195770c99..34dd4e4d2 100644 --- a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs @@ -30,16 +30,9 @@ public interface IReproOperations : IStatefulOrm { } public class ReproOperations : StatefulOrm, IReproOperations { - private static readonly Dictionary DEFAULT_OS = new() - { - { Os.Linux, "Canonical:UbuntuServer:18.04-LTS:latest" }, - { Os.Windows, "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest" } - }; const string DEFAULT_SKU = "Standard_DS1_v2"; - - public ReproOperations(ILogTracer log, IOnefuzzContext context) : base(log, context) { @@ -57,16 +50,22 @@ public class ReproOperations : StatefulOrm, IRe throw new Exception($"previous existing task missing: {repro.TaskId}"); } + Dictionary default_os = new() + { + { Os.Linux, config.DefaultLinuxVmImage }, + { Os.Windows, config.DefaultWindowsVmImage } + }; + var vmConfig = await taskOperations.GetReproVmConfig(task); if (vmConfig == null) { - if (!DEFAULT_OS.ContainsKey(task.Os)) { + if (!default_os.ContainsKey(task.Os)) { throw new NotSupportedException($"unsupport OS for repro {task.Os}"); } vmConfig = new TaskVm( await _context.Creds.GetBaseRegion(), DEFAULT_SKU, - DEFAULT_OS[task.Os], + default_os[task.Os], null ); } diff --git a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs index 5998a1d45..be685d5c6 100644 --- a/src/ApiService/FunctionalTests/1f-api/Scaleset.cs +++ b/src/ApiService/FunctionalTests/1f-api/Scaleset.cs @@ -57,9 +57,6 @@ public class Scaleset : IFromJsonElement { public class ScalesetApi : ApiBase { - public const string Image_Ubuntu_20_04 = "Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest"; - public const string ImageWindows = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"; - public ScalesetApi(Uri endpoint, Microsoft.OneFuzz.Service.Request request, ITestOutputHelper output) : base(endpoint, "/api/Scaleset", request, output) { } @@ -73,7 +70,7 @@ public class ScalesetApi : ApiBase { return IEnumerableResult(res); } - public async Task> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string image = Image_Ubuntu_20_04, bool spotInstance = false) { + public async Task> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string? image = null, bool spotInstance = false) { _output.WriteLine($"Creating scaleset in pool {poolName}, size: {size}"); var rootScalesetCreate = new JsonObject() diff --git a/src/ApiService/FunctionalTests/Helpers.cs b/src/ApiService/FunctionalTests/Helpers.cs index caf3e8bdf..b6e28fd5c 100644 --- a/src/ApiService/FunctionalTests/Helpers.cs +++ b/src/ApiService/FunctionalTests/Helpers.cs @@ -3,14 +3,13 @@ namespace FunctionalTests { public class Helpers { public static async Task<(Pool, Scaleset)> CreatePoolAndScaleset(PoolApi poolApi, ScalesetApi scalesetApi, string os = "linux", string? region = null, int numNodes = 2) { - var image = (os == "linux") ? ScalesetApi.Image_Ubuntu_20_04 : ScalesetApi.ImageWindows; var newPoolId = Guid.NewGuid().ToString(); var newPoolName = PoolApi.TestPoolPrefix + newPoolId; var newPool = await poolApi.Create(newPoolName, os); Assert.True(newPool.IsOk, $"failed to create new pool: {newPool.ErrorV}"); - var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region, image: image); + var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region); Assert.True(newScalesetResult.IsOk, $"failed to crate new scaleset: {newScalesetResult.ErrorV}"); var newScaleset = newScalesetResult.OkV!; diff --git a/src/api-service/__app__/onefuzzlib/proxy.py b/src/api-service/__app__/onefuzzlib/proxy.py index 1ae03cb63..53660a68a 100644 --- a/src/api-service/__app__/onefuzzlib/proxy.py +++ b/src/api-service/__app__/onefuzzlib/proxy.py @@ -43,7 +43,6 @@ from .extension import proxy_manager_extensions from .orm import ORMMixin, QueryFilter from .proxy_forward import ProxyForward -PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest" PROXY_LOG_PREFIX = "scaleset-proxy: " PROXY_LIFESPAN = datetime.timedelta(days=7) @@ -70,6 +69,7 @@ class Proxy(ORMMixin): return ("region", "proxy_id") def get_vm(self, config: InstanceConfig) -> VM: + config = InstanceConfig.fetch() sku = config.proxy_vm_sku tags = None if config.vm_tags: @@ -78,7 +78,7 @@ class Proxy(ORMMixin): name="proxy-%s" % base58.b58encode(self.proxy_id.bytes).decode(), region=self.region, sku=sku, - image=PROXY_IMAGE, + image=config.default_linux_vm_image, auth=self.auth, tags=tags, ) diff --git a/src/api-service/__app__/onefuzzlib/repro.py b/src/api-service/__app__/onefuzzlib/repro.py index 68f6a0772..621a62381 100644 --- a/src/api-service/__app__/onefuzzlib/repro.py +++ b/src/api-service/__app__/onefuzzlib/repro.py @@ -27,11 +27,6 @@ from .orm import ORMMixin, QueryFilter from .reports import get_report from .tasks.main import Task -DEFAULT_OS = { - OS.linux: "Canonical:UbuntuServer:18.04-LTS:latest", - OS.windows: "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest", -} - DEFAULT_SKU = "Standard_DS1_v2" @@ -56,14 +51,19 @@ class Repro(BASE_REPRO, ORMMixin): if isinstance(task, Error): raise Exception("previously existing task missing: %s" % self.task_id) + config = InstanceConfig.fetch() + default_os = { + OS.linux: config.default_linux_vm_image, + OS.windows: config.default_windows_vm_image, + } vm_config = task.get_repro_vm_config() if vm_config is None: # if using a pool without any scalesets defined yet, use reasonable defaults - if task.os not in DEFAULT_OS: + if task.os not in default_os: raise NotImplementedError("unsupported OS for repro %s" % task.os) vm_config = TaskVm( - region=get_base_region(), sku=DEFAULT_SKU, image=DEFAULT_OS[task.os] + region=get_base_region(), sku=DEFAULT_SKU, image=default_os[task.os] ) if self.auth is None: diff --git a/src/api-service/__app__/scaleset/__init__.py b/src/api-service/__app__/scaleset/__init__.py index a3969559b..32ca011c9 100644 --- a/src/api-service/__app__/scaleset/__init__.py +++ b/src/api-service/__app__/scaleset/__init__.py @@ -4,7 +4,7 @@ # Licensed under the MIT License. import azure.functions as func -from onefuzztypes.enums import ErrorCode, ScalesetState +from onefuzztypes.enums import OS, ErrorCode, ScalesetState from onefuzztypes.models import Error from onefuzztypes.requests import ( ScalesetCreate, @@ -78,6 +78,14 @@ def post(req: func.HttpRequest) -> func.HttpResponse: region = request.region + if request.image is None: + if pool.os == OS.windows: + image = instance_config.default_windows_vm_image + else: + image = instance_config.default_linux_vm_image + else: + image = request.image + if request.vm_sku not in list_available_skus(region): return not_ok( Error( @@ -97,7 +105,7 @@ def post(req: func.HttpRequest) -> func.HttpResponse: scaleset = Scaleset.create( pool_name=request.pool_name, vm_sku=request.vm_sku, - image=request.image, + image=image, region=region, size=request.size, spot_instances=request.spot_instances, diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 5ccc53c67..17be517f3 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -48,9 +48,6 @@ ONEFUZZ_GUID_NAMESPACE = uuid.UUID("27f25e3f-6544-4b69-b309-9b096c5a9cbc") ONE_HOUR_IN_SECONDS = 3600 -DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest" -DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest" - REPRO_SSH_FORWARD = "1337:127.0.0.1:1337" UUID_RE = r"^[a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12}\Z" @@ -1424,15 +1421,6 @@ class Scaleset(Endpoint): if tags is None: tags = {} - if image is None: - pool = self.onefuzz.pools.get(pool_name) - if pool.os == enums.OS.linux: - image = DEFAULT_LINUX_IMAGE - elif pool.os == enums.OS.windows: - image = DEFAULT_WINDOWS_IMAGE - else: - raise NotImplementedError - auto_scale = requests.AutoScaleOptions( min=min_instances, max=max_size, diff --git a/src/cli/onefuzz/templates/__init__.py b/src/cli/onefuzz/templates/__init__.py index 596887c9b..2c9e3a792 100644 --- a/src/cli/onefuzz/templates/__init__.py +++ b/src/cli/onefuzz/templates/__init__.py @@ -16,8 +16,6 @@ from onefuzztypes.primitives import Container, Directory, File from ..job_templates.job_monitor import JobMonitor ELF_MAGIC = b"\x7fELF" -DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest" -DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest" class StoppedEarly(Exception): @@ -177,13 +175,6 @@ class JobHelper: self.containers[ContainerType.inputs], Directory(tmp_dir) ) - @classmethod - def get_image(_cls, platform: OS) -> str: - if platform == OS.linux: - return DEFAULT_LINUX_IMAGE - else: - return DEFAULT_WINDOWS_IMAGE - @classmethod def get_platform(_cls, target_exe: File) -> OS: with open(target_exe, "rb") as handle: diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index a136dc0e1..72c99d259 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -880,6 +880,12 @@ class InstanceConfig(BaseModel): default_factory=NetworkSecurityGroupConfig ) extensions: Optional[AzureVmExtensionConfig] + default_windows_vm_image: str = Field( + default="MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest" + ) + default_linux_vm_image: str = Field( + default="Canonical:UbuntuServer:18.04-LTS:latest" + ) proxy_vm_sku: str = Field(default="Standard_B2s") api_access_rules: Optional[Dict[Endpoint, ApiAccessRule]] = None group_membership: Optional[Dict[PrincipalID, List[GroupId]]] = None diff --git a/src/pytypes/onefuzztypes/requests.py b/src/pytypes/onefuzztypes/requests.py index 82e8cc80b..54bad0ade 100644 --- a/src/pytypes/onefuzztypes/requests.py +++ b/src/pytypes/onefuzztypes/requests.py @@ -182,7 +182,7 @@ class AutoScaleOptions(BaseModel): class ScalesetCreate(BaseRequest): pool_name: PoolName vm_sku: str - image: str + image: Optional[str] region: Optional[Region] size: int = Field(ge=1) spot_instances: bool