Adding New Default Image Config Value to IC. (#2434)

* Adding New Default Image Config Value to IC.

* Removing forced image setting.

* Updating Webhook Events.

* Removing typo.

* Updating webhook_events again.

* Syncing webhook events.

* Fixing check for os type.

* Fixing import.

* PR Suggestions.

* Fix C# Model Typo.

* Removing other refs to images.

* Removing remaining refs to images outside of models.

* Removing hardcoded image values from tests.

* Update Default Proxy and Repro Images.

Co-authored-by: Marc Greisen <mgreisen@microsoft.com>

docs/webhook_events.md

@@ -682,6 +682,8 @@ If webhook is set to have Event Grid message format then the payload will look a
         "allowed_aad_tenants": [
             "00000000-0000-0000-0000-000000000000"
         ],
+        "default_linux_vm_image": "Canonical:UbuntuServer:18.04-LTS:latest",
+        "default_windows_vm_image": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
         "network_config": {
             "address_space": "10.0.0.0/8",
             "subnet": "10.0.0.0/16"
@@ -822,6 +824,16 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Api Access Rules",
                     "type": "object"
                 },
+                "default_linux_vm_image": {
+                    "default": "Canonical:UbuntuServer:18.04-LTS:latest",
+                    "title": "Default Linux Vm Image",
+                    "type": "string"
+                },
+                "default_windows_vm_image": {
+                    "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
+                    "title": "Default Windows Vm Image",
+                    "type": "string"
+                },
                 "extensions": {
                     "$ref": "#/definitions/AzureVmExtensionConfig"
                 },
@@ -6046,6 +6058,16 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Api Access Rules",
                     "type": "object"
                 },
+                "default_linux_vm_image": {
+                    "default": "Canonical:UbuntuServer:18.04-LTS:latest",
+                    "title": "Default Linux Vm Image",
+                    "type": "string"
+                },
+                "default_windows_vm_image": {
+                    "default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
+                    "title": "Default Windows Vm Image",
+                    "type": "string"
+                },
                 "extensions": {
                     "$ref": "#/definitions/AzureVmExtensionConfig"
                 },

src/ApiService/ApiService/Functions/Scaleset.cs

@@ -75,6 +75,18 @@ public class Scaleset {
                 context: "ScalesetCreate");
         }
 
+        string image;
+        if (create.Image is null) {
+            var config = await _context.ConfigOperations.Fetch();
+            if (pool.Os == Os.Windows) {
+                image = config.DefaultWindowsVmImage;
+            } else {
+                image = config.DefaultLinuxVmImage;
+            }
+        } else {
+            image = create.Image;
+        }
+
         Region region;
         if (create.Region is null) {
             region = await _context.Creds.GetBaseRegion();
@@ -117,7 +129,7 @@ public class Scaleset {
             Auth: await Auth.BuildAuth(_log),
             PoolName: create.PoolName,
             VmSku: create.VmSku,
-            Image: create.Image,
+            Image: image,
             Region: region,
             Size: create.Size,
             SpotInstances: create.SpotInstances,

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -329,6 +329,8 @@ public record InstanceConfig
     [DefaultValue(InitMethod.DefaultConstructor)] NetworkConfig NetworkConfig,
     [DefaultValue(InitMethod.DefaultConstructor)] NetworkSecurityGroupConfig ProxyNsgConfig,
     AzureVmExtensionConfig? Extensions,
+    string DefaultWindowsVmImage = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
+    string DefaultLinuxVmImage = "Canonical:UbuntuServer:18.04-LTS:latest",
     string ProxyVmSku = "Standard_B2s",
     bool RequireAdminPrivileges = false,
     IDictionary<Endpoint, ApiAccessRule>? ApiAccessRules = null,
@@ -343,6 +345,8 @@ public record InstanceConfig
         new NetworkConfig(),
         new NetworkSecurityGroupConfig(),
         null,
+        "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
+        "Canonical:UbuntuServer:18.04-LTS:latest",
         "Standard_B2s",
         false
         ) { }

src/ApiService/ApiService/OneFuzzTypes/Requests.cs

@@ -190,7 +190,7 @@ public record ProxyReset(
 public record ScalesetCreate(
     [property: Required] PoolName PoolName,
     [property: Required] string VmSku,
-    [property: Required] string Image,
+    string? Image,
     Region? Region,
     [property: Range(1, long.MaxValue), Required] long Size,
     [property: Required] bool SpotInstances,

src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs

@@ -236,7 +236,7 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
     public static Vm GetVm(Proxy proxy, InstanceConfig config) {
         var tags = config.VmssTags;
         string proxyVmSku;
-        const string PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest";
+        string proxyImage = config.DefaultLinuxVmImage;
         if (config.ProxyVmSku is null) {
             proxyVmSku = "Standard_B2s";
         } else {
@@ -247,7 +247,7 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
             Name: $"proxy-{proxy.ProxyId:N}",
             Region: proxy.Region,
             Sku: proxyVmSku,
-            Image: PROXY_IMAGE,
+            Image: proxyImage,
             Auth: proxy.Auth,
             Tags: tags,
             Nsg: null

src/ApiService/ApiService/onefuzzlib/ReproOperations.cs

@@ -30,16 +30,9 @@ public interface IReproOperations : IStatefulOrm<Repro, VmState> {
 }
 
 public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IReproOperations {
-    private static readonly Dictionary<Os, string> DEFAULT_OS = new()
-    {
-        { Os.Linux, "Canonical:UbuntuServer:18.04-LTS:latest" },
-        { Os.Windows, "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest" }
-    };
 
     const string DEFAULT_SKU = "Standard_DS1_v2";
 
-
-
     public ReproOperations(ILogTracer log, IOnefuzzContext context)
         : base(log, context) {
 
@@ -57,16 +50,22 @@ public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IRe
             throw new Exception($"previous existing task missing: {repro.TaskId}");
         }
 
+        Dictionary<Os, string> default_os = new()
+        {
+            { Os.Linux, config.DefaultLinuxVmImage },
+            { Os.Windows, config.DefaultWindowsVmImage }
+        };
+
         var vmConfig = await taskOperations.GetReproVmConfig(task);
         if (vmConfig == null) {
-            if (!DEFAULT_OS.ContainsKey(task.Os)) {
+            if (!default_os.ContainsKey(task.Os)) {
                 throw new NotSupportedException($"unsupport OS for repro {task.Os}");
             }
 
             vmConfig = new TaskVm(
                 await _context.Creds.GetBaseRegion(),
                 DEFAULT_SKU,
-                DEFAULT_OS[task.Os],
+                default_os[task.Os],
                 null
             );
         }

src/ApiService/FunctionalTests/1f-api/Scaleset.cs

@@ -57,9 +57,6 @@ public class Scaleset : IFromJsonElement<Scaleset> {
 
 public class ScalesetApi : ApiBase {
 
-    public const string Image_Ubuntu_20_04 = "Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest";
-    public const string ImageWindows = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest";
-
     public ScalesetApi(Uri endpoint, Microsoft.OneFuzz.Service.Request request, ITestOutputHelper output) :
         base(endpoint, "/api/Scaleset", request, output) { }
 
@@ -73,7 +70,7 @@ public class ScalesetApi : ApiBase {
         return IEnumerableResult<Scaleset>(res);
     }
 
-    public async Task<Result<Scaleset, Error>> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string image = Image_Ubuntu_20_04, bool spotInstance = false) {
+    public async Task<Result<Scaleset, Error>> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string? image = null, bool spotInstance = false) {
         _output.WriteLine($"Creating scaleset in pool {poolName}, size: {size}");
 
         var rootScalesetCreate = new JsonObject()

src/ApiService/FunctionalTests/Helpers.cs

@@ -3,14 +3,13 @@
 namespace FunctionalTests {
     public class Helpers {
         public static async Task<(Pool, Scaleset)> CreatePoolAndScaleset(PoolApi poolApi, ScalesetApi scalesetApi, string os = "linux", string? region = null, int numNodes = 2) {
-            var image = (os == "linux") ? ScalesetApi.Image_Ubuntu_20_04 : ScalesetApi.ImageWindows;
 
             var newPoolId = Guid.NewGuid().ToString();
             var newPoolName = PoolApi.TestPoolPrefix + newPoolId;
             var newPool = await poolApi.Create(newPoolName, os);
 
             Assert.True(newPool.IsOk, $"failed to create new pool: {newPool.ErrorV}");
-            var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region, image: image);
+            var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region);
 
             Assert.True(newScalesetResult.IsOk, $"failed to crate new scaleset: {newScalesetResult.ErrorV}");
             var newScaleset = newScalesetResult.OkV!;

src/api-service/__app__/onefuzzlib/proxy.py

@@ -43,7 +43,6 @@ from .extension import proxy_manager_extensions
 from .orm import ORMMixin, QueryFilter
 from .proxy_forward import ProxyForward
 
-PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"
 PROXY_LOG_PREFIX = "scaleset-proxy: "
 PROXY_LIFESPAN = datetime.timedelta(days=7)
 
@@ -70,6 +69,7 @@ class Proxy(ORMMixin):
         return ("region", "proxy_id")
 
     def get_vm(self, config: InstanceConfig) -> VM:
+        config = InstanceConfig.fetch()
         sku = config.proxy_vm_sku
         tags = None
         if config.vm_tags:
@@ -78,7 +78,7 @@ class Proxy(ORMMixin):
             name="proxy-%s" % base58.b58encode(self.proxy_id.bytes).decode(),
             region=self.region,
             sku=sku,
-            image=PROXY_IMAGE,
+            image=config.default_linux_vm_image,
             auth=self.auth,
             tags=tags,
         )

src/api-service/__app__/onefuzzlib/repro.py

@@ -27,11 +27,6 @@ from .orm import ORMMixin, QueryFilter
 from .reports import get_report
 from .tasks.main import Task
 
-DEFAULT_OS = {
-    OS.linux: "Canonical:UbuntuServer:18.04-LTS:latest",
-    OS.windows: "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest",
-}
-
 DEFAULT_SKU = "Standard_DS1_v2"
 
 
@@ -56,14 +51,19 @@ class Repro(BASE_REPRO, ORMMixin):
         if isinstance(task, Error):
             raise Exception("previously existing task missing: %s" % self.task_id)
 
+        config = InstanceConfig.fetch()
+        default_os = {
+            OS.linux: config.default_linux_vm_image,
+            OS.windows: config.default_windows_vm_image,
+        }
         vm_config = task.get_repro_vm_config()
         if vm_config is None:
             # if using a pool without any scalesets defined yet, use reasonable defaults
-            if task.os not in DEFAULT_OS:
+            if task.os not in default_os:
                 raise NotImplementedError("unsupported OS for repro %s" % task.os)
 
             vm_config = TaskVm(
-                region=get_base_region(), sku=DEFAULT_SKU, image=DEFAULT_OS[task.os]
+                region=get_base_region(), sku=DEFAULT_SKU, image=default_os[task.os]
             )
 
         if self.auth is None:

src/api-service/__app__/scaleset/__init__.py

@@ -4,7 +4,7 @@
 # Licensed under the MIT License.
 
 import azure.functions as func
-from onefuzztypes.enums import ErrorCode, ScalesetState
+from onefuzztypes.enums import OS, ErrorCode, ScalesetState
 from onefuzztypes.models import Error
 from onefuzztypes.requests import (
     ScalesetCreate,
@@ -78,6 +78,14 @@ def post(req: func.HttpRequest) -> func.HttpResponse:
 
         region = request.region
 
+    if request.image is None:
+        if pool.os == OS.windows:
+            image = instance_config.default_windows_vm_image
+        else:
+            image = instance_config.default_linux_vm_image
+    else:
+        image = request.image
+
     if request.vm_sku not in list_available_skus(region):
         return not_ok(
             Error(
@@ -97,7 +105,7 @@ def post(req: func.HttpRequest) -> func.HttpResponse:
     scaleset = Scaleset.create(
         pool_name=request.pool_name,
         vm_sku=request.vm_sku,
-        image=request.image,
+        image=image,
         region=region,
         size=request.size,
         spot_instances=request.spot_instances,

src/cli/onefuzz/api.py

@@ -48,9 +48,6 @@ ONEFUZZ_GUID_NAMESPACE = uuid.UUID("27f25e3f-6544-4b69-b309-9b096c5a9cbc")
 
 ONE_HOUR_IN_SECONDS = 3600
 
-DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"
-DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"
-
 REPRO_SSH_FORWARD = "1337:127.0.0.1:1337"
 
 UUID_RE = r"^[a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12}\Z"
@@ -1424,15 +1421,6 @@ class Scaleset(Endpoint):
         if tags is None:
             tags = {}
 
-        if image is None:
-            pool = self.onefuzz.pools.get(pool_name)
-            if pool.os == enums.OS.linux:
-                image = DEFAULT_LINUX_IMAGE
-            elif pool.os == enums.OS.windows:
-                image = DEFAULT_WINDOWS_IMAGE
-            else:
-                raise NotImplementedError
-
         auto_scale = requests.AutoScaleOptions(
             min=min_instances,
             max=max_size,

src/cli/onefuzz/templates/__init__.py

@@ -16,8 +16,6 @@ from onefuzztypes.primitives import Container, Directory, File
 from ..job_templates.job_monitor import JobMonitor
 
 ELF_MAGIC = b"\x7fELF"
-DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"
-DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"
 
 
 class StoppedEarly(Exception):
@@ -177,13 +175,6 @@ class JobHelper:
                 self.containers[ContainerType.inputs], Directory(tmp_dir)
             )
 
-    @classmethod
-    def get_image(_cls, platform: OS) -> str:
-        if platform == OS.linux:
-            return DEFAULT_LINUX_IMAGE
-        else:
-            return DEFAULT_WINDOWS_IMAGE
-
     @classmethod
     def get_platform(_cls, target_exe: File) -> OS:
         with open(target_exe, "rb") as handle:

src/pytypes/onefuzztypes/models.py

@@ -880,6 +880,12 @@ class InstanceConfig(BaseModel):
         default_factory=NetworkSecurityGroupConfig
     )
     extensions: Optional[AzureVmExtensionConfig]
+    default_windows_vm_image: str = Field(
+        default="MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"
+    )
+    default_linux_vm_image: str = Field(
+        default="Canonical:UbuntuServer:18.04-LTS:latest"
+    )
     proxy_vm_sku: str = Field(default="Standard_B2s")
     api_access_rules: Optional[Dict[Endpoint, ApiAccessRule]] = None
     group_membership: Optional[Dict[PrincipalID, List[GroupId]]] = None

src/pytypes/onefuzztypes/requests.py

@@ -182,7 +182,7 @@ class AutoScaleOptions(BaseModel):
 class ScalesetCreate(BaseRequest):
     pool_name: PoolName
     vm_sku: str
-    image: str
+    image: Optional[str]
     region: Optional[Region]
     size: int = Field(ge=1)
     spot_instances: bool