ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-11 17:51:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding New Default Image Config Value to IC. (#2434)

Browse Source 
* Adding New Default Image Config Value to IC.

* Removing forced image setting.

* Updating Webhook Events.

* Removing typo.

* Updating webhook_events again.

* Syncing webhook events.

* Fixing check for os type.

* Fixing import.

* PR Suggestions.

* Fix C# Model Typo.

* Removing other refs to images.

* Removing remaining refs to images outside of models.

* Removing hardcoded image values from tests.

* Update Default Proxy and Repro Images.

Co-authored-by: Marc Greisen <mgreisen@microsoft.com>
This commit is contained in:
Noah McGregor Harper 2022-09-23 10:40:44 -07:00 committed by GitHub
parent dc2c4649c8
commit 3f35d81f4b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 78 additions and 52 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
docs/webhook_events.md
View File

@ -682,6 +682,8 @@ If webhook is set to have Event Grid message format then the payload will look a
"allowed_aad_tenants": [
"00000000-0000-0000-0000-000000000000"
],
"default_linux_vm_image": "Canonical:UbuntuServer:18.04-LTS:latest",
"default_windows_vm_image": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
"network_config": {
"address_space": "10.0.0.0/8",
"subnet": "10.0.0.0/16"
@ -822,6 +824,16 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Api Access Rules",
"type": "object"
},
"default_linux_vm_image": {
"default": "Canonical:UbuntuServer:18.04-LTS:latest",
"title": "Default Linux Vm Image",
"type": "string"
},
"default_windows_vm_image": {
"default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
"title": "Default Windows Vm Image",
"type": "string"
},
"extensions": {
"$ref": "#/definitions/AzureVmExtensionConfig"
},
@ -6046,6 +6058,16 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Api Access Rules",
"type": "object"
},
"default_linux_vm_image": {
"default": "Canonical:UbuntuServer:18.04-LTS:latest",
"title": "Default Linux Vm Image",
"type": "string"
},
"default_windows_vm_image": {
"default": "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
"title": "Default Windows Vm Image",
"type": "string"
},
"extensions": {
"$ref": "#/definitions/AzureVmExtensionConfig"
},

14
src/ApiService/ApiService/Functions/Scaleset.cs
View File

@ -75,6 +75,18 @@ public class Scaleset {
context: "ScalesetCreate");
}
string image;
if (create.Image is null) {
var config = await _context.ConfigOperations.Fetch();
if (pool.Os == Os.Windows) {
image = config.DefaultWindowsVmImage;
} else {
image = config.DefaultLinuxVmImage;
}
} else {
image = create.Image;
}
Region region;
if (create.Region is null) {
region = await _context.Creds.GetBaseRegion();
@ -117,7 +129,7 @@ public class Scaleset {
Auth: await Auth.BuildAuth(_log),
PoolName: create.PoolName,
VmSku: create.VmSku,
Image: create.Image,
Image: image,
Region: region,
Size: create.Size,
SpotInstances: create.SpotInstances,

4
src/ApiService/ApiService/OneFuzzTypes/Model.cs
View File

@ -329,6 +329,8 @@ public record InstanceConfig
[DefaultValue(InitMethod.DefaultConstructor)] NetworkConfig NetworkConfig,
[DefaultValue(InitMethod.DefaultConstructor)] NetworkSecurityGroupConfig ProxyNsgConfig,
AzureVmExtensionConfig? Extensions,
string DefaultWindowsVmImage = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
string DefaultLinuxVmImage = "Canonical:UbuntuServer:18.04-LTS:latest",
string ProxyVmSku = "Standard_B2s",
bool RequireAdminPrivileges = false,
IDictionary<Endpoint, ApiAccessRule>? ApiAccessRules = null,
@ -343,6 +345,8 @@ public record InstanceConfig
new NetworkConfig(),
new NetworkSecurityGroupConfig(),
null,
"MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest",
"Canonical:UbuntuServer:18.04-LTS:latest",
"Standard_B2s",
false
) { }

2
src/ApiService/ApiService/OneFuzzTypes/Requests.cs
View File

@ -190,7 +190,7 @@ public record ProxyReset(
public record ScalesetCreate(
[property: Required] PoolName PoolName,
[property: Required] string VmSku,
[property: Required] string Image,
string? Image,
Region? Region,
[property: Range(1, long.MaxValue), Required] long Size,
[property: Required] bool SpotInstances,

4
src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs
View File

@ -236,7 +236,7 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
public static Vm GetVm(Proxy proxy, InstanceConfig config) {
var tags = config.VmssTags;
string proxyVmSku;
const string PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest";
string proxyImage = config.DefaultLinuxVmImage;
if (config.ProxyVmSku is null) {
proxyVmSku = "Standard_B2s";
} else {
@ -247,7 +247,7 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
Name: $"proxy-{proxy.ProxyId:N}",
Region: proxy.Region,
Sku: proxyVmSku,
Image: PROXY_IMAGE,
Image: proxyImage,
Auth: proxy.Auth,
Tags: tags,
Nsg: null

17
src/ApiService/ApiService/onefuzzlib/ReproOperations.cs
View File

@ -30,16 +30,9 @@ public interface IReproOperations : IStatefulOrm<Repro, VmState> {
}
public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IReproOperations {
private static readonly Dictionary<Os, string> DEFAULT_OS = new()
{
{ Os.Linux, "Canonical:UbuntuServer:18.04-LTS:latest" },
{ Os.Windows, "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest" }
};
const string DEFAULT_SKU = "Standard_DS1_v2";
public ReproOperations(ILogTracer log, IOnefuzzContext context)
: base(log, context) {
@ -57,16 +50,22 @@ public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IRe
throw new Exception($"previous existing task missing: {repro.TaskId}");
}
Dictionary<Os, string> default_os = new()
{
{ Os.Linux, config.DefaultLinuxVmImage },
{ Os.Windows, config.DefaultWindowsVmImage }
};
var vmConfig = await taskOperations.GetReproVmConfig(task);
if (vmConfig == null) {
if (!DEFAULT_OS.ContainsKey(task.Os)) {
if (!default_os.ContainsKey(task.Os)) {
throw new NotSupportedException($"unsupport OS for repro {task.Os}");
}
vmConfig = new TaskVm(
await _context.Creds.GetBaseRegion(),
DEFAULT_SKU,
DEFAULT_OS[task.Os],
default_os[task.Os],
null
);
}

5
src/ApiService/FunctionalTests/1f-api/Scaleset.cs
View File

@ -57,9 +57,6 @@ public class Scaleset : IFromJsonElement<Scaleset> {
public class ScalesetApi : ApiBase {
public const string Image_Ubuntu_20_04 = "Canonical:0001-com-ubuntu-server-focal:20_04-lts:latest";
public const string ImageWindows = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest";
public ScalesetApi(Uri endpoint, Microsoft.OneFuzz.Service.Request request, ITestOutputHelper output) :
base(endpoint, "/api/Scaleset", request, output) { }
@ -73,7 +70,7 @@ public class ScalesetApi : ApiBase {
return IEnumerableResult<Scaleset>(res);
}
public async Task<Result<Scaleset, Error>> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string image = Image_Ubuntu_20_04, bool spotInstance = false) {
public async Task<Result<Scaleset, Error>> Create(string poolName, int size, string? region = null, string vmSku = "Standard_D2s_v3", string? image = null, bool spotInstance = false) {
_output.WriteLine($"Creating scaleset in pool {poolName}, size: {size}");
var rootScalesetCreate = new JsonObject()

3
src/ApiService/FunctionalTests/Helpers.cs
View File

@ -3,14 +3,13 @@
namespace FunctionalTests {
public class Helpers {
public static async Task<(Pool, Scaleset)> CreatePoolAndScaleset(PoolApi poolApi, ScalesetApi scalesetApi, string os = "linux", string? region = null, int numNodes = 2) {
var image = (os == "linux") ? ScalesetApi.Image_Ubuntu_20_04 : ScalesetApi.ImageWindows;
var newPoolId = Guid.NewGuid().ToString();
var newPoolName = PoolApi.TestPoolPrefix + newPoolId;
var newPool = await poolApi.Create(newPoolName, os);
Assert.True(newPool.IsOk, $"failed to create new pool: {newPool.ErrorV}");
var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region, image: image);
var newScalesetResult = await scalesetApi.Create(newPool.OkV!.Name, numNodes, region: region);
Assert.True(newScalesetResult.IsOk, $"failed to crate new scaleset: {newScalesetResult.ErrorV}");
var newScaleset = newScalesetResult.OkV!;

4
src/api-service/__app__/onefuzzlib/proxy.py
View File

@ -43,7 +43,6 @@ from .extension import proxy_manager_extensions
from .orm import ORMMixin, QueryFilter
from .proxy_forward import ProxyForward
PROXY_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"
PROXY_LOG_PREFIX = "scaleset-proxy: "
PROXY_LIFESPAN = datetime.timedelta(days=7)
@ -70,6 +69,7 @@ class Proxy(ORMMixin):
return ("region", "proxy_id")
def get_vm(self, config: InstanceConfig) -> VM:
config = InstanceConfig.fetch()
sku = config.proxy_vm_sku
tags = None
if config.vm_tags:
@ -78,7 +78,7 @@ class Proxy(ORMMixin):
name="proxy-%s" % base58.b58encode(self.proxy_id.bytes).decode(),
region=self.region,
sku=sku,
image=PROXY_IMAGE,
image=config.default_linux_vm_image,
auth=self.auth,
tags=tags,
)

14
src/api-service/__app__/onefuzzlib/repro.py
View File

@ -27,11 +27,6 @@ from .orm import ORMMixin, QueryFilter
from .reports import get_report
from .tasks.main import Task
DEFAULT_OS = {
OS.linux: "Canonical:UbuntuServer:18.04-LTS:latest",
OS.windows: "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest",
}
DEFAULT_SKU = "Standard_DS1_v2"
@ -56,14 +51,19 @@ class Repro(BASE_REPRO, ORMMixin):
if isinstance(task, Error):
raise Exception("previously existing task missing: %s" % self.task_id)
config = InstanceConfig.fetch()
default_os = {
OS.linux: config.default_linux_vm_image,
OS.windows: config.default_windows_vm_image,
}
vm_config = task.get_repro_vm_config()
if vm_config is None:
# if using a pool without any scalesets defined yet, use reasonable defaults
if task.os not in DEFAULT_OS:
if task.os not in default_os:
raise NotImplementedError("unsupported OS for repro %s" % task.os)
vm_config = TaskVm(
region=get_base_region(), sku=DEFAULT_SKU, image=DEFAULT_OS[task.os]
region=get_base_region(), sku=DEFAULT_SKU, image=default_os[task.os]
)
if self.auth is None:

12
src/api-service/__app__/scaleset/__init__.py
View File

@ -4,7 +4,7 @@
# Licensed under the MIT License.
import azure.functions as func
from onefuzztypes.enums import ErrorCode, ScalesetState
from onefuzztypes.enums import OS, ErrorCode, ScalesetState
from onefuzztypes.models import Error
from onefuzztypes.requests import (
ScalesetCreate,
@ -78,6 +78,14 @@ def post(req: func.HttpRequest) -> func.HttpResponse:
region = request.region
if request.image is None:
if pool.os == OS.windows:
image = instance_config.default_windows_vm_image
else:
image = instance_config.default_linux_vm_image
else:
image = request.image
if request.vm_sku not in list_available_skus(region):
return not_ok(
Error(
@ -97,7 +105,7 @@ def post(req: func.HttpRequest) -> func.HttpResponse:
scaleset = Scaleset.create(
pool_name=request.pool_name,
vm_sku=request.vm_sku,
image=request.image,
image=image,
region=region,
size=request.size,
spot_instances=request.spot_instances,

12
src/cli/onefuzz/api.py
View File

@ -48,9 +48,6 @@ ONEFUZZ_GUID_NAMESPACE = uuid.UUID("27f25e3f-6544-4b69-b309-9b096c5a9cbc")
ONE_HOUR_IN_SECONDS = 3600
DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"
DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"
REPRO_SSH_FORWARD = "1337:127.0.0.1:1337"
UUID_RE = r"^[a-f0-9]{8}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{4}-?[a-f0-9]{12}\Z"
@ -1424,15 +1421,6 @@ class Scaleset(Endpoint):
if tags is None:
tags = {}
if image is None:
pool = self.onefuzz.pools.get(pool_name)
if pool.os == enums.OS.linux:
image = DEFAULT_LINUX_IMAGE
elif pool.os == enums.OS.windows:
image = DEFAULT_WINDOWS_IMAGE
else:
raise NotImplementedError
auto_scale = requests.AutoScaleOptions(
min=min_instances,
max=max_size,

9
src/cli/onefuzz/templates/__init__.py
View File

@ -16,8 +16,6 @@ from onefuzztypes.primitives import Container, Directory, File
from ..job_templates.job_monitor import JobMonitor
ELF_MAGIC = b"\x7fELF"
DEFAULT_LINUX_IMAGE = "Canonical:UbuntuServer:18.04-LTS:latest"
DEFAULT_WINDOWS_IMAGE = "MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"
class StoppedEarly(Exception):
@ -177,13 +175,6 @@ class JobHelper:
self.containers[ContainerType.inputs], Directory(tmp_dir)
)
@classmethod
def get_image(_cls, platform: OS) -> str:
if platform == OS.linux:
return DEFAULT_LINUX_IMAGE
else:
return DEFAULT_WINDOWS_IMAGE
@classmethod
def get_platform(_cls, target_exe: File) -> OS:
with open(target_exe, "rb") as handle:

6
src/pytypes/onefuzztypes/models.py
View File

@ -880,6 +880,12 @@ class InstanceConfig(BaseModel):
default_factory=NetworkSecurityGroupConfig
)
extensions: Optional[AzureVmExtensionConfig]
default_windows_vm_image: str = Field(
default="MicrosoftWindowsDesktop:Windows-10:win10-21h2-pro:latest"
)
default_linux_vm_image: str = Field(
default="Canonical:UbuntuServer:18.04-LTS:latest"
)
proxy_vm_sku: str = Field(default="Standard_B2s")
api_access_rules: Optional[Dict[Endpoint, ApiAccessRule]] = None
group_membership: Optional[Dict[PrincipalID, List[GroupId]]] = None

2
src/pytypes/onefuzztypes/requests.py
View File

@ -182,7 +182,7 @@ class AutoScaleOptions(BaseModel):
class ScalesetCreate(BaseRequest):
pool_name: PoolName
vm_sku: str
image: str
image: Optional[str]
region: Optional[Region]
size: int = Field(ge=1)
spot_instances: bool