ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-10 17:21:34 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' into kanan/expand-all

Browse Source
This commit is contained in:
Kanan B 2023-10-18 16:00:17 -07:00 committed by GitHub
commit 7d6337709b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
19 changed files with 196 additions and 92 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
docs/webhook_events.md
View File

@ -445,6 +445,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"
@ -2332,6 +2337,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"
@ -3054,6 +3064,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"
@ -3572,6 +3587,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"
@ -4056,6 +4076,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"
@ -4514,6 +4539,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"
@ -4999,6 +5029,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"
@ -6743,6 +6778,11 @@ If webhook is set to have Event Grid message format then the payload will look a
"title": "Generator Options",
"type": "array"
},
"min_available_memory_mb": {
"minimum": 0,
"title": "Min Available Memory Mb",
"type": "integer"
},
"minimized_stack_depth": {
"title": "Minimized Stack Depth",
"type": "integer"

6
src/ApiService/ApiService/ApiService.csproj
View File

@ -12,7 +12,7 @@
<PackageReference Include="Microsoft.Extensions.Logging.ApplicationInsights" Version="2.21.0" />
<PackageReference Include="Microsoft.Azure.Functions.Worker.ApplicationInsights" Version="1.0.0-preview4" />
<PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.24" />
<PackageReference Include="Semver" Version="2.3.0" />
<PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.3.0" />
<PackageReference Include="Microsoft.Azure.AppConfiguration.Functions.Worker" Version="6.0.0" />
@ -31,7 +31,7 @@
<PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.18.0" />
<PackageReference Include="Azure.Data.Tables" Version="12.8.0" />
<PackageReference Include="Azure.ResourceManager.Compute" Version="1.0.0-beta.8" />
<PackageReference Include="Azure.Identity" Version="1.8.2" />
<PackageReference Include="Azure.Identity" Version="1.10.2" />
<PackageReference Include="Azure.Messaging.EventGrid" Version="4.15.0" />
<PackageReference Include="Azure.ResourceManager" Version="1.6.0" />
<PackageReference Include="Azure.ResourceManager.Network" Version="1.0.0" />
@ -40,7 +40,7 @@
<PackageReference Include="Azure.Storage.Queues" Version="12.11.0" />
<PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
<PackageReference Include="Microsoft.Graph" Version="4.37.0" />
<PackageReference Include="Microsoft.Identity.Client" Version="4.52.0" />
<PackageReference Include="Microsoft.Identity.Client" Version="4.54.1" />
<PackageReference Include="Microsoft.Identity.Web.TokenCache" Version="2.7.0" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.29.0" />
<PackageReference Include="System.Linq.Async" Version="6.0.1" />

2
src/ApiService/ApiService/OneFuzzTypes/Model.cs
View File

@ -239,6 +239,7 @@ public record TaskDetails(
List<string>? ReportList = null,
long? MinimizedStackDepth = null,
Dictionary<string, string>? TaskEnv = null,
ulong? MinAvailableMemoryMb = null,
// Deprecated. Retained for processing old table data.
string? CoverageFilter = null,
@ -1164,6 +1165,7 @@ public record TaskUnitConfig(
public IContainerDef? RegressionReports { get; set; }
public IContainerDef? ExtraSetup { get; set; }
public IContainerDef? ExtraOutput { get; set; }
public ulong? MinAvailableMemoryMb { get; set; }
}
public record NodeCommandEnvelope(

5
src/ApiService/ApiService/onefuzzlib/Config.cs
View File

@ -73,7 +73,10 @@ public class Config : IConfig {
HeartbeatQueue: await _queue.GetQueueSas("task-heartbeat", StorageType.Config, QueueSasPermissions.Add) ?? throw new Exception("unable to get heartbeat queue sas"),
JobResultQueue: await _queue.GetQueueSas("job-result", StorageType.Config, QueueSasPermissions.Add) ?? throw new Exception("unable to get heartbeat queue sas"),
Tags: task.Config.Tags ?? new Dictionary<string, string>()
);
) {
// It's okay if this is null because the agent will use a default value if so.
MinAvailableMemoryMb = task.Config.Task.MinAvailableMemoryMb,
};
if (definition.MonitorQueue != null) {
config.inputQueue = await _queue.GetQueueSas(task.TaskId.ToString(), StorageType.Corpus, QueueSasPermissions.All);

1
src/ApiService/ApiService/onefuzzlib/notifications/JinjaTemplateAdapter.cs
View File

@ -166,6 +166,7 @@ public class JinjaTemplateAdapter {
targetOptions,
1,
new Dictionary<string, string>(),
null,
"coverage filter",
"module allow list",
"source allow list",

30
src/ApiService/ApiService/packages.lock.json
View File

@ -14,13 +14,13 @@
},
"Azure.Identity": {
"type": "Direct",
"requested": "[1.8.2, )",
"resolved": "1.8.2",
"contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==",
"requested": "[1.10.2, )",
"resolved": "1.10.2",
"contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==",
"dependencies": {
"Azure.Core": "1.25.0",
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.25.3",
"Azure.Core": "1.35.0",
"Microsoft.Identity.Client": "4.54.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.31.0",
"System.Memory": "4.5.4",
"System.Security.Cryptography.ProtectedData": "4.7.0",
"System.Text.Json": "4.7.2",
@ -299,9 +299,9 @@
},
"Microsoft.Identity.Client": {
"type": "Direct",
"requested": "[4.52.0, )",
"resolved": "4.52.0",
"contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==",
"requested": "[4.54.1, )",
"resolved": "4.54.1",
"contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==",
"dependencies": {
"Microsoft.IdentityModel.Abstractions": "6.22.0"
}
@ -446,11 +446,11 @@
},
"Azure.Core": {
"type": "Transitive",
"resolved": "1.32.0",
"contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==",
"resolved": "1.35.0",
"contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==",
"dependencies": {
"Microsoft.Bcl.AsyncInterfaces": "1.1.1",
"System.Diagnostics.DiagnosticSource": "4.6.0",
"System.Diagnostics.DiagnosticSource": "6.0.1",
"System.Memory.Data": "1.0.2",
"System.Numerics.Vectors": "4.5.0",
"System.Text.Encodings.Web": "4.7.2",
@ -1031,10 +1031,10 @@
},
"Microsoft.Identity.Client.Extensions.Msal": {
"type": "Transitive",
"resolved": "2.25.3",
"contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==",
"resolved": "2.31.0",
"contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==",
"dependencies": {
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client": "4.54.1",
"System.IO.FileSystem.AccessControl": "5.0.0",
"System.Security.Cryptography.ProtectedData": "4.5.0"
}

2
src/ApiService/FunctionalTests/FunctionalTests.csproj
View File

@ -19,7 +19,7 @@
<PrivateAssets>all</PrivateAssets>
</PackageReference>
<PackageReference Include="Microsoft.Azure.Functions.Worker.Extensions.Http" Version="3.0.13" />
<PackageReference Include="Microsoft.Identity.Client" Version="4.52.0" />
<PackageReference Include="Microsoft.Identity.Client" Version="4.54.1" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.2" />
<PackageReference Include="System.Net.Http" Version="4.3.4" />
<PackageReference Include="System.Security.Cryptography.Pkcs" Version="7.0.2" />

32
src/ApiService/FunctionalTests/packages.lock.json
View File

@ -34,9 +34,9 @@
},
"Microsoft.Identity.Client": {
"type": "Direct",
"requested": "[4.52.0, )",
"resolved": "4.52.0",
"contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==",
"requested": "[4.54.1, )",
"resolved": "4.54.1",
"contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==",
"dependencies": {
"Microsoft.IdentityModel.Abstractions": "6.22.0"
}
@ -122,11 +122,11 @@
},
"Azure.Core": {
"type": "Transitive",
"resolved": "1.32.0",
"contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==",
"resolved": "1.35.0",
"contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==",
"dependencies": {
"Microsoft.Bcl.AsyncInterfaces": "1.1.1",
"System.Diagnostics.DiagnosticSource": "4.6.0",
"System.Diagnostics.DiagnosticSource": "6.0.1",
"System.Memory.Data": "1.0.2",
"System.Numerics.Vectors": "4.5.0",
"System.Text.Encodings.Web": "4.7.2",
@ -155,12 +155,12 @@
},
"Azure.Identity": {
"type": "Transitive",
"resolved": "1.8.2",
"contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==",
"resolved": "1.10.2",
"contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==",
"dependencies": {
"Azure.Core": "1.25.0",
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.25.3",
"Azure.Core": "1.35.0",
"Microsoft.Identity.Client": "4.54.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.31.0",
"System.Memory": "4.5.4",
"System.Security.Cryptography.ProtectedData": "4.7.0",
"System.Text.Json": "4.7.2",
@ -971,10 +971,10 @@
},
"Microsoft.Identity.Client.Extensions.Msal": {
"type": "Transitive",
"resolved": "2.25.3",
"contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==",
"resolved": "2.31.0",
"contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==",
"dependencies": {
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client": "4.54.1",
"System.IO.FileSystem.AccessControl": "5.0.0",
"System.Security.Cryptography.ProtectedData": "4.5.0"
}
@ -2376,7 +2376,7 @@
"type": "Project",
"dependencies": {
"Azure.Data.Tables": "[12.8.0, )",
"Azure.Identity": "[1.8.2, )",
"Azure.Identity": "[1.10.2, )",
"Azure.Messaging.EventGrid": "[4.15.0, )",
"Azure.ResourceManager": "[1.6.0, )",
"Azure.ResourceManager.Compute": "[1.0.0-beta.8, )",
@ -2403,7 +2403,7 @@
"Microsoft.Extensions.Logging.ApplicationInsights": "[2.21.0, )",
"Microsoft.FeatureManagement": "[2.5.1, )",
"Microsoft.Graph": "[4.37.0, )",
"Microsoft.Identity.Client": "[4.52.0, )",
"Microsoft.Identity.Client": "[4.54.1, )",
"Microsoft.Identity.Web.TokenCache": "[2.7.0, )",
"Microsoft.Rest.ClientRuntime": "[2.3.24, )",
"Microsoft.TeamFoundationServer.Client": "[19.219.0-preview, )",

30
src/ApiService/IntegrationTests/packages.lock.json
View File

@ -61,11 +61,11 @@
},
"Azure.Core": {
"type": "Transitive",
"resolved": "1.32.0",
"contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==",
"resolved": "1.35.0",
"contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==",
"dependencies": {
"Microsoft.Bcl.AsyncInterfaces": "1.1.1",
"System.Diagnostics.DiagnosticSource": "4.6.0",
"System.Diagnostics.DiagnosticSource": "6.0.1",
"System.Memory.Data": "1.0.2",
"System.Numerics.Vectors": "4.5.0",
"System.Text.Encodings.Web": "4.7.2",
@ -94,12 +94,12 @@
},
"Azure.Identity": {
"type": "Transitive",
"resolved": "1.8.2",
"contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==",
"resolved": "1.10.2",
"contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==",
"dependencies": {
"Azure.Core": "1.25.0",
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.25.3",
"Azure.Core": "1.35.0",
"Microsoft.Identity.Client": "4.54.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.31.0",
"System.Memory": "4.5.4",
"System.Security.Cryptography.ProtectedData": "4.7.0",
"System.Text.Json": "4.7.2",
@ -973,18 +973,18 @@
},
"Microsoft.Identity.Client": {
"type": "Transitive",
"resolved": "4.52.0",
"contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==",
"resolved": "4.54.1",
"contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==",
"dependencies": {
"Microsoft.IdentityModel.Abstractions": "6.22.0"
}
},
"Microsoft.Identity.Client.Extensions.Msal": {
"type": "Transitive",
"resolved": "2.25.3",
"contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==",
"resolved": "2.31.0",
"contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==",
"dependencies": {
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client": "4.54.1",
"System.IO.FileSystem.AccessControl": "5.0.0",
"System.Security.Cryptography.ProtectedData": "4.5.0"
}
@ -2508,7 +2508,7 @@
"type": "Project",
"dependencies": {
"Azure.Data.Tables": "[12.8.0, )",
"Azure.Identity": "[1.8.2, )",
"Azure.Identity": "[1.10.2, )",
"Azure.Messaging.EventGrid": "[4.15.0, )",
"Azure.ResourceManager": "[1.6.0, )",
"Azure.ResourceManager.Compute": "[1.0.0-beta.8, )",
@ -2535,7 +2535,7 @@
"Microsoft.Extensions.Logging.ApplicationInsights": "[2.21.0, )",
"Microsoft.FeatureManagement": "[2.5.1, )",
"Microsoft.Graph": "[4.37.0, )",
"Microsoft.Identity.Client": "[4.52.0, )",
"Microsoft.Identity.Client": "[4.54.1, )",
"Microsoft.Identity.Web.TokenCache": "[2.7.0, )",
"Microsoft.Rest.ClientRuntime": "[2.3.24, )",
"Microsoft.TeamFoundationServer.Client": "[19.219.0-preview, )",

30
src/ApiService/Tests/packages.lock.json
View File

@ -89,11 +89,11 @@
},
"Azure.Core": {
"type": "Transitive",
"resolved": "1.32.0",
"contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==",
"resolved": "1.35.0",
"contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==",
"dependencies": {
"Microsoft.Bcl.AsyncInterfaces": "1.1.1",
"System.Diagnostics.DiagnosticSource": "4.6.0",
"System.Diagnostics.DiagnosticSource": "6.0.1",
"System.Memory.Data": "1.0.2",
"System.Numerics.Vectors": "4.5.0",
"System.Text.Encodings.Web": "4.7.2",
@ -122,12 +122,12 @@
},
"Azure.Identity": {
"type": "Transitive",
"resolved": "1.8.2",
"contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==",
"resolved": "1.10.2",
"contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==",
"dependencies": {
"Azure.Core": "1.25.0",
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.25.3",
"Azure.Core": "1.35.0",
"Microsoft.Identity.Client": "4.54.1",
"Microsoft.Identity.Client.Extensions.Msal": "2.31.0",
"System.Memory": "4.5.4",
"System.Security.Cryptography.ProtectedData": "4.7.0",
"System.Text.Json": "4.7.2",
@ -984,18 +984,18 @@
},
"Microsoft.Identity.Client": {
"type": "Transitive",
"resolved": "4.52.0",
"contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==",
"resolved": "4.54.1",
"contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==",
"dependencies": {
"Microsoft.IdentityModel.Abstractions": "6.22.0"
}
},
"Microsoft.Identity.Client.Extensions.Msal": {
"type": "Transitive",
"resolved": "2.25.3",
"contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==",
"resolved": "2.31.0",
"contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==",
"dependencies": {
"Microsoft.Identity.Client": "4.49.1",
"Microsoft.Identity.Client": "4.54.1",
"System.IO.FileSystem.AccessControl": "5.0.0",
"System.Security.Cryptography.ProtectedData": "4.5.0"
}
@ -2511,7 +2511,7 @@
"type": "Project",
"dependencies": {
"Azure.Data.Tables": "[12.8.0, )",
"Azure.Identity": "[1.8.2, )",
"Azure.Identity": "[1.10.2, )",
"Azure.Messaging.EventGrid": "[4.15.0, )",
"Azure.ResourceManager": "[1.6.0, )",
"Azure.ResourceManager.Compute": "[1.0.0-beta.8, )",
@ -2538,7 +2538,7 @@
"Microsoft.Extensions.Logging.ApplicationInsights": "[2.21.0, )",
"Microsoft.FeatureManagement": "[2.5.1, )",
"Microsoft.Graph": "[4.37.0, )",
"Microsoft.Identity.Client": "[4.52.0, )",
"Microsoft.Identity.Client": "[4.54.1, )",
"Microsoft.Identity.Web.TokenCache": "[2.7.0, )",
"Microsoft.Rest.ClientRuntime": "[2.3.24, )",
"Microsoft.TeamFoundationServer.Client": "[19.219.0-preview, )",

38
src/agent/onefuzz-task/src/managed/cmd.rs
View File

@ -2,7 +2,7 @@
// Licensed under the MIT License.
use std::path::PathBuf;
use anyhow::{bail, Result};
use anyhow::Result;
use clap::{value_parser, Arg, Command};
use ipc_channel::ipc::{self, IpcReceiver, IpcSender};
@ -96,21 +96,29 @@ pub async fn run(args: &clap::ArgMatches) -> Result<()> {
let min_available_memory_bytes = 1_000_000 * config.common().min_available_memory_mb;
// If the memory limit is 0, this will resolve immediately with an error.
let check_oom = out_of_memory(min_available_memory_bytes);
let result = match min_available_memory_bytes {
0 => {
log::info!("memory watchdog is disabled: this task may fail suddenly if it runs out of memory.");
config.run().await
}
_ => {
// If the memory limit is 0, this will never return.
let check_oom = out_of_memory(min_available_memory_bytes);
let result = tokio::select! {
result = config.run() => result,
tokio::select! {
result = config.run() => result,
// Ignore this task if it returns due to a querying error.
Ok(oom) = check_oom => {
// Convert the OOM notification to an error, so we can log it below.
let err = anyhow::format_err!("out of memory: {} bytes available, {} required", oom.available_bytes, oom.min_bytes);
Err(err)
},
// Ignore this task if it returns due to a querying error.
Ok(oom) = check_oom => {
// Convert the OOM notification to an error, so we can log it below.
let err = anyhow::format_err!("out of memory: {} bytes available, {} required", oom.available_bytes, oom.min_bytes);
Err(err)
},
_shutdown = shutdown_listener => {
Ok(())
_shutdown = shutdown_listener => {
Ok(())
}
}
}
};
@ -131,9 +139,7 @@ const MAX_OOM_QUERY_ERRORS: usize = 5;
//
// Parameterized to enable future configuration by VMSS.
async fn out_of_memory(min_bytes: u64) -> Result<OutOfMemory> {
if min_bytes == 0 {
bail!("available memory minimum is unreachable");
}
log::info!("memory watchdog is enabled: this task will fail informatively if there are {} bytes ({:.2}MB) or fewer of usable memory left.", min_bytes, min_bytes as f64 / 1_000_000f64);
let mut consecutive_query_errors = 0;

2
src/cli/onefuzz/api.py
View File

@ -1061,6 +1061,7 @@ class Tasks(Endpoint):
module_allowlist: Optional[str] = None,
source_allowlist: Optional[str] = None,
task_env: Optional[Dict[str, str]] = None,
min_available_memory_mb: Optional[int] = None,
) -> models.Task:
"""
Create a task
@ -1138,6 +1139,7 @@ class Tasks(Endpoint):
module_allowlist=module_allowlist,
source_allowlist=source_allowlist,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
),
)

36
src/cli/onefuzz/debug.py
View File

@ -19,7 +19,7 @@ from azure.applicationinsights import ApplicationInsightsDataClient
from azure.applicationinsights.models import QueryBody
from azure.identity import AzureCliCredential
from azure.storage.blob import ContainerClient
from onefuzztypes import models, requests, responses
from onefuzztypes import models, primitives, requests, responses
from onefuzztypes.enums import ContainerType, TaskType
from onefuzztypes.models import (
BlobRef,
@ -635,20 +635,26 @@ class DebugLog(Command):
job_id: Optional[str],
task_id: Optional[str],
machine_id: Optional[str],
last: Optional[int] = 1,
out_dir: Optional[primitives.Directory],
last: Optional[int] = None,
all: bool = False,
) -> None:
"""
Download the latest agent logs.
Download all of the agent logs.
Make sure you have Storage Blob Data Reader permission.
:param str job_id: Which job you would like the logs for.
:param str task_id: Which task you would like the logs for.
:param str machine_id: Which machine you would like the logs for.
:param int last: The logs are split in files. Starting with the newest files, how many files you would you like to download.
:param bool all: Download all log files.
:param str out_dir: The directory where you would like to download the logs.
:param int last: (DEPRECATED) This option is a no-op. Now that machines have just one log file, getting the most recent logs implies downloading all of the log files.
:param bool all: (DEPRECATED) This option is a no-op. Now that machines have just one log file, getting the most recent logs implies downloading all of the log files.
"""
# Appease mypy while these options are still part of the API
del last
del all
from typing import cast
from urllib import parse
@ -667,11 +673,14 @@ class DebugLog(Command):
f"Job with id {job_id} does not have a logging location configured"
)
granularity = "job"
file_path = None
if task_id is not None:
granularity = "task"
file_path = f"{task_id}/"
if machine_id is not None:
granularity = "machine"
file_path += f"{machine_id}/"
container_name = parse.urlsplit(container_url).path[1:]
@ -707,14 +716,23 @@ class DebugLog(Command):
self.logger.info("Did not find any matching files to download")
return None
if not all:
self.logger.info(f"Downloading only the {last} most recent files")
files = files[:last]
if granularity == "job":
self.logger.info(
f"Downloading all of the log files for each task associated with the job with id {job_id}"
)
elif granularity == "task":
self.logger.info(
f"Downloading the log file for each machine associated with the task with id {task_id}"
)
else:
self.logger.info(
f"Downloading the log file for the machine with id {machine_id}"
)
for f in files:
self.logger.info(f"Downloading {f.name}")
local_path = os.path.join(os.getcwd(), f.name)
local_path = os.path.join(out_dir or os.getcwd(), f.name)
local_directory = os.path.dirname(local_path)
if not os.path.exists(local_directory):
os.makedirs(local_directory)

2
src/cli/onefuzz/templates/afl.py
View File

@ -55,6 +55,7 @@ class AFL(Command):
debug: Optional[List[TaskDebugFlag]] = None,
ensemble_sync_delay: Optional[int] = None,
extra_setup_container: Optional[Container] = None,
min_available_memory_mb: Optional[int] = None,
) -> Optional[Job]:
"""
Basic AFL job
@ -166,6 +167,7 @@ class AFL(Command):
target_env=target_env,
debug=debug,
ensemble_sync_delay=ensemble_sync_delay,
min_available_memory_mb=min_available_memory_mb,
)
report_containers = [

17
src/cli/onefuzz/templates/libfuzzer.py
View File

@ -83,6 +83,7 @@ class Libfuzzer(Command):
analyzer_env: Optional[Dict[str, str]] = None,
tools: Optional[Container] = None,
task_env: Optional[Dict[str, str]] = None,
min_available_memory_mb: Optional[int] = None,
) -> None:
target_options = target_options or []
regression_containers: List[Tuple[ContainerType, Container]] = [
@ -127,6 +128,7 @@ class Libfuzzer(Command):
colocate=colocate_all_tasks or colocate_secondary_tasks,
minimized_stack_depth=minimized_stack_depth,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
fuzzer_containers: List[Tuple[ContainerType, Container]] = [
@ -180,6 +182,7 @@ class Libfuzzer(Command):
check_fuzzer_help=check_fuzzer_help,
expect_crash_on_failure=expect_crash_on_failure,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
prereq_tasks = [fuzzer_task.task_id, regression_task.task_id]
@ -243,6 +246,7 @@ class Libfuzzer(Command):
module_allowlist=module_allowlist,
source_allowlist=source_allowlist,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
report_containers: List[Tuple[ContainerType, Container]] = [
@ -280,6 +284,7 @@ class Libfuzzer(Command):
colocate=colocate_all_tasks or colocate_secondary_tasks,
minimized_stack_depth=minimized_stack_depth,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
if analyzer_exe is not None:
@ -320,6 +325,7 @@ class Libfuzzer(Command):
debug=debug,
target_timeout=target_timeout,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
def basic(
@ -369,6 +375,7 @@ class Libfuzzer(Command):
extra_output_container: Optional[Container] = None,
crashes: Optional[Container] = None,
task_env: Optional[Dict[str, str]] = None,
min_available_memory_mb: Optional[int] = None,
) -> Optional[Job]:
"""
Basic libfuzzer job
@ -507,6 +514,7 @@ class Libfuzzer(Command):
tools=tools,
task_env=task_env,
target_timeout=target_timeout,
min_available_memory_mb=min_available_memory_mb,
)
self.logger.info("done creating tasks")
@ -543,6 +551,7 @@ class Libfuzzer(Command):
no_check_fuzzer_help: bool = False,
extra_setup_container: Optional[Container] = None,
task_env: Optional[Dict[str, str]] = None,
min_available_memory_mb: Optional[int] = None,
) -> Optional[Job]:
"""
libfuzzer merge task
@ -657,6 +666,7 @@ class Libfuzzer(Command):
preserve_existing_outputs=preserve_existing_outputs,
check_fuzzer_help=check_fuzzer_help,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
self.logger.info("done creating tasks")
@ -698,6 +708,7 @@ class Libfuzzer(Command):
extra_setup_container: Optional[Container] = None,
crashes: Optional[Container] = None,
task_env: Optional[Dict[str, str]] = None,
min_available_memory_mb: Optional[int] = None,
) -> Optional[Job]:
pool = self.onefuzz.pools.get(pool_name)
@ -820,6 +831,7 @@ class Libfuzzer(Command):
expect_crash_on_failure=expect_crash_on_failure,
check_fuzzer_help=False,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
# Ensure the fuzzing task starts before we schedule the coverage and
@ -874,6 +886,7 @@ class Libfuzzer(Command):
debug=debug,
colocate=colocate_all_tasks or colocate_secondary_tasks,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
report_containers = [
@ -915,6 +928,7 @@ class Libfuzzer(Command):
debug=debug,
colocate=colocate_all_tasks or colocate_secondary_tasks,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
self.logger.info("done creating tasks")
@ -956,6 +970,7 @@ class Libfuzzer(Command):
crashes: Optional[Container] = None,
readonly_inputs: Optional[Container] = None,
task_env: Optional[Dict[str, str]] = None,
min_available_memory_mb: Optional[int] = None,
) -> Optional[Job]:
"""
libfuzzer tasks, wrapped via qemu-user (PREVIEW FEATURE)
@ -1125,6 +1140,7 @@ class Libfuzzer(Command):
expect_crash_on_failure=False,
check_fuzzer_help=check_fuzzer_help,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
report_containers = [
@ -1167,6 +1183,7 @@ class Libfuzzer(Command):
expect_crash_on_failure=False,
check_fuzzer_help=check_fuzzer_help,
task_env=task_env,
min_available_memory_mb=min_available_memory_mb,
)
self.logger.info("done creating tasks")

2
src/cli/onefuzz/templates/ossfuzz.py
View File

@ -120,6 +120,7 @@ class OssFuzz(Command):
debug: Optional[List[TaskDebugFlag]] = None,
ensemble_sync_delay: Optional[int] = None,
extra_setup_container: Optional[Container] = None,
min_available_memory_mb: Optional[int] = None,
) -> None:
"""
OssFuzz style libfuzzer jobs
@ -257,6 +258,7 @@ class OssFuzz(Command):
tags=helper.tags,
debug=debug,
ensemble_sync_delay=ensemble_sync_delay,
min_available_memory_mb=min_available_memory_mb,
)
helpers.append(helper)
base_helper.wait()

4
src/cli/onefuzz/templates/radamsa.py
View File

@ -51,6 +51,7 @@ class Radamsa(Command):
ensemble_sync_delay: Optional[int] = None,
target_timeout: Optional[int] = None,
extra_setup_container: Optional[Container] = None,
min_available_memory_mb: Optional[int] = None,
) -> Optional[Job]:
"""
Basic radamsa job
@ -187,6 +188,7 @@ class Radamsa(Command):
debug=debug,
ensemble_sync_delay=ensemble_sync_delay,
target_timeout=target_timeout,
min_available_memory_mb=min_available_memory_mb,
)
report_containers = [
@ -227,6 +229,7 @@ class Radamsa(Command):
prereq_tasks=[fuzzer_task.task_id],
debug=debug,
target_timeout=target_timeout,
min_available_memory_mb=min_available_memory_mb,
)
if helper.platform == OS.windows:
@ -277,6 +280,7 @@ class Radamsa(Command):
prereq_tasks=[fuzzer_task.task_id],
debug=debug,
target_timeout=target_timeout,
min_available_memory_mb=min_available_memory_mb,
)
self.logger.info("done creating tasks")

6
src/cli/onefuzz/templates/regression.py
View File

@ -57,6 +57,7 @@ class Regression(Command):
delete_input_container: bool = True,
check_regressions: bool = False,
extra_setup_container: Optional[Container] = None,
min_available_memory_mb: Optional[int] = None,
) -> None:
"""
generic regression task
@ -91,6 +92,7 @@ class Regression(Command):
delete_input_container=delete_input_container,
check_regressions=check_regressions,
extra_setup_container=extra_setup_container,
min_available_memory_mb=min_available_memory_mb,
)
def libfuzzer(
@ -118,6 +120,7 @@ class Regression(Command):
delete_input_container: bool = True,
check_regressions: bool = False,
extra_setup_container: Optional[Container] = None,
min_available_memory_mb: Optional[int] = None,
) -> None:
"""
libfuzzer regression task
@ -152,6 +155,7 @@ class Regression(Command):
delete_input_container=delete_input_container,
check_regressions=check_regressions,
extra_setup_container=extra_setup_container,
min_available_memory_mb=min_available_memory_mb,
)
def _create_job(
@ -180,6 +184,7 @@ class Regression(Command):
delete_input_container: bool = True,
check_regressions: bool = False,
extra_setup_container: Optional[Container] = None,
min_available_memory_mb: Optional[int] = None,
) -> None:
if dryrun:
return None
@ -270,6 +275,7 @@ class Regression(Command):
debug=debug,
check_fuzzer_help=check_fuzzer_help,
report_list=reports,
min_available_memory_mb=min_available_memory_mb,
)
helper.wait_for_stopped = check_regressions

3
src/pytypes/onefuzztypes/models.py
View File

@ -42,7 +42,7 @@ class UserInfo(BaseModel):
upn: Optional[str]
# Stores the address of a secret
# Store the address of a secret
class SecretAddress(BaseModel):
# keyvault address of a secret
url: str
@ -173,6 +173,7 @@ class TaskDetails(BaseModel):
target_class: Optional[str]
target_method: Optional[str]
task_env: Optional[Dict[str, str]]
min_available_memory_mb: Optional[int] = Field(ge=0)
class TaskPool(BaseModel):