From d9ac2a7da93408740249e82a2f1e68e3d49740c6 Mon Sep 17 00:00:00 2001 From: Kanan B <32438208+kananb@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:09:55 -0700 Subject: [PATCH 1/3] Make `min_available_memory_mb` configurable (#3577) * Add min_available_memory_mb to cli and service configs to be passed to agent * Add min_available_memory_mb to task create and template parameters * Add logging to indicate the configured min_available_memory * Add missing parameter * Remove unnecessary import * Update webhook_events.py * test change * original version of docs. * Update webhook_events.py again * Don't start memory checking task when min_available_memory_mb is zero * cargo fmt * Remove comments * Add min_available_memory_mb to libfuzzer's _create_tasks() * Add a megabyte converted value for min available memory logs * Use float instead of int for division --------- Co-authored-by: Noah Harper --- docs/webhook_events.md | 40 +++++++++++++++++++ .../ApiService/OneFuzzTypes/Model.cs | 2 + .../ApiService/onefuzzlib/Config.cs | 5 ++- .../notifications/JinjaTemplateAdapter.cs | 1 + src/agent/onefuzz-task/src/managed/cmd.rs | 38 ++++++++++-------- src/cli/onefuzz/api.py | 2 + src/cli/onefuzz/templates/afl.py | 2 + src/cli/onefuzz/templates/libfuzzer.py | 17 ++++++++ src/cli/onefuzz/templates/ossfuzz.py | 2 + src/cli/onefuzz/templates/radamsa.py | 4 ++ src/cli/onefuzz/templates/regression.py | 6 +++ src/pytypes/onefuzztypes/models.py | 3 +- 12 files changed, 104 insertions(+), 18 deletions(-) diff --git a/docs/webhook_events.md b/docs/webhook_events.md index cd8c5932f..16f9b6a20 100644 --- a/docs/webhook_events.md +++ b/docs/webhook_events.md @@ -445,6 +445,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" @@ -2332,6 +2337,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" @@ -3054,6 +3064,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" @@ -3572,6 +3587,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" @@ -4056,6 +4076,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" @@ -4514,6 +4539,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" @@ -4999,6 +5029,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" @@ -6743,6 +6778,11 @@ If webhook is set to have Event Grid message format then the payload will look a "title": "Generator Options", "type": "array" }, + "min_available_memory_mb": { + "minimum": 0, + "title": "Min Available Memory Mb", + "type": "integer" + }, "minimized_stack_depth": { "title": "Minimized Stack Depth", "type": "integer" diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index 23811e9fe..54847d0e1 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -239,6 +239,7 @@ public record TaskDetails( List? ReportList = null, long? MinimizedStackDepth = null, Dictionary? TaskEnv = null, + ulong? MinAvailableMemoryMb = null, // Deprecated. Retained for processing old table data. string? CoverageFilter = null, @@ -1164,6 +1165,7 @@ public record TaskUnitConfig( public IContainerDef? RegressionReports { get; set; } public IContainerDef? ExtraSetup { get; set; } public IContainerDef? ExtraOutput { get; set; } + public ulong? MinAvailableMemoryMb { get; set; } } public record NodeCommandEnvelope( diff --git a/src/ApiService/ApiService/onefuzzlib/Config.cs b/src/ApiService/ApiService/onefuzzlib/Config.cs index 872cedbc0..01c9f9442 100644 --- a/src/ApiService/ApiService/onefuzzlib/Config.cs +++ b/src/ApiService/ApiService/onefuzzlib/Config.cs @@ -73,7 +73,10 @@ public class Config : IConfig { HeartbeatQueue: await _queue.GetQueueSas("task-heartbeat", StorageType.Config, QueueSasPermissions.Add) ?? throw new Exception("unable to get heartbeat queue sas"), JobResultQueue: await _queue.GetQueueSas("job-result", StorageType.Config, QueueSasPermissions.Add) ?? throw new Exception("unable to get heartbeat queue sas"), Tags: task.Config.Tags ?? new Dictionary() - ); + ) { + // It's okay if this is null because the agent will use a default value if so. + MinAvailableMemoryMb = task.Config.Task.MinAvailableMemoryMb, + }; if (definition.MonitorQueue != null) { config.inputQueue = await _queue.GetQueueSas(task.TaskId.ToString(), StorageType.Corpus, QueueSasPermissions.All); diff --git a/src/ApiService/ApiService/onefuzzlib/notifications/JinjaTemplateAdapter.cs b/src/ApiService/ApiService/onefuzzlib/notifications/JinjaTemplateAdapter.cs index 17dc8873a..2d7e61ebd 100644 --- a/src/ApiService/ApiService/onefuzzlib/notifications/JinjaTemplateAdapter.cs +++ b/src/ApiService/ApiService/onefuzzlib/notifications/JinjaTemplateAdapter.cs @@ -166,6 +166,7 @@ public class JinjaTemplateAdapter { targetOptions, 1, new Dictionary(), + null, "coverage filter", "module allow list", "source allow list", diff --git a/src/agent/onefuzz-task/src/managed/cmd.rs b/src/agent/onefuzz-task/src/managed/cmd.rs index 1640fffd1..7c7bc815a 100644 --- a/src/agent/onefuzz-task/src/managed/cmd.rs +++ b/src/agent/onefuzz-task/src/managed/cmd.rs @@ -2,7 +2,7 @@ // Licensed under the MIT License. use std::path::PathBuf; -use anyhow::{bail, Result}; +use anyhow::Result; use clap::{value_parser, Arg, Command}; use ipc_channel::ipc::{self, IpcReceiver, IpcSender}; @@ -96,21 +96,29 @@ pub async fn run(args: &clap::ArgMatches) -> Result<()> { let min_available_memory_bytes = 1_000_000 * config.common().min_available_memory_mb; - // If the memory limit is 0, this will resolve immediately with an error. - let check_oom = out_of_memory(min_available_memory_bytes); + let result = match min_available_memory_bytes { + 0 => { + log::info!("memory watchdog is disabled: this task may fail suddenly if it runs out of memory."); + config.run().await + } + _ => { + // If the memory limit is 0, this will never return. + let check_oom = out_of_memory(min_available_memory_bytes); - let result = tokio::select! { - result = config.run() => result, + tokio::select! { + result = config.run() => result, - // Ignore this task if it returns due to a querying error. - Ok(oom) = check_oom => { - // Convert the OOM notification to an error, so we can log it below. - let err = anyhow::format_err!("out of memory: {} bytes available, {} required", oom.available_bytes, oom.min_bytes); - Err(err) - }, + // Ignore this task if it returns due to a querying error. + Ok(oom) = check_oom => { + // Convert the OOM notification to an error, so we can log it below. + let err = anyhow::format_err!("out of memory: {} bytes available, {} required", oom.available_bytes, oom.min_bytes); + Err(err) + }, - _shutdown = shutdown_listener => { - Ok(()) + _shutdown = shutdown_listener => { + Ok(()) + } + } } }; @@ -131,9 +139,7 @@ const MAX_OOM_QUERY_ERRORS: usize = 5; // // Parameterized to enable future configuration by VMSS. async fn out_of_memory(min_bytes: u64) -> Result { - if min_bytes == 0 { - bail!("available memory minimum is unreachable"); - } + log::info!("memory watchdog is enabled: this task will fail informatively if there are {} bytes ({:.2}MB) or fewer of usable memory left.", min_bytes, min_bytes as f64 / 1_000_000f64); let mut consecutive_query_errors = 0; diff --git a/src/cli/onefuzz/api.py b/src/cli/onefuzz/api.py index 696819264..64cad8c36 100644 --- a/src/cli/onefuzz/api.py +++ b/src/cli/onefuzz/api.py @@ -1061,6 +1061,7 @@ class Tasks(Endpoint): module_allowlist: Optional[str] = None, source_allowlist: Optional[str] = None, task_env: Optional[Dict[str, str]] = None, + min_available_memory_mb: Optional[int] = None, ) -> models.Task: """ Create a task @@ -1138,6 +1139,7 @@ class Tasks(Endpoint): module_allowlist=module_allowlist, source_allowlist=source_allowlist, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ), ) diff --git a/src/cli/onefuzz/templates/afl.py b/src/cli/onefuzz/templates/afl.py index e4d49233d..339b45765 100644 --- a/src/cli/onefuzz/templates/afl.py +++ b/src/cli/onefuzz/templates/afl.py @@ -55,6 +55,7 @@ class AFL(Command): debug: Optional[List[TaskDebugFlag]] = None, ensemble_sync_delay: Optional[int] = None, extra_setup_container: Optional[Container] = None, + min_available_memory_mb: Optional[int] = None, ) -> Optional[Job]: """ Basic AFL job @@ -166,6 +167,7 @@ class AFL(Command): target_env=target_env, debug=debug, ensemble_sync_delay=ensemble_sync_delay, + min_available_memory_mb=min_available_memory_mb, ) report_containers = [ diff --git a/src/cli/onefuzz/templates/libfuzzer.py b/src/cli/onefuzz/templates/libfuzzer.py index f48737212..4b32cac0c 100644 --- a/src/cli/onefuzz/templates/libfuzzer.py +++ b/src/cli/onefuzz/templates/libfuzzer.py @@ -83,6 +83,7 @@ class Libfuzzer(Command): analyzer_env: Optional[Dict[str, str]] = None, tools: Optional[Container] = None, task_env: Optional[Dict[str, str]] = None, + min_available_memory_mb: Optional[int] = None, ) -> None: target_options = target_options or [] regression_containers: List[Tuple[ContainerType, Container]] = [ @@ -127,6 +128,7 @@ class Libfuzzer(Command): colocate=colocate_all_tasks or colocate_secondary_tasks, minimized_stack_depth=minimized_stack_depth, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) fuzzer_containers: List[Tuple[ContainerType, Container]] = [ @@ -180,6 +182,7 @@ class Libfuzzer(Command): check_fuzzer_help=check_fuzzer_help, expect_crash_on_failure=expect_crash_on_failure, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) prereq_tasks = [fuzzer_task.task_id, regression_task.task_id] @@ -243,6 +246,7 @@ class Libfuzzer(Command): module_allowlist=module_allowlist, source_allowlist=source_allowlist, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) report_containers: List[Tuple[ContainerType, Container]] = [ @@ -280,6 +284,7 @@ class Libfuzzer(Command): colocate=colocate_all_tasks or colocate_secondary_tasks, minimized_stack_depth=minimized_stack_depth, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) if analyzer_exe is not None: @@ -320,6 +325,7 @@ class Libfuzzer(Command): debug=debug, target_timeout=target_timeout, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) def basic( @@ -369,6 +375,7 @@ class Libfuzzer(Command): extra_output_container: Optional[Container] = None, crashes: Optional[Container] = None, task_env: Optional[Dict[str, str]] = None, + min_available_memory_mb: Optional[int] = None, ) -> Optional[Job]: """ Basic libfuzzer job @@ -507,6 +514,7 @@ class Libfuzzer(Command): tools=tools, task_env=task_env, target_timeout=target_timeout, + min_available_memory_mb=min_available_memory_mb, ) self.logger.info("done creating tasks") @@ -543,6 +551,7 @@ class Libfuzzer(Command): no_check_fuzzer_help: bool = False, extra_setup_container: Optional[Container] = None, task_env: Optional[Dict[str, str]] = None, + min_available_memory_mb: Optional[int] = None, ) -> Optional[Job]: """ libfuzzer merge task @@ -657,6 +666,7 @@ class Libfuzzer(Command): preserve_existing_outputs=preserve_existing_outputs, check_fuzzer_help=check_fuzzer_help, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) self.logger.info("done creating tasks") @@ -698,6 +708,7 @@ class Libfuzzer(Command): extra_setup_container: Optional[Container] = None, crashes: Optional[Container] = None, task_env: Optional[Dict[str, str]] = None, + min_available_memory_mb: Optional[int] = None, ) -> Optional[Job]: pool = self.onefuzz.pools.get(pool_name) @@ -820,6 +831,7 @@ class Libfuzzer(Command): expect_crash_on_failure=expect_crash_on_failure, check_fuzzer_help=False, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) # Ensure the fuzzing task starts before we schedule the coverage and @@ -874,6 +886,7 @@ class Libfuzzer(Command): debug=debug, colocate=colocate_all_tasks or colocate_secondary_tasks, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) report_containers = [ @@ -915,6 +928,7 @@ class Libfuzzer(Command): debug=debug, colocate=colocate_all_tasks or colocate_secondary_tasks, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) self.logger.info("done creating tasks") @@ -956,6 +970,7 @@ class Libfuzzer(Command): crashes: Optional[Container] = None, readonly_inputs: Optional[Container] = None, task_env: Optional[Dict[str, str]] = None, + min_available_memory_mb: Optional[int] = None, ) -> Optional[Job]: """ libfuzzer tasks, wrapped via qemu-user (PREVIEW FEATURE) @@ -1125,6 +1140,7 @@ class Libfuzzer(Command): expect_crash_on_failure=False, check_fuzzer_help=check_fuzzer_help, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) report_containers = [ @@ -1167,6 +1183,7 @@ class Libfuzzer(Command): expect_crash_on_failure=False, check_fuzzer_help=check_fuzzer_help, task_env=task_env, + min_available_memory_mb=min_available_memory_mb, ) self.logger.info("done creating tasks") diff --git a/src/cli/onefuzz/templates/ossfuzz.py b/src/cli/onefuzz/templates/ossfuzz.py index 94024add8..2b7696956 100644 --- a/src/cli/onefuzz/templates/ossfuzz.py +++ b/src/cli/onefuzz/templates/ossfuzz.py @@ -120,6 +120,7 @@ class OssFuzz(Command): debug: Optional[List[TaskDebugFlag]] = None, ensemble_sync_delay: Optional[int] = None, extra_setup_container: Optional[Container] = None, + min_available_memory_mb: Optional[int] = None, ) -> None: """ OssFuzz style libfuzzer jobs @@ -257,6 +258,7 @@ class OssFuzz(Command): tags=helper.tags, debug=debug, ensemble_sync_delay=ensemble_sync_delay, + min_available_memory_mb=min_available_memory_mb, ) helpers.append(helper) base_helper.wait() diff --git a/src/cli/onefuzz/templates/radamsa.py b/src/cli/onefuzz/templates/radamsa.py index d9ec34e15..369851fb1 100644 --- a/src/cli/onefuzz/templates/radamsa.py +++ b/src/cli/onefuzz/templates/radamsa.py @@ -51,6 +51,7 @@ class Radamsa(Command): ensemble_sync_delay: Optional[int] = None, target_timeout: Optional[int] = None, extra_setup_container: Optional[Container] = None, + min_available_memory_mb: Optional[int] = None, ) -> Optional[Job]: """ Basic radamsa job @@ -187,6 +188,7 @@ class Radamsa(Command): debug=debug, ensemble_sync_delay=ensemble_sync_delay, target_timeout=target_timeout, + min_available_memory_mb=min_available_memory_mb, ) report_containers = [ @@ -227,6 +229,7 @@ class Radamsa(Command): prereq_tasks=[fuzzer_task.task_id], debug=debug, target_timeout=target_timeout, + min_available_memory_mb=min_available_memory_mb, ) if helper.platform == OS.windows: @@ -277,6 +280,7 @@ class Radamsa(Command): prereq_tasks=[fuzzer_task.task_id], debug=debug, target_timeout=target_timeout, + min_available_memory_mb=min_available_memory_mb, ) self.logger.info("done creating tasks") diff --git a/src/cli/onefuzz/templates/regression.py b/src/cli/onefuzz/templates/regression.py index 0aa2550da..c8f47ea80 100644 --- a/src/cli/onefuzz/templates/regression.py +++ b/src/cli/onefuzz/templates/regression.py @@ -57,6 +57,7 @@ class Regression(Command): delete_input_container: bool = True, check_regressions: bool = False, extra_setup_container: Optional[Container] = None, + min_available_memory_mb: Optional[int] = None, ) -> None: """ generic regression task @@ -91,6 +92,7 @@ class Regression(Command): delete_input_container=delete_input_container, check_regressions=check_regressions, extra_setup_container=extra_setup_container, + min_available_memory_mb=min_available_memory_mb, ) def libfuzzer( @@ -118,6 +120,7 @@ class Regression(Command): delete_input_container: bool = True, check_regressions: bool = False, extra_setup_container: Optional[Container] = None, + min_available_memory_mb: Optional[int] = None, ) -> None: """ libfuzzer regression task @@ -152,6 +155,7 @@ class Regression(Command): delete_input_container=delete_input_container, check_regressions=check_regressions, extra_setup_container=extra_setup_container, + min_available_memory_mb=min_available_memory_mb, ) def _create_job( @@ -180,6 +184,7 @@ class Regression(Command): delete_input_container: bool = True, check_regressions: bool = False, extra_setup_container: Optional[Container] = None, + min_available_memory_mb: Optional[int] = None, ) -> None: if dryrun: return None @@ -270,6 +275,7 @@ class Regression(Command): debug=debug, check_fuzzer_help=check_fuzzer_help, report_list=reports, + min_available_memory_mb=min_available_memory_mb, ) helper.wait_for_stopped = check_regressions diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 70e9831e0..53d8fa84a 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -42,7 +42,7 @@ class UserInfo(BaseModel): upn: Optional[str] -# Stores the address of a secret +# Store the address of a secret class SecretAddress(BaseModel): # keyvault address of a secret url: str @@ -173,6 +173,7 @@ class TaskDetails(BaseModel): target_class: Optional[str] target_method: Optional[str] task_env: Optional[Dict[str, str]] + min_available_memory_mb: Optional[int] = Field(ge=0) class TaskPool(BaseModel): From 09c857904747c180df8e347a0896d963cb4ec618 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 22:27:54 +0000 Subject: [PATCH 2/3] Bump Azure.Identity from 1.8.2 to 1.10.2 in /src/ApiService/ApiService (#3584) * Bump Azure.Identity from 1.8.2 to 1.10.2 in /src/ApiService/ApiService Bumps [Azure.Identity](https://github.com/Azure/azure-sdk-for-net) from 1.8.2 to 1.10.2. - [Release notes](https://github.com/Azure/azure-sdk-for-net/releases) - [Commits](https://github.com/Azure/azure-sdk-for-net/compare/Azure.Identity_1.8.2...Azure.Identity_1.10.2) --- updated-dependencies: - dependency-name: Azure.Identity dependency-type: direct:production ... Signed-off-by: dependabot[bot] * fix dependencies --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Cheick Keita --- src/ApiService/ApiService/ApiService.csproj | 6 ++-- src/ApiService/ApiService/packages.lock.json | 30 ++++++++--------- .../FunctionalTests/FunctionalTests.csproj | 2 +- .../FunctionalTests/packages.lock.json | 32 +++++++++---------- .../IntegrationTests/packages.lock.json | 30 ++++++++--------- src/ApiService/Tests/packages.lock.json | 30 ++++++++--------- 6 files changed, 65 insertions(+), 65 deletions(-) diff --git a/src/ApiService/ApiService/ApiService.csproj b/src/ApiService/ApiService/ApiService.csproj index 1a80d69bc..59ded11fa 100644 --- a/src/ApiService/ApiService/ApiService.csproj +++ b/src/ApiService/ApiService/ApiService.csproj @@ -12,7 +12,7 @@ - + @@ -31,7 +31,7 @@ - + @@ -40,7 +40,7 @@ - + diff --git a/src/ApiService/ApiService/packages.lock.json b/src/ApiService/ApiService/packages.lock.json index 1f912aa01..28cbc3f3d 100644 --- a/src/ApiService/ApiService/packages.lock.json +++ b/src/ApiService/ApiService/packages.lock.json @@ -14,13 +14,13 @@ }, "Azure.Identity": { "type": "Direct", - "requested": "[1.8.2, )", - "resolved": "1.8.2", - "contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==", + "requested": "[1.10.2, )", + "resolved": "1.10.2", + "contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==", "dependencies": { - "Azure.Core": "1.25.0", - "Microsoft.Identity.Client": "4.49.1", - "Microsoft.Identity.Client.Extensions.Msal": "2.25.3", + "Azure.Core": "1.35.0", + "Microsoft.Identity.Client": "4.54.1", + "Microsoft.Identity.Client.Extensions.Msal": "2.31.0", "System.Memory": "4.5.4", "System.Security.Cryptography.ProtectedData": "4.7.0", "System.Text.Json": "4.7.2", @@ -299,9 +299,9 @@ }, "Microsoft.Identity.Client": { "type": "Direct", - "requested": "[4.52.0, )", - "resolved": "4.52.0", - "contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==", + "requested": "[4.54.1, )", + "resolved": "4.54.1", + "contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==", "dependencies": { "Microsoft.IdentityModel.Abstractions": "6.22.0" } @@ -446,11 +446,11 @@ }, "Azure.Core": { "type": "Transitive", - "resolved": "1.32.0", - "contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==", + "resolved": "1.35.0", + "contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==", "dependencies": { "Microsoft.Bcl.AsyncInterfaces": "1.1.1", - "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Diagnostics.DiagnosticSource": "6.0.1", "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", "System.Text.Encodings.Web": "4.7.2", @@ -1031,10 +1031,10 @@ }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.25.3", - "contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==", + "resolved": "2.31.0", + "contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==", "dependencies": { - "Microsoft.Identity.Client": "4.49.1", + "Microsoft.Identity.Client": "4.54.1", "System.IO.FileSystem.AccessControl": "5.0.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } diff --git a/src/ApiService/FunctionalTests/FunctionalTests.csproj b/src/ApiService/FunctionalTests/FunctionalTests.csproj index 5b702705a..4108bc944 100644 --- a/src/ApiService/FunctionalTests/FunctionalTests.csproj +++ b/src/ApiService/FunctionalTests/FunctionalTests.csproj @@ -19,7 +19,7 @@ all - + diff --git a/src/ApiService/FunctionalTests/packages.lock.json b/src/ApiService/FunctionalTests/packages.lock.json index d138ad07c..685aa0c39 100644 --- a/src/ApiService/FunctionalTests/packages.lock.json +++ b/src/ApiService/FunctionalTests/packages.lock.json @@ -34,9 +34,9 @@ }, "Microsoft.Identity.Client": { "type": "Direct", - "requested": "[4.52.0, )", - "resolved": "4.52.0", - "contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==", + "requested": "[4.54.1, )", + "resolved": "4.54.1", + "contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==", "dependencies": { "Microsoft.IdentityModel.Abstractions": "6.22.0" } @@ -122,11 +122,11 @@ }, "Azure.Core": { "type": "Transitive", - "resolved": "1.32.0", - "contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==", + "resolved": "1.35.0", + "contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==", "dependencies": { "Microsoft.Bcl.AsyncInterfaces": "1.1.1", - "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Diagnostics.DiagnosticSource": "6.0.1", "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", "System.Text.Encodings.Web": "4.7.2", @@ -155,12 +155,12 @@ }, "Azure.Identity": { "type": "Transitive", - "resolved": "1.8.2", - "contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==", + "resolved": "1.10.2", + "contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==", "dependencies": { - "Azure.Core": "1.25.0", - "Microsoft.Identity.Client": "4.49.1", - "Microsoft.Identity.Client.Extensions.Msal": "2.25.3", + "Azure.Core": "1.35.0", + "Microsoft.Identity.Client": "4.54.1", + "Microsoft.Identity.Client.Extensions.Msal": "2.31.0", "System.Memory": "4.5.4", "System.Security.Cryptography.ProtectedData": "4.7.0", "System.Text.Json": "4.7.2", @@ -971,10 +971,10 @@ }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.25.3", - "contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==", + "resolved": "2.31.0", + "contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==", "dependencies": { - "Microsoft.Identity.Client": "4.49.1", + "Microsoft.Identity.Client": "4.54.1", "System.IO.FileSystem.AccessControl": "5.0.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } @@ -2376,7 +2376,7 @@ "type": "Project", "dependencies": { "Azure.Data.Tables": "[12.8.0, )", - "Azure.Identity": "[1.8.2, )", + "Azure.Identity": "[1.10.2, )", "Azure.Messaging.EventGrid": "[4.15.0, )", "Azure.ResourceManager": "[1.6.0, )", "Azure.ResourceManager.Compute": "[1.0.0-beta.8, )", @@ -2403,7 +2403,7 @@ "Microsoft.Extensions.Logging.ApplicationInsights": "[2.21.0, )", "Microsoft.FeatureManagement": "[2.5.1, )", "Microsoft.Graph": "[4.37.0, )", - "Microsoft.Identity.Client": "[4.52.0, )", + "Microsoft.Identity.Client": "[4.54.1, )", "Microsoft.Identity.Web.TokenCache": "[2.7.0, )", "Microsoft.Rest.ClientRuntime": "[2.3.24, )", "Microsoft.TeamFoundationServer.Client": "[19.219.0-preview, )", diff --git a/src/ApiService/IntegrationTests/packages.lock.json b/src/ApiService/IntegrationTests/packages.lock.json index 497cf663a..32119423e 100644 --- a/src/ApiService/IntegrationTests/packages.lock.json +++ b/src/ApiService/IntegrationTests/packages.lock.json @@ -61,11 +61,11 @@ }, "Azure.Core": { "type": "Transitive", - "resolved": "1.32.0", - "contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==", + "resolved": "1.35.0", + "contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==", "dependencies": { "Microsoft.Bcl.AsyncInterfaces": "1.1.1", - "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Diagnostics.DiagnosticSource": "6.0.1", "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", "System.Text.Encodings.Web": "4.7.2", @@ -94,12 +94,12 @@ }, "Azure.Identity": { "type": "Transitive", - "resolved": "1.8.2", - "contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==", + "resolved": "1.10.2", + "contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==", "dependencies": { - "Azure.Core": "1.25.0", - "Microsoft.Identity.Client": "4.49.1", - "Microsoft.Identity.Client.Extensions.Msal": "2.25.3", + "Azure.Core": "1.35.0", + "Microsoft.Identity.Client": "4.54.1", + "Microsoft.Identity.Client.Extensions.Msal": "2.31.0", "System.Memory": "4.5.4", "System.Security.Cryptography.ProtectedData": "4.7.0", "System.Text.Json": "4.7.2", @@ -973,18 +973,18 @@ }, "Microsoft.Identity.Client": { "type": "Transitive", - "resolved": "4.52.0", - "contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==", + "resolved": "4.54.1", + "contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==", "dependencies": { "Microsoft.IdentityModel.Abstractions": "6.22.0" } }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.25.3", - "contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==", + "resolved": "2.31.0", + "contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==", "dependencies": { - "Microsoft.Identity.Client": "4.49.1", + "Microsoft.Identity.Client": "4.54.1", "System.IO.FileSystem.AccessControl": "5.0.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } @@ -2508,7 +2508,7 @@ "type": "Project", "dependencies": { "Azure.Data.Tables": "[12.8.0, )", - "Azure.Identity": "[1.8.2, )", + "Azure.Identity": "[1.10.2, )", "Azure.Messaging.EventGrid": "[4.15.0, )", "Azure.ResourceManager": "[1.6.0, )", "Azure.ResourceManager.Compute": "[1.0.0-beta.8, )", @@ -2535,7 +2535,7 @@ "Microsoft.Extensions.Logging.ApplicationInsights": "[2.21.0, )", "Microsoft.FeatureManagement": "[2.5.1, )", "Microsoft.Graph": "[4.37.0, )", - "Microsoft.Identity.Client": "[4.52.0, )", + "Microsoft.Identity.Client": "[4.54.1, )", "Microsoft.Identity.Web.TokenCache": "[2.7.0, )", "Microsoft.Rest.ClientRuntime": "[2.3.24, )", "Microsoft.TeamFoundationServer.Client": "[19.219.0-preview, )", diff --git a/src/ApiService/Tests/packages.lock.json b/src/ApiService/Tests/packages.lock.json index d3ad57006..d15590a24 100644 --- a/src/ApiService/Tests/packages.lock.json +++ b/src/ApiService/Tests/packages.lock.json @@ -89,11 +89,11 @@ }, "Azure.Core": { "type": "Transitive", - "resolved": "1.32.0", - "contentHash": "NmnJxaNqKjPwnHXngVg63SrkwbJXrkT0mcK8uCx9rSq0nK6Q3Q+/GZRCaTWcdcECoRP5XK0lr3Ce8PZkHkuHNg==", + "resolved": "1.35.0", + "contentHash": "hENcx03Jyuqv05F4RBEPbxz29UrM3Nbhnr6Wl6NQpoU9BCIbL3XLentrxDCTrH54NLS11Exxi/o8MYgT/cnKFA==", "dependencies": { "Microsoft.Bcl.AsyncInterfaces": "1.1.1", - "System.Diagnostics.DiagnosticSource": "4.6.0", + "System.Diagnostics.DiagnosticSource": "6.0.1", "System.Memory.Data": "1.0.2", "System.Numerics.Vectors": "4.5.0", "System.Text.Encodings.Web": "4.7.2", @@ -122,12 +122,12 @@ }, "Azure.Identity": { "type": "Transitive", - "resolved": "1.8.2", - "contentHash": "ywnpn9MLhNTtBG12WOxSaomx0Dwu5HK5PyhHH/CApGrd1BCrhgEwdy4Uwy5IfAznJzVJKZRyKR9cp4aa61xYvA==", + "resolved": "1.10.2", + "contentHash": "jfq07QnxB7Rx15DWHxIfZbdbgICL1IARncBPIYmnmF+1Xqn6KqiF6ijlKv2hj82WFr9kUi+jzU8zVqrBocJZ8A==", "dependencies": { - "Azure.Core": "1.25.0", - "Microsoft.Identity.Client": "4.49.1", - "Microsoft.Identity.Client.Extensions.Msal": "2.25.3", + "Azure.Core": "1.35.0", + "Microsoft.Identity.Client": "4.54.1", + "Microsoft.Identity.Client.Extensions.Msal": "2.31.0", "System.Memory": "4.5.4", "System.Security.Cryptography.ProtectedData": "4.7.0", "System.Text.Json": "4.7.2", @@ -984,18 +984,18 @@ }, "Microsoft.Identity.Client": { "type": "Transitive", - "resolved": "4.52.0", - "contentHash": "6/qdhyE+nmbtoBxwmeMvTCWfin3KLoADNx+XwgDVuju7n6kiAVwjhJj4M9aXvVJ6caZzzteuahUbsHBhLYq8Ag==", + "resolved": "4.54.1", + "contentHash": "YkQkV3IRaA1W36HD4NRD1cq+QFr+4QPKK3SgTSpx+RiobXnLZ6E9anOjDi2TS7okOEofBbjR6GyTPp4IR0MnEQ==", "dependencies": { "Microsoft.IdentityModel.Abstractions": "6.22.0" } }, "Microsoft.Identity.Client.Extensions.Msal": { "type": "Transitive", - "resolved": "2.25.3", - "contentHash": "I6/Od0d3OMD9b7RPxW1l25A8oA94H+r9ZtrOe4Ogk49Ftxhs9RS+pbzPE5dLe0i7nQy+1aob7mR22YsNcc0BiQ==", + "resolved": "2.31.0", + "contentHash": "IhGSqN0szneKC5Qk3/okJQJbDpQfLW/+mvslhzJPox4t2UuIkA2ZHe4w/z62ASye46G9sQWF9qqLXTgNacE2xQ==", "dependencies": { - "Microsoft.Identity.Client": "4.49.1", + "Microsoft.Identity.Client": "4.54.1", "System.IO.FileSystem.AccessControl": "5.0.0", "System.Security.Cryptography.ProtectedData": "4.5.0" } @@ -2511,7 +2511,7 @@ "type": "Project", "dependencies": { "Azure.Data.Tables": "[12.8.0, )", - "Azure.Identity": "[1.8.2, )", + "Azure.Identity": "[1.10.2, )", "Azure.Messaging.EventGrid": "[4.15.0, )", "Azure.ResourceManager": "[1.6.0, )", "Azure.ResourceManager.Compute": "[1.0.0-beta.8, )", @@ -2538,7 +2538,7 @@ "Microsoft.Extensions.Logging.ApplicationInsights": "[2.21.0, )", "Microsoft.FeatureManagement": "[2.5.1, )", "Microsoft.Graph": "[4.37.0, )", - "Microsoft.Identity.Client": "[4.52.0, )", + "Microsoft.Identity.Client": "[4.54.1, )", "Microsoft.Identity.Web.TokenCache": "[2.7.0, )", "Microsoft.Rest.ClientRuntime": "[2.3.24, )", "Microsoft.TeamFoundationServer.Client": "[19.219.0-preview, )", From 86a642be8d2f345704b34af9560edf30c2f34a14 Mon Sep 17 00:00:00 2001 From: Kanan B <32438208+kananb@users.noreply.github.com> Date: Wed, 18 Oct 2023 15:58:45 -0700 Subject: [PATCH 3/3] Make `debug logs get` behave more intuitively (#3573) * Update the printed message for debug logs get to be more accurate * Fix min() that was being passed an Optional[int] * Negate equality operator * Change != to 'is not' * Make --all and --last noops, add --out_dir, and always download all log files * Add del statements for unused parameters --- src/cli/onefuzz/debug.py | 36 +++++++++++++++++++++++++++--------- 1 file changed, 27 insertions(+), 9 deletions(-) diff --git a/src/cli/onefuzz/debug.py b/src/cli/onefuzz/debug.py index 27aee5a0e..638bab05c 100644 --- a/src/cli/onefuzz/debug.py +++ b/src/cli/onefuzz/debug.py @@ -19,7 +19,7 @@ from azure.applicationinsights import ApplicationInsightsDataClient from azure.applicationinsights.models import QueryBody from azure.identity import AzureCliCredential from azure.storage.blob import ContainerClient -from onefuzztypes import models, requests, responses +from onefuzztypes import models, primitives, requests, responses from onefuzztypes.enums import ContainerType, TaskType from onefuzztypes.models import ( BlobRef, @@ -635,20 +635,26 @@ class DebugLog(Command): job_id: Optional[str], task_id: Optional[str], machine_id: Optional[str], - last: Optional[int] = 1, + out_dir: Optional[primitives.Directory], + last: Optional[int] = None, all: bool = False, ) -> None: """ - Download the latest agent logs. + Download all of the agent logs. Make sure you have Storage Blob Data Reader permission. :param str job_id: Which job you would like the logs for. :param str task_id: Which task you would like the logs for. :param str machine_id: Which machine you would like the logs for. - :param int last: The logs are split in files. Starting with the newest files, how many files you would you like to download. - :param bool all: Download all log files. + :param str out_dir: The directory where you would like to download the logs. + :param int last: (DEPRECATED) This option is a no-op. Now that machines have just one log file, getting the most recent logs implies downloading all of the log files. + :param bool all: (DEPRECATED) This option is a no-op. Now that machines have just one log file, getting the most recent logs implies downloading all of the log files. """ + # Appease mypy while these options are still part of the API + del last + del all + from typing import cast from urllib import parse @@ -667,11 +673,14 @@ class DebugLog(Command): f"Job with id {job_id} does not have a logging location configured" ) + granularity = "job" file_path = None if task_id is not None: + granularity = "task" file_path = f"{task_id}/" if machine_id is not None: + granularity = "machine" file_path += f"{machine_id}/" container_name = parse.urlsplit(container_url).path[1:] @@ -707,14 +716,23 @@ class DebugLog(Command): self.logger.info("Did not find any matching files to download") return None - if not all: - self.logger.info(f"Downloading only the {last} most recent files") - files = files[:last] + if granularity == "job": + self.logger.info( + f"Downloading all of the log files for each task associated with the job with id {job_id}" + ) + elif granularity == "task": + self.logger.info( + f"Downloading the log file for each machine associated with the task with id {task_id}" + ) + else: + self.logger.info( + f"Downloading the log file for the machine with id {machine_id}" + ) for f in files: self.logger.info(f"Downloading {f.name}") - local_path = os.path.join(os.getcwd(), f.name) + local_path = os.path.join(out_dir or os.getcwd(), f.name) local_directory = os.path.dirname(local_path) if not os.path.exists(local_directory): os.makedirs(local_directory)