Add extra_output container, rename extra container (#3064)

## Summary of the Pull Request

- **Breaking** (but as far as I know this feature is not yet in use): rename the `extra_container` to `extra_setup_container`.
- **Add**: the `extra_output_container`, which pushes its outputs continually.
  - We may also want a type of container which both pushes & pulls? See discussion below.
- **Improved**: if `onefuzz-task` fails upon launch, we will log its output for diagnosis (might close #3113)

---

Some thoughts for the future:

We might want to redesign the containers so that we have something like the following which is passed to the agent, and the agent doesn't need to know the specifics of the containers supplied:

```jsonc
{
    // ...
    "containers": {
        "extra_setup_dir": {
            "mode": "pull",
            "container_name": "yyy",
        },
        "extra_output_dir": {
            "mode": "push",
            "continuous": true, // keep pushing while job is running
            "container_name": "xxx"
        }
    }
}
```

At the moment the agent needs to know what each container is for, for each task type. A more generic and flexible method might be simpler overall.

src/cli/onefuzz/templates/afl.py

@@ -53,7 +53,7 @@ class AFL(Command):
         notification_config: Optional[NotificationConfig] = None,
         debug: Optional[List[TaskDebugFlag]] = None,
         ensemble_sync_delay: Optional[int] = None,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
     ) -> Optional[Job]:
         """
         Basic AFL job
@@ -135,9 +135,12 @@ class AFL(Command):
             (ContainerType.inputs, helper.containers[ContainerType.inputs]),
         ]
 
-        if extra_container is not None:
+        if extra_setup_container is not None:
             containers.append(
-                (ContainerType.extra, helper.containers[ContainerType.extra])
+                (
+                    ContainerType.extra_setup,
+                    helper.containers[ContainerType.extra_setup],
+                )
             )
 
         self.logger.info("creating afl fuzz task")
@@ -173,9 +176,12 @@ class AFL(Command):
             ),
         ]
 
-        if extra_container is not None:
+        if extra_setup_container is not None:
             report_containers.append(
-                (ContainerType.extra, helper.containers[ContainerType.extra])
+                (
+                    ContainerType.extra_setup,
+                    helper.containers[ContainerType.extra_setup],
+                )
             )
 
         self.logger.info("creating generic_crash_report task")

src/cli/onefuzz/templates/libfuzzer.py

@@ -6,7 +6,7 @@
 import os
 import tempfile
 from enum import Enum
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 
 from onefuzztypes.enums import OS, ContainerType, TaskDebugFlag, TaskType
 from onefuzztypes.models import Job, NotificationConfig
@@ -43,6 +43,14 @@ class Libfuzzer(Command):
         if LIBFUZZER_MAGIC_STRING not in data:
             raise Exception("not a libfuzzer binary: %s" % target_exe)
 
+    def _add_optional_containers(
+        self,
+        dest: List[Tuple[ContainerType, Container]],
+        source: Dict[ContainerType, Container],
+        types: List[ContainerType],
+    ) -> None:
+        dest.extend([(c, source[c]) for c in types if c in source])
+
     def _create_tasks(
         self,
         *,
@@ -86,10 +94,11 @@ class Libfuzzer(Command):
             ),
         ]
 
-        if ContainerType.extra in containers:
-            regression_containers.append(
-                (ContainerType.extra, containers[ContainerType.extra])
-            )
+        self._add_optional_containers(
+            regression_containers,
+            containers,
+            [ContainerType.extra_setup, ContainerType.extra_output],
+        )
 
         # We don't really need a separate timeout for crash reporting, and we could just
         # use `target_timeout`. But `crash_report_timeout` was introduced first, so we
@@ -124,18 +133,15 @@ class Libfuzzer(Command):
             (ContainerType.inputs, containers[ContainerType.inputs]),
         ]
 
-        if ContainerType.readonly_inputs in containers:
-            fuzzer_containers.append(
-                (
-                    ContainerType.readonly_inputs,
-                    containers[ContainerType.readonly_inputs],
-                )
-            )
-
-        if ContainerType.extra in containers:
-            fuzzer_containers.append(
-                (ContainerType.extra, containers[ContainerType.extra])
-            )
+        self._add_optional_containers(
+            fuzzer_containers,
+            containers,
+            [
+                ContainerType.extra_setup,
+                ContainerType.extra_output,
+                ContainerType.readonly_inputs,
+            ],
+        )
 
         self.logger.info("creating libfuzzer task")
 
@@ -180,18 +186,15 @@ class Libfuzzer(Command):
             (ContainerType.readonly_inputs, containers[ContainerType.inputs]),
         ]
 
-        if ContainerType.extra in containers:
-            coverage_containers.append(
-                (ContainerType.extra, containers[ContainerType.extra])
-            )
-
-        if ContainerType.readonly_inputs in containers:
-            coverage_containers.append(
-                (
-                    ContainerType.readonly_inputs,
-                    containers[ContainerType.readonly_inputs],
-                )
-            )
+        self._add_optional_containers(
+            coverage_containers,
+            containers,
+            [
+                ContainerType.extra_setup,
+                ContainerType.extra_output,
+                ContainerType.readonly_inputs,
+            ],
+        )
 
         self.logger.info("creating coverage task")
 
@@ -245,10 +248,11 @@ class Libfuzzer(Command):
             (ContainerType.no_repro, containers[ContainerType.no_repro]),
         ]
 
-        if ContainerType.extra in containers:
-            report_containers.append(
-                (ContainerType.extra, containers[ContainerType.extra])
-            )
+        self._add_optional_containers(
+            report_containers,
+            containers,
+            [ContainerType.extra_setup, ContainerType.extra_output],
+        )
 
         self.logger.info("creating libfuzzer_crash_report task")
         self.onefuzz.tasks.create(
@@ -288,10 +292,11 @@ class Libfuzzer(Command):
                 (ContainerType.crashes, containers[ContainerType.crashes]),
             ]
 
-            if ContainerType.extra in containers:
-                analysis_containers.append(
-                    (ContainerType.extra, containers[ContainerType.extra])
-                )
+            self._add_optional_containers(
+                analysis_containers,
+                containers,
+                [ContainerType.extra_setup, ContainerType.extra_output],
+            )
 
             self.onefuzz.tasks.create(
                 job.job_id,
@@ -357,7 +362,8 @@ class Libfuzzer(Command):
         analyzer_options: Optional[List[str]] = None,
         analyzer_env: Optional[Dict[str, str]] = None,
         tools: Optional[Container] = None,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
+        extra_output_container: Optional[Container] = None,
         crashes: Optional[Container] = None,
     ) -> Optional[Job]:
         """
@@ -456,8 +462,11 @@ class Libfuzzer(Command):
 
         containers = helper.containers
 
-        if extra_container is not None:
-            containers[ContainerType.extra] = extra_container
+        if extra_setup_container is not None:
+            containers[ContainerType.extra_setup] = extra_setup_container
+
+        if extra_output_container is not None:
+            containers[ContainerType.extra_output] = extra_output_container
 
         self._create_tasks(
             job=helper.job,
@@ -521,7 +530,7 @@ class Libfuzzer(Command):
         preserve_existing_outputs: bool = False,
         check_fuzzer_help: bool = False,
         no_check_fuzzer_help: bool = False,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
     ) -> Optional[Job]:
         """
         libfuzzer merge task
@@ -590,8 +599,8 @@ class Libfuzzer(Command):
             ),
         ]
 
-        if extra_container is not None:
-            merge_containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            merge_containers.append((ContainerType.extra_setup, extra_setup_container))
 
         if inputs:
             merge_containers.append(
@@ -657,7 +666,7 @@ class Libfuzzer(Command):
         colocate_secondary_tasks: bool = True,
         expect_crash_on_failure: bool = False,
         notification_config: Optional[NotificationConfig] = None,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
         crashes: Optional[Container] = None,
     ) -> Optional[Job]:
         pool = self.onefuzz.pools.get(pool_name)
@@ -740,8 +749,8 @@ class Libfuzzer(Command):
             (ContainerType.tools, fuzzer_tools_container),
         ]
 
-        if extra_container is not None:
-            fuzzer_containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            fuzzer_containers.append((ContainerType.extra_setup, extra_setup_container))
 
         helper.create_containers()
         helper.setup_notifications(notification_config)
@@ -798,8 +807,10 @@ class Libfuzzer(Command):
             (ContainerType.tools, fuzzer_tools_container),
         ]
 
-        if extra_container is not None:
-            coverage_containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            coverage_containers.append(
+                (ContainerType.extra_setup, extra_setup_container)
+            )
 
         self.logger.info("creating `dotnet_coverage` task")
         self.onefuzz.tasks.create(
@@ -829,8 +840,8 @@ class Libfuzzer(Command):
             (ContainerType.tools, fuzzer_tools_container),
         ]
 
-        if extra_container is not None:
-            report_containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            report_containers.append((ContainerType.extra_setup, extra_setup_container))
 
         self.logger.info("creating `dotnet_crash_report` task")
         self.onefuzz.tasks.create(
@@ -886,7 +897,7 @@ class Libfuzzer(Command):
         check_retry_count: Optional[int] = 300,
         check_fuzzer_help: bool = False,
         no_check_fuzzer_help: bool = False,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
         crashes: Optional[Container] = None,
         readonly_inputs: Optional[Container] = None,
     ) -> Optional[Job]:
@@ -957,8 +968,8 @@ class Libfuzzer(Command):
             (ContainerType.inputs, helper.containers[ContainerType.inputs]),
         ]
 
-        if extra_container is not None:
-            fuzzer_containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            fuzzer_containers.append((ContainerType.extra_setup, extra_setup_container))
 
         if readonly_inputs is not None:
             self.onefuzz.containers.get(readonly_inputs)  # ensure it exists
@@ -1057,8 +1068,8 @@ class Libfuzzer(Command):
             (ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
         ]
 
-        if extra_container is not None:
-            report_containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            report_containers.append((ContainerType.extra_setup, extra_setup_container))
 
         self.logger.info("creating libfuzzer_crash_report task")
         self.onefuzz.tasks.create(

src/cli/onefuzz/templates/ossfuzz.py

@@ -119,7 +119,7 @@ class OssFuzz(Command):
         notification_config: Optional[NotificationConfig] = None,
         debug: Optional[List[TaskDebugFlag]] = None,
         ensemble_sync_delay: Optional[int] = None,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
     ) -> None:
         """
         OssFuzz style libfuzzer jobs
@@ -214,8 +214,8 @@ class OssFuzz(Command):
                 ContainerType.coverage,
             )
 
-            if extra_container is not None:
-                helper.containers[ContainerType.extra] = extra_container
+            if extra_setup_container is not None:
+                helper.containers[ContainerType.extra_setup] = extra_setup_container
 
             helper.create_containers()
             helper.setup_notifications(notification_config)

src/cli/onefuzz/templates/radamsa.py

@@ -50,7 +50,7 @@ class Radamsa(Command):
         debug: Optional[List[TaskDebugFlag]] = None,
         ensemble_sync_delay: Optional[int] = None,
         target_timeout: Optional[int] = None,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
     ) -> Optional[Job]:
         """
         Basic radamsa job
@@ -157,8 +157,8 @@ class Radamsa(Command):
             ),
         ]
 
-        if extra_container is not None:
-            containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            containers.append((ContainerType.extra_setup, extra_setup_container))
 
         fuzzer_task = self.onefuzz.tasks.create(
             helper.job.job_id,
@@ -193,8 +193,8 @@ class Radamsa(Command):
             (ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
         ]
 
-        if extra_container is not None:
-            report_containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container is not None:
+            report_containers.append((ContainerType.extra_setup, extra_setup_container))
 
         self.logger.info("creating generic_crash_report task")
         self.onefuzz.tasks.create(
@@ -239,8 +239,10 @@ class Radamsa(Command):
                 (ContainerType.crashes, helper.containers[ContainerType.crashes]),
             ]
 
-            if extra_container is not None:
-                analysis_containers.append((ContainerType.extra, extra_container))
+            if extra_setup_container is not None:
+                analysis_containers.append(
+                    (ContainerType.extra_setup, extra_setup_container)
+                )
 
             self.onefuzz.tasks.create(
                 helper.job.job_id,

src/cli/onefuzz/templates/regression.py

@@ -56,7 +56,7 @@ class Regression(Command):
         check_fuzzer_help: bool = True,
         delete_input_container: bool = True,
         check_regressions: bool = False,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
     ) -> None:
         """
         generic regression task
@@ -90,7 +90,7 @@ class Regression(Command):
             check_fuzzer_help=check_fuzzer_help,
             delete_input_container=delete_input_container,
             check_regressions=check_regressions,
-            extra_container=extra_container,
+            extra_setup_container=extra_setup_container,
         )
 
     def libfuzzer(
@@ -117,7 +117,7 @@ class Regression(Command):
         check_fuzzer_help: bool = True,
         delete_input_container: bool = True,
         check_regressions: bool = False,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
     ) -> None:
         """
         libfuzzer regression task
@@ -151,7 +151,7 @@ class Regression(Command):
             check_fuzzer_help=check_fuzzer_help,
             delete_input_container=delete_input_container,
             check_regressions=check_regressions,
-            extra_container=extra_container,
+            extra_setup_container=extra_setup_container,
         )
 
     def _create_job(
@@ -179,7 +179,7 @@ class Regression(Command):
         check_fuzzer_help: bool = True,
         delete_input_container: bool = True,
         check_regressions: bool = False,
-        extra_container: Optional[Container] = None,
+        extra_setup_container: Optional[Container] = None,
     ) -> None:
         if dryrun:
             return None
@@ -221,8 +221,8 @@ class Regression(Command):
             ),
         ]
 
-        if extra_container:
-            containers.append((ContainerType.extra, extra_container))
+        if extra_setup_container:
+            containers.append((ContainerType.extra_setup, extra_setup_container))
 
         if crashes:
             helper.containers[