ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-17 20:38:06 +00:00
Code Issues Packages Projects Releases Wiki Activity

Adding extra container to tasks (#2847)

Browse Source 
* adding extra container to tasks

* setup expand

* build fix

* generate docs

* build fix

* build fix

* build fix

* format

* format

* build fix

* fix extra container references

* format

* Update "Needs Triage" label to the one we use. (#2845)

* Report extension errors (#2846)

Old failure message:
```
failed to launch extension
```

New failure message:

```
failed to launch extension(s): Errors for extension 'CustomScriptExtension':
:Error: ProvisioningState/failed/3 (Provisioning failed) - Failed to download all specified files. Exiting. Error Message: The remote server returned an error: (400) Bad Request.
```

* Sematically validate notification configs (#2850)

* Add new command

* Update remaining jinja templates and references to use scriban

* Add ado template validation

* Validate ado and github templates

* Remove unnecessary function

* Update src/ApiService/ApiService/OneFuzzTypes/Model.cs

Co-authored-by: Cheick Keita <kcheick@gmail.com>

---------

Co-authored-by: Cheick Keita <kcheick@gmail.com>

* adding extra container to integration tests

* adding doc

* update tests

* format

* build and clippy fix

* Update src/agent/onefuzz-task/src/tasks/report/generic.rs

Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>

---------

Co-authored-by: Marc Greisen <mgreisen@microsoft.com>
Co-authored-by: George Pollard <gpollard@microsoft.com>
Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>
This commit is contained in:
Cheick Keita
2023-02-23 11:08:01 -08:00
committed by GitHub
parent dfb0db87c1
commit b84896802c
47 changed files with 367 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
src/cli/onefuzz/templates/afl.py
View File

@ -53,6 +53,7 @@ class AFL(Command):
notification_config: Optional[NotificationConfig] = None,
debug: Optional[List[TaskDebugFlag]] = None,
ensemble_sync_delay: Optional[int] = None,
extra_container: Optional[Container] = None,
) -> Optional[Job]:
"""
Basic AFL job
@ -93,6 +94,7 @@ class AFL(Command):
ContainerType.reports,
ContainerType.unique_reports,
)
if existing_inputs:
self.onefuzz.containers.get(existing_inputs)
helper.containers[ContainerType.inputs] = existing_inputs
@ -133,6 +135,11 @@ class AFL(Command):
(ContainerType.inputs, helper.containers[ContainerType.inputs]),
]
if extra_container is not None:
containers.append(
(ContainerType.extra, helper.containers[ContainerType.extra])
)
self.logger.info("creating afl fuzz task")
fuzzer_task = self.onefuzz.tasks.create(
helper.job.job_id,
@ -166,6 +173,11 @@ class AFL(Command):
),
]
if extra_container is not None:
report_containers.append(
(ContainerType.extra, helper.containers[ContainerType.extra])
)
self.logger.info("creating generic_crash_report task")
self.onefuzz.tasks.create(
helper.job.job_id,

31
src/cli/onefuzz/templates/libfuzzer.py
View File

@ -74,6 +74,7 @@ class Libfuzzer(Command):
analyzer_options: Optional[List[str]] = None,
analyzer_env: Optional[Dict[str, str]] = None,
tools: Optional[Container] = None,
extra_container: Optional[Container] = None,
) -> None:
target_options = target_options or []
@ -331,6 +332,7 @@ class Libfuzzer(Command):
analyzer_options: Optional[List[str]] = None,
analyzer_env: Optional[Dict[str, str]] = None,
tools: Optional[Container] = None,
extra_container: Optional[Container] = None,
) -> Optional[Job]:
"""
Basic libfuzzer job
@ -413,9 +415,14 @@ class Libfuzzer(Command):
else:
source_allowlist_blob_name = None
containers = helper.containers
if extra_container is not None:
containers[ContainerType.extra] = extra_container
self._create_tasks(
job=helper.job,
containers=helper.containers,
containers=containers,
pool_name=pool_name,
target_exe=target_exe_blob_name,
vm_count=vm_count,
@ -474,6 +481,7 @@ class Libfuzzer(Command):
debug: Optional[List[TaskDebugFlag]] = None,
preserve_existing_outputs: bool = False,
check_fuzzer_help: bool = True,
extra_container: Optional[Container] = None,
) -> Optional[Job]:
"""
libfuzzer merge task
@ -510,6 +518,7 @@ class Libfuzzer(Command):
helper.define_containers(
ContainerType.setup,
)
if inputs:
helper.define_containers(ContainerType.inputs)
@ -535,6 +544,9 @@ class Libfuzzer(Command):
),
]
if extra_container is not None:
merge_containers.append((ContainerType.extra, extra_container))
if inputs:
merge_containers.append(
(ContainerType.inputs, helper.containers[ContainerType.inputs])
@ -598,6 +610,7 @@ class Libfuzzer(Command):
colocate_secondary_tasks: bool = True,
expect_crash_on_failure: bool = False,
notification_config: Optional[NotificationConfig] = None,
extra_container: Optional[Container] = None,
) -> Optional[Job]:
pool = self.onefuzz.pools.get(pool_name)
@ -673,6 +686,9 @@ class Libfuzzer(Command):
(ContainerType.tools, fuzzer_tools_container),
]
if extra_container is not None:
fuzzer_containers.append((ContainerType.extra, extra_container))
helper.create_containers()
helper.setup_notifications(notification_config)
@ -728,6 +744,9 @@ class Libfuzzer(Command):
(ContainerType.tools, fuzzer_tools_container),
]
if extra_container is not None:
coverage_containers.append((ContainerType.extra, extra_container))
self.logger.info("creating `dotnet_coverage` task")
self.onefuzz.tasks.create(
helper.job.job_id,
@ -756,6 +775,9 @@ class Libfuzzer(Command):
(ContainerType.tools, fuzzer_tools_container),
]
if extra_container is not None:
report_containers.append((ContainerType.extra, extra_container))
self.logger.info("creating `dotnet_crash_report` task")
self.onefuzz.tasks.create(
helper.job.job_id,
@ -808,6 +830,7 @@ class Libfuzzer(Command):
crash_report_timeout: Optional[int] = 1,
check_retry_count: Optional[int] = 300,
check_fuzzer_help: bool = True,
extra_container: Optional[Container] = None,
) -> Optional[Job]:
"""
libfuzzer tasks, wrapped via qemu-user (PREVIEW FEATURE)
@ -866,6 +889,9 @@ class Libfuzzer(Command):
(ContainerType.inputs, helper.containers[ContainerType.inputs]),
]
if extra_container is not None:
fuzzer_containers.append((ContainerType.extra, extra_container))
helper.create_containers()
target_exe_blob_name = helper.setup_relative_blob_name(target_exe, None)
@ -959,6 +985,9 @@ class Libfuzzer(Command):
(ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
]
if extra_container is not None:
report_containers.append((ContainerType.extra, extra_container))
self.logger.info("creating libfuzzer_crash_report task")
self.onefuzz.tasks.create(
helper.job.job_id,

7
src/cli/onefuzz/templates/ossfuzz.py
View File

@ -11,7 +11,7 @@ from typing import Dict, List, Optional, Tuple
from onefuzztypes.enums import OS, ContainerType, TaskDebugFlag
from onefuzztypes.models import NotificationConfig
from onefuzztypes.primitives import File, PoolName
from onefuzztypes.primitives import Container, File, PoolName
from onefuzz.api import Command
from onefuzz.backend import container_file_path
@ -119,6 +119,7 @@ class OssFuzz(Command):
notification_config: Optional[NotificationConfig] = None,
debug: Optional[List[TaskDebugFlag]] = None,
ensemble_sync_delay: Optional[int] = None,
extra_container: Optional[Container] = None,
) -> None:
"""
OssFuzz style libfuzzer jobs
@ -212,6 +213,10 @@ class OssFuzz(Command):
ContainerType.no_repro,
ContainerType.coverage,
)
if extra_container is not None:
helper.containers[ContainerType.extra] = extra_container
helper.create_containers()
helper.setup_notifications(notification_config)

11
src/cli/onefuzz/templates/radamsa.py
View File

@ -50,6 +50,7 @@ class Radamsa(Command):
debug: Optional[List[TaskDebugFlag]] = None,
ensemble_sync_delay: Optional[int] = None,
target_timeout: Optional[int] = None,
extra_container: Optional[Container] = None,
) -> Optional[Job]:
"""
Basic radamsa job
@ -90,6 +91,7 @@ class Radamsa(Command):
ContainerType.no_repro,
ContainerType.analysis,
)
if existing_inputs:
self.onefuzz.containers.get(existing_inputs)
helper.containers[ContainerType.readonly_inputs] = existing_inputs
@ -155,6 +157,9 @@ class Radamsa(Command):
),
]
if extra_container is not None:
containers.append((ContainerType.extra, extra_container))
fuzzer_task = self.onefuzz.tasks.create(
helper.job.job_id,
TaskType.generic_generator,
@ -188,6 +193,9 @@ class Radamsa(Command):
(ContainerType.no_repro, helper.containers[ContainerType.no_repro]),
]
if extra_container is not None:
report_containers.append((ContainerType.extra, extra_container))
self.logger.info("creating generic_crash_report task")
self.onefuzz.tasks.create(
helper.job.job_id,
@ -231,6 +239,9 @@ class Radamsa(Command):
(ContainerType.crashes, helper.containers[ContainerType.crashes]),
]
if extra_container is not None:
analysis_containers.append((ContainerType.extra, extra_container))
self.onefuzz.tasks.create(
helper.job.job_id,
TaskType.generic_analysis,

8
src/cli/onefuzz/templates/regression.py
View File

@ -56,6 +56,7 @@ class Regression(Command):
check_fuzzer_help: bool = True,
delete_input_container: bool = True,
check_regressions: bool = False,
extra_container: Optional[Container] = None,
) -> None:
"""
generic regression task
@ -89,6 +90,7 @@ class Regression(Command):
check_fuzzer_help=check_fuzzer_help,
delete_input_container=delete_input_container,
check_regressions=check_regressions,
extra_container=extra_container,
)
def libfuzzer(
@ -115,6 +117,7 @@ class Regression(Command):
check_fuzzer_help: bool = True,
delete_input_container: bool = True,
check_regressions: bool = False,
extra_container: Optional[Container] = None,
) -> None:
"""
libfuzzer regression task
@ -148,6 +151,7 @@ class Regression(Command):
check_fuzzer_help=check_fuzzer_help,
delete_input_container=delete_input_container,
check_regressions=check_regressions,
extra_container=extra_container,
)
def _create_job(
@ -175,6 +179,7 @@ class Regression(Command):
check_fuzzer_help: bool = True,
delete_input_container: bool = True,
check_regressions: bool = False,
extra_container: Optional[Container] = None,
) -> None:
if dryrun:
return None
@ -216,6 +221,9 @@ class Regression(Command):
),
]
if extra_container:
containers.append((ContainerType.extra, extra_container))
if crashes:
helper.containers[
ContainerType.readonly_inputs