mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-21 13:51:19 +00:00
Add mariner support (#3306)
* Updating setup.sh * logger works * Install omi * syntax * Add option to create mariner pool in checkpr * . * Need to install sudo * . * Include both logging extensions * Revert because we already isntall the azure monitor for linux extension * Downgrade type handler version * Add data collection rules and fuzzing articacts for integration test * TODOs * Fix linux jobs getting sent to mariner * Fix linux jobs going to marienr pool * Fix pools * Remove the old logging extension on linux * try to retain syslog * Value to be set was not clear * Trying to route logs * Maybe we need to specify properties even if we don't set anything * Start adding a data collection association * Create association * update packages.lock * . * Remove auto config and add dependency map * Bring back GCS autoconfig * Undo DCR stuff * Undo package version bump * Fix up files * Remove TODO
This commit is contained in:
Teo Voinea
GitHub
@ -88,6 +88,7 @@ class Integration(BaseModel):
|
||||
target_method: Optional[str]
|
||||
setup_dir: Optional[str]
|
||||
target_env: Optional[Dict[str, str]]
|
||||
pool: PoolName
|
||||
|
||||
|
||||
TARGETS: Dict[str, Integration] = {
|
||||
@ -97,6 +98,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
target_exe="fuzz.exe",
|
||||
inputs="seeds",
|
||||
wait_for_files={ContainerType.unique_reports: 1},
|
||||
pool="linux",
|
||||
),
|
||||
"linux-libfuzzer": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -124,6 +126,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
"--only_asan_failures",
|
||||
"--write_test_file={extra_output_dir}/test.txt",
|
||||
],
|
||||
pool="linux",
|
||||
),
|
||||
"linux-libfuzzer-with-options": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -137,6 +140,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
},
|
||||
reboot_after_setup=True,
|
||||
fuzzing_target_options=["-runs=10000000"],
|
||||
pool="linux",
|
||||
),
|
||||
"linux-libfuzzer-dlopen": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -150,6 +154,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
},
|
||||
reboot_after_setup=True,
|
||||
use_setup=True,
|
||||
pool="linux",
|
||||
),
|
||||
"linux-libfuzzer-linked-library": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -163,6 +168,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
},
|
||||
reboot_after_setup=True,
|
||||
use_setup=True,
|
||||
pool="linux",
|
||||
),
|
||||
"linux-libfuzzer-dotnet": Integration(
|
||||
template=TemplateType.libfuzzer_dotnet,
|
||||
@ -180,6 +186,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
ContainerType.unique_reports: 1,
|
||||
},
|
||||
test_repro=False,
|
||||
pool="linux",
|
||||
),
|
||||
"linux-libfuzzer-aarch64-crosscompile": Integration(
|
||||
template=TemplateType.libfuzzer_qemu_user,
|
||||
@ -189,6 +196,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
use_setup=True,
|
||||
wait_for_files={ContainerType.inputs: 2, ContainerType.crashes: 1},
|
||||
test_repro=False,
|
||||
pool="linux",
|
||||
),
|
||||
"linux-libfuzzer-rust": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -196,6 +204,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
target_exe="fuzz_target_1",
|
||||
wait_for_files={ContainerType.unique_reports: 1, ContainerType.coverage: 1},
|
||||
fuzzing_target_options=["--test:{extra_setup_dir}"],
|
||||
pool="linux",
|
||||
),
|
||||
"linux-trivial-crash": Integration(
|
||||
template=TemplateType.radamsa,
|
||||
@ -204,6 +213,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
inputs="seeds",
|
||||
wait_for_files={ContainerType.unique_reports: 1},
|
||||
inject_fake_regression=True,
|
||||
pool="linux",
|
||||
),
|
||||
"linux-trivial-crash-asan": Integration(
|
||||
template=TemplateType.radamsa,
|
||||
@ -213,6 +223,28 @@ TARGETS: Dict[str, Integration] = {
|
||||
wait_for_files={ContainerType.unique_reports: 1},
|
||||
check_asan_log=True,
|
||||
disable_check_debugger=True,
|
||||
pool="linux",
|
||||
),
|
||||
# TODO: Don't install OMS extension on linux anymore
|
||||
# TODO: Figure out why non mariner work is being scheduled to the mariner pool
|
||||
"mariner-libfuzzer": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
os=OS.linux,
|
||||
target_exe="fuzz.exe",
|
||||
inputs="seeds",
|
||||
wait_for_files={
|
||||
ContainerType.unique_reports: 1,
|
||||
ContainerType.coverage: 1,
|
||||
ContainerType.inputs: 2,
|
||||
ContainerType.extra_output: 1,
|
||||
},
|
||||
reboot_after_setup=True,
|
||||
inject_fake_regression=True,
|
||||
fuzzing_target_options=[
|
||||
"--test:{extra_setup_dir}",
|
||||
"--write_test_file={extra_output_dir}/test.txt",
|
||||
],
|
||||
pool=PoolName("mariner")
|
||||
),
|
||||
"windows-libfuzzer": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -234,6 +266,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
"--only_asan_failures",
|
||||
"--write_test_file={extra_output_dir}/test.txt",
|
||||
],
|
||||
pool="windows",
|
||||
),
|
||||
"windows-libfuzzer-linked-library": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -246,6 +279,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
ContainerType.coverage: 1,
|
||||
},
|
||||
use_setup=True,
|
||||
pool="windows",
|
||||
),
|
||||
"windows-libfuzzer-load-library": Integration(
|
||||
template=TemplateType.libfuzzer,
|
||||
@ -258,6 +292,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
ContainerType.coverage: 1,
|
||||
},
|
||||
use_setup=True,
|
||||
pool="windows",
|
||||
),
|
||||
"windows-libfuzzer-dotnet": Integration(
|
||||
template=TemplateType.libfuzzer_dotnet,
|
||||
@ -275,6 +310,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
ContainerType.unique_reports: 1,
|
||||
},
|
||||
test_repro=False,
|
||||
pool="windows",
|
||||
),
|
||||
"windows-trivial-crash": Integration(
|
||||
template=TemplateType.radamsa,
|
||||
@ -283,6 +319,7 @@ TARGETS: Dict[str, Integration] = {
|
||||
inputs="seeds",
|
||||
wait_for_files={ContainerType.unique_reports: 1},
|
||||
inject_fake_regression=True,
|
||||
pool="windows",
|
||||
),
|
||||
}
|
||||
|
||||
@ -351,7 +388,7 @@ class TestOnefuzz:
|
||||
|
||||
self.inject_log(self.start_log_marker)
|
||||
for entry in os_list:
|
||||
name = PoolName(f"testpool-{entry.name}-{self.test_id}")
|
||||
name = self.build_pool_name(entry.name)
|
||||
self.logger.info("creating pool: %s:%s", entry.name, name)
|
||||
self.of.pools.create(name, entry)
|
||||
self.logger.info("creating scaleset for pool: %s", name)
|
||||
@ -359,6 +396,15 @@ class TestOnefuzz:
|
||||
name, pool_size, region=region, initial_size=pool_size
|
||||
)
|
||||
|
||||
name = self.build_pool_name("mariner")
|
||||
self.logger.info("creating pool: %s:%s", "mariner", name)
|
||||
self.of.pools.create(name, OS.linux)
|
||||
self.logger.info("creating scaleset for pool: %s", name)
|
||||
self.of.scalesets.create(
|
||||
name, pool_size, region=region, initial_size=pool_size, image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest"
|
||||
)
|
||||
|
||||
|
||||
class UnmanagedPool:
|
||||
def __init__(
|
||||
self,
|
||||
@ -560,12 +606,9 @@ class TestOnefuzz:
|
||||
) -> List[UUID]:
|
||||
"""Launch all of the fuzzing templates"""
|
||||
|
||||
pools: Dict[OS, Pool] = {}
|
||||
pool = None
|
||||
if unmanaged_pool is not None:
|
||||
pools[unmanaged_pool.the_os] = self.of.pools.get(unmanaged_pool.pool_name)
|
||||
else:
|
||||
for pool in self.of.pools.list():
|
||||
pools[pool.os] = pool
|
||||
pool = unmanaged_pool.pool_name
|
||||
|
||||
job_ids = []
|
||||
|
||||
@ -576,8 +619,8 @@ class TestOnefuzz:
|
||||
if config.os not in os_list:
|
||||
continue
|
||||
|
||||
if config.os not in pools.keys():
|
||||
raise Exception(f"No pool for target: {target} ,os: {config.os}")
|
||||
if pool is None:
|
||||
pool = self.build_pool_name(config.pool)
|
||||
|
||||
self.logger.info("launching: %s", target)
|
||||
|
||||
@ -601,8 +644,9 @@ class TestOnefuzz:
|
||||
setup = Directory(os.path.join(setup, config.nested_setup_dir))
|
||||
|
||||
job: Optional[Job] = None
|
||||
|
||||
job = self.build_job(
|
||||
duration, pools, target, config, setup, target_exe, inputs
|
||||
duration, pool, target, config, setup, target_exe, inputs
|
||||
)
|
||||
|
||||
if config.inject_fake_regression and job is not None:
|
||||
@ -618,7 +662,7 @@ class TestOnefuzz:
|
||||
def build_job(
|
||||
self,
|
||||
duration: int,
|
||||
pools: Dict[OS, Pool],
|
||||
pool: PoolName,
|
||||
target: str,
|
||||
config: Integration,
|
||||
setup: Optional[Directory],
|
||||
@ -634,7 +678,7 @@ class TestOnefuzz:
|
||||
self.project,
|
||||
target,
|
||||
BUILD,
|
||||
pools[config.os].name,
|
||||
pool,
|
||||
target_exe=target_exe,
|
||||
inputs=inputs,
|
||||
setup_dir=setup,
|
||||
@ -659,7 +703,7 @@ class TestOnefuzz:
|
||||
self.project,
|
||||
target,
|
||||
BUILD,
|
||||
pools[config.os].name,
|
||||
pool,
|
||||
target_dll=File(config.target_exe),
|
||||
inputs=inputs,
|
||||
setup_dir=setup,
|
||||
@ -675,7 +719,7 @@ class TestOnefuzz:
|
||||
self.project,
|
||||
target,
|
||||
BUILD,
|
||||
pools[config.os].name,
|
||||
pool,
|
||||
inputs=inputs,
|
||||
target_exe=target_exe,
|
||||
duration=duration,
|
||||
@ -688,7 +732,7 @@ class TestOnefuzz:
|
||||
self.project,
|
||||
target,
|
||||
BUILD,
|
||||
pool_name=pools[config.os].name,
|
||||
pool_name=pool,
|
||||
target_exe=target_exe,
|
||||
inputs=inputs,
|
||||
setup_dir=setup,
|
||||
@ -703,7 +747,7 @@ class TestOnefuzz:
|
||||
self.project,
|
||||
target,
|
||||
BUILD,
|
||||
pool_name=pools[config.os].name,
|
||||
pool_name=pool,
|
||||
target_exe=target_exe,
|
||||
inputs=inputs,
|
||||
setup_dir=setup,
|
||||
@ -1233,6 +1277,9 @@ class TestOnefuzz:
|
||||
|
||||
if seen_errors:
|
||||
raise Exception("logs included errors")
|
||||
|
||||
def build_pool_name(self, os_type: str) -> PoolName:
|
||||
return PoolName(f"testpool-{os_type}-{self.test_id}")
|
||||
|
||||
|
||||
class Run(Command):
|
||||
|
||||
Reference in New Issue
Block a user