ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-21 13:51:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add mariner support (#3306)

Browse Source 
* Updating setup.sh

* logger works

* Install omi

* syntax

* Add option to create mariner pool in checkpr

* .

* Need to install sudo

* .

* Include both logging extensions

* Revert because we already isntall the azure monitor for linux extension

* Downgrade type handler version

* Add data collection rules and fuzzing articacts for integration test

* TODOs

* Fix linux jobs getting sent to mariner

* Fix linux jobs going to marienr pool

* Fix pools

* Remove the old logging extension on linux

* try to retain syslog

* Value to be set was not clear

* Trying to route logs

* Maybe we need to specify properties even if we don't set anything

* Start adding a data collection association

* Create association

* update packages.lock

* .

* Remove auto config and add dependency map

* Bring back GCS autoconfig

* Undo DCR stuff

* Undo package version bump

* Fix up files

* Remove TODO
This commit is contained in:
Teo Voinea
2023-08-18 09:52:53 -04:00
committed by GitHub
parent 4e2b3fc648
commit bdb2f1337d
4 changed files with 126 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/ci.yml vendored
View File

@ -538,9 +538,11 @@ jobs:
mkdir -p artifacts/linux-libfuzzer
mkdir -p artifacts/linux-libfuzzer-with-options
mkdir -p artifacts/mariner-libfuzzer
(cd libfuzzer ; make )
cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer
cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer-with-options
cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/mariner-libfuzzer
mkdir -p artifacts/linux-libfuzzer-regression
(cd libfuzzer-regression ; make )

22
src/ApiService/ApiService/onefuzzlib/Extension.cs
View File

@ -36,7 +36,9 @@ public class Extensions : IExtensions {
var extensions = new List<VMExtensionWrapper>();
var instanceConfig = await _context.ConfigOperations.Fetch();
extensions.Add(await MonitorExtension(region, vmOs));
if (vmOs == Os.Windows) {
extensions.Add(await MonitorExtension(region));
}
var depenency = DependencyExtension(region, vmOs);
if (depenency is not null) {
@ -329,11 +331,10 @@ public class Extensions : IExtensions {
throw new NotSupportedException($"unsupported OS: {vmOs}");
}
public async Async.Task<VMExtensionWrapper> MonitorExtension(AzureLocation region, Os vmOs) {
public async Async.Task<VMExtensionWrapper> MonitorExtension(AzureLocation region) {
var settings = await _context.LogAnalytics.GetMonitorSettings();
var extensionSettings = JsonSerializer.Serialize(new { WorkspaceId = settings.Id }, _extensionSerializerOptions);
var protectedExtensionSettings = JsonSerializer.Serialize(new { WorkspaceKey = settings.Key }, _extensionSerializerOptions);
if (vmOs == Os.Windows) {
return new VMExtensionWrapper {
Location = region,
Name = "OMSExtension",
@ -345,21 +346,6 @@ public class Extensions : IExtensions {
ProtectedSettings = new BinaryData(protectedExtensionSettings),
EnableAutomaticUpgrade = false
};
} else if (vmOs == Os.Linux) {
return new VMExtensionWrapper {
Location = region,
Name = "OmsAgentForLinux",
TypePropertiesType = "OmsAgentForLinux",
Publisher = "Microsoft.EnterpriseCloud.Monitoring",
TypeHandlerVersion = "1.0",
AutoUpgradeMinorVersion = true,
Settings = new BinaryData(extensionSettings),
ProtectedSettings = new BinaryData(protectedExtensionSettings),
EnableAutomaticUpgrade = false
};
} else {
throw new NotSupportedException($"unsupported os: {vmOs}");
}
}

77
src/integration-tests/integration-test.py
View File

@ -88,6 +88,7 @@ class Integration(BaseModel):
target_method: Optional[str]
setup_dir: Optional[str]
target_env: Optional[Dict[str, str]]
pool: PoolName
TARGETS: Dict[str, Integration] = {
@ -97,6 +98,7 @@ TARGETS: Dict[str, Integration] = {
target_exe="fuzz.exe",
inputs="seeds",
wait_for_files={ContainerType.unique_reports: 1},
pool="linux",
),
"linux-libfuzzer": Integration(
template=TemplateType.libfuzzer,
@ -124,6 +126,7 @@ TARGETS: Dict[str, Integration] = {
"--only_asan_failures",
"--write_test_file={extra_output_dir}/test.txt",
],
pool="linux",
),
"linux-libfuzzer-with-options": Integration(
template=TemplateType.libfuzzer,
@ -137,6 +140,7 @@ TARGETS: Dict[str, Integration] = {
},
reboot_after_setup=True,
fuzzing_target_options=["-runs=10000000"],
pool="linux",
),
"linux-libfuzzer-dlopen": Integration(
template=TemplateType.libfuzzer,
@ -150,6 +154,7 @@ TARGETS: Dict[str, Integration] = {
},
reboot_after_setup=True,
use_setup=True,
pool="linux",
),
"linux-libfuzzer-linked-library": Integration(
template=TemplateType.libfuzzer,
@ -163,6 +168,7 @@ TARGETS: Dict[str, Integration] = {
},
reboot_after_setup=True,
use_setup=True,
pool="linux",
),
"linux-libfuzzer-dotnet": Integration(
template=TemplateType.libfuzzer_dotnet,
@ -180,6 +186,7 @@ TARGETS: Dict[str, Integration] = {
ContainerType.unique_reports: 1,
},
test_repro=False,
pool="linux",
),
"linux-libfuzzer-aarch64-crosscompile": Integration(
template=TemplateType.libfuzzer_qemu_user,
@ -189,6 +196,7 @@ TARGETS: Dict[str, Integration] = {
use_setup=True,
wait_for_files={ContainerType.inputs: 2, ContainerType.crashes: 1},
test_repro=False,
pool="linux",
),
"linux-libfuzzer-rust": Integration(
template=TemplateType.libfuzzer,
@ -196,6 +204,7 @@ TARGETS: Dict[str, Integration] = {
target_exe="fuzz_target_1",
wait_for_files={ContainerType.unique_reports: 1, ContainerType.coverage: 1},
fuzzing_target_options=["--test:{extra_setup_dir}"],
pool="linux",
),
"linux-trivial-crash": Integration(
template=TemplateType.radamsa,
@ -204,6 +213,7 @@ TARGETS: Dict[str, Integration] = {
inputs="seeds",
wait_for_files={ContainerType.unique_reports: 1},
inject_fake_regression=True,
pool="linux",
),
"linux-trivial-crash-asan": Integration(
template=TemplateType.radamsa,
@ -213,6 +223,28 @@ TARGETS: Dict[str, Integration] = {
wait_for_files={ContainerType.unique_reports: 1},
check_asan_log=True,
disable_check_debugger=True,
pool="linux",
),
# TODO: Don't install OMS extension on linux anymore
# TODO: Figure out why non mariner work is being scheduled to the mariner pool
"mariner-libfuzzer": Integration(
template=TemplateType.libfuzzer,
os=OS.linux,
target_exe="fuzz.exe",
inputs="seeds",
wait_for_files={
ContainerType.unique_reports: 1,
ContainerType.coverage: 1,
ContainerType.inputs: 2,
ContainerType.extra_output: 1,
},
reboot_after_setup=True,
inject_fake_regression=True,
fuzzing_target_options=[
"--test:{extra_setup_dir}",
"--write_test_file={extra_output_dir}/test.txt",
],
pool=PoolName("mariner")
),
"windows-libfuzzer": Integration(
template=TemplateType.libfuzzer,
@ -234,6 +266,7 @@ TARGETS: Dict[str, Integration] = {
"--only_asan_failures",
"--write_test_file={extra_output_dir}/test.txt",
],
pool="windows",
),
"windows-libfuzzer-linked-library": Integration(
template=TemplateType.libfuzzer,
@ -246,6 +279,7 @@ TARGETS: Dict[str, Integration] = {
ContainerType.coverage: 1,
},
use_setup=True,
pool="windows",
),
"windows-libfuzzer-load-library": Integration(
template=TemplateType.libfuzzer,
@ -258,6 +292,7 @@ TARGETS: Dict[str, Integration] = {
ContainerType.coverage: 1,
},
use_setup=True,
pool="windows",
),
"windows-libfuzzer-dotnet": Integration(
template=TemplateType.libfuzzer_dotnet,
@ -275,6 +310,7 @@ TARGETS: Dict[str, Integration] = {
ContainerType.unique_reports: 1,
},
test_repro=False,
pool="windows",
),
"windows-trivial-crash": Integration(
template=TemplateType.radamsa,
@ -283,6 +319,7 @@ TARGETS: Dict[str, Integration] = {
inputs="seeds",
wait_for_files={ContainerType.unique_reports: 1},
inject_fake_regression=True,
pool="windows",
),
}
@ -351,7 +388,7 @@ class TestOnefuzz:
self.inject_log(self.start_log_marker)
for entry in os_list:
name = PoolName(f"testpool-{entry.name}-{self.test_id}")
name = self.build_pool_name(entry.name)
self.logger.info("creating pool: %s:%s", entry.name, name)
self.of.pools.create(name, entry)
self.logger.info("creating scaleset for pool: %s", name)
@ -359,6 +396,15 @@ class TestOnefuzz:
name, pool_size, region=region, initial_size=pool_size
)
name = self.build_pool_name("mariner")
self.logger.info("creating pool: %s:%s", "mariner", name)
self.of.pools.create(name, OS.linux)
self.logger.info("creating scaleset for pool: %s", name)
self.of.scalesets.create(
name, pool_size, region=region, initial_size=pool_size, image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest"
)
class UnmanagedPool:
def __init__(
self,
@ -560,12 +606,9 @@ class TestOnefuzz:
) -> List[UUID]:
"""Launch all of the fuzzing templates"""
pools: Dict[OS, Pool] = {}
pool = None
if unmanaged_pool is not None:
pools[unmanaged_pool.the_os] = self.of.pools.get(unmanaged_pool.pool_name)
else:
for pool in self.of.pools.list():
pools[pool.os] = pool
pool = unmanaged_pool.pool_name
job_ids = []
@ -576,8 +619,8 @@ class TestOnefuzz:
if config.os not in os_list:
continue
if config.os not in pools.keys():
raise Exception(f"No pool for target: {target} ,os: {config.os}")
if pool is None:
pool = self.build_pool_name(config.pool)
self.logger.info("launching: %s", target)
@ -601,8 +644,9 @@ class TestOnefuzz:
setup = Directory(os.path.join(setup, config.nested_setup_dir))
job: Optional[Job] = None
job = self.build_job(
duration, pools, target, config, setup, target_exe, inputs
duration, pool, target, config, setup, target_exe, inputs
)
if config.inject_fake_regression and job is not None:
@ -618,7 +662,7 @@ class TestOnefuzz:
def build_job(
self,
duration: int,
pools: Dict[OS, Pool],
pool: PoolName,
target: str,
config: Integration,
setup: Optional[Directory],
@ -634,7 +678,7 @@ class TestOnefuzz:
self.project,
target,
BUILD,
pools[config.os].name,
pool,
target_exe=target_exe,
inputs=inputs,
setup_dir=setup,
@ -659,7 +703,7 @@ class TestOnefuzz:
self.project,
target,
BUILD,
pools[config.os].name,
pool,
target_dll=File(config.target_exe),
inputs=inputs,
setup_dir=setup,
@ -675,7 +719,7 @@ class TestOnefuzz:
self.project,
target,
BUILD,
pools[config.os].name,
pool,
inputs=inputs,
target_exe=target_exe,
duration=duration,
@ -688,7 +732,7 @@ class TestOnefuzz:
self.project,
target,
BUILD,
pool_name=pools[config.os].name,
pool_name=pool,
target_exe=target_exe,
inputs=inputs,
setup_dir=setup,
@ -703,7 +747,7 @@ class TestOnefuzz:
self.project,
target,
BUILD,
pool_name=pools[config.os].name,
pool_name=pool,
target_exe=target_exe,
inputs=inputs,
setup_dir=setup,
@ -1234,6 +1278,9 @@ class TestOnefuzz:
if seen_errors:
raise Exception("logs included errors")
def build_pool_name(self, os_type: str) -> PoolName:
return PoolName(f"testpool-{os_type}-{self.test_id}")
class Run(Command):
def check_jobs(

34
src/runtime-tools/linux/setup.sh Executable file → Normal file
View File

@ -18,6 +18,14 @@ export DOTNET_CLI_HOME="$DOTNET_ROOT"
export ONEFUZZ_ROOT=/onefuzz
export LLVM_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer
# `logger` won't work on mariner unless we install this package first
if type yum > /dev/null 2> /dev/null; then
until yum install -y util-linux sudo; do
echo "yum failed. sleep 10s, then retrying"
sleep 10
done
fi
logger "onefuzz: making directories"
sudo mkdir -p /onefuzz/downloaded
sudo chown -R $(whoami) /onefuzz
@ -134,12 +142,35 @@ if type apt > /dev/null 2> /dev/null; then
sudo ln -f -s $(which llvm-symbolizer-12) $LLVM_SYMBOLIZER_PATH
fi
# Install dotnet
# Needed to install dotnet
until sudo apt install -y curl libicu-dev; do
logger "apt failed, sleeping 10s then retrying"
sleep 10
done
elif type yum > /dev/null 2> /dev/null; then
until yum install -y gdb gdb-gdbserver libunwind awk ca-certificates tar yum-utils shadow-utils cronie procps; do
echo "yum failed. sleep 10s, then retrying"
sleep 10
done
# Install updated Microsoft Open Management Infrastructure - github.com/microsoft/omi
yum-config-manager --add-repo=https://packages.microsoft.com/config/rhel/8/prod.repo 2>&1 | logger -s -i -t 'onefuzz-OMI-add-MS-repo'
yum install -y omi 2>&1 | logger -s -i -t 'onefuzz-OMI-install'
if ! [ -f ${LLVM_SYMBOLIZER_PATH} ]; then
until yum install -y llvm-12.0.1; do
echo "yum failed, sleeping 10s then retrying"
sleep 10
done
# If specifying symbolizer, exe name must be a "known symbolizer".
# Using `llvm-symbolizer` works for clang 8 .. 12.
sudo ln -f -s $(which llvm-symbolizer-12) $LLVM_SYMBOLIZER_PATH
fi
fi
# Install dotnet
logger "downloading dotnet install"
curl --retry 10 -sSL https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh 2>&1 | logger -s -i -t 'onefuzz-curl-dotnet-install'
chmod +x dotnet-install.sh
@ -157,7 +188,6 @@ if type apt > /dev/null 2> /dev/null; then
"$DOTNET_ROOT"/dotnet tool install dotnet-coverage --version 17.5 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
"$DOTNET_ROOT"/dotnet tool install dotnet-sos --version 6.0.351802 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
popd
fi
if [ -v DOCKER_BUILD ]; then
echo "building for docker"