Add mariner support (#3306)

* Updating setup.sh

* logger works

* Install omi

* syntax

* Add option to create mariner pool in checkpr

* .

* Need to install sudo

* .

* Include both logging extensions

* Revert because we already isntall the azure monitor for linux extension

* Downgrade type handler version

* Add data collection rules and fuzzing articacts for integration test

* TODOs

* Fix linux jobs getting sent to mariner

* Fix linux jobs going to marienr pool

* Fix pools

* Remove the old logging extension on linux

* try to retain syslog

* Value to be set was not clear

* Trying to route logs

* Maybe we need to specify properties even if we don't set anything

* Start adding a data collection association

* Create association

* update packages.lock

* .

* Remove auto config and add dependency map

* Bring back GCS autoconfig

* Undo DCR stuff

* Undo package version bump

* Fix up files

* Remove TODO

.github/workflows/ci.yml

@@ -538,9 +538,11 @@ jobs:
 
           mkdir -p artifacts/linux-libfuzzer
           mkdir -p artifacts/linux-libfuzzer-with-options
+          mkdir -p artifacts/mariner-libfuzzer
           (cd libfuzzer ; make )
           cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer
           cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/linux-libfuzzer-with-options
+          cp -r libfuzzer/fuzz.exe libfuzzer/seeds artifacts/mariner-libfuzzer
 
           mkdir -p artifacts/linux-libfuzzer-regression
           (cd libfuzzer-regression ; make )

src/ApiService/ApiService/onefuzzlib/Extension.cs

@@ -36,7 +36,9 @@ public class Extensions : IExtensions {
         var extensions = new List<VMExtensionWrapper>();
 
         var instanceConfig = await _context.ConfigOperations.Fetch();
-        extensions.Add(await MonitorExtension(region, vmOs));
+        if (vmOs == Os.Windows) {
+            extensions.Add(await MonitorExtension(region));
+        }
 
         var depenency = DependencyExtension(region, vmOs);
         if (depenency is not null) {
@@ -329,37 +331,21 @@ public class Extensions : IExtensions {
         throw new NotSupportedException($"unsupported OS: {vmOs}");
     }
 
-    public async Async.Task<VMExtensionWrapper> MonitorExtension(AzureLocation region, Os vmOs) {
+    public async Async.Task<VMExtensionWrapper> MonitorExtension(AzureLocation region) {
         var settings = await _context.LogAnalytics.GetMonitorSettings();
         var extensionSettings = JsonSerializer.Serialize(new { WorkspaceId = settings.Id }, _extensionSerializerOptions);
         var protectedExtensionSettings = JsonSerializer.Serialize(new { WorkspaceKey = settings.Key }, _extensionSerializerOptions);
-        if (vmOs == Os.Windows) {
+        return new VMExtensionWrapper {
-            return new VMExtensionWrapper {
+            Location = region,
-                Location = region,
+            Name = "OMSExtension",
-                Name = "OMSExtension",
+            TypePropertiesType = "MicrosoftMonitoringAgent",
-                TypePropertiesType = "MicrosoftMonitoringAgent",
+            Publisher = "Microsoft.EnterpriseCloud.Monitoring",
-                Publisher = "Microsoft.EnterpriseCloud.Monitoring",
+            TypeHandlerVersion = "1.0",
-                TypeHandlerVersion = "1.0",
+            AutoUpgradeMinorVersion = true,
-                AutoUpgradeMinorVersion = true,
+            Settings = new BinaryData(extensionSettings),
-                Settings = new BinaryData(extensionSettings),
+            ProtectedSettings = new BinaryData(protectedExtensionSettings),
-                ProtectedSettings = new BinaryData(protectedExtensionSettings),
+            EnableAutomaticUpgrade = false
-                EnableAutomaticUpgrade = false
+        };
-            };
-        } else if (vmOs == Os.Linux) {
-            return new VMExtensionWrapper {
-                Location = region,
-                Name = "OmsAgentForLinux",
-                TypePropertiesType = "OmsAgentForLinux",
-                Publisher = "Microsoft.EnterpriseCloud.Monitoring",
-                TypeHandlerVersion = "1.0",
-                AutoUpgradeMinorVersion = true,
-                Settings = new BinaryData(extensionSettings),
-                ProtectedSettings = new BinaryData(protectedExtensionSettings),
-                EnableAutomaticUpgrade = false
-            };
-        } else {
-            throw new NotSupportedException($"unsupported os: {vmOs}");
-        }
     }
 
 

src/integration-tests/integration-test.py

@@ -88,6 +88,7 @@ class Integration(BaseModel):
     target_method: Optional[str]
     setup_dir: Optional[str]
     target_env: Optional[Dict[str, str]]
+    pool: PoolName
 
 
 TARGETS: Dict[str, Integration] = {
@@ -97,6 +98,7 @@ TARGETS: Dict[str, Integration] = {
         target_exe="fuzz.exe",
         inputs="seeds",
         wait_for_files={ContainerType.unique_reports: 1},
+        pool="linux",
     ),
     "linux-libfuzzer": Integration(
         template=TemplateType.libfuzzer,
@@ -124,6 +126,7 @@ TARGETS: Dict[str, Integration] = {
             "--only_asan_failures",
             "--write_test_file={extra_output_dir}/test.txt",
         ],
+        pool="linux",
     ),
     "linux-libfuzzer-with-options": Integration(
         template=TemplateType.libfuzzer,
@@ -137,6 +140,7 @@ TARGETS: Dict[str, Integration] = {
         },
         reboot_after_setup=True,
         fuzzing_target_options=["-runs=10000000"],
+        pool="linux",
     ),
     "linux-libfuzzer-dlopen": Integration(
         template=TemplateType.libfuzzer,
@@ -150,6 +154,7 @@ TARGETS: Dict[str, Integration] = {
         },
         reboot_after_setup=True,
         use_setup=True,
+        pool="linux",
     ),
     "linux-libfuzzer-linked-library": Integration(
         template=TemplateType.libfuzzer,
@@ -163,6 +168,7 @@ TARGETS: Dict[str, Integration] = {
         },
         reboot_after_setup=True,
         use_setup=True,
+        pool="linux",
     ),
     "linux-libfuzzer-dotnet": Integration(
         template=TemplateType.libfuzzer_dotnet,
@@ -180,6 +186,7 @@ TARGETS: Dict[str, Integration] = {
             ContainerType.unique_reports: 1,
         },
         test_repro=False,
+        pool="linux",
     ),
     "linux-libfuzzer-aarch64-crosscompile": Integration(
         template=TemplateType.libfuzzer_qemu_user,
@@ -189,6 +196,7 @@ TARGETS: Dict[str, Integration] = {
         use_setup=True,
         wait_for_files={ContainerType.inputs: 2, ContainerType.crashes: 1},
         test_repro=False,
+        pool="linux",
     ),
     "linux-libfuzzer-rust": Integration(
         template=TemplateType.libfuzzer,
@@ -196,6 +204,7 @@ TARGETS: Dict[str, Integration] = {
         target_exe="fuzz_target_1",
         wait_for_files={ContainerType.unique_reports: 1, ContainerType.coverage: 1},
         fuzzing_target_options=["--test:{extra_setup_dir}"],
+        pool="linux",
     ),
     "linux-trivial-crash": Integration(
         template=TemplateType.radamsa,
@@ -204,6 +213,7 @@ TARGETS: Dict[str, Integration] = {
         inputs="seeds",
         wait_for_files={ContainerType.unique_reports: 1},
         inject_fake_regression=True,
+        pool="linux",
     ),
     "linux-trivial-crash-asan": Integration(
         template=TemplateType.radamsa,
@@ -213,6 +223,28 @@ TARGETS: Dict[str, Integration] = {
         wait_for_files={ContainerType.unique_reports: 1},
         check_asan_log=True,
         disable_check_debugger=True,
+        pool="linux",
+    ),
+    # TODO: Don't install OMS extension on linux anymore
+    # TODO: Figure out why non mariner work is being scheduled to the mariner pool
+    "mariner-libfuzzer": Integration(
+        template=TemplateType.libfuzzer,
+        os=OS.linux,
+        target_exe="fuzz.exe",
+        inputs="seeds",
+        wait_for_files={
+            ContainerType.unique_reports: 1,
+            ContainerType.coverage: 1,
+            ContainerType.inputs: 2,
+            ContainerType.extra_output: 1,
+        },
+        reboot_after_setup=True,
+        inject_fake_regression=True,
+        fuzzing_target_options=[
+            "--test:{extra_setup_dir}",
+            "--write_test_file={extra_output_dir}/test.txt",
+        ],
+        pool=PoolName("mariner")
     ),
     "windows-libfuzzer": Integration(
         template=TemplateType.libfuzzer,
@@ -234,6 +266,7 @@ TARGETS: Dict[str, Integration] = {
             "--only_asan_failures",
             "--write_test_file={extra_output_dir}/test.txt",
         ],
+        pool="windows",
     ),
     "windows-libfuzzer-linked-library": Integration(
         template=TemplateType.libfuzzer,
@@ -246,6 +279,7 @@ TARGETS: Dict[str, Integration] = {
             ContainerType.coverage: 1,
         },
         use_setup=True,
+        pool="windows",
     ),
     "windows-libfuzzer-load-library": Integration(
         template=TemplateType.libfuzzer,
@@ -258,6 +292,7 @@ TARGETS: Dict[str, Integration] = {
             ContainerType.coverage: 1,
         },
         use_setup=True,
+        pool="windows",
     ),
     "windows-libfuzzer-dotnet": Integration(
         template=TemplateType.libfuzzer_dotnet,
@@ -275,6 +310,7 @@ TARGETS: Dict[str, Integration] = {
             ContainerType.unique_reports: 1,
         },
         test_repro=False,
+        pool="windows",
     ),
     "windows-trivial-crash": Integration(
         template=TemplateType.radamsa,
@@ -283,6 +319,7 @@ TARGETS: Dict[str, Integration] = {
         inputs="seeds",
         wait_for_files={ContainerType.unique_reports: 1},
         inject_fake_regression=True,
+        pool="windows",
     ),
 }
 
@@ -351,7 +388,7 @@ class TestOnefuzz:
 
         self.inject_log(self.start_log_marker)
         for entry in os_list:
-            name = PoolName(f"testpool-{entry.name}-{self.test_id}")
+            name = self.build_pool_name(entry.name)
             self.logger.info("creating pool: %s:%s", entry.name, name)
             self.of.pools.create(name, entry)
             self.logger.info("creating scaleset for pool: %s", name)
@@ -359,6 +396,15 @@ class TestOnefuzz:
                 name, pool_size, region=region, initial_size=pool_size
             )
 
+        name = self.build_pool_name("mariner")
+        self.logger.info("creating pool: %s:%s", "mariner", name)
+        self.of.pools.create(name, OS.linux)
+        self.logger.info("creating scaleset for pool: %s", name)
+        self.of.scalesets.create(
+            name, pool_size, region=region, initial_size=pool_size, image="MicrosoftCBLMariner:cbl-mariner:cbl-mariner-2-gen2:latest"
+        )
+
+
     class UnmanagedPool:
         def __init__(
             self,
@@ -560,12 +606,9 @@ class TestOnefuzz:
     ) -> List[UUID]:
         """Launch all of the fuzzing templates"""
 
-        pools: Dict[OS, Pool] = {}
+        pool = None
         if unmanaged_pool is not None:
-            pools[unmanaged_pool.the_os] = self.of.pools.get(unmanaged_pool.pool_name)
+            pool = unmanaged_pool.pool_name
-        else:
-            for pool in self.of.pools.list():
-                pools[pool.os] = pool
 
         job_ids = []
 
@@ -576,8 +619,8 @@ class TestOnefuzz:
             if config.os not in os_list:
                 continue
 
-            if config.os not in pools.keys():
+            if pool is None:
-                raise Exception(f"No pool for target: {target} ,os: {config.os}")
+                pool = self.build_pool_name(config.pool)
 
             self.logger.info("launching: %s", target)
 
@@ -601,8 +644,9 @@ class TestOnefuzz:
                 setup = Directory(os.path.join(setup, config.nested_setup_dir))
 
             job: Optional[Job] = None
+                
             job = self.build_job(
-                duration, pools, target, config, setup, target_exe, inputs
+                duration, pool, target, config, setup, target_exe, inputs
             )
 
             if config.inject_fake_regression and job is not None:
@@ -618,7 +662,7 @@ class TestOnefuzz:
     def build_job(
         self,
         duration: int,
-        pools: Dict[OS, Pool],
+        pool: PoolName,
         target: str,
         config: Integration,
         setup: Optional[Directory],
@@ -634,7 +678,7 @@ class TestOnefuzz:
                 self.project,
                 target,
                 BUILD,
-                pools[config.os].name,
+                pool,
                 target_exe=target_exe,
                 inputs=inputs,
                 setup_dir=setup,
@@ -659,7 +703,7 @@ class TestOnefuzz:
                 self.project,
                 target,
                 BUILD,
-                pools[config.os].name,
+                pool,
                 target_dll=File(config.target_exe),
                 inputs=inputs,
                 setup_dir=setup,
@@ -675,7 +719,7 @@ class TestOnefuzz:
                 self.project,
                 target,
                 BUILD,
-                pools[config.os].name,
+                pool,
                 inputs=inputs,
                 target_exe=target_exe,
                 duration=duration,
@@ -688,7 +732,7 @@ class TestOnefuzz:
                 self.project,
                 target,
                 BUILD,
-                pool_name=pools[config.os].name,
+                pool_name=pool,
                 target_exe=target_exe,
                 inputs=inputs,
                 setup_dir=setup,
@@ -703,7 +747,7 @@ class TestOnefuzz:
                 self.project,
                 target,
                 BUILD,
-                pool_name=pools[config.os].name,
+                pool_name=pool,
                 target_exe=target_exe,
                 inputs=inputs,
                 setup_dir=setup,
@@ -1234,6 +1278,9 @@ class TestOnefuzz:
         if seen_errors:
             raise Exception("logs included errors")
         
+    def build_pool_name(self, os_type: str) -> PoolName:
+        return PoolName(f"testpool-{os_type}-{self.test_id}")
+
 
 class Run(Command):
     def check_jobs(

src/runtime-tools/linux/setup.sh

@@ -18,6 +18,14 @@ export DOTNET_CLI_HOME="$DOTNET_ROOT"
 export ONEFUZZ_ROOT=/onefuzz
 export LLVM_SYMBOLIZER_PATH=/onefuzz/bin/llvm-symbolizer
 
+# `logger` won't work on mariner unless we install this package first
+if type yum > /dev/null 2> /dev/null; then
+    until yum install -y util-linux sudo; do
+        echo "yum failed.  sleep 10s, then retrying"
+        sleep 10
+    done
+fi
+
 logger "onefuzz: making directories"
 sudo mkdir -p /onefuzz/downloaded
 sudo chown -R $(whoami) /onefuzz
@@ -134,31 +142,53 @@ if type apt > /dev/null 2> /dev/null; then
         sudo ln -f -s $(which llvm-symbolizer-12) $LLVM_SYMBOLIZER_PATH
     fi
 
-    # Install dotnet
+    # Needed to install dotnet
     until sudo apt install -y curl libicu-dev; do
         logger "apt failed, sleeping 10s then retrying"
         sleep 10
     done
-
+elif type yum > /dev/null 2> /dev/null; then
-    logger "downloading dotnet install"
+    until yum install -y gdb gdb-gdbserver libunwind awk ca-certificates tar yum-utils shadow-utils cronie procps; do
-    curl --retry 10 -sSL https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh 2>&1 | logger -s -i -t 'onefuzz-curl-dotnet-install'
+        echo "yum failed.  sleep 10s, then retrying"
-    chmod +x dotnet-install.sh
+        sleep 10
-
-    for version in "${DOTNET_VERSIONS[@]}"; do
-        logger "running dotnet install $version"
-        /bin/bash ./dotnet-install.sh --channel "$version" --install-dir "$DOTNET_ROOT" 2>&1 | logger -s -i -t 'onefuzz-dotnet-setup'
     done
-    rm dotnet-install.sh
 
-    logger "install dotnet tools"
+    # Install updated Microsoft Open Management Infrastructure - github.com/microsoft/omi
-    pushd "$DOTNET_ROOT"
+    yum-config-manager --add-repo=https://packages.microsoft.com/config/rhel/8/prod.repo 2>&1 | logger -s -i -t 'onefuzz-OMI-add-MS-repo'
-    ls -lah 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+    yum install -y omi 2>&1 | logger -s -i -t 'onefuzz-OMI-install'
-    "$DOTNET_ROOT"/dotnet tool install dotnet-dump --version 6.0.351802 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+
-    "$DOTNET_ROOT"/dotnet tool install dotnet-coverage --version 17.5 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+
-    "$DOTNET_ROOT"/dotnet tool install dotnet-sos --version 6.0.351802 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+    if ! [ -f ${LLVM_SYMBOLIZER_PATH} ]; then
-    popd
+        until yum install -y llvm-12.0.1; do
+            echo "yum failed, sleeping 10s then retrying"
+            sleep 10
+        done
+
+        # If specifying symbolizer, exe name must be a "known symbolizer".
+        # Using `llvm-symbolizer` works for clang 8 .. 12.
+        sudo ln -f -s $(which llvm-symbolizer-12) $LLVM_SYMBOLIZER_PATH
+    fi   
 fi
 
+# Install dotnet
+logger "downloading dotnet install"
+curl --retry 10 -sSL https://dot.net/v1/dotnet-install.sh -o dotnet-install.sh 2>&1 | logger -s -i -t 'onefuzz-curl-dotnet-install'
+chmod +x dotnet-install.sh
+
+for version in "${DOTNET_VERSIONS[@]}"; do
+    logger "running dotnet install $version"
+    /bin/bash ./dotnet-install.sh --channel "$version" --install-dir "$DOTNET_ROOT" 2>&1 | logger -s -i -t 'onefuzz-dotnet-setup'
+done
+rm dotnet-install.sh
+
+logger "install dotnet tools"
+pushd "$DOTNET_ROOT"
+ls -lah 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+"$DOTNET_ROOT"/dotnet tool install dotnet-dump --version 6.0.351802 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+"$DOTNET_ROOT"/dotnet tool install dotnet-coverage --version 17.5 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+"$DOTNET_ROOT"/dotnet tool install dotnet-sos --version 6.0.351802 --tool-path /onefuzz/tools 2>&1 | logger -s -i -t 'onefuzz-dotnet-tools'
+popd
+
 if  [ -v DOCKER_BUILD ]; then
     echo "building for docker"
 elif [ -d /etc/systemd/system ]; then