add regression testing tasks (#664)

src/integration-tests/check-regression.py

@@ -0,0 +1,156 @@
+#!/usr/bin/env python
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+
+import json
+import logging
+import os
+import sys
+import time
+from typing import Optional
+from uuid import UUID, uuid4
+
+from onefuzz.api import Command, Onefuzz
+from onefuzz.cli import execute_api
+from onefuzztypes.enums import OS, ContainerType, TaskState, TaskType
+from onefuzztypes.models import Job, RegressionReport
+from onefuzztypes.primitives import Container, Directory, File, PoolName
+
+
+class Run(Command):
+    def cleanup(self, test_id: UUID):
+        for pool in self.onefuzz.pools.list():
+            if str(test_id) in pool.name:
+                self.onefuzz.pools.shutdown(pool.name, now=True)
+
+        self.onefuzz.template.stop(
+            str(test_id), "linux-libfuzzer", build=None, delete_containers=True
+        )
+
+    def _wait_for_regression_task(self, job: Job) -> None:
+        while True:
+            self.logger.info("waiting for regression task to finish")
+            for task in self.onefuzz.jobs.tasks.list(job.job_id):
+                if task.config.task.type not in [
+                    TaskType.libfuzzer_regression,
+                    TaskType.generic_regression,
+                ]:
+                    continue
+                if task.state != TaskState.stopped:
+                    continue
+                return
+            time.sleep(10)
+
+    def _check_regression(self, job: Job) -> bool:
+        # get the regression reports containers for the job
+        results = self.onefuzz.jobs.containers.list(
+            job.job_id, ContainerType.regression_reports
+        )
+
+        # expect one and only one regression report container
+        if len(results) != 1:
+            raise Exception(f"unexpected regression containers: {results}")
+        container = list(results.keys())[0]
+
+        # expect one and only one file in the container
+        if len(results[container]) != 1:
+            raise Exception(f"unexpected regression container output: {results}")
+        file = results[container][0]
+
+        # get the regression report
+        content = self.onefuzz.containers.files.get(Container(container), file)
+        as_str = content.decode()
+        as_obj = json.loads(as_str)
+        report = RegressionReport.parse_obj(as_obj)
+
+        if report.crash_test_result.crash_report is not None:
+            self.logger.info("regression report has crash report")
+            return True
+
+        if report.crash_test_result.no_repro is not None:
+            self.logger.info("regression report has no-repro")
+            return False
+
+        raise Exception(f"unexpected report: {report}")
+
+    def _run_job(
+        self, test_id: UUID, pool: PoolName, target: str, exe: File, build: int
+    ) -> Job:
+        if build == 1:
+            wait_for_files = [ContainerType.unique_reports]
+        else:
+            wait_for_files = [ContainerType.regression_reports]
+        job = self.onefuzz.template.libfuzzer.basic(
+            str(test_id),
+            target,
+            str(build),
+            pool,
+            target_exe=exe,
+            duration=1,
+            vm_count=1,
+            wait_for_files=wait_for_files,
+        )
+        if job is None:
+            raise Exception(f"invalid job: {target} {build}")
+
+        if build > 1:
+            self._wait_for_regression_task(job)
+        self.onefuzz.template.stop(str(test_id), target, str(build))
+        return job
+
+    def _run(self, target_os: OS, test_id: UUID, base: Directory, target: str) -> None:
+        pool = PoolName(f"{target}-{target_os.name}-{test_id}")
+        self.onefuzz.pools.create(pool, target_os)
+        self.onefuzz.scalesets.create(pool, 5)
+        broken = File(os.path.join(base, target, "broken.exe"))
+        fixed = File(os.path.join(base, target, "fixed.exe"))
+
+        self.logger.info("starting first build")
+        self._run_job(test_id, pool, target, broken, 1)
+
+        self.logger.info("starting second build")
+        job = self._run_job(test_id, pool, target, fixed, 2)
+        if self._check_regression(job):
+            raise Exception("fixed binary should be a no repro")
+
+        self.logger.info("starting third build")
+        job = self._run_job(test_id, pool, target, broken, 3)
+        if not self._check_regression(job):
+            raise Exception("broken binary should be a crash report")
+
+        self.onefuzz.pools.shutdown(pool, now=True)
+
+    def test(
+        self,
+        samples: Directory,
+        *,
+        endpoint: Optional[str] = None,
+    ):
+        test_id = uuid4()
+        self.logger.info(f"launch test {test_id}")
+        self.onefuzz.__setup__(endpoint=endpoint)
+        error: Optional[Exception] = None
+        try:
+            self._run(OS.linux, test_id, samples, "linux-libfuzzer-regression")
+        except Exception as err:
+            error = err
+        except KeyboardInterrupt:
+            self.logger.warning("interruptted")
+        finally:
+            self.logger.info("cleaning up tests")
+            self.cleanup(test_id)
+
+        if error:
+            raise error
+
+
+def main() -> int:
+    return execute_api(
+        Run(Onefuzz(), logging.getLogger("regression")), [Command], "0.0.1"
+    )
+
+
+if __name__ == "__main__":
+    sys.exit(main())

src/integration-tests/git-bisect/README.md

@@ -0,0 +1,21 @@
+# git-bisect regression source
+
+This assumes you have a working clang with libfuzzer, bash, and git.
+
+This makes a git repo `test` with 9 commits.  Each commit after the first adds a bug.
+
+* `commit 0` has no bugs.
+* `commit 1` will additionally cause an abort if the input is `1`.
+* `commit 2` will additionally cause an abort if the input is `2`.
+* `commit 3` will additionally cause an abort if the input is `3`.
+* etc.
+
+This directory provides exemplar scripts that demonstrate how to perform
+ `git bisect` with libfuzzer.
+
+ * [run-local.sh](run-local.sh) builds & runs the libfuzzer target locally.  It uses [src/bisect-local.sh](src/bisect-local.sh) as the `git bisect run` command.
+ * [run-onefuzz.sh](run-onefuzz.sh) builds the libfuzzer target locally, but uses OneFuzz to run the regression tasks.  It uses [src/bisect-onefuzz.sh](src/bisect-onefuzz.sh) as the `git bisect run` command.
+
+With each project having their own unique paradigm for building, this model
+allows plugging OneFuzz as a `bisect` command in whatever fashion your
+project requires.

src/integration-tests/git-bisect/build.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+set -e
+
+rm -rf test
+
+git init test
+(cd test; git config user.name "Example"; git config user.email example@contoso.com)
+(cp src/Makefile test; cd test; git add Makefile)
+for i in $(seq 0 8); do
+    cp src/fuzz.c test/fuzz.c
+    for j in $(seq $i 8); do
+        if [ $i != $j ]; then
+            sed -i /TEST$j/d test/fuzz.c
+        fi
+    done
+    (cd test; git add fuzz.c; git commit -m "commit $i")
+done

src/integration-tests/git-bisect/run-local.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+# build our git repo with our samples in `test`
+# (note, we don't care about the output of this script)
+./build.sh 2>/dev/null > /dev/null
+
+# create our crashing input
+echo -n '3' > test/test.txt
+
+cd test
+
+# start the bisect, looking from HEAD backwards 8 commits
+git bisect start HEAD HEAD~8 --
+git bisect run ../src/bisect-local.sh test.txt
+git bisect reset

src/integration-tests/git-bisect/run-onefuzz.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+set -e
+
+# build our git repo with our samples in `test`
+# (note, we don't care about the output of this script)
+./build.sh 2>/dev/null > /dev/null
+
+# create our crashing input
+echo -n '3' > test/test.txt
+
+cd test
+
+# start the bisect, looking from HEAD backwards 8 commits
+git bisect start HEAD HEAD~8 --
+git bisect run ../src/bisect-onefuzz.sh test.txt
+git bisect reset

src/integration-tests/git-bisect/src/Makefile

@@ -0,0 +1,13 @@
+CC=clang
+
+CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address
+
+all: fuzz.exe
+
+fuzz.exe: fuzz.c
+	$(CC) $(CFLAGS) fuzz.c -o fuzz.exe
+
+.PHONY: clean
+
+clean:
+	@rm -f fuzz.exe

src/integration-tests/git-bisect/src/bisect-local.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+set -ex
+
+make clean
+make
+./fuzz.exe $*

src/integration-tests/git-bisect/src/bisect-onefuzz.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+set -e
+
+PROJECT=${PROJECT:-regression-test}
+TARGET=${TARGET:-$(uuidgen)}
+BUILD=regression-$(git rev-parse HEAD)
+POOL=${ONEFUZZ_POOL:-linux}
+
+make clean
+make
+onefuzz template regression libfuzzer ${PROJECT} ${TARGET} ${BUILD} ${POOL} --check_regressions --delete_input_container --reports --crashes $*

src/integration-tests/git-bisect/src/fuzz.c

@@ -0,0 +1,13 @@
+#include <stdlib.h>
+int LLVMFuzzerTestOneInput(char *data, size_t len) { 
+   if (len != 1) { return 0; }  
+   if (data[0] == '1') { abort(); } // TEST1
+   if (data[0] == '2') { abort(); } // TEST2
+   if (data[0] == '3') { abort(); } // TEST3
+   if (data[0] == '4') { abort(); } // TEST4
+   if (data[0] == '5') { abort(); } // TEST5
+   if (data[0] == '6') { abort(); } // TEST6
+   if (data[0] == '7') { abort(); } // TEST7
+   if (data[0] == '8') { abort(); } // TEST8
+   return 0;
+}

src/integration-tests/libfuzzer-regression/Makefile

@@ -0,0 +1,16 @@
+CC=clang
+
+CFLAGS=-g3 -fsanitize=fuzzer -fsanitize=address
+
+all: broken.exe fixed.exe
+
+broken.exe: simple.c
+	$(CC) $(CFLAGS) simple.c -o broken.exe
+
+fixed.exe: simple.c
+	$(CC) $(CFLAGS) simple.c -o fixed.exe -DFIXED
+
+.PHONY: clean
+
+clean:
+	rm -f broken.exe fixed.exe

src/integration-tests/libfuzzer-regression/seeds/good.txt

@@ -0,0 +1 @@
+good

src/integration-tests/libfuzzer-regression/simple.c

@@ -0,0 +1,26 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include <stdint.h>
+#include <stdlib.h>
+
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t len) {
+  int cnt = 0;
+
+  if (len < 3) {
+    return 0;
+  }
+
+  if (data[0] == 'x') { cnt++; }
+  if (data[1] == 'y') { cnt++; }
+  if (data[2] == 'z') { cnt++; }
+
+#ifndef FIXED
+  if (cnt >= 3) {
+     abort();
+  }
+#endif
+
+  return 0;
+}