add regression testing tasks (#664)

src/pytypes/extra/generate-docs.py

@@ -31,6 +31,7 @@ from onefuzztypes.events import (
     EventProxyCreated,
     EventProxyDeleted,
     EventProxyFailed,
+    EventRegressionReported,
     EventScalesetCreated,
     EventScalesetDeleted,
     EventScalesetFailed,
@@ -45,8 +46,10 @@ from onefuzztypes.events import (
 )
 from onefuzztypes.models import (
     BlobRef,
+    CrashTestResult,
     Error,
     JobConfig,
+    RegressionReport,
     Report,
     TaskConfig,
     TaskContainers,
@@ -87,6 +90,24 @@ def main() -> None:
         ],
         tags={},
     )
+    report = Report(
+        input_blob=BlobRef(
+            account="contoso-storage-account",
+            container=Container("crashes"),
+            name="input.txt",
+        ),
+        executable="fuzz.exe",
+        crash_type="example crash report type",
+        crash_site="example crash site",
+        call_stack=["#0 line", "#1 line", "#2 line"],
+        call_stack_sha256=ZERO_SHA256,
+        input_sha256=EMPTY_SHA256,
+        asan_log="example asan log",
+        task_id=UUID(int=0),
+        job_id=UUID(int=0),
+        scariness_score=10,
+        scariness_description="example-scariness",
+    )
     examples: List[Event] = [
         EventPing(ping_id=UUID(int=0)),
         EventTaskCreated(
@@ -193,27 +214,18 @@ def main() -> None:
             pool_name=PoolName("example"),
             state=NodeState.setting_up,
         ),
+        EventRegressionReported(
+            regression_report=RegressionReport(
+                crash_test_result=CrashTestResult(crash_report=report),
+                original_crash_test_result=CrashTestResult(crash_report=report),
+            ),
+            container=Container("container-name"),
+            filename="example.json",
+        ),
         EventCrashReported(
             container=Container("container-name"),
             filename="example.json",
-            report=Report(
-                input_blob=BlobRef(
-                    account="contoso-storage-account",
-                    container=Container("crashes"),
-                    name="input.txt",
-                ),
-                executable="fuzz.exe",
-                crash_type="example crash report type",
-                crash_site="example crash site",
-                call_stack=["#0 line", "#1 line", "#2 line"],
-                call_stack_sha256=ZERO_SHA256,
-                input_sha256=EMPTY_SHA256,
-                asan_log="example asan log",
-                task_id=UUID(int=0),
-                job_id=UUID(int=0),
-                scariness_score=10,
-                scariness_description="example-scariness",
-            ),
+            report=report,
         ),
         EventFileAdded(container=Container("container-name"), filename="example.txt"),
         EventNodeHeartbeat(machine_id=UUID(int=0), pool_name=PoolName("example")),

src/pytypes/onefuzztypes/enums.py

@@ -78,6 +78,7 @@ class TaskFeature(Enum):
     preserve_existing_outputs = "preserve_existing_outputs"
     check_fuzzer_help = "check_fuzzer_help"
     expect_crash_on_failure = "expect_crash_on_failure"
+    report_list = "report_list"
 
 
 # Permissions for an Azure Blob Storage Container.
@@ -149,11 +150,13 @@ class TaskType(Enum):
     libfuzzer_coverage = "libfuzzer_coverage"
     libfuzzer_crash_report = "libfuzzer_crash_report"
     libfuzzer_merge = "libfuzzer_merge"
+    libfuzzer_regression = "libfuzzer_regression"
     generic_analysis = "generic_analysis"
     generic_supervisor = "generic_supervisor"
     generic_merge = "generic_merge"
     generic_generator = "generic_generator"
     generic_crash_report = "generic_crash_report"
+    generic_regression = "generic_regression"
 
 
 class VmState(Enum):
@@ -207,6 +210,7 @@ class ContainerType(Enum):
     tools = "tools"
     unique_inputs = "unique_inputs"
     unique_reports = "unique_reports"
+    regression_reports = "regression_reports"
 
     @classmethod
     def reset_defaults(cls) -> List["ContainerType"]:
@@ -219,8 +223,9 @@ class ContainerType(Enum):
             cls.readonly_inputs,
             cls.reports,
             cls.setup,
-            cls.unique_reports,
             cls.unique_inputs,
+            cls.unique_reports,
+            cls.regression_reports,
         ]
 
     @classmethod

src/pytypes/onefuzztypes/events.py

@@ -11,7 +11,15 @@ from uuid import UUID, uuid4
 from pydantic import BaseModel, Extra, Field
 
 from .enums import OS, Architecture, NodeState, TaskState, TaskType
-from .models import AutoScaleConfig, Error, JobConfig, Report, TaskConfig, UserInfo
+from .models import (
+    AutoScaleConfig,
+    Error,
+    JobConfig,
+    RegressionReport,
+    Report,
+    TaskConfig,
+    UserInfo,
+)
 from .primitives import Container, PoolName, Region
 from .responses import BaseResponse
 
@@ -156,6 +164,12 @@ class EventCrashReported(BaseEvent):
     filename: str
 
 
+class EventRegressionReported(BaseEvent):
+    regression_report: RegressionReport
+    container: Container
+    filename: str
+
+
 class EventFileAdded(BaseEvent):
     container: Container
     filename: str
@@ -183,6 +197,7 @@ Event = Union[
     EventTaskStopped,
     EventTaskHeartbeat,
     EventCrashReported,
+    EventRegressionReported,
     EventFileAdded,
 ]
 
@@ -207,6 +222,7 @@ class EventType(Enum):
     task_state_updated = "task_state_updated"
     task_stopped = "task_stopped"
     crash_reported = "crash_reported"
+    regression_reported = "regression_reported"
     file_added = "file_added"
     task_heartbeat = "task_heartbeat"
     node_heartbeat = "node_heartbeat"
@@ -234,6 +250,7 @@ EventTypeMap = {
     EventType.task_heartbeat: EventTaskHeartbeat,
     EventType.task_stopped: EventTaskStopped,
     EventType.crash_reported: EventCrashReported,
+    EventType.regression_reported: EventRegressionReported,
     EventType.file_added: EventFileAdded,
 }
 

src/pytypes/onefuzztypes/models.py

@@ -170,6 +170,7 @@ class TaskDetails(BaseModel):
     target_timeout: Optional[int]
     ensemble_sync_delay: Optional[int]
     preserve_existing_outputs: Optional[bool]
+    report_list: Optional[List[str]]
 
     @validator("check_retry_count", allow_reuse=True)
     def validate_check_retry_count(cls, value: int) -> int:
@@ -237,7 +238,7 @@ class BlobRef(BaseModel):
 
 class Report(BaseModel):
     input_url: Optional[str]
-    input_blob: BlobRef
+    input_blob: Optional[BlobRef]
     executable: str
     crash_type: str
     crash_site: str
@@ -251,6 +252,26 @@ class Report(BaseModel):
     scariness_description: Optional[str]
 
 
+class NoReproReport(BaseModel):
+    input_sha256: str
+    input_blob: Optional[BlobRef]
+    executable: str
+    task_id: UUID
+    job_id: UUID
+    tries: int
+    error: Optional[str]
+
+
+class CrashTestResult(BaseModel):
+    crash_report: Optional[Report]
+    no_repro: Optional[NoReproReport]
+
+
+class RegressionReport(BaseModel):
+    crash_test_result: CrashTestResult
+    original_crash_test_result: Optional[CrashTestResult]
+
+
 class ADODuplicateTemplate(BaseModel):
     increment: List[str]
     comment: Optional[str]
@@ -377,6 +398,7 @@ class TaskUnitConfig(BaseModel):
     stats_file: Optional[str]
     stats_format: Optional[StatsFormat]
     ensemble_sync_delay: Optional[int]
+    report_list: Optional[List[str]]
 
     # from here forwards are Container definitions.  These need to be inline
     # with TaskDefinitions and ContainerTypes
@@ -390,6 +412,7 @@ class TaskUnitConfig(BaseModel):
     tools: CONTAINER_DEF
     unique_inputs: CONTAINER_DEF
     unique_reports: CONTAINER_DEF
+    regression_reports: CONTAINER_DEF
 
 
 class Forward(BaseModel):