handle messages that are too big to fit in a queue message (#2020)

* handle messages that are too big to fit in a queue message

* tests

Co-authored-by: stas <statis@microsoft.com>

src/api-service/__app__/onefuzzlib/events.py

@@ -76,6 +76,6 @@ def send_event(event: Event) -> None:
     if event_message.event != event:
         event_message.event = event.copy(deep=True)
 
+    log_event(event, event_type)
     queue_signalr_event(event_message)
     Webhook.send_event(event_message)
-    log_event(event, event_type)

src/api-service/__app__/onefuzzlib/reports.py

@@ -5,6 +5,7 @@
 
 import json
 import logging
+from sys import getsizeof
 from typing import Optional, Union
 
 from memoization import cached
@@ -16,6 +17,65 @@ from .azure.containers import get_blob
 from .azure.storage import StorageType
 
 
+# This is fix for the following error:
+# Exception while executing function:
+# Functions.queue_file_changes Result: Failure
+# Exception: AzureHttpError: Bad Request
+# "The property value exceeds the maximum allowed size (64KB).
+# If the property value is a string, it is UTF-16 encoded and
+# the maximum number of characters should be 32K or less.
+def fix_report_size(
+    content: str,
+    report: Report,
+    acceptable_report_length_kb: int = 24,
+    keep_num_entries: int = 10,
+    keep_string_len: int = 256,
+) -> Report:
+    logging.info(f"report content length {getsizeof(content)}")
+    if getsizeof(content) > acceptable_report_length_kb * 1024:
+        msg = f"report data exceeds {acceptable_report_length_kb}K {getsizeof(content)}"
+        if len(report.call_stack) > keep_num_entries:
+            msg = msg + "; removing some of stack frames from the report"
+            report.call_stack = report.call_stack[0:keep_num_entries] + ["..."]
+
+        if report.asan_log and len(report.asan_log) > keep_string_len:
+            msg = msg + "; removing some of asan log entries from the report"
+            report.asan_log = report.asan_log[0:keep_string_len] + "..."
+
+        if report.minimized_stack and len(report.minimized_stack) > keep_num_entries:
+            msg = msg + "; removing some of minimized stack frames from the report"
+            report.minimized_stack = report.minimized_stack[0:keep_num_entries] + [
+                "..."
+            ]
+
+        if (
+            report.minimized_stack_function_names
+            and len(report.minimized_stack_function_names) > keep_num_entries
+        ):
+            msg = (
+                msg
+                + "; removing some of minimized stack function names from the report"
+            )
+            report.minimized_stack_function_names = (
+                report.minimized_stack_function_names[0:keep_num_entries] + ["..."]
+            )
+
+        if (
+            report.minimized_stack_function_lines
+            and len(report.minimized_stack_function_lines) > keep_num_entries
+        ):
+            msg = (
+                msg
+                + "; removing some of minimized stack function lines from the report"
+            )
+            report.minimized_stack_function_lines = (
+                report.minimized_stack_function_lines[0:keep_num_entries] + ["..."]
+            )
+
+        logging.info(msg)
+    return report
+
+
 def parse_report_or_regression(
     content: Union[str, bytes],
     file_path: Optional[str] = None,
@@ -43,12 +103,30 @@ def parse_report_or_regression(
 
     regression_err = None
     try:
-        return RegressionReport.parse_obj(data)
+        regression_report = RegressionReport.parse_obj(data)
+
+        if (
+            regression_report.crash_test_result is not None
+            and regression_report.crash_test_result.crash_report is not None
+        ):
+            regression_report.crash_test_result.crash_report = fix_report_size(
+                content, regression_report.crash_test_result.crash_report
+            )
+
+        if (
+            regression_report.original_crash_test_result is not None
+            and regression_report.original_crash_test_result.crash_report is not None
+        ):
+            regression_report.original_crash_test_result.crash_report = fix_report_size(
+                content, regression_report.original_crash_test_result.crash_report
+            )
+        return regression_report
     except ValidationError as err:
         regression_err = err
 
     try:
-        return Report.parse_obj(data)
+        report = Report.parse_obj(data)
+        return fix_report_size(content, report)
     except ValidationError as err:
         if expect_reports:
             logging.error(

src/api-service/tests/data/report-long.json

@@ -0,0 +1,151 @@
+{
+    "call_stack_sha256": "972a371a291ed5668a77576368ead0c46c2bac9f9a16b7fa7c0b48aec5b059b1",
+    "input_url": "https://fuzzxkbh6uhuuke4m.blob.core.windows.net/oft-asan-crashes/crash",
+    "executable": "setup/fuzz.exe",
+    "crash_type": "double-free",
+    "crash_site": "double-free (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+    "call_stack": [
+        "#0 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#1 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#2 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#3 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#4 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#5 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#6 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#7 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#8 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#9 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#10 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#11 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#12 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#13 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#14 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#15 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#16 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#17 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#18 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#19 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#20 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#21 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#22 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#23 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#24 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#25 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#26 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#27 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#28 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#29 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#30 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#31 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#32 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#33 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#34 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#35 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#36 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#37 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#38 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#39 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#40 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#41 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#42 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#43 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#44 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#45 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#46 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#47 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#48 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#49 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#50 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#51 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#52 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#53 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#54 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#55 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#56 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#57 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#58 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#59 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#60 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#61 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#62 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#63 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#64 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#65 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#66 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#67 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#68 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#69 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#70 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#71 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#72 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#73 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#74 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#75 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#76 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#77 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#78 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#79 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#80 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#81 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#82 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#83 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#84 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#85 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#86 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#87 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#88 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#89 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#90 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#91 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#92 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#93 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#94 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#95 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#96 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#97 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#98 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#99 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#100 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#101 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#102 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#103 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#104 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#105 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#106 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#107 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#108 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#109 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#110 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#111 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#112 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#113 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#114 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#115 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#116 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#117 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#118 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#119 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#120 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)",
+        "#121 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)",
+        "#122 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)",
+        "#123 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)",
+        "#124 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)",
+        "#125 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)",
+        "#126 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#127 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)",
+        "#128 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
+        "#129 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)"
+    ],
+    "asan_log": "INFO: Seed: 1720627312\nINFO: Loaded 1 modules   (21 inline 8-bit counters): 21 [0x766ef0, 0x766f05), \nINFO: Loaded 1 PC tables (21 PCs): 21 [0x542fd0,0x543120), \nsetup/fuzz.exe: Running 1 inputs 1 time(s) each.\nRunning: ./tmp/crash-66e9fe527ddb160d75f8c2cc373479e841f7999c\n=================================================================\n==16771==ERROR: AddressSanitizer: attempting double-free on 0x602000000050 in thread T0:\n==16771==WARNING: invalid path to external symbolizer!\n==16771==WARNING: Failed to use and restart external symbolizer!\n    #0 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)\n    #1 0x5273f0  (/onefuzz/setup/fuzz.exe+0x5273f0)\n    #2 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)\n    #3 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)\n    #4 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)\n    #5 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)\n    #6 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)\n    #7 0x41d879  (/onefuzz/setup/fuzz.exe+0x41d879)\n\n0x602000000050 is located 0 bytes inside of 4-byte region [0x602000000050,0x602000000054)\nfreed by thread T0 here:\n    #0 0x4f72e2  (/onefuzz/setup/fuzz.exe+0x4f72e2)\n    #1 0x5273e1  (/onefuzz/setup/fuzz.exe+0x5273e1)\n    #2 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)\n    #3 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)\n    #4 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)\n    #5 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)\n    #6 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)\n\npreviously allocated by thread T0 here:\n    #0 0x4f7663  (/onefuzz/setup/fuzz.exe+0x4f7663)\n    #1 0x5273cb  (/onefuzz/setup/fuzz.exe+0x5273cb)\n    #2 0x42fb3a  (/onefuzz/setup/fuzz.exe+0x42fb3a)\n    #3 0x41ef87  (/onefuzz/setup/fuzz.exe+0x41ef87)\n    #4 0x424ba1  (/onefuzz/setup/fuzz.exe+0x424ba1)\n    #5 0x44bd72  (/onefuzz/setup/fuzz.exe+0x44bd72)\n    #6 0x7ffff6a9bb96  (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)\n\nSUMMARY: AddressSanitizer: double-free (/onefuzz/setup/fuzz.exe+0x4f72e2) \n==16771==ABORTING\n",
+    "task_id": "218e1cdb-529a-45dd-b45b-1966d42b652c",
+    "job_id": "218e1cdb-529a-45dd-b45b-1966d42b652c",
+    "input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+    "input_blob": {
+        "account": "fuzzxkbh6uhuuke4m",
+        "container": "oft-asn-crashes",
+        "name": "crash"
+    },
+    "tool_name": "libfuzzer",
+    "tool_version": "1.2.3",
+	"onefuzz_version": "1.2.3"
+}

src/api-service/tests/test_report_parse.py

@@ -10,7 +10,7 @@ from pathlib import Path
 
 from onefuzztypes.models import Report
 
-from __app__.onefuzzlib.reports import parse_report_or_regression
+from __app__.onefuzzlib.reports import fix_report_size, parse_report_or_regression
 
 
 class TestReportParse(unittest.TestCase):
@@ -34,6 +34,28 @@ class TestReportParse(unittest.TestCase):
             )
             self.assertTrue(any(["unable to parse report" in x for x in logs.output]))
 
+    def test_report_no_resize(self) -> None:
+        report_path = Path(__file__).parent / "data" / "report.json"
+        with open(report_path, "r") as handle:
+            content = handle.read()
+            data = json.loads(content)
+            report = Report.parse_obj(data)
+            fixed_report = fix_report_size(content, report)
+            self.assertEqual(report, fixed_report)
+
+    def test_report_resize(self) -> None:
+        report_path = Path(__file__).parent / "data" / "report-long.json"
+        with open(report_path, "r") as handle:
+            content = handle.read()
+            data = json.loads(content)
+            report = Report.parse_obj(data)
+            fixed_report = fix_report_size(
+                content, report, acceptable_report_length_kb=10, keep_num_entries=10
+            )
+            self.assertEqual(len(fixed_report.call_stack), 11)  # extra item is "..."
+            report.call_stack = report.call_stack[0:10] + ["..."]
+            self.assertEqual(report, fixed_report)
+
 
 if __name__ == "__main__":
     unittest.main()