ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-22 14:19:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

handle messages that are too big to fit in a queue message (#2020)

Browse Source 
* handle messages that are too big to fit in a queue message

* tests

Co-authored-by: stas <statis@microsoft.com>
This commit is contained in:
Stas
2022-06-06 12:16:47 -07:00
committed by GitHub
parent e27eccc39f
commit 60b304a220
4 changed files with 255 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/api-service/__app__/onefuzzlib/events.py
View File

@ -76,6 +76,6 @@ def send_event(event: Event) -> None:
if event_message.event != event:
event_message.event = event.copy(deep=True)
log_event(event, event_type)
queue_signalr_event(event_message)
Webhook.send_event(event_message)
log_event(event, event_type)

82
src/api-service/__app__/onefuzzlib/reports.py
View File

@ -5,6 +5,7 @@
import json
import logging
from sys import getsizeof
from typing import Optional, Union
from memoization import cached
@ -16,6 +17,65 @@ from .azure.containers import get_blob
from .azure.storage import StorageType
# This is fix for the following error:
# Exception while executing function:
# Functions.queue_file_changes Result: Failure
# Exception: AzureHttpError: Bad Request
# "The property value exceeds the maximum allowed size (64KB).
# If the property value is a string, it is UTF-16 encoded and
# the maximum number of characters should be 32K or less.
def fix_report_size(
content: str,
report: Report,
acceptable_report_length_kb: int = 24,
keep_num_entries: int = 10,
keep_string_len: int = 256,
) -> Report:
logging.info(f"report content length {getsizeof(content)}")
if getsizeof(content) > acceptable_report_length_kb * 1024:
msg = f"report data exceeds {acceptable_report_length_kb}K {getsizeof(content)}"
if len(report.call_stack) > keep_num_entries:
msg = msg + "; removing some of stack frames from the report"
report.call_stack = report.call_stack[0:keep_num_entries] + ["..."]
if report.asan_log and len(report.asan_log) > keep_string_len:
msg = msg + "; removing some of asan log entries from the report"
report.asan_log = report.asan_log[0:keep_string_len] + "..."
if report.minimized_stack and len(report.minimized_stack) > keep_num_entries:
msg = msg + "; removing some of minimized stack frames from the report"
report.minimized_stack = report.minimized_stack[0:keep_num_entries] + [
"..."
]
if (
report.minimized_stack_function_names
and len(report.minimized_stack_function_names) > keep_num_entries
):
msg = (
msg
+ "; removing some of minimized stack function names from the report"
)
report.minimized_stack_function_names = (
report.minimized_stack_function_names[0:keep_num_entries] + ["..."]
)
if (
report.minimized_stack_function_lines
and len(report.minimized_stack_function_lines) > keep_num_entries
):
msg = (
msg
+ "; removing some of minimized stack function lines from the report"
)
report.minimized_stack_function_lines = (
report.minimized_stack_function_lines[0:keep_num_entries] + ["..."]
)
logging.info(msg)
return report
def parse_report_or_regression(
content: Union[str, bytes],
file_path: Optional[str] = None,
@ -43,12 +103,30 @@ def parse_report_or_regression(
regression_err = None
try:
return RegressionReport.parse_obj(data)
regression_report = RegressionReport.parse_obj(data)
if (
regression_report.crash_test_result is not None
and regression_report.crash_test_result.crash_report is not None
):
regression_report.crash_test_result.crash_report = fix_report_size(
content, regression_report.crash_test_result.crash_report
)
if (
regression_report.original_crash_test_result is not None
and regression_report.original_crash_test_result.crash_report is not None
):
regression_report.original_crash_test_result.crash_report = fix_report_size(
content, regression_report.original_crash_test_result.crash_report
)
return regression_report
except ValidationError as err:
regression_err = err
try:
return Report.parse_obj(data)
report = Report.parse_obj(data)
return fix_report_size(content, report)
except ValidationError as err:
if expect_reports:
logging.error(

151
src/api-service/tests/data/report-long.json Normal file
View File

@ -0,0 +1,151 @@
{
"call_stack_sha256": "972a371a291ed5668a77576368ead0c46c2bac9f9a16b7fa7c0b48aec5b059b1",
"input_url": "https://fuzzxkbh6uhuuke4m.blob.core.windows.net/oft-asan-crashes/crash",
"executable": "setup/fuzz.exe",
"crash_type": "double-free",
"crash_site": "double-free (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"call_stack": [
"#0 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#1 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#2 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#3 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#4 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#5 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#6 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#7 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#8 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#9 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#10 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#11 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#12 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#13 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#14 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#15 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#16 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#17 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#18 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#19 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#20 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#21 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#22 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#23 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#24 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#25 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#26 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#27 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#28 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#29 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#30 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#31 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#32 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#33 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#34 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#35 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#36 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#37 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#38 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#39 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#40 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#41 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#42 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#43 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#44 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#45 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#46 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#47 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#48 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#49 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#50 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#51 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#52 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#53 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#54 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#55 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#56 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#57 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#58 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#59 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#60 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#61 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#62 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#63 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#64 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#65 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#66 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#67 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#68 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#69 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#70 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#71 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#72 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#73 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#74 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#75 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#76 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#77 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#78 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#79 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#80 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#81 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#82 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#83 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#84 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#85 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#86 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#87 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#88 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#89 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#90 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#91 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#92 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#93 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#94 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#95 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#96 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#97 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#98 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#99 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#100 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#101 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#102 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#103 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#104 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#105 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#106 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#107 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#108 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#109 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#110 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#111 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#112 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#113 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#114 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#115 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#116 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#117 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#118 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#119 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#120 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)",
"#121 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)",
"#122 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)",
"#123 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)",
"#124 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)",
"#125 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)",
"#126 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#127 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)",
"#128 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)",
"#129 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)"
],
"asan_log": "INFO: Seed: 1720627312\nINFO: Loaded 1 modules (21 inline 8-bit counters): 21 [0x766ef0, 0x766f05), \nINFO: Loaded 1 PC tables (21 PCs): 21 [0x542fd0,0x543120), \nsetup/fuzz.exe: Running 1 inputs 1 time(s) each.\nRunning: ./tmp/crash-66e9fe527ddb160d75f8c2cc373479e841f7999c\n=================================================================\n==16771==ERROR: AddressSanitizer: attempting double-free on 0x602000000050 in thread T0:\n==16771==WARNING: invalid path to external symbolizer!\n==16771==WARNING: Failed to use and restart external symbolizer!\n #0 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)\n #1 0x5273f0 (/onefuzz/setup/fuzz.exe+0x5273f0)\n #2 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)\n #3 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)\n #4 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)\n #5 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)\n #6 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)\n #7 0x41d879 (/onefuzz/setup/fuzz.exe+0x41d879)\n\n0x602000000050 is located 0 bytes inside of 4-byte region [0x602000000050,0x602000000054)\nfreed by thread T0 here:\n #0 0x4f72e2 (/onefuzz/setup/fuzz.exe+0x4f72e2)\n #1 0x5273e1 (/onefuzz/setup/fuzz.exe+0x5273e1)\n #2 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)\n #3 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)\n #4 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)\n #5 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)\n #6 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)\n\npreviously allocated by thread T0 here:\n #0 0x4f7663 (/onefuzz/setup/fuzz.exe+0x4f7663)\n #1 0x5273cb (/onefuzz/setup/fuzz.exe+0x5273cb)\n #2 0x42fb3a (/onefuzz/setup/fuzz.exe+0x42fb3a)\n #3 0x41ef87 (/onefuzz/setup/fuzz.exe+0x41ef87)\n #4 0x424ba1 (/onefuzz/setup/fuzz.exe+0x424ba1)\n #5 0x44bd72 (/onefuzz/setup/fuzz.exe+0x44bd72)\n #6 0x7ffff6a9bb96 (/lib/x86_64-linux-gnu/libc.so.6+0x21b96)\n\nSUMMARY: AddressSanitizer: double-free (/onefuzz/setup/fuzz.exe+0x4f72e2) \n==16771==ABORTING\n",
"task_id": "218e1cdb-529a-45dd-b45b-1966d42b652c",
"job_id": "218e1cdb-529a-45dd-b45b-1966d42b652c",
"input_sha256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
"input_blob": {
"account": "fuzzxkbh6uhuuke4m",
"container": "oft-asn-crashes",
"name": "crash"
},
"tool_name": "libfuzzer",
"tool_version": "1.2.3",
"onefuzz_version": "1.2.3"
}

24
src/api-service/tests/test_report_parse.py
View File

@ -10,7 +10,7 @@ from pathlib import Path
from onefuzztypes.models import Report
from __app__.onefuzzlib.reports import parse_report_or_regression
from __app__.onefuzzlib.reports import fix_report_size, parse_report_or_regression
class TestReportParse(unittest.TestCase):
@ -34,6 +34,28 @@ class TestReportParse(unittest.TestCase):
)
self.assertTrue(any(["unable to parse report" in x for x in logs.output]))
def test_report_no_resize(self) -> None:
report_path = Path(__file__).parent / "data" / "report.json"
with open(report_path, "r") as handle:
content = handle.read()
data = json.loads(content)
report = Report.parse_obj(data)
fixed_report = fix_report_size(content, report)
self.assertEqual(report, fixed_report)
def test_report_resize(self) -> None:
report_path = Path(__file__).parent / "data" / "report-long.json"
with open(report_path, "r") as handle:
content = handle.read()
data = json.loads(content)
report = Report.parse_obj(data)
fixed_report = fix_report_size(
content, report, acceptable_report_length_kb=10, keep_num_entries=10
)
self.assertEqual(len(fixed_report.call_stack), 11) # extra item is "..."
report.call_stack = report.call_stack[0:10] + ["..."]
self.assertEqual(report, fixed_report)
if __name__ == "__main__":
unittest.main()