ExternalVendorCode/onefuzz
1
0
Fork 0
mirror of https://github.com/microsoft/onefuzz.git synced 2025-06-17 04:18:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

add regression testing tasks (#664)

Browse Source
This commit is contained in:
bmc-msft
2021-03-18 15:37:19 -04:00
committed by GitHub
parent 34b2a739cb
commit 6e60a8cf10
50 changed files with 2141 additions and 203 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
src/api-service/__app__/onefuzzlib/extension.py
View File

@ -271,6 +271,9 @@ def repro_extensions(
if report is None:
raise Exception("invalid report: %s" % repro_config)
if report.input_blob is None:
raise Exception("unable to perform reproduction without an input blob")
commands = []
if setup_container:
commands += [

24
src/api-service/__app__/onefuzzlib/notifications/ado.py
View File

@ -4,7 +4,7 @@
# Licensed under the MIT License.
import logging
from typing import Iterator, List, Optional
from typing import Iterator, List, Optional, Union
from azure.devops.connection import Connection
from azure.devops.credentials import BasicAuthentication
@ -24,7 +24,7 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import (
WorkItemTrackingClient,
)
from memoization import cached
from onefuzztypes.models import ADOTemplate, Report
from onefuzztypes.models import ADOTemplate, RegressionReport, Report
from onefuzztypes.primitives import Container
from ..secrets import get_secret_string_value
@ -51,7 +51,11 @@ def get_valid_fields(
class ADO:
def __init__(
self, container: Container, filename: str, config: ADOTemplate, report: Report
self,
container: Container,
filename: str,
config: ADOTemplate,
report: Report,
):
self.config = config
self.renderer = Render(container, filename, report)
@ -203,8 +207,20 @@ class ADO:
def notify_ado(
config: ADOTemplate, container: Container, filename: str, report: Report
config: ADOTemplate,
container: Container,
filename: str,
report: Union[Report, RegressionReport],
) -> None:
if isinstance(report, RegressionReport):
logging.info(
"ado integration does not support regression reports. "
"container:%s filename:%s",
container,
filename,
)
return
logging.info(
"notify ado: job_id:%s task_id:%s container:%s filename:%s",
report.job_id,

19
src/api-service/__app__/onefuzzlib/notifications/github_issues.py
View File

@ -4,13 +4,18 @@
# Licensed under the MIT License.
import logging
from typing import List, Optional
from typing import List, Optional, Union
from github3 import login
from github3.exceptions import GitHubException
from github3.issues import Issue
from onefuzztypes.enums import GithubIssueSearchMatch
from onefuzztypes.models import GithubAuth, GithubIssueTemplate, Report
from onefuzztypes.models import (
GithubAuth,
GithubIssueTemplate,
RegressionReport,
Report,
)
from onefuzztypes.primitives import Container
from ..secrets import get_secret_obj
@ -107,10 +112,18 @@ def github_issue(
config: GithubIssueTemplate,
container: Container,
filename: str,
report: Optional[Report],
report: Optional[Union[Report, RegressionReport]],
) -> None:
if report is None:
return
if isinstance(report, RegressionReport):
logging.info(
"github issue integration does not support regression reports. "
"container:%s filename:%s",
container,
filename,
)
return
try:
handler = GithubIssue(config, container, filename, report)

20
src/api-service/__app__/onefuzzlib/notifications/main.py
View File

@ -10,12 +10,18 @@ from uuid import UUID
from memoization import cached
from onefuzztypes import models
from onefuzztypes.enums import ErrorCode, TaskState
from onefuzztypes.events import EventCrashReported, EventFileAdded
from onefuzztypes.events import (
EventCrashReported,
EventFileAdded,
EventRegressionReported,
)
from onefuzztypes.models import (
ADOTemplate,
Error,
GithubIssueTemplate,
NotificationTemplate,
RegressionReport,
Report,
Result,
TeamsTemplate,
)
@ -26,7 +32,7 @@ from ..azure.queue import send_message
from ..azure.storage import StorageType
from ..events import send_event
from ..orm import ORMMixin
from ..reports import get_report
from ..reports import get_report_or_regression
from ..tasks.config import get_input_container_queues
from ..tasks.main import Task
from .ado import notify_ado
@ -102,7 +108,7 @@ def get_queue_tasks() -> Sequence[Tuple[Task, Sequence[str]]]:
def new_files(container: Container, filename: str) -> None:
report = get_report(container, filename)
report = get_report_or_regression(container, filename)
notifications = get_notifications(container)
if notifications:
@ -134,9 +140,15 @@ def new_files(container: Container, filename: str) -> None:
)
send_message(task.task_id, bytes(url, "utf-8"), StorageType.corpus)
if report:
if isinstance(report, Report):
send_event(
EventCrashReported(report=report, container=container, filename=filename)
)
elif isinstance(report, RegressionReport):
send_event(
EventRegressionReported(
regression_report=report, container=container, filename=filename
)
)
else:
send_event(EventFileAdded(container=container, filename=filename))

11
src/api-service/__app__/onefuzzlib/notifications/teams.py
View File

@ -4,10 +4,10 @@
# Licensed under the MIT License.
import logging
from typing import Any, Dict, List, Optional
from typing import Any, Dict, List, Optional, Union
import requests
from onefuzztypes.models import Report, TeamsTemplate
from onefuzztypes.models import RegressionReport, Report, TeamsTemplate
from onefuzztypes.primitives import Container
from ..azure.containers import auth_download_url
@ -54,12 +54,15 @@ def send_teams_webhook(
def notify_teams(
config: TeamsTemplate, container: Container, filename: str, report: Optional[Report]
config: TeamsTemplate,
container: Container,
filename: str,
report: Optional[Union[Report, RegressionReport]],
) -> None:
text = None
facts: List[Dict[str, str]] = []
if report:
if isinstance(report, Report):
task = Task.get(report.job_id, report.task_id)
if not task:
logging.error(

41
src/api-service/__app__/onefuzzlib/reports.py
View File

@ -8,7 +8,7 @@ import logging
from typing import Optional, Union
from memoization import cached
from onefuzztypes.models import Report
from onefuzztypes.models import RegressionReport, Report
from onefuzztypes.primitives import Container
from pydantic import ValidationError
@ -16,17 +16,16 @@ from .azure.containers import get_blob
from .azure.storage import StorageType
def parse_report(
def parse_report_or_regression(
content: Union[str, bytes], file_path: Optional[str] = None
) -> Optional[Report]:
) -> Optional[Union[Report, RegressionReport]]:
if isinstance(content, bytes):
try:
content = content.decode()
except UnicodeDecodeError as err:
logging.error(
"unable to parse report (%s): unicode decode of report failed - %s",
file_path,
err,
f"unable to parse report ({file_path}): "
f"unicode decode of report failed - {err}"
)
return None
@ -34,22 +33,31 @@ def parse_report(
data = json.loads(content)
except json.decoder.JSONDecodeError as err:
logging.error(
"unable to parse report (%s): json decoding failed - %s", file_path, err
f"unable to parse report ({file_path}): json decoding failed - {err}"
)
return None
regression_err = None
try:
entry = Report.parse_obj(data)
return RegressionReport.parse_obj(data)
except ValidationError as err:
logging.error("unable to parse report (%s): %s", file_path, err)
return None
regression_err = err
return entry
try:
return Report.parse_obj(data)
except ValidationError as err:
logging.error(
f"unable to parse report ({file_path}) as a report or regression. "
f"regression error: {regression_err} report error: {err}"
)
return None
# cache the last 1000 reports
@cached(max_size=1000)
def get_report(container: Container, filename: str) -> Optional[Report]:
def get_report_or_regression(
container: Container, filename: str
) -> Optional[Union[Report, RegressionReport]]:
file_path = "/".join([container, filename])
if not filename.endswith(".json"):
logging.error("get_report invalid extension: %s", file_path)
@ -60,4 +68,11 @@ def get_report(container: Container, filename: str) -> Optional[Report]:
logging.error("get_report invalid blob: %s", file_path)
return None
return parse_report(blob, file_path=file_path)
return parse_report_or_regression(blob, file_path=file_path)
def get_report(container: Container, filename: str) -> Optional[Report]:
result = get_report_or_regression(container, filename)
if isinstance(result, Report):
return result
return None

6
src/api-service/__app__/onefuzzlib/repro.py
View File

@ -164,6 +164,12 @@ class Repro(BASE_REPRO, ORMMixin):
if report is None:
return Error(code=ErrorCode.VM_CREATE_FAILED, errors=["missing report"])
if report.input_blob is None:
return Error(
code=ErrorCode.VM_CREATE_FAILED,
errors=["unable to perform repro for crash reports without inputs"],
)
files = {}
if task.os == OS.windows:

3
src/api-service/__app__/onefuzzlib/tasks/config.py
View File

@ -348,6 +348,9 @@ def build_task_config(
else True
)
if TaskFeature.report_list in definition.features:
config.report_list = task_config.task.report_list
if TaskFeature.expect_crash_on_failure in definition.features:
config.expect_crash_on_failure = (
task_config.task.expect_crash_on_failure

127
src/api-service/__app__/onefuzzlib/tasks/defs.py
View File

@ -414,4 +414,131 @@ TASK_DEFINITIONS = {
],
monitor_queue=ContainerType.crashes,
),
TaskType.generic_regression: TaskDefinition(
features=[
TaskFeature.target_exe,
TaskFeature.target_env,
TaskFeature.target_options,
TaskFeature.target_timeout,
TaskFeature.check_asan_log,
TaskFeature.check_debugger,
TaskFeature.check_retry_count,
TaskFeature.report_list,
],
vm=VmDefinition(compare=Compare.AtLeast, value=1),
containers=[
ContainerDefinition(
type=ContainerType.setup,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.regression_reports,
compare=Compare.Equal,
value=1,
permissions=[
ContainerPermission.Write,
ContainerPermission.Read,
ContainerPermission.List,
],
),
ContainerDefinition(
type=ContainerType.crashes,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.unique_reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.no_repro,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.readonly_inputs,
compare=Compare.AtMost,
value=1,
permissions=[
ContainerPermission.Read,
ContainerPermission.List,
],
),
],
),
TaskType.libfuzzer_regression: TaskDefinition(
features=[
TaskFeature.target_exe,
TaskFeature.target_env,
TaskFeature.target_options,
TaskFeature.target_timeout,
TaskFeature.check_fuzzer_help,
TaskFeature.check_retry_count,
TaskFeature.report_list,
],
vm=VmDefinition(compare=Compare.AtLeast, value=1),
containers=[
ContainerDefinition(
type=ContainerType.setup,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.regression_reports,
compare=Compare.Equal,
value=1,
permissions=[
ContainerPermission.Write,
ContainerPermission.Read,
ContainerPermission.List,
],
),
ContainerDefinition(
type=ContainerType.crashes,
compare=Compare.Equal,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.unique_reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.reports,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.no_repro,
compare=Compare.AtMost,
value=1,
permissions=[ContainerPermission.Read, ContainerPermission.List],
),
ContainerDefinition(
type=ContainerType.readonly_inputs,
compare=Compare.AtMost,
value=1,
permissions=[
ContainerPermission.Read,
ContainerPermission.List,
],
),
],
),
}