mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-15 03:18:07 +00:00
initial public release
This commit is contained in:
Brian Caswell
206
src/api-service/__app__/onefuzzlib/notifications/ado.py
Normal file
206
src/api-service/__app__/onefuzzlib/notifications/ado.py
Normal file
@ -0,0 +1,206 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import logging
|
||||
from typing import Iterator, List, Optional
|
||||
|
||||
from azure.devops.connection import Connection
|
||||
from azure.devops.credentials import BasicAuthentication
|
||||
from azure.devops.exceptions import AzureDevOpsServiceError
|
||||
from azure.devops.v6_0.work_item_tracking.models import (
|
||||
CommentCreate,
|
||||
JsonPatchOperation,
|
||||
Wiql,
|
||||
WorkItem,
|
||||
)
|
||||
from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import (
|
||||
WorkItemTrackingClient,
|
||||
)
|
||||
from memoization import cached
|
||||
from onefuzztypes.models import ADOTemplate, Report
|
||||
|
||||
from .common import Render
|
||||
|
||||
|
||||
@cached(ttl=60)
|
||||
def get_ado_client(base_url: str, token: str) -> WorkItemTrackingClient:
|
||||
connection = Connection(base_url=base_url, creds=BasicAuthentication("PAT", token))
|
||||
client = connection.clients_v6_0.get_work_item_tracking_client()
|
||||
return client
|
||||
|
||||
|
||||
@cached(ttl=60)
|
||||
def get_valid_fields(
|
||||
client: WorkItemTrackingClient, project: Optional[str] = None
|
||||
) -> List[str]:
|
||||
valid_fields = [
|
||||
x.reference_name.lower()
|
||||
for x in client.get_fields(project=project, expand="ExtensionFields")
|
||||
]
|
||||
return valid_fields
|
||||
|
||||
|
||||
class ADO:
|
||||
def __init__(
|
||||
self, container: str, filename: str, config: ADOTemplate, report: Report
|
||||
):
|
||||
self.config = config
|
||||
self.renderer = Render(container, filename, report)
|
||||
self.client = get_ado_client(self.config.base_url, self.config.auth_token)
|
||||
self.project = self.render(self.config.project)
|
||||
|
||||
def render(self, template: str) -> str:
|
||||
return self.renderer.render(template)
|
||||
|
||||
def existing_work_items(self) -> Iterator[WorkItem]:
|
||||
filters = {}
|
||||
for key in self.config.unique_fields:
|
||||
if key == "System.TeamProject":
|
||||
value = self.render(self.config.project)
|
||||
else:
|
||||
value = self.render(self.config.ado_fields[key])
|
||||
filters[key.lower()] = value
|
||||
|
||||
valid_fields = get_valid_fields(
|
||||
self.client, project=filters.get("system.teamproject")
|
||||
)
|
||||
|
||||
post_query_filter = {}
|
||||
|
||||
# WIQL (Work Item Query Language) is an SQL like query language that
|
||||
# doesn't support query params, safe quoting, or any other SQL-injection
|
||||
# protection mechanisms.
|
||||
#
|
||||
# As such, build the WIQL with a those fields we can pre-determine are
|
||||
# "safe" and otherwise use post-query filtering.
|
||||
parts = []
|
||||
for k, v in filters.items():
|
||||
# Only add pre-system approved fields to the query
|
||||
if k not in valid_fields:
|
||||
post_query_filter[k] = v
|
||||
continue
|
||||
|
||||
# WIQL supports wrapping values in ' or " and escaping ' by doubling it
|
||||
#
|
||||
# For this System.Title: hi'there
|
||||
# use this query fragment: [System.Title] = 'hi''there'
|
||||
#
|
||||
# For this System.Title: hi"there
|
||||
# use this query fragment: [System.Title] = 'hi"there'
|
||||
#
|
||||
# For this System.Title: hi'"there
|
||||
# use this query fragment: [System.Title] = 'hi''"there'
|
||||
SINGLE = "'"
|
||||
parts.append("[%s] = '%s'" % (k, v.replace(SINGLE, SINGLE + SINGLE)))
|
||||
|
||||
query = "select [System.Id] from WorkItems"
|
||||
if parts:
|
||||
query += " where " + " AND ".join(parts)
|
||||
|
||||
wiql = Wiql(query=query)
|
||||
for entry in self.client.query_by_wiql(wiql).work_items:
|
||||
item = self.client.get_work_item(entry.id, expand="Fields")
|
||||
lowered_fields = {x.lower(): str(y) for (x, y) in item.fields.items()}
|
||||
if post_query_filter and not all(
|
||||
[
|
||||
k.lower() in lowered_fields and lowered_fields[k.lower()] == v
|
||||
for (k, v) in post_query_filter.items()
|
||||
]
|
||||
):
|
||||
continue
|
||||
yield item
|
||||
|
||||
def update_existing(self, item: WorkItem) -> None:
|
||||
if self.config.on_duplicate.comment:
|
||||
comment = self.render(self.config.on_duplicate.comment)
|
||||
self.client.add_comment(
|
||||
CommentCreate(comment),
|
||||
self.project,
|
||||
item.id,
|
||||
)
|
||||
|
||||
document = []
|
||||
for field in self.config.on_duplicate.increment:
|
||||
value = int(item.fields[field]) if field in item.fields else 0
|
||||
value += 1
|
||||
document.append(
|
||||
JsonPatchOperation(
|
||||
op="Replace", path="/fields/%s" % field, value=str(value)
|
||||
)
|
||||
)
|
||||
|
||||
for field in self.config.on_duplicate.ado_fields:
|
||||
field_value = self.render(self.config.on_duplicate.ado_fields[field])
|
||||
document.append(
|
||||
JsonPatchOperation(
|
||||
op="Replace", path="/fields/%s" % field, value=field_value
|
||||
)
|
||||
)
|
||||
|
||||
if item.fields["System.State"] in self.config.on_duplicate.set_state:
|
||||
document.append(
|
||||
JsonPatchOperation(
|
||||
op="Replace",
|
||||
path="/fields/System.State",
|
||||
value=self.config.on_duplicate.set_state[
|
||||
item.fields["System.State"]
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
if document:
|
||||
self.client.update_work_item(document, item.id, project=self.project)
|
||||
|
||||
def create_new(self) -> None:
|
||||
task_type = self.render(self.config.type)
|
||||
|
||||
document = []
|
||||
if "System.Tags" not in self.config.ado_fields:
|
||||
document.append(
|
||||
JsonPatchOperation(
|
||||
op="Add", path="/fields/System.Tags", value="Onefuzz"
|
||||
)
|
||||
)
|
||||
|
||||
for field in self.config.ado_fields:
|
||||
value = self.render(self.config.ado_fields[field])
|
||||
if field == "System.Tags":
|
||||
value += ";Onefuzz"
|
||||
document.append(
|
||||
JsonPatchOperation(op="Add", path="/fields/%s" % field, value=value)
|
||||
)
|
||||
|
||||
entry = self.client.create_work_item(
|
||||
document=document, project=self.project, type=task_type
|
||||
)
|
||||
|
||||
if self.config.comment:
|
||||
comment = self.render(self.config.comment)
|
||||
self.client.add_comment(
|
||||
CommentCreate(comment),
|
||||
self.project,
|
||||
entry.id,
|
||||
)
|
||||
|
||||
def process(self) -> None:
|
||||
seen = False
|
||||
for work_item in self.existing_work_items():
|
||||
self.update_existing(work_item)
|
||||
seen = True
|
||||
|
||||
if not seen:
|
||||
self.create_new()
|
||||
|
||||
|
||||
def notify_ado(
|
||||
config: ADOTemplate, container: str, filename: str, report: Report
|
||||
) -> None:
|
||||
try:
|
||||
ado = ADO(container, filename, config, report)
|
||||
ado.process()
|
||||
except AzureDevOpsServiceError as err:
|
||||
logging.error("ADO report failed: %s", err)
|
||||
except ValueError as err:
|
||||
logging.error("ADO report value error: %s", err)
|
||||
61
src/api-service/__app__/onefuzzlib/notifications/common.py
Normal file
61
src/api-service/__app__/onefuzzlib/notifications/common.py
Normal file
@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
from typing import Optional
|
||||
|
||||
from jinja2.sandbox import SandboxedEnvironment
|
||||
from onefuzztypes.models import Report
|
||||
|
||||
from ..azure.containers import auth_download_url
|
||||
from ..jobs import Job
|
||||
from ..tasks.config import get_setup_container
|
||||
from ..tasks.main import Task
|
||||
|
||||
|
||||
class Render:
|
||||
def __init__(self, container: str, filename: str, report: Report):
|
||||
self.report = report
|
||||
self.container = container
|
||||
self.filename = filename
|
||||
task = Task.get(report.job_id, report.task_id)
|
||||
if not task:
|
||||
raise ValueError
|
||||
job = Job.get(report.job_id)
|
||||
if not job:
|
||||
raise ValueError
|
||||
|
||||
self.task_config = task.config
|
||||
self.job_config = job.config
|
||||
self.env = SandboxedEnvironment()
|
||||
|
||||
self.target_url: Optional[str] = None
|
||||
setup_container = get_setup_container(task.config)
|
||||
if setup_container:
|
||||
self.target_url = auth_download_url(
|
||||
setup_container, self.report.executable.replace("setup/", "", 1)
|
||||
)
|
||||
|
||||
self.report_url = auth_download_url(container, filename)
|
||||
self.input_url: Optional[str] = None
|
||||
if self.report.input_blob:
|
||||
self.input_url = auth_download_url(
|
||||
self.report.input_blob.container, self.report.input_blob.name
|
||||
)
|
||||
|
||||
def render(self, template: str) -> str:
|
||||
return self.env.from_string(template).render(
|
||||
{
|
||||
"report": self.report,
|
||||
"task": self.task_config,
|
||||
"job": self.job_config,
|
||||
"report_url": self.report_url,
|
||||
"input_url": self.input_url,
|
||||
"target_url": self.target_url,
|
||||
"report_container": self.container,
|
||||
"report_filename": self.filename,
|
||||
"repro_cmd": "onefuzz repro create_and_connect %s %s"
|
||||
% (self.container, self.filename),
|
||||
}
|
||||
)
|
||||
122
src/api-service/__app__/onefuzzlib/notifications/main.py
Normal file
122
src/api-service/__app__/onefuzzlib/notifications/main.py
Normal file
@ -0,0 +1,122 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Sequence, Tuple, Union
|
||||
from uuid import UUID
|
||||
|
||||
from memoization import cached
|
||||
from onefuzztypes import models
|
||||
from onefuzztypes.enums import ErrorCode, TaskState
|
||||
from onefuzztypes.models import ADOTemplate, Error, NotificationTemplate, TeamsTemplate
|
||||
from onefuzztypes.primitives import Container, Event
|
||||
|
||||
from ..azure.containers import get_container_metadata, get_file_sas_url
|
||||
from ..azure.creds import get_fuzz_storage
|
||||
from ..azure.queue import send_message
|
||||
from ..dashboard import add_event
|
||||
from ..orm import ORMMixin
|
||||
from ..reports import get_report
|
||||
from ..tasks.config import get_input_container_queues
|
||||
from ..tasks.main import Task
|
||||
from .ado import notify_ado
|
||||
from .teams import notify_teams
|
||||
|
||||
|
||||
class Notification(models.Notification, ORMMixin):
|
||||
@classmethod
|
||||
def get_by_id(cls, notification_id: UUID) -> Union[Error, "Notification"]:
|
||||
notifications = cls.search(query={"notification_id": [notification_id]})
|
||||
if not notifications:
|
||||
return Error(
|
||||
code=ErrorCode.INVALID_REQUEST, errors=["unable to find Notification"]
|
||||
)
|
||||
|
||||
if len(notifications) != 1:
|
||||
return Error(
|
||||
code=ErrorCode.INVALID_REQUEST,
|
||||
errors=["error identifying Notification"],
|
||||
)
|
||||
notification = notifications[0]
|
||||
return notification
|
||||
|
||||
@classmethod
|
||||
def get_existing(
|
||||
cls, container: Container, config: NotificationTemplate
|
||||
) -> Optional["Notification"]:
|
||||
notifications = Notification.search(query={"container": [container]})
|
||||
for notification in notifications:
|
||||
if notification.config == config:
|
||||
return notification
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def key_fields(cls) -> Tuple[str, str]:
|
||||
return ("notification_id", "container")
|
||||
|
||||
|
||||
@cached(ttl=10)
|
||||
def get_notifications(container: Container) -> List[Notification]:
|
||||
return Notification.search(query={"container": [container]})
|
||||
|
||||
|
||||
@cached(ttl=10)
|
||||
def get_queue_tasks() -> Sequence[Tuple[Task, Sequence[str]]]:
|
||||
results = []
|
||||
for task in Task.search_states(states=TaskState.available()):
|
||||
containers = get_input_container_queues(task.config)
|
||||
if containers:
|
||||
results.append((task, containers))
|
||||
return results
|
||||
|
||||
|
||||
@cached(ttl=60)
|
||||
def container_metadata(container: Container) -> Optional[Dict[str, str]]:
|
||||
return get_container_metadata(container)
|
||||
|
||||
|
||||
def new_files(container: Container, filename: str) -> None:
|
||||
results: Dict[str, Event] = {"container": container, "file": filename}
|
||||
|
||||
metadata = container_metadata(container)
|
||||
if metadata:
|
||||
results["metadata"] = metadata
|
||||
|
||||
notifications = get_notifications(container)
|
||||
if notifications:
|
||||
report = get_report(container, filename)
|
||||
if report:
|
||||
results["executable"] = report.executable
|
||||
results["crash_type"] = report.crash_type
|
||||
results["crash_site"] = report.crash_site
|
||||
results["job_id"] = report.job_id
|
||||
results["task_id"] = report.task_id
|
||||
|
||||
logging.info("notifications for %s %s %s", container, filename, notifications)
|
||||
done = []
|
||||
for notification in notifications:
|
||||
# ignore duplicate configurations
|
||||
if notification.config in done:
|
||||
continue
|
||||
done.append(notification.config)
|
||||
|
||||
if isinstance(notification.config, TeamsTemplate):
|
||||
notify_teams(notification.config, container, filename, report)
|
||||
|
||||
if not report:
|
||||
continue
|
||||
|
||||
if isinstance(notification.config, ADOTemplate):
|
||||
notify_ado(notification.config, container, filename, report)
|
||||
|
||||
for (task, containers) in get_queue_tasks():
|
||||
if container in containers:
|
||||
logging.info("queuing input %s %s %s", container, filename, task.task_id)
|
||||
url = get_file_sas_url(container, filename, read=True, delete=True)
|
||||
send_message(
|
||||
task.task_id, bytes(url, "utf-8"), account_id=get_fuzz_storage()
|
||||
)
|
||||
|
||||
add_event("new_file", results)
|
||||
127
src/api-service/__app__/onefuzzlib/notifications/teams.py
Normal file
127
src/api-service/__app__/onefuzzlib/notifications/teams.py
Normal file
@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env python
|
||||
#
|
||||
# Copyright (c) Microsoft Corporation.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
import logging
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
import requests
|
||||
from onefuzztypes.models import Report, TeamsTemplate
|
||||
|
||||
from ..azure.containers import auth_download_url
|
||||
from ..tasks.config import get_setup_container
|
||||
from ..tasks.main import Task
|
||||
|
||||
|
||||
def markdown_escape(data: str) -> str:
|
||||
values = "\\*_{}[]()#+-.!"
|
||||
for value in values:
|
||||
data = data.replace(value, "\\" + value)
|
||||
data = data.replace("`", "``")
|
||||
return data
|
||||
|
||||
|
||||
def code_block(data: str) -> str:
|
||||
data = data.replace("`", "``")
|
||||
return "\n```%s\n```\n" % data
|
||||
|
||||
|
||||
def send_teams_webhook(
|
||||
config: TeamsTemplate,
|
||||
title: str,
|
||||
facts: List[Dict[str, str]],
|
||||
text: Optional[str],
|
||||
) -> None:
|
||||
title = markdown_escape(title)
|
||||
|
||||
message: Dict[str, Any] = {
|
||||
"@type": "MessageCard",
|
||||
"@context": "https://schema.org/extensions",
|
||||
"summary": title,
|
||||
"sections": [{"activityTitle": title, "facts": facts}],
|
||||
}
|
||||
|
||||
if text:
|
||||
message["sections"].append({"text": text})
|
||||
|
||||
response = requests.post(config.url, json=message)
|
||||
if not response.ok:
|
||||
logging.error("webhook failed %s %s", response.status_code, response.content)
|
||||
|
||||
|
||||
def notify_teams(
|
||||
config: TeamsTemplate, container: str, filename: str, report: Optional[Report]
|
||||
) -> None:
|
||||
text = None
|
||||
facts: List[Dict[str, str]] = []
|
||||
|
||||
if report:
|
||||
task = Task.get(report.job_id, report.task_id)
|
||||
if not task:
|
||||
logging.error(
|
||||
"report with invalid task %s:%s", report.job_id, report.task_id
|
||||
)
|
||||
return
|
||||
|
||||
title = "new crash in %s: %s @ %s" % (
|
||||
report.executable,
|
||||
report.crash_type,
|
||||
report.crash_site,
|
||||
)
|
||||
|
||||
links = [
|
||||
"[report](%s)" % auth_download_url(container, filename),
|
||||
]
|
||||
|
||||
setup_container = get_setup_container(task.config)
|
||||
if setup_container:
|
||||
links.append(
|
||||
"[executable](%s)"
|
||||
% auth_download_url(
|
||||
setup_container,
|
||||
report.executable.replace("setup/", "", 1),
|
||||
),
|
||||
)
|
||||
|
||||
if report.input_blob:
|
||||
links.append(
|
||||
"[input](%s)"
|
||||
% auth_download_url(
|
||||
report.input_blob.container, report.input_blob.name
|
||||
),
|
||||
)
|
||||
|
||||
facts += [
|
||||
{"name": "Files", "value": " | ".join(links)},
|
||||
{
|
||||
"name": "Task",
|
||||
"value": markdown_escape(
|
||||
"job_id: %s task_id: %s" % (report.job_id, report.task_id)
|
||||
),
|
||||
},
|
||||
{
|
||||
"name": "Repro",
|
||||
"value": code_block(
|
||||
"onefuzz repro create_and_connect %s %s" % (container, filename)
|
||||
),
|
||||
},
|
||||
]
|
||||
|
||||
text = "## Call Stack\n" + "\n".join(code_block(x) for x in report.call_stack)
|
||||
|
||||
else:
|
||||
title = "new file found"
|
||||
facts += [
|
||||
{
|
||||
"name": "file",
|
||||
"value": "[%s/%s](%s)"
|
||||
% (
|
||||
markdown_escape(container),
|
||||
markdown_escape(filename),
|
||||
auth_download_url(container, filename),
|
||||
),
|
||||
}
|
||||
]
|
||||
|
||||
send_teams_webhook(config, title, facts, text)
|
||||
Reference in New Issue
Block a user