diff --git a/docs/notifications.md b/docs/notifications.md index b3600c262..f3dc7d5c5 100644 --- a/docs/notifications.md +++ b/docs/notifications.md @@ -38,4 +38,5 @@ JSON via a string, such as `'{"config":{...}}'` ## Supported integrations * [Microsoft Teams](notifications/teams.md) -* [Azure Devops Work Items](notifications/ado.md) \ No newline at end of file +* [Azure Devops Work Items](notifications/ado.md) +* [Github Issues](notifications/github.md) \ No newline at end of file diff --git a/docs/notifications/github.md b/docs/notifications/github.md new file mode 100644 index 000000000..a8b541b0b --- /dev/null +++ b/docs/notifications/github.md @@ -0,0 +1,79 @@ +# Notifications via Github Issues + +OneFuzz can create or update [Github Issues](https://guides.github.com/features/issues/) +upon creation of crash reports in OneFuzz managed [containers](../containers.md). + +Nearly every field can be customized using [jinja2](https://jinja.palletsprojects.com/) +templates. There are multiple python objects provided via the template engine that +can be used such that any arbitrary component can be used to flesh out the configuration: + +* task (See [TaskConfig](../../src/pytypes/onefuzztypes/models.py)) +* report (See [Report](../../src/pytypes/onefuzztypes/models.py)) +* job (See [JobConfig](../../src/pytypes/onefuzztypes/models.py)) + +Using these objects allows dynamic configuration. As an example, the `repository` +could be specified directly, or dynamically pulled from the task configuration: + +```json +{ + "repository": "{{ task.tags['repository'] }}" +} +``` + +There are additional values that can be used in any template: + +* report_url: This will link to an authenticated download link for the report +* input_url: This will link to an authenticated download link for crashing input +* target_url: This will link to an authenticated download link for the target + executable +* repro_cmd: This will give an example command to initiate a live reproduction + of a crash +* report_container: This will give the name of the report storage container +* report_filename: This will give the container relative path to the report + +## Example Config + +```json +{ + "config": { + "auth": { + "user": "INSERT_YOUR_USERNAME_HERE", + "personal_access_token": "INSERT_YOUR_PERSONAL_ACCESS_TOKEN_HERE" + }, + "organization": "contoso", + "repository": "sample-project", + "title": "{{ report.executable }} - {{report.crash_site}}", + "body": "## Files\n\n* input: [{{ report.input_blob.name }}]({{ input_url }})\n* exe: [{{ report.executable }}]( {{ target_url }})\n* report: [{{ report_filename }}]({{ report_url }})\n\n## Repro\n\n `{{ repro_cmd }}`\n\n## Call Stack\n\n```{% for item in report.call_stack %}{{ item }}\n{% endfor %}```\n\n## ASAN Log\n\n```{{ report.asan_log }}```", + "unique_search": { + "field_match": ["title"], + "string": "{{ report.executable }}" + }, + "assignees": [], + "labels": ["bug", "{{ report.crash_type }}"], + "on_duplicate": { + "comment": "Duplicate found.\n\n* input: [{{ report.input_blob.name }}]({{ input_url }})\n* exe: [{{ report.executable }}]( {{ target_url }})\n* report: [{{ report_filename }}]({{ report_url }})", + "labels": ["{{ report.crash_type }}"], + "reopen": true + } + } +} +``` + +For full documentation on the syntax, see [GithubIssueTemplate](../../src/pytypes/onefuzztypes/models.py)) + +## Integration + +1. Create a [Personal access token](https://github.com/settings/tokens). +2. Update your config to specify your user and personal access token. +1. Add a notification to your OneFuzz instance. + + ``` + onefuzz notifications create @./config.json + ``` + +Until the integration is deleted, when a crash report is written to the indicated container, +issues will be created and updated based on the reports. + +The OneFuzz SDK provides an example tool [fake-report.py](../../src/cli/examples/fake-report.py), +which can be used to generate a synthetic crash report to verify the integration +is functional. \ No newline at end of file diff --git a/src/api-service/__app__/notifications/__init__.py b/src/api-service/__app__/notifications/__init__.py index 4d1c797a5..e8ca50091 100644 --- a/src/api-service/__app__/notifications/__init__.py +++ b/src/api-service/__app__/notifications/__init__.py @@ -17,6 +17,9 @@ from ..onefuzzlib.request import not_ok, ok, parse_request def get(req: func.HttpRequest) -> func.HttpResponse: entries = Notification.search() + for entry in entries: + entry.config.redact() + return ok(entries) @@ -52,6 +55,7 @@ def delete(req: func.HttpRequest) -> func.HttpResponse: return not_ok(entry, context="notification delete") entry.delete() + entry.config.redact() return ok(entry) diff --git a/src/api-service/__app__/onefuzzlib/notifications/ado.py b/src/api-service/__app__/onefuzzlib/notifications/ado.py index 91186bfac..7fa78e745 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/ado.py +++ b/src/api-service/__app__/onefuzzlib/notifications/ado.py @@ -24,11 +24,9 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import ( WorkItemTrackingClient, ) from memoization import cached -from onefuzztypes.enums import ErrorCode -from onefuzztypes.models import ADOTemplate, Error, Report +from onefuzztypes.models import ADOTemplate, Report -from ..tasks.main import Task -from .common import Render +from .common import Render, fail_task @cached(ttl=60) @@ -201,21 +199,6 @@ class ADO: self.create_new() -def fail_task(report: Report, error: Exception) -> None: - logging.error( - "ADO report failed: job_id:%s task_id:%s err:%s", - report.job_id, - report.task_id, - error, - ) - - task = Task.get(report.job_id, report.task_id) - if task: - task.mark_failed( - Error(code=ErrorCode.NOTIFICATION_FAILURE, errors=[str(error)]) - ) - - def notify_ado( config: ADOTemplate, container: str, filename: str, report: Report ) -> None: diff --git a/src/api-service/__app__/onefuzzlib/notifications/common.py b/src/api-service/__app__/onefuzzlib/notifications/common.py index f13424464..3ae30b629 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/common.py +++ b/src/api-service/__app__/onefuzzlib/notifications/common.py @@ -3,10 +3,12 @@ # Copyright (c) Microsoft Corporation. # Licensed under the MIT License. +import logging from typing import Optional from jinja2.sandbox import SandboxedEnvironment -from onefuzztypes.models import Report +from onefuzztypes.enums import ErrorCode +from onefuzztypes.models import Error, Report from ..azure.containers import auth_download_url from ..jobs import Job @@ -14,6 +16,21 @@ from ..tasks.config import get_setup_container from ..tasks.main import Task +def fail_task(report: Report, error: Exception) -> None: + logging.error( + "notification failed: job_id:%s task_id:%s err:%s", + report.job_id, + report.task_id, + error, + ) + + task = Task.get(report.job_id, report.task_id) + if task: + task.mark_failed( + Error(code=ErrorCode.NOTIFICATION_FAILURE, errors=[str(error)]) + ) + + class Render: def __init__(self, container: str, filename: str, report: Report): self.report = report diff --git a/src/api-service/__app__/onefuzzlib/notifications/github_issues.py b/src/api-service/__app__/onefuzzlib/notifications/github_issues.py new file mode 100644 index 000000000..def869181 --- /dev/null +++ b/src/api-service/__app__/onefuzzlib/notifications/github_issues.py @@ -0,0 +1,102 @@ +import logging +from typing import List, Optional + +from github3 import login +from github3.exceptions import GitHubException +from github3.issues import Issue +from onefuzztypes.enums import GithubIssueSearchMatch +from onefuzztypes.models import GithubIssueTemplate, Report + +from .common import Render, fail_task + + +class GithubIssue: + def __init__( + self, config: GithubIssueTemplate, container: str, filename: str, report: Report + ): + self.config = config + self.report = report + self.gh = login( + username=config.auth.user, password=config.auth.personal_access_token + ) + self.renderer = Render(container, filename, report) + + def render(self, field: str) -> str: + return self.renderer.render(field) + + def existing(self) -> List[Issue]: + query = [ + self.render(self.config.unique_search.string), + "repo:%s/%s" + % ( + self.render(self.config.organization), + self.render(self.config.repository), + ), + ] + if self.config.unique_search.author: + query.append("author:%s" % self.render(self.config.unique_search.author)) + + if self.config.unique_search.state: + query.append("state:%s" % self.config.unique_search.state.name) + + issues = [] + title = self.render(self.config.title) + body = self.render(self.config.body) + for issue in self.gh.search_issues(" ".join(query)): + skip = False + for field in self.config.unique_search.field_match: + if field == GithubIssueSearchMatch.title and issue.title != title: + skip = True + break + if field == GithubIssueSearchMatch.body and issue.body != body: + skip = True + break + if not skip: + issues.append(issue) + + return issues + + def update(self, issue: Issue) -> None: + logging.info("updating issue: %s", issue) + if self.config.on_duplicate.comment: + issue.issue.create_comment(self.render(self.config.on_duplicate.comment)) + if self.config.on_duplicate.labels: + labels = [self.render(x) for x in self.config.on_duplicate.labels] + issue.issue.edit(labels=labels) + if self.config.on_duplicate.reopen and issue.state != "open": + issue.issue.edit(state="open") + + def create(self) -> None: + logging.info("creating issue") + + assignees = [self.render(x) for x in self.config.assignees] + labels = list(set(["OneFuzz"] + [self.render(x) for x in self.config.labels])) + + self.gh.create_issue( + self.render(self.config.organization), + self.render(self.config.repository), + self.render(self.config.title), + body=self.render(self.config.body), + labels=labels, + assignees=assignees, + ) + + def process(self) -> None: + issues = self.existing() + if issues: + self.update(issues[0]) + else: + self.create() + + +def github_issue( + config: GithubIssueTemplate, container: str, filename: str, report: Optional[Report] +) -> None: + if report is None: + return + + try: + handler = GithubIssue(config, container, filename, report) + handler.process() + except GitHubException as err: + fail_task(report, err) diff --git a/src/api-service/__app__/onefuzzlib/notifications/main.py b/src/api-service/__app__/onefuzzlib/notifications/main.py index 655b98087..ca8f453c5 100644 --- a/src/api-service/__app__/onefuzzlib/notifications/main.py +++ b/src/api-service/__app__/onefuzzlib/notifications/main.py @@ -10,7 +10,13 @@ from uuid import UUID from memoization import cached from onefuzztypes import models from onefuzztypes.enums import ErrorCode, TaskState -from onefuzztypes.models import ADOTemplate, Error, NotificationTemplate, TeamsTemplate +from onefuzztypes.models import ( + ADOTemplate, + Error, + GithubIssueTemplate, + NotificationTemplate, + TeamsTemplate, +) from onefuzztypes.primitives import Container, Event from ..azure.containers import get_container_metadata, get_file_sas_url @@ -22,6 +28,7 @@ from ..reports import get_report from ..tasks.config import get_input_container_queues from ..tasks.main import Task from .ado import notify_ado +from .github_issues import github_issue from .teams import notify_teams @@ -111,6 +118,9 @@ def new_files(container: Container, filename: str) -> None: if isinstance(notification.config, ADOTemplate): notify_ado(notification.config, container, filename, report) + if isinstance(notification.config, GithubIssueTemplate): + github_issue(notification.config, container, filename, report) + for (task, containers) in get_queue_tasks(): if container in containers: logging.info("queuing input %s %s %s", container, filename, task.task_id) diff --git a/src/api-service/__app__/onefuzzlib/versions.py b/src/api-service/__app__/onefuzzlib/versions.py index 246909dba..7b822fe10 100644 --- a/src/api-service/__app__/onefuzzlib/versions.py +++ b/src/api-service/__app__/onefuzzlib/versions.py @@ -16,8 +16,8 @@ from .__version__ import __version__ def read_local_file(filename: str) -> str: path = os.path.join(os.path.dirname(os.path.realpath(__file__)), filename) if os.path.exists(path): - with open(path, "rb") as handle: - return handle.read().strip().decode("utf-16") + with open(path, "r") as handle: + return handle.read().strip() else: return "UNKNOWN" diff --git a/src/api-service/__app__/requirements.txt b/src/api-service/__app__/requirements.txt index 108c5d2ea..ffc792689 100644 --- a/src/api-service/__app__/requirements.txt +++ b/src/api-service/__app__/requirements.txt @@ -31,5 +31,6 @@ pydantic~=1.6.1 PyJWT~=1.7.1 requests~=2.24.0 memoization~=0.3.1 +github3.py~=1.3.0 # onefuzz types version is set during build onefuzztypes==0.0.0 diff --git a/src/api-service/mypy.ini b/src/api-service/mypy.ini index 8c24005ed..78b0a9684 100644 --- a/src/api-service/mypy.ini +++ b/src/api-service/mypy.ini @@ -32,5 +32,5 @@ ignore_missing_imports = True [mypy-memoization.*] ignore_missing_imports = True -[mypy-github.*] -ignore_missing_imports = True +[mypy-github3.*] +ignore_missing_imports = True \ No newline at end of file diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py index d4f46da13..6e6b366c5 100644 --- a/src/pytypes/onefuzztypes/enums.py +++ b/src/pytypes/onefuzztypes/enums.py @@ -318,3 +318,13 @@ class NodeState(Enum): # If Node is in one of these states, ignore updates # from the agent. return [cls.done, cls.shutdown, cls.halt] + + +class GithubIssueState(Enum): + open = "open" + closed = "closed" + + +class GithubIssueSearchMatch(Enum): + title = "title" + body = "body" diff --git a/src/pytypes/onefuzztypes/models.py b/src/pytypes/onefuzztypes/models.py index 42b372f19..e87705c48 100644 --- a/src/pytypes/onefuzztypes/models.py +++ b/src/pytypes/onefuzztypes/models.py @@ -17,6 +17,8 @@ from .enums import ( ContainerPermission, ContainerType, ErrorCode, + GithubIssueSearchMatch, + GithubIssueState, HeartbeatType, JobState, NodeState, @@ -213,10 +215,16 @@ class ADOTemplate(BaseModel): ado_fields: Dict[str, str] on_duplicate: ADODuplicateTemplate + def redact(self) -> None: + self.auth_token = "***" + class TeamsTemplate(BaseModel): url: str + def redact(self) -> None: + self.url = "***" + class ContainerDefinition(BaseModel): type: ContainerType @@ -368,7 +376,41 @@ class WorkSetSummary(BaseModel): work_units: List[WorkUnitSummary] -NotificationTemplate = Union[ADOTemplate, TeamsTemplate] +class GithubIssueDuplicate(BaseModel): + comment: Optional[str] + labels: List[str] + reopen: bool + + +class GithubIssueSearch(BaseModel): + author: Optional[str] + state: Optional[GithubIssueState] + field_match: List[GithubIssueSearchMatch] + string: str + + +class GithubAuth(BaseModel): + user: str + personal_access_token: str + + +class GithubIssueTemplate(BaseModel): + auth: GithubAuth + organization: str + repository: str + title: str + body: str + unique_search: GithubIssueSearch + assignees: List[str] + labels: List[str] + on_duplicate: GithubIssueDuplicate + + def redact(self) -> None: + self.auth.user = "***" + self.auth.personal_access_token = "***" + + +NotificationTemplate = Union[ADOTemplate, TeamsTemplate, GithubIssueTemplate] class Notification(BaseModel):