Add github issues integration (#110)

docs/notifications.md

@@ -38,4 +38,5 @@ JSON via a string, such as `'{"config":{...}}'`
 ## Supported integrations
 
 * [Microsoft Teams](notifications/teams.md)
-* [Azure Devops Work Items](notifications/ado.md)
+* [Azure Devops Work Items](notifications/ado.md)
+* [Github Issues](notifications/github.md)

docs/notifications/github.md

@@ -0,0 +1,79 @@
+# Notifications via Github Issues
+
+OneFuzz can create or update [Github Issues](https://guides.github.com/features/issues/) 
+upon creation of crash reports in OneFuzz managed [containers](../containers.md).
+
+Nearly every field can be customized using [jinja2](https://jinja.palletsprojects.com/)
+templates.   There are multiple python objects provided via the template engine that
+can be used such that any arbitrary component can be used to flesh out the configuration:
+
+* task (See [TaskConfig](../../src/pytypes/onefuzztypes/models.py))
+* report (See [Report](../../src/pytypes/onefuzztypes/models.py))
+* job (See [JobConfig](../../src/pytypes/onefuzztypes/models.py))
+
+Using these objects allows dynamic configuration. As an example, the `repository`
+could be specified directly, or dynamically pulled from the task configuration:
+
+```json
+{
+  "repository": "{{ task.tags['repository'] }}"
+}
+```
+
+There are additional values that can be used in any template:
+
+* report_url: This will link to an authenticated download link for the report
+* input_url: This will link to an authenticated download link for crashing input
+* target_url: This will link to an authenticated download link for the target
+  executable
+* repro_cmd: This will give an example command to initiate a live reproduction
+  of a crash
+* report_container: This will give the name of the report storage container
+* report_filename: This will give the container relative path to the report
+
+## Example Config
+
+```json
+{
+   "config": {
+      "auth": {
+         "user": "INSERT_YOUR_USERNAME_HERE",
+         "personal_access_token": "INSERT_YOUR_PERSONAL_ACCESS_TOKEN_HERE"
+      },
+      "organization": "contoso",
+      "repository": "sample-project",
+      "title": "{{ report.executable }} - {{report.crash_site}}",
+      "body": "## Files\n\n* input: [{{ report.input_blob.name }}]({{ input_url }})\n* exe: [{{ report.executable }}]( {{ target_url }})\n* report: [{{ report_filename }}]({{ report_url }})\n\n## Repro\n\n `{{ repro_cmd }}`\n\n## Call Stack\n\n```{% for item in report.call_stack %}{{ item }}\n{% endfor %}```\n\n## ASAN Log\n\n```{{ report.asan_log }}```",
+      "unique_search": {
+         "field_match": ["title"],
+         "string": "{{ report.executable }}"
+      },
+      "assignees": [],
+      "labels": ["bug", "{{ report.crash_type }}"],
+      "on_duplicate": {
+         "comment": "Duplicate found.\n\n* input: [{{ report.input_blob.name }}]({{ input_url }})\n* exe: [{{ report.executable }}]( {{ target_url }})\n* report: [{{ report_filename }}]({{ report_url }})",
+         "labels": ["{{ report.crash_type }}"],
+         "reopen": true
+      }
+   }
+}
+```
+
+For full documentation on the syntax, see [GithubIssueTemplate](../../src/pytypes/onefuzztypes/models.py))
+
+## Integration
+
+1. Create a [Personal access token](https://github.com/settings/tokens).
+2. Update your config to specify your user and personal access token.
+1. Add a notification to your OneFuzz instance.
+
+    ```
+    onefuzz notifications create <CONTAINER> @./config.json
+    ```
+
+Until the integration is deleted, when a crash report is written to the indicated container, 
+issues will be created and updated based on the reports.
+
+The OneFuzz SDK provides an example tool [fake-report.py](../../src/cli/examples/fake-report.py),
+which can be used to generate a synthetic crash report to verify the integration
+is functional.

src/api-service/__app__/notifications/__init__.py

@@ -17,6 +17,9 @@ from ..onefuzzlib.request import not_ok, ok, parse_request
 
 def get(req: func.HttpRequest) -> func.HttpResponse:
     entries = Notification.search()
+    for entry in entries:
+        entry.config.redact()
+
     return ok(entries)
 
 
@@ -52,6 +55,7 @@ def delete(req: func.HttpRequest) -> func.HttpResponse:
         return not_ok(entry, context="notification delete")
 
     entry.delete()
+    entry.config.redact()
     return ok(entry)
 
 

src/api-service/__app__/onefuzzlib/notifications/ado.py

@@ -24,11 +24,9 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import (
     WorkItemTrackingClient,
 )
 from memoization import cached
-from onefuzztypes.enums import ErrorCode
+from onefuzztypes.models import ADOTemplate, Report
-from onefuzztypes.models import ADOTemplate, Error, Report
 
-from ..tasks.main import Task
+from .common import Render, fail_task
-from .common import Render
 
 
 @cached(ttl=60)
@@ -201,21 +199,6 @@ class ADO:
             self.create_new()
 
 
-def fail_task(report: Report, error: Exception) -> None:
-    logging.error(
-        "ADO report failed: job_id:%s task_id:%s err:%s",
-        report.job_id,
-        report.task_id,
-        error,
-    )
-
-    task = Task.get(report.job_id, report.task_id)
-    if task:
-        task.mark_failed(
-            Error(code=ErrorCode.NOTIFICATION_FAILURE, errors=[str(error)])
-        )
-
-
 def notify_ado(
     config: ADOTemplate, container: str, filename: str, report: Report
 ) -> None:

src/api-service/__app__/onefuzzlib/notifications/common.py

@@ -3,10 +3,12 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 
+import logging
 from typing import Optional
 
 from jinja2.sandbox import SandboxedEnvironment
-from onefuzztypes.models import Report
+from onefuzztypes.enums import ErrorCode
+from onefuzztypes.models import Error, Report
 
 from ..azure.containers import auth_download_url
 from ..jobs import Job
@@ -14,6 +16,21 @@ from ..tasks.config import get_setup_container
 from ..tasks.main import Task
 
 
+def fail_task(report: Report, error: Exception) -> None:
+    logging.error(
+        "notification failed: job_id:%s task_id:%s err:%s",
+        report.job_id,
+        report.task_id,
+        error,
+    )
+
+    task = Task.get(report.job_id, report.task_id)
+    if task:
+        task.mark_failed(
+            Error(code=ErrorCode.NOTIFICATION_FAILURE, errors=[str(error)])
+        )
+
+
 class Render:
     def __init__(self, container: str, filename: str, report: Report):
         self.report = report

src/api-service/__app__/onefuzzlib/notifications/github_issues.py

@@ -0,0 +1,102 @@
+import logging
+from typing import List, Optional
+
+from github3 import login
+from github3.exceptions import GitHubException
+from github3.issues import Issue
+from onefuzztypes.enums import GithubIssueSearchMatch
+from onefuzztypes.models import GithubIssueTemplate, Report
+
+from .common import Render, fail_task
+
+
+class GithubIssue:
+    def __init__(
+        self, config: GithubIssueTemplate, container: str, filename: str, report: Report
+    ):
+        self.config = config
+        self.report = report
+        self.gh = login(
+            username=config.auth.user, password=config.auth.personal_access_token
+        )
+        self.renderer = Render(container, filename, report)
+
+    def render(self, field: str) -> str:
+        return self.renderer.render(field)
+
+    def existing(self) -> List[Issue]:
+        query = [
+            self.render(self.config.unique_search.string),
+            "repo:%s/%s"
+            % (
+                self.render(self.config.organization),
+                self.render(self.config.repository),
+            ),
+        ]
+        if self.config.unique_search.author:
+            query.append("author:%s" % self.render(self.config.unique_search.author))
+
+        if self.config.unique_search.state:
+            query.append("state:%s" % self.config.unique_search.state.name)
+
+        issues = []
+        title = self.render(self.config.title)
+        body = self.render(self.config.body)
+        for issue in self.gh.search_issues(" ".join(query)):
+            skip = False
+            for field in self.config.unique_search.field_match:
+                if field == GithubIssueSearchMatch.title and issue.title != title:
+                    skip = True
+                    break
+                if field == GithubIssueSearchMatch.body and issue.body != body:
+                    skip = True
+                    break
+            if not skip:
+                issues.append(issue)
+
+        return issues
+
+    def update(self, issue: Issue) -> None:
+        logging.info("updating issue: %s", issue)
+        if self.config.on_duplicate.comment:
+            issue.issue.create_comment(self.render(self.config.on_duplicate.comment))
+        if self.config.on_duplicate.labels:
+            labels = [self.render(x) for x in self.config.on_duplicate.labels]
+            issue.issue.edit(labels=labels)
+        if self.config.on_duplicate.reopen and issue.state != "open":
+            issue.issue.edit(state="open")
+
+    def create(self) -> None:
+        logging.info("creating issue")
+
+        assignees = [self.render(x) for x in self.config.assignees]
+        labels = list(set(["OneFuzz"] + [self.render(x) for x in self.config.labels]))
+
+        self.gh.create_issue(
+            self.render(self.config.organization),
+            self.render(self.config.repository),
+            self.render(self.config.title),
+            body=self.render(self.config.body),
+            labels=labels,
+            assignees=assignees,
+        )
+
+    def process(self) -> None:
+        issues = self.existing()
+        if issues:
+            self.update(issues[0])
+        else:
+            self.create()
+
+
+def github_issue(
+    config: GithubIssueTemplate, container: str, filename: str, report: Optional[Report]
+) -> None:
+    if report is None:
+        return
+
+    try:
+        handler = GithubIssue(config, container, filename, report)
+        handler.process()
+    except GitHubException as err:
+        fail_task(report, err)

src/api-service/__app__/onefuzzlib/notifications/main.py

@@ -10,7 +10,13 @@ from uuid import UUID
 from memoization import cached
 from onefuzztypes import models
 from onefuzztypes.enums import ErrorCode, TaskState
-from onefuzztypes.models import ADOTemplate, Error, NotificationTemplate, TeamsTemplate
+from onefuzztypes.models import (
+    ADOTemplate,
+    Error,
+    GithubIssueTemplate,
+    NotificationTemplate,
+    TeamsTemplate,
+)
 from onefuzztypes.primitives import Container, Event
 
 from ..azure.containers import get_container_metadata, get_file_sas_url
@@ -22,6 +28,7 @@ from ..reports import get_report
 from ..tasks.config import get_input_container_queues
 from ..tasks.main import Task
 from .ado import notify_ado
+from .github_issues import github_issue
 from .teams import notify_teams
 
 
@@ -111,6 +118,9 @@ def new_files(container: Container, filename: str) -> None:
             if isinstance(notification.config, ADOTemplate):
                 notify_ado(notification.config, container, filename, report)
 
+            if isinstance(notification.config, GithubIssueTemplate):
+                github_issue(notification.config, container, filename, report)
+
     for (task, containers) in get_queue_tasks():
         if container in containers:
             logging.info("queuing input %s %s %s", container, filename, task.task_id)

src/api-service/__app__/onefuzzlib/versions.py

@@ -16,8 +16,8 @@ from .__version__ import __version__
 def read_local_file(filename: str) -> str:
     path = os.path.join(os.path.dirname(os.path.realpath(__file__)), filename)
     if os.path.exists(path):
-        with open(path, "rb") as handle:
+        with open(path, "r") as handle:
-            return handle.read().strip().decode("utf-16")
+            return handle.read().strip()
     else:
         return "UNKNOWN"
 

src/api-service/__app__/requirements.txt

@@ -31,5 +31,6 @@ pydantic~=1.6.1
 PyJWT~=1.7.1
 requests~=2.24.0
 memoization~=0.3.1
+github3.py~=1.3.0
 # onefuzz types version is set during build
 onefuzztypes==0.0.0

src/api-service/mypy.ini

@@ -32,5 +32,5 @@ ignore_missing_imports = True
 [mypy-memoization.*]
 ignore_missing_imports = True
 
-[mypy-github.*]
+[mypy-github3.*]
-ignore_missing_imports = True
+ignore_missing_imports = True

src/pytypes/onefuzztypes/enums.py

@@ -318,3 +318,13 @@ class NodeState(Enum):
         # If Node is in one of these states, ignore updates
         # from the agent.
         return [cls.done, cls.shutdown, cls.halt]
+
+
+class GithubIssueState(Enum):
+    open = "open"
+    closed = "closed"
+
+
+class GithubIssueSearchMatch(Enum):
+    title = "title"
+    body = "body"

src/pytypes/onefuzztypes/models.py

@@ -17,6 +17,8 @@ from .enums import (
     ContainerPermission,
     ContainerType,
     ErrorCode,
+    GithubIssueSearchMatch,
+    GithubIssueState,
     HeartbeatType,
     JobState,
     NodeState,
@@ -213,10 +215,16 @@ class ADOTemplate(BaseModel):
     ado_fields: Dict[str, str]
     on_duplicate: ADODuplicateTemplate
 
+    def redact(self) -> None:
+        self.auth_token = "***"
+
 
 class TeamsTemplate(BaseModel):
     url: str
 
+    def redact(self) -> None:
+        self.url = "***"
+
 
 class ContainerDefinition(BaseModel):
     type: ContainerType
@@ -368,7 +376,41 @@ class WorkSetSummary(BaseModel):
     work_units: List[WorkUnitSummary]
 
 
-NotificationTemplate = Union[ADOTemplate, TeamsTemplate]
+class GithubIssueDuplicate(BaseModel):
+    comment: Optional[str]
+    labels: List[str]
+    reopen: bool
+
+
+class GithubIssueSearch(BaseModel):
+    author: Optional[str]
+    state: Optional[GithubIssueState]
+    field_match: List[GithubIssueSearchMatch]
+    string: str
+
+
+class GithubAuth(BaseModel):
+    user: str
+    personal_access_token: str
+
+
+class GithubIssueTemplate(BaseModel):
+    auth: GithubAuth
+    organization: str
+    repository: str
+    title: str
+    body: str
+    unique_search: GithubIssueSearch
+    assignees: List[str]
+    labels: List[str]
+    on_duplicate: GithubIssueDuplicate
+
+    def redact(self) -> None:
+        self.auth.user = "***"
+        self.auth.personal_access_token = "***"
+
+
+NotificationTemplate = Union[ADOTemplate, TeamsTemplate, GithubIssueTemplate]
 
 
 class Notification(BaseModel):