From 8b6ea9f66ba4750f325f723adf257dcb75111da9 Mon Sep 17 00:00:00 2001
From: bmc-msft <41130664+bmc-msft@users.noreply.github.com>
Date: Thu, 1 Oct 2020 22:00:08 -0400
Subject: [PATCH] mark task as failued if ADO notification fails (#77)

---
 .../__app__/onefuzzlib/notifications/ado.py   | 31 +++++++++++++++++--
 src/pytypes/onefuzztypes/enums.py             |  1 +
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/src/api-service/__app__/onefuzzlib/notifications/ado.py b/src/api-service/__app__/onefuzzlib/notifications/ado.py
index f00f60cd8..28fb62deb 100644
--- a/src/api-service/__app__/onefuzzlib/notifications/ado.py
+++ b/src/api-service/__app__/onefuzzlib/notifications/ado.py
@@ -19,8 +19,10 @@ from azure.devops.v6_0.work_item_tracking.work_item_tracking_client import (
     WorkItemTrackingClient,
 )
 from memoization import cached
-from onefuzztypes.models import ADOTemplate, Report
+from onefuzztypes.enums import ErrorCode, TaskState
+from onefuzztypes.models import ADOTemplate, Error, Report
 
+from ..tasks.main import Task
 from .common import Render
 
 
@@ -194,13 +196,36 @@ class ADO:
             self.create_new()
 
 
+def fail_task(report: Report, error: Exception) -> None:
+    logging.error(
+        "ADO report failed: job_id:%s task_id:%s err:%s",
+        report.job_id,
+        report.task_id,
+        error,
+    )
+
+    task = Task.get(report.job_id, report.task_id)
+    if task:
+        task.error = Error(code=ErrorCode.NOTIFICATION_FAILURE, errors=[str(error)])
+        task.state = TaskState.stopping
+        task.save()
+
+
 def notify_ado(
     config: ADOTemplate, container: str, filename: str, report: Report
 ) -> None:
+    logging.info(
+        "notify ado: job_id:%s task_id:%s container:%s filename:%s",
+        report.job_id,
+        report.task_id,
+        container,
+        filename,
+    )
+
     try:
         ado = ADO(container, filename, config, report)
         ado.process()
     except AzureDevOpsServiceError as err:
-        logging.error("ADO report failed: %s", err)
+        fail_task(report, err)
     except ValueError as err:
-        logging.error("ADO report value error: %s", err)
+        fail_task(report, err)
diff --git a/src/pytypes/onefuzztypes/enums.py b/src/pytypes/onefuzztypes/enums.py
index 8934332e8..d4f46da13 100644
--- a/src/pytypes/onefuzztypes/enums.py
+++ b/src/pytypes/onefuzztypes/enums.py
@@ -227,6 +227,7 @@ class ErrorCode(Enum):
     UNABLE_TO_FIND = 467
     TASK_FAILED = 468
     INVALID_NODE = 469
+    NOTIFICATION_FAILURE = 470
 
 
 class HeartbeatType(Enum):