Allow notifications to be retried when an error occurs (#1026)

src/api-service/__app__/onefuzzlib/notifications/ado.py

@@ -31,6 +31,10 @@ from ..secrets import get_secret_string_value
 from .common import Render, fail_task
 
 
+class AdoNotificationException(Exception):
+    pass
+
+
 @cached(ttl=60)
 def get_ado_client(base_url: str, token: str) -> WorkItemTrackingClient:
     connection = Connection(base_url=base_url, creds=BasicAuthentication("PAT", token))
@@ -206,11 +210,26 @@ class ADO:
             self.create_new()
 
 
+def is_transient(err: Exception) -> bool:
+    error_codes = [
+        # "TF401349: An unexpected error has occurred, please verify your request and try again." # noqa: E501
+        "TF401349",
+        # TF26071: This work item has been changed by someone else since you opened it. You will need to refresh it and discard your changes. # noqa: E501
+        "TF26071",
+    ]
+    error_str = str(err)
+    for code in error_codes:
+        if code in error_str:
+            return True
+    return False
+
+
 def notify_ado(
     config: ADOTemplate,
     container: Container,
     filename: str,
     report: Union[Report, RegressionReport],
+    fail_task_on_transient_error: bool,
 ) -> None:
     if isinstance(report, RegressionReport):
         logging.info(
@@ -232,13 +251,17 @@ def notify_ado(
     try:
         ado = ADO(container, filename, config, report)
         ado.process()
-    except AzureDevOpsAuthenticationError as err:
-        fail_task(report, err)
-    except AzureDevOpsClientError as err:
-        fail_task(report, err)
-    except AzureDevOpsServiceError as err:
-        fail_task(report, err)
-    except AzureDevOpsClientRequestError as err:
-        fail_task(report, err)
-    except ValueError as err:
-        fail_task(report, err)
+    except (
+        AzureDevOpsAuthenticationError,
+        AzureDevOpsClientError,
+        AzureDevOpsServiceError,
+        AzureDevOpsClientRequestError,
+        ValueError,
+    ) as err:
+
+        if not fail_task_on_transient_error and is_transient(err):
+            raise AdoNotificationException(
+                "transient ADO notification failure"
+            ) from err
+        else:
+            fail_task(report, err)

src/api-service/__app__/onefuzzlib/notifications/github_issues.py

@@ -128,7 +128,5 @@ def github_issue(
     try:
         handler = GithubIssue(config, container, filename, report)
         handler.process()
-    except GitHubException as err:
-        fail_task(report, err)
-    except ValueError as err:
+    except (GitHubException, ValueError) as err:
         fail_task(report, err)

src/api-service/__app__/onefuzzlib/notifications/main.py

@@ -127,7 +127,9 @@ def get_queue_tasks() -> Sequence[Tuple[Task, Sequence[str]]]:
     return results
 
 
-def new_files(container: Container, filename: str) -> None:
+def new_files(
+    container: Container, filename: str, fail_task_on_transient_error: bool
+) -> None:
     notifications = get_notifications(container)
 
     report = get_report_or_regression(
@@ -149,7 +151,13 @@ def new_files(container: Container, filename: str) -> None:
                 continue
 
             if isinstance(notification.config, ADOTemplate):
-                notify_ado(notification.config, container, filename, report)
+                notify_ado(
+                    notification.config,
+                    container,
+                    filename,
+                    report,
+                    fail_task_on_transient_error,
+                )
 
             if isinstance(notification.config, GithubIssueTemplate):
                 github_issue(notification.config, container, filename, report)