Allow notifications to be retried when an error occurs (#1026)

src/api-service/__app__/onefuzzlib/notifications/ado.py

@@ -31,6 +31,10 @@ from ..secrets import get_secret_string_value
 from .common import Render, fail_task
 
 
+class AdoNotificationException(Exception):
+    pass
+
+
 @cached(ttl=60)
 def get_ado_client(base_url: str, token: str) -> WorkItemTrackingClient:
     connection = Connection(base_url=base_url, creds=BasicAuthentication("PAT", token))
@@ -206,11 +210,26 @@ class ADO:
             self.create_new()
 
 
+def is_transient(err: Exception) -> bool:
+    error_codes = [
+        # "TF401349: An unexpected error has occurred, please verify your request and try again." # noqa: E501
+        "TF401349",
+        # TF26071: This work item has been changed by someone else since you opened it. You will need to refresh it and discard your changes. # noqa: E501
+        "TF26071",
+    ]
+    error_str = str(err)
+    for code in error_codes:
+        if code in error_str:
+            return True
+    return False
+
+
 def notify_ado(
     config: ADOTemplate,
     container: Container,
     filename: str,
     report: Union[Report, RegressionReport],
+    fail_task_on_transient_error: bool,
 ) -> None:
     if isinstance(report, RegressionReport):
         logging.info(
@@ -232,13 +251,17 @@ def notify_ado(
     try:
         ado = ADO(container, filename, config, report)
         ado.process()
-    except AzureDevOpsAuthenticationError as err:
-        fail_task(report, err)
-    except AzureDevOpsClientError as err:
-        fail_task(report, err)
-    except AzureDevOpsServiceError as err:
-        fail_task(report, err)
-    except AzureDevOpsClientRequestError as err:
-        fail_task(report, err)
-    except ValueError as err:
-        fail_task(report, err)
+    except (
+        AzureDevOpsAuthenticationError,
+        AzureDevOpsClientError,
+        AzureDevOpsServiceError,
+        AzureDevOpsClientRequestError,
+        ValueError,
+    ) as err:
+
+        if not fail_task_on_transient_error and is_transient(err):
+            raise AdoNotificationException(
+                "transient ADO notification failure"
+            ) from err
+        else:
+            fail_task(report, err)