Requeue ado notification on ff (#3358)

* Requeue ado notification on ff

* Fix test

src/ApiService/ApiService/Functions/QueueFileChanges.cs

@@ -61,14 +61,17 @@ public class QueueFileChanges {
             // requeuing ourselves because azure functions doesn't support retry policies
             // for queue based functions.
 
-            await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false);
+            if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
+                await RequeueMessage(msg, TimeSpan.FromDays(1));
+            }
         } catch (Exception e) {
             _log.LogError(e, "File Added failed");
             await RequeueMessage(msg);
         }
     }
 
-    private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
+    private async Async.Task<OneFuzzResultVoid> FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) {
         var data = fileChangeEvent.RootElement.GetProperty("data");
         var url = data.GetProperty("url").GetString()!;
         var parts = url.Split("/").Skip(3).ToList();
@@ -77,10 +80,10 @@ public class QueueFileChanges {
         var path = string.Join('/', parts.Skip(1));
 
         _log.LogInformation("file added : {Container} - {Path}", container, path);
-        await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
+        return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt);
     }
 
-    private async Async.Task RequeueMessage(string msg) {
+    private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {
         var json = JsonNode.Parse(msg);
 
         // Messages that are 'manually' requeued by us as opposed to being requeued by the azure functions runtime
@@ -103,7 +106,7 @@ public class QueueFileChanges {
             queueName,
             json,
             StorageType.Config,
-            CalculateExponentialBackoff(newCustomDequeueCount))
+            visibilityTimeout ?? CalculateExponentialBackoff(newCustomDequeueCount))
             .IgnoreResult();
     }
 

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -47,6 +47,7 @@ public enum ErrorCode {
     ADO_VALIDATION_UNEXPECTED_HTTP_EXCEPTION = 490,
     ADO_VALIDATION_UNEXPECTED_ERROR = 491,
     ADO_VALIDATION_MISSING_PAT_SCOPES = 492,
+    ADO_WORKITEM_PROCESSING_DISABLED = 494,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs

@@ -31,7 +31,7 @@ namespace ApiService.TestHooks {
             var fileName = query["fileName"];
             var isLastRetryAttempt = UriExtension.GetBool("isLastRetryAttempt", query, true);
 
-            await _notificationOps.NewFiles(Container.Parse(container), fileName, isLastRetryAttempt);
+            _ = await _notificationOps.NewFiles(Container.Parse(container), fileName, isLastRetryAttempt);
             var resp = req.CreateResponse(HttpStatusCode.OK);
             return resp;
         }

src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs

@@ -5,7 +5,7 @@ using Microsoft.Extensions.Logging;
 namespace Microsoft.OneFuzz.Service;
 
 public interface INotificationOperations : IOrm<Notification> {
-    Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt);
+    Async.Task<OneFuzzResultVoid> NewFiles(Container container, string filename, bool isLastRetryAttempt);
     IAsyncEnumerable<Notification> GetNotifications(Container container);
     IAsyncEnumerable<(Task, IEnumerable<Container>)> GetQueueTasks();
     Async.Task<OneFuzzResult<Notification>> Create(Container container, NotificationTemplate config, bool replaceExisting);
@@ -21,16 +21,18 @@ public class NotificationOperations : Orm<Notification>, INotificationOperations
         : base(log, context) {
 
     }
-    public async Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt) {
+    public async Async.Task<OneFuzzResultVoid> NewFiles(Container container, string filename, bool isLastRetryAttempt) {
+        var result = OneFuzzResultVoid.Ok;
+
         // We don't want to store file added events for the events container because that causes an infinite loop
         if (container == WellKnownContainers.Events) {
-            return;
+            return result;
         }
 
         var notifications = GetNotifications(container);
         var hasNotifications = await notifications.AnyAsync();
         var reportOrRegression = await _context.Reports.GetReportOrRegression(container, filename, expectReports: hasNotifications);
-        if (hasNotifications && await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableWorkItemCreation)) {
+        if (hasNotifications) {
             var done = new List<NotificationTemplate>();
             await foreach (var notification in notifications) {
                 if (done.Contains(notification.Config)) {
@@ -38,7 +40,10 @@ public class NotificationOperations : Orm<Notification>, INotificationOperations
                 }
 
                 done.Add(notification.Config);
-                _ = await TriggerNotification(container, notification, reportOrRegression, isLastRetryAttempt);
+                var notificationResult = await TriggerNotification(container, notification, reportOrRegression, isLastRetryAttempt);
+                if (result.IsOk && !notificationResult.IsOk) {
+                    result = notificationResult;
+                }
             }
         }
 
@@ -77,6 +82,8 @@ public class NotificationOperations : Orm<Notification>, INotificationOperations
         } else {
             await _context.Events.SendEvent(new EventFileAdded(container, filename));
         }
+
+        return result;
     }
 
     public async System.Threading.Tasks.Task<OneFuzzResultVoid> TriggerNotification(Container container,
@@ -87,8 +94,12 @@ public class NotificationOperations : Orm<Notification>, INotificationOperations
                     notification.NotificationId);
                 break;
             case AdoTemplate adoTemplate when reportOrRegression is not null:
+                if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableWorkItemCreation)) {
                     return await _context.Ado.NotifyAdo(adoTemplate, container, reportOrRegression, isLastRetryAttempt,
                         notification.NotificationId);
+                } else {
+                    return OneFuzzResultVoid.Error(ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED, "Work item processing is currently disabled");
+                }
             case GithubIssuesTemplate githubIssuesTemplate when reportOrRegression is not null:
                 await _context.GithubIssues.GithubIssue(githubIssuesTemplate, container, reportOrRegression,
                     notification.NotificationId);