diff --git a/src/ApiService/ApiService/Functions/QueueFileChanges.cs b/src/ApiService/ApiService/Functions/QueueFileChanges.cs index bae0b3a62..acdd3e328 100644 --- a/src/ApiService/ApiService/Functions/QueueFileChanges.cs +++ b/src/ApiService/ApiService/Functions/QueueFileChanges.cs @@ -61,14 +61,17 @@ public class QueueFileChanges { // requeuing ourselves because azure functions doesn't support retry policies // for queue based functions. - await FileAdded(fileChangeEvent, isLastRetryAttempt: false); + var result = await FileAdded(fileChangeEvent, isLastRetryAttempt: false); + if (!result.IsOk && result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) { + await RequeueMessage(msg, TimeSpan.FromDays(1)); + } } catch (Exception e) { _log.LogError(e, "File Added failed"); await RequeueMessage(msg); } } - private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) { + private async Async.Task FileAdded(JsonDocument fileChangeEvent, bool isLastRetryAttempt) { var data = fileChangeEvent.RootElement.GetProperty("data"); var url = data.GetProperty("url").GetString()!; var parts = url.Split("/").Skip(3).ToList(); @@ -77,10 +80,10 @@ public class QueueFileChanges { var path = string.Join('/', parts.Skip(1)); _log.LogInformation("file added : {Container} - {Path}", container, path); - await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt); + return await _notificationOperations.NewFiles(Container.Parse(container), path, isLastRetryAttempt); } - private async Async.Task RequeueMessage(string msg) { + private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) { var json = JsonNode.Parse(msg); // Messages that are 'manually' requeued by us as opposed to being requeued by the azure functions runtime @@ -103,7 +106,7 @@ public class QueueFileChanges { queueName, json, StorageType.Config, - CalculateExponentialBackoff(newCustomDequeueCount)) + visibilityTimeout ?? CalculateExponentialBackoff(newCustomDequeueCount)) .IgnoreResult(); } diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index f98ac8dd0..bd636caa1 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -47,6 +47,7 @@ public enum ErrorCode { ADO_VALIDATION_UNEXPECTED_HTTP_EXCEPTION = 490, ADO_VALIDATION_UNEXPECTED_ERROR = 491, ADO_VALIDATION_MISSING_PAT_SCOPES = 492, + ADO_WORKITEM_PROCESSING_DISABLED = 494, // NB: if you update this enum, also update enums.py } diff --git a/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs b/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs index 247403fdf..528c590f7 100644 --- a/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs +++ b/src/ApiService/ApiService/TestHooks/NotificationOperationsTestHooks.cs @@ -31,7 +31,7 @@ namespace ApiService.TestHooks { var fileName = query["fileName"]; var isLastRetryAttempt = UriExtension.GetBool("isLastRetryAttempt", query, true); - await _notificationOps.NewFiles(Container.Parse(container), fileName, isLastRetryAttempt); + _ = await _notificationOps.NewFiles(Container.Parse(container), fileName, isLastRetryAttempt); var resp = req.CreateResponse(HttpStatusCode.OK); return resp; } diff --git a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs index ac5cea011..9fd87f766 100644 --- a/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs @@ -5,7 +5,7 @@ using Microsoft.Extensions.Logging; namespace Microsoft.OneFuzz.Service; public interface INotificationOperations : IOrm { - Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt); + Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt); IAsyncEnumerable GetNotifications(Container container); IAsyncEnumerable<(Task, IEnumerable)> GetQueueTasks(); Async.Task> Create(Container container, NotificationTemplate config, bool replaceExisting); @@ -21,16 +21,18 @@ public class NotificationOperations : Orm, INotificationOperations : base(log, context) { } - public async Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt) { + public async Async.Task NewFiles(Container container, string filename, bool isLastRetryAttempt) { + var result = OneFuzzResultVoid.Ok; + // We don't want to store file added events for the events container because that causes an infinite loop if (container == WellKnownContainers.Events) { - return; + return result; } var notifications = GetNotifications(container); var hasNotifications = await notifications.AnyAsync(); var reportOrRegression = await _context.Reports.GetReportOrRegression(container, filename, expectReports: hasNotifications); - if (hasNotifications && await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableWorkItemCreation)) { + if (hasNotifications) { var done = new List(); await foreach (var notification in notifications) { if (done.Contains(notification.Config)) { @@ -38,7 +40,10 @@ public class NotificationOperations : Orm, INotificationOperations } done.Add(notification.Config); - _ = await TriggerNotification(container, notification, reportOrRegression, isLastRetryAttempt); + var notificationResult = await TriggerNotification(container, notification, reportOrRegression, isLastRetryAttempt); + if (result.IsOk && !notificationResult.IsOk) { + result = notificationResult; + } } } @@ -77,6 +82,8 @@ public class NotificationOperations : Orm, INotificationOperations } else { await _context.Events.SendEvent(new EventFileAdded(container, filename)); } + + return result; } public async System.Threading.Tasks.Task TriggerNotification(Container container, @@ -87,8 +94,12 @@ public class NotificationOperations : Orm, INotificationOperations notification.NotificationId); break; case AdoTemplate adoTemplate when reportOrRegression is not null: - return await _context.Ado.NotifyAdo(adoTemplate, container, reportOrRegression, isLastRetryAttempt, - notification.NotificationId); + if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableWorkItemCreation)) { + return await _context.Ado.NotifyAdo(adoTemplate, container, reportOrRegression, isLastRetryAttempt, + notification.NotificationId); + } else { + return OneFuzzResultVoid.Error(ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED, "Work item processing is currently disabled"); + } case GithubIssuesTemplate githubIssuesTemplate when reportOrRegression is not null: await _context.GithubIssues.GithubIssue(githubIssuesTemplate, container, reportOrRegression, notification.NotificationId);