Implement new_files (#1794)

* Checkpoint

* Checkpoint

* More merge resolving

* Code complete

* Tested that it works

* Keep the queue name different for now

* Query was wrong, should be and

* Style

* Fix compile issue

* Change report to use string instead of SHA, fixes tests as well

* PR comments

* Comments and formatting

src/ApiService/ApiService/ApiService.csproj

@@ -25,6 +25,7 @@
     <PackageReference Include="Azure.ResourceManager.Resources" Version="1.0.0" />
     <PackageReference Include="Azure.ResourceManager.Storage" Version="1.0.0-beta.8" />
     <PackageReference Include="Azure.Storage.Queues" Version="12.9.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.11.0" />
     <PackageReference Include="Microsoft.Graph" Version="4.24.0" />
     <PackageReference Include="Microsoft.Identity.Client" Version="4.43.0" />
     <PackageReference Include="Microsoft.Identity.Web.TokenCache" Version="1.23.1" />

src/ApiService/ApiService/EnvironmentVariables.cs

@@ -1,5 +1,4 @@
-using System;
+namespace Microsoft.OneFuzz.Service;
-namespace Microsoft.OneFuzz.Service;
 
 public enum LogDestination
 {

src/ApiService/ApiService/HttpClient.cs

@@ -1,6 +1,4 @@
-using System;
+using System.Net.Http;
-using System.Collections.Generic;
-using System.Net.Http;
 using System.Threading.Tasks;
 using System.Net.Http.Headers;
 

src/ApiService/ApiService/Log.cs

@@ -1,6 +1,4 @@
-using System;
+using Microsoft.ApplicationInsights;
-using System.Collections.Generic;
-using Microsoft.ApplicationInsights;
 using Microsoft.ApplicationInsights.Extensibility;
 using Microsoft.ApplicationInsights.DataContracts;
 

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -178,6 +178,24 @@ public static class ScalesetStateHelper
             };
         });
     }
-
+}
-
+
+public static class TaskStateHelper
+{
+    static ConcurrentDictionary<string, TaskState[]> _states = new ConcurrentDictionary<string, TaskState[]>();
+    public static TaskState[] Available()
+    {
+        return
+        _states.GetOrAdd("Available", k =>
+        {
+            return
+                 new[]{
+                    TaskState.Waiting,
+                    TaskState.Scheduled,
+                    TaskState.SettingUp,
+                    TaskState.Running,
+                    TaskState.WaitJob
+                 };
+        });
+    }
 }

src/ApiService/ApiService/OneFuzzTypes/Events.cs

@@ -1,5 +1,4 @@
-using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using PoolName = System.String;
@@ -37,7 +36,7 @@ public enum EventType
     FileAdded,
     TaskHeartbeat,
     NodeHeartbeat,
-    InstanceConfigUpdated
+    InstanceConfigUpdated,
 }
 
 public abstract record BaseEvent()
@@ -50,6 +49,9 @@ public abstract record BaseEvent()
                 EventNodeHeartbeat _ => EventType.NodeHeartbeat,
                 EventTaskHeartbeat _ => EventType.TaskHeartbeat,
                 EventInstanceConfigUpdated _ => EventType.InstanceConfigUpdated,
+                EventCrashReported _ => EventType.CrashReported,
+                EventRegressionReported _ => EventType.RegressionReported,
+                EventFileAdded _ => EventType.FileAdded,
                 _ => throw new NotImplementedException(),
             };
 
@@ -62,6 +64,9 @@ public abstract record BaseEvent()
             EventType.NodeHeartbeat => typeof(EventNodeHeartbeat),
             EventType.InstanceConfigUpdated => typeof(EventInstanceConfigUpdated),
             EventType.TaskHeartbeat => typeof(EventTaskHeartbeat),
+            EventType.CrashReported => typeof(EventCrashReported),
+            EventType.RegressionReported => typeof(EventRegressionReported),
+            EventType.FileAdded => typeof(EventFileAdded),
             _ => throw new ArgumentException($"invalid input {eventType}"),
 
         };
@@ -249,25 +254,25 @@ public record EventNodeHeartbeat(
 //        NodeState state
 //        ) : BaseEvent();
 
-//    record EventCrashReported(
+record EventCrashReported(
-//        Report Report,
+    Report Report,
-//        Container Container,
+    Container Container,
-//        [property: JsonPropertyName("filename")] String FileName,
+    [property: JsonPropertyName("filename")] String FileName,
-//        TaskConfig? TaskConfig
+    TaskConfig? TaskConfig
-//        ) : BaseEvent();
+) : BaseEvent();
 
-//    record EventRegressionReported(
+record EventRegressionReported(
-//        RegressionReport RegressionReport,
+    RegressionReport RegressionReport,
-//        Container Container,
+    Container Container,
-//        [property: JsonPropertyName("filename")] String FileName,
+    [property: JsonPropertyName("filename")] String FileName,
-//        TaskConfig? TaskConfig
+    TaskConfig? TaskConfig
-//        ) : BaseEvent();
+) : BaseEvent();
 
 
-//    record EventFileAdded(
+record EventFileAdded(
-//        Container Container,
+    Container Container,
-//        [property: JsonPropertyName("filename")] String FileName
+    [property: JsonPropertyName("filename")] String FileName
-//        ) : BaseEvent();
+) : BaseEvent();
 
 
 public record EventInstanceConfigUpdated(

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -1,9 +1,5 @@
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
-using System.Collections.Generic;
 using System.Text.Json.Serialization;
-
-using Container = System.String;
 using Region = System.String;
 using PoolName = System.String;
 using Endpoint = System.String;
@@ -397,32 +393,74 @@ public record Scaleset(
 
 ) : EntityBase();
 
+public record Container(string ContainerName)
+{
+    public string ContainerName { get; } = ContainerName.All(c => char.IsLetterOrDigit(c) || c == '-') ? ContainerName : throw new ArgumentException("Container name must have only numbers, letters or dashes");
+}
+
+public record Notification(
+    DateTime? Timestamp,
+    Container Container,
+    Guid NotificationId,
+    NotificationTemplate Config
+) : EntityBase();
 
 public record BlobRef(
     string Account,
-    Container Container,
+    Container container,
-    string Name
+    string name
 );
 
-
 public record Report(
-    string? InputURL,
+    string? InputUrl,
     BlobRef? InputBlob,
-    string? Executable,
+    string Executable,
     string CrashType,
     string CrashSite,
     List<string> CallStack,
     string CallStackSha256,
     string InputSha256,
     string? AsanLog,
-    Guid TaskID,
+    Guid TaskId,
-    Guid JobID,
+    Guid JobId,
     int? ScarinessScore,
     string? ScarinessDescription,
-    List<string> MinimizedStack,
+    List<string>? MinimizedStack,
     string? MinimizedStackSha256,
-    List<string> MinimizedStackFunctionNames,
+    List<string>? MinimizedStackFunctionNames,
     string? MinimizedStackFunctionNamesSha256,
-    List<string> MinimizedStackFunctionLines,
+    List<string>? MinimizedStackFunctionLines,
     string? MinimizedStackFunctionLinesSha256
 );
+
+public record NoReproReport(
+    string InputSha,
+    BlobRef? InputBlob,
+    string? Executable,
+    Guid TaskId,
+    Guid JobId,
+    int Tries,
+    string? Error
+);
+
+public record CrashTestResult(
+    Report? CrashReport,
+    NoReproReport? NoReproReport
+);
+
+public record RegressionReport(
+    CrashTestResult CrashTestResult,
+    CrashTestResult? OriginalCrashTestResult
+);
+
+public record NotificationTemplate(
+    AdoTemplate? AdoTemplate,
+    TeamsTemplate? TeamsTemplate,
+    GithubIssuesTemplate? GithubIssuesTemplate
+);
+
+public record AdoTemplate();
+
+public record TeamsTemplate();
+
+public record GithubIssuesTemplate();

src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs

@@ -1,6 +1,4 @@
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
-using System.Collections.Generic;
 using System.Text.Json.Serialization;
 
 namespace Microsoft.OneFuzz.Service;

src/ApiService/ApiService/Program.cs

@@ -1,8 +1,10 @@
 // to avoid collision with Task in model.cs
 global using Async = System.Threading.Tasks;
 
-using System;
+global using System;
-using System.Collections.Generic;
+global using System.Collections.Generic;
+global using System.Linq;
+
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Azure.Functions.Worker.Middleware;
@@ -76,6 +78,9 @@ public class Program
             .AddScoped<IProxyOperations, ProxyOperations>()
             .AddScoped<IConfigOperations, ConfigOperations>()
             .AddScoped<IScalesetOperations, ScalesetOperations>()
+            .AddScoped<IContainers, Containers>()
+            .AddScoped<IReports, Reports>()
+            .AddScoped<INotificationOperations, NotificationOperations>()
 
         //TODO: move out expensive resources into separate class, and add those as Singleton
         // ArmClient, Table Client(s), Queue Client(s), HttpClient, etc.

src/ApiService/ApiService/QueueFileChanges.cs

@@ -1,9 +1,6 @@
-using System;
 using Microsoft.Azure.Functions.Worker;
-using System.Collections.Generic;
 using System.Text.Json;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System.Linq;
 
 namespace Microsoft.OneFuzz.Service;
 
@@ -17,51 +14,53 @@ public class QueueFileChanges
 
     private readonly IStorage _storage;
 
-    public QueueFileChanges(ILogTracer log, IStorage storage)
+    private readonly INotificationOperations _notificationOperations;
+
+    public QueueFileChanges(ILogTracer log, IStorage storage, INotificationOperations notificationOperations)
     {
         _log = log;
         _storage = storage;
+        _notificationOperations = notificationOperations;
     }
 
     [Function("QueueFileChanges")]
-    public Async.Task Run(
+    public async Async.Task Run(
         [QueueTrigger("file-changes-refactored", Connection = "AzureWebJobsStorage")] string msg,
         int dequeueCount)
     {
-        var fileChangeEvent = JsonSerializer.Deserialize<Dictionary<string, string>>(msg, EntityConverter.GetJsonSerializerOptions());
+        var fileChangeEvent = JsonSerializer.Deserialize<JsonDocument>(msg, EntityConverter.GetJsonSerializerOptions());
         var lastTry = dequeueCount == MAX_DEQUEUE_COUNT;
 
         var _ = fileChangeEvent ?? throw new ArgumentException("Unable to parse queue trigger as JSON");
 
         // check type first before calling Azure APIs
         const string eventType = "eventType";
-        if (!fileChangeEvent.ContainsKey(eventType)
+        if (!fileChangeEvent.RootElement.TryGetProperty(eventType, out var eventTypeElement)
-            || fileChangeEvent[eventType] != "Microsoft.Storage.BlobCreated")
+            || eventTypeElement.GetString() != "Microsoft.Storage.BlobCreated")
         {
-            return Async.Task.CompletedTask;
+            return;
         }
 
         const string topic = "topic";
-        if (!fileChangeEvent.ContainsKey(topic)
+        if (!fileChangeEvent.RootElement.TryGetProperty(topic, out var topicElement)
-            || !_storage.CorpusAccounts().Contains(fileChangeEvent[topic]))
+            || !_storage.CorpusAccounts().Contains(topicElement.GetString()))
         {
-            return Async.Task.CompletedTask;
+            return;
         }
 
-        file_added(_log, fileChangeEvent, lastTry);
+        await file_added(_log, fileChangeEvent, lastTry);
-        return Async.Task.CompletedTask;
     }
 
-    private void file_added(ILogTracer log, Dictionary<string, string> fileChangeEvent, bool failTaskOnTransientError)
+    private async Async.Task file_added(ILogTracer log, JsonDocument fileChangeEvent, bool failTaskOnTransientError)
     {
-        var data = JsonSerializer.Deserialize<Dictionary<string, string>>(fileChangeEvent["data"])!;
+        var data = fileChangeEvent.RootElement.GetProperty("data");
-        var url = data["url"];
+        var url = data.GetProperty("url").GetString()!;
         var parts = url.Split("/").Skip(3).ToList();
 
         var container = parts[0];
         var path = string.Join('/', parts.Skip(1));
 
         log.Info($"file added container: {container} - path: {path}");
-        // TODO: new_files(container, path, fail_task_on_transient_error)
+        await _notificationOperations.NewFiles(new Container(container), path, failTaskOnTransientError);
     }
 }

src/ApiService/ApiService/QueueNodeHearbeat.cs

@@ -1,4 +1,3 @@
-using System;
 using Microsoft.Azure.Functions.Worker;
 using System.Text.Json;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;

src/ApiService/ApiService/QueueProxyHeartbeat.cs

@@ -1,4 +1,3 @@
-using System;
 using Microsoft.Azure.Functions.Worker;
 using System.Text.Json;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;

src/ApiService/ApiService/QueueTaskHearbeat.cs

@@ -1,4 +1,3 @@
-using System;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Extensions.Logging;
 using System.Text.Json;

src/ApiService/ApiService/TestHooks.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Net;
-using System.Net;
 using System.Threading.Tasks;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Azure.Functions.Worker.Http;

src/ApiService/ApiService/UserCredentials.cs

@@ -1,6 +1,4 @@
-using System;
+using System.Net.Http.Headers;
-using System.Linq;
-using System.Net.Http.Headers;
 using System.Threading.Tasks;
 using Microsoft.Azure.Functions.Worker.Http;
 using Microsoft.IdentityModel.Tokens;

src/ApiService/ApiService/onefuzzlib/Containers.cs

@@ -0,0 +1,93 @@
+using System.Threading.Tasks;
+using Azure.ResourceManager;
+using Azure.Storage.Blobs;
+using Azure.Storage;
+using Azure;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IContainers
+{
+    public Task<IEnumerable<byte>?> GetBlob(Container container, string name, StorageType storageType);
+
+    public Async.Task<BlobContainerClient?> FindContainer(Container container, StorageType storageType);
+
+    public Uri GetFileSasUrl(Container container, string name, StorageType storageType, bool read = false, bool add = false, bool create = false, bool write = false, bool delete = false, bool delete_previous_version = false, bool tag = false, int days = 30, int hours = 0, int minutes = 0);
+
+}
+
+public class Containers : IContainers
+{
+    private ILogTracer _log;
+    private IStorage _storage;
+    private ICreds _creds;
+    private ArmClient _armClient;
+    public Containers(ILogTracer log, IStorage storage, ICreds creds)
+    {
+        _log = log;
+        _storage = storage;
+        _creds = creds;
+        _armClient = new ArmClient(credential: _creds.GetIdentity(), defaultSubscriptionId: _creds.GetSubcription());
+    }
+    public async Task<IEnumerable<byte>?> GetBlob(Container container, string name, StorageType storageType)
+    {
+        var client = await FindContainer(container, storageType);
+
+        if (client == null)
+        {
+            return null;
+        }
+
+        try
+        {
+            return (await client.GetBlobClient(name).DownloadContentAsync())
+                .Value.Content.ToArray();
+        }
+        catch (RequestFailedException)
+        {
+            return null;
+        }
+    }
+
+    public async Async.Task<BlobContainerClient?> FindContainer(Container container, StorageType storageType)
+    {
+        // # check secondary accounts first by searching in reverse.
+        // #
+        // # By implementation, the primary account is specified first, followed by
+        // # any secondary accounts.
+        // #
+        // # Secondary accounts, if they exist, are preferred for containers and have
+        // # increased IOP rates, this should be a slight optimization
+        return await _storage.GetAccounts(storageType)
+            .Reverse()
+            .Select(account => GetBlobService(account)?.GetBlobContainerClient(container.ContainerName))
+            .ToAsyncEnumerable()
+            .WhereAwait(async client => client != null && (await client.ExistsAsync()).Value)
+            .FirstOrDefaultAsync();
+    }
+
+    private BlobServiceClient? GetBlobService(string accountId)
+    {
+        _log.Info($"getting blob container (account_id: {accountId}");
+        var (accountName, accountKey) = _storage.GetStorageAccountNameAndKey(accountId);
+        if (accountName == null)
+        {
+            _log.Error("Failed to get storage account name");
+            return null;
+        }
+        var storageKeyCredential = new StorageSharedKeyCredential(accountName, accountKey);
+        var accountUrl = GetUrl(accountName);
+        return new BlobServiceClient(accountUrl, storageKeyCredential);
+    }
+
+    private static Uri GetUrl(string accountName)
+    {
+        return new Uri($"https://{accountName}.blob.core.windows.net/");
+    }
+
+    public Uri GetFileSasUrl(Container container, string name, StorageType storageType, bool read = false, bool add = false, bool create = false, bool write = false, bool delete = false, bool delete_previous_version = false, bool tag = false, int days = 30, int hours = 0, int minutes = 0)
+    {
+        throw new NotImplementedException();
+    }
+}
+

src/ApiService/ApiService/onefuzzlib/Events.cs

@@ -1,6 +1,4 @@
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
-using System.Collections.Generic;
 using System.Text;
 using System.Text.Json;
 using System.Text.Json.Serialization;

src/ApiService/ApiService/onefuzzlib/InstanceConfig.cs

@@ -1,5 +1,4 @@
 using ApiService.OneFuzzLib.Orm;
-using System;
 using System.Threading.Tasks;
 
 namespace Microsoft.OneFuzz.Service;

src/ApiService/ApiService/onefuzzlib/NodeOperations.cs

@@ -1,6 +1,4 @@
 using ApiService.OneFuzzLib.Orm;
-using System;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace Microsoft.OneFuzz.Service;

src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs

@@ -0,0 +1,146 @@
+using System.Text.Json;
+using ApiService.OneFuzzLib.Orm;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface INotificationOperations
+{
+    Async.Task NewFiles(Container container, string filename, bool failTaskOnTransientError);
+}
+
+public class NotificationOperations : Orm<Notification>, INotificationOperations
+{
+    private ILogTracer _log;
+    private IReports _reports;
+    private ITaskOperations _taskOperations;
+
+    private IContainers _containers;
+
+    private IQueue _queue;
+
+    private IEvents _events;
+
+    public NotificationOperations(ILogTracer log, IStorage storage, IReports reports, ITaskOperations taskOperations, IContainers containers, IQueue queue, IEvents events)
+        : base(storage)
+    {
+        _log = log;
+        _reports = reports;
+        _taskOperations = taskOperations;
+        _containers = containers;
+        _queue = queue;
+        _events = events;
+    }
+    public async Async.Task NewFiles(Container container, string filename, bool failTaskOnTransientError)
+    {
+        var notifications = GetNotifications(container);
+        var hasNotifications = await notifications.AnyAsync();
+        var report = await _reports.GetReportOrRegression(container, filename, expectReports: hasNotifications);
+
+        if (!hasNotifications)
+        {
+            return;
+        }
+
+        var done = new List<NotificationTemplate>();
+        await foreach (var notification in notifications)
+        {
+            if (done.Contains(notification.Config))
+            {
+                continue;
+            }
+
+            done.Add(notification.Config);
+
+            if (notification.Config.TeamsTemplate != null)
+            {
+                NotifyTeams(notification.Config.TeamsTemplate, container, filename, report);
+            }
+
+            if (report == null)
+            {
+                continue;
+            }
+
+            if (notification.Config.AdoTemplate != null)
+            {
+                NotifyAdo(notification.Config.AdoTemplate, container, filename, report, failTaskOnTransientError);
+            }
+
+            if (notification.Config.GithubIssuesTemplate != null)
+            {
+                GithubIssue(notification.Config.GithubIssuesTemplate, container, filename, report);
+            }
+        }
+
+        await foreach (var (task, containers) in GetQueueTasks())
+        {
+            if (containers.Contains(container.ContainerName))
+            {
+                _log.Info($"queuing input {container.ContainerName} {filename} {task.TaskId}");
+                var url = _containers.GetFileSasUrl(container, filename, StorageType.Corpus, read: true, delete: true);
+                await _queue.SendMessage(task.TaskId.ToString(), System.Text.Encoding.UTF8.GetBytes(url.ToString()), StorageType.Corpus);
+            }
+        }
+
+        if (report == null)
+        {
+            await _events.SendEvent(new EventFileAdded(container, filename));
+        }
+        else if (report.Report != null)
+        {
+            var reportTask = await _taskOperations.GetByJobIdAndTaskId(report.Report.JobId, report.Report.TaskId);
+
+            var crashReportedEvent = new EventCrashReported(report.Report, container, filename, reportTask?.Config);
+            await _events.SendEvent(crashReportedEvent);
+        }
+        else if (report.RegressionReport != null)
+        {
+            var reportTask = await GetRegressionReportTask(report.RegressionReport);
+
+            var regressionEvent = new EventRegressionReported(report.RegressionReport, container, filename, reportTask?.Config);
+        }
+    }
+
+    public IAsyncEnumerable<Notification> GetNotifications(Container container)
+    {
+        return QueryAsync(filter: $"container eq '{container.ContainerName}'");
+    }
+
+    public IAsyncEnumerable<(Task, IEnumerable<string>)> GetQueueTasks()
+    {
+        // Nullability mismatch: We filter tuples where the containers are null
+        return _taskOperations.SearchStates(states: TaskStateHelper.Available())
+            .Select(task => (task, _taskOperations.GetInputContainerQueues(task.Config)))
+            .Where(taskTuple => taskTuple.Item2 != null)!;
+    }
+
+    private async Async.Task<Task?> GetRegressionReportTask(RegressionReport report)
+    {
+        if (report.CrashTestResult.CrashReport != null)
+        {
+            return await _taskOperations.GetByJobIdAndTaskId(report.CrashTestResult.CrashReport.JobId, report.CrashTestResult.CrashReport.TaskId);
+        }
+        if (report.CrashTestResult.NoReproReport != null)
+        {
+            return await _taskOperations.GetByJobIdAndTaskId(report.CrashTestResult.NoReproReport.JobId, report.CrashTestResult.NoReproReport.TaskId);
+        }
+
+        _log.Error($"unable to find crash_report or no repro entry for report: {JsonSerializer.Serialize(report)}");
+        return null;
+    }
+
+    private void GithubIssue(GithubIssuesTemplate config, Container container, string filename, RegressionReportOrReport? report)
+    {
+        throw new NotImplementedException();
+    }
+
+    private void NotifyAdo(AdoTemplate config, Container container, string filename, RegressionReportOrReport report, bool failTaskOnTransientError)
+    {
+        throw new NotImplementedException();
+    }
+
+    private void NotifyTeams(TeamsTemplate config, Container container, string filename, RegressionReportOrReport? report)
+    {
+        throw new NotImplementedException();
+    }
+}

src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs

@@ -1,6 +1,4 @@
 using ApiService.OneFuzzLib.Orm;
-using System;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace Microsoft.OneFuzz.Service;

src/ApiService/ApiService/onefuzzlib/Queue.cs

@@ -1,7 +1,6 @@
 using Azure.Storage;
 using Azure.Storage.Queues;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
 using System.Text.Json;
 using System.Threading.Tasks;
 

src/ApiService/ApiService/onefuzzlib/Reports.cs

@@ -0,0 +1,98 @@
+using System.Text.Json;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IReports
+{
+    public Async.Task<RegressionReportOrReport?> GetReportOrRegression(Container container, string fileName, bool expectReports = false, params string[] args);
+}
+
+public class Reports : IReports
+{
+    private ILogTracer _log;
+    private IContainers _containers;
+    public Reports(ILogTracer log, IContainers containers)
+    {
+        _log = log;
+        _containers = containers;
+    }
+
+    public async Async.Task<RegressionReportOrReport?> GetReportOrRegression(Container container, string fileName, bool expectReports = false, params string[] args)
+    {
+        var filePath = String.Join("/", new[] { container.ContainerName, fileName });
+        if (!fileName.EndsWith(".json"))
+        {
+            if (expectReports)
+            {
+                _log.Error($"get_report invalid extension: {filePath}");
+            }
+            return null;
+        }
+
+        var blob = await _containers.GetBlob(container, fileName, StorageType.Corpus);
+
+        if (blob == null)
+        {
+            if (expectReports)
+            {
+                _log.Error($"get_report invalid blob: {filePath}");
+            }
+            return null;
+        }
+
+        return ParseReportOrRegression(blob, filePath, expectReports);
+    }
+
+    private RegressionReportOrReport? ParseReportOrRegression(string content, string? filePath, bool expectReports = false)
+    {
+        try
+        {
+            return new RegressionReportOrReport
+            {
+                RegressionReport = JsonSerializer.Deserialize<RegressionReport>(content, EntityConverter.GetJsonSerializerOptions())
+            };
+        }
+        catch (JsonException e)
+        {
+            try
+            {
+                return new RegressionReportOrReport
+                {
+                    Report = JsonSerializer.Deserialize<Report>(content, EntityConverter.GetJsonSerializerOptions())
+                };
+            }
+            catch (JsonException e2)
+            {
+                if (expectReports)
+                {
+                    _log.Error($"unable to parse report ({filePath}) as a report or regression. regression error: {e.Message} report error: {e2.Message}");
+                }
+                return null;
+            }
+        }
+    }
+
+    private RegressionReportOrReport? ParseReportOrRegression(IEnumerable<byte> content, string? filePath, bool expectReports = false)
+    {
+        try
+        {
+            var str = System.Text.Encoding.UTF8.GetString(content.ToArray());
+            return ParseReportOrRegression(str, filePath, expectReports);
+        }
+        catch (Exception e)
+        {
+            if (expectReports)
+            {
+                _log.Error($"unable to parse report ({filePath}): unicode decode of report failed - {e.Message} {e.StackTrace}");
+            }
+            return null;
+        }
+    }
+}
+
+public class RegressionReportOrReport
+{
+    public RegressionReport? RegressionReport { get; set; }
+    public Report? Report { get; set; }
+}

src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs

@@ -1,5 +1,4 @@
 using ApiService.OneFuzzLib.Orm;
-using System.Collections.Generic;
 
 namespace Microsoft.OneFuzz.Service;
 

src/ApiService/ApiService/onefuzzlib/Storage.cs

@@ -1,10 +1,7 @@
-using System.Collections.Generic;
-using System;
 using Azure.ResourceManager;
 using Azure.ResourceManager.Storage;
 using Azure.Core;
 using System.Text.Json;
-using System.Linq;
 
 namespace Microsoft.OneFuzz.Service;
 
@@ -21,6 +18,8 @@ public interface IStorage
     public IEnumerable<string> CorpusAccounts();
     string GetPrimaryAccount(StorageType storageType);
     public (string?, string?) GetStorageAccountNameAndKey(string accountId);
+
+    public IEnumerable<string> GetAccounts(StorageType storageType);
 }
 
 public class Storage : IStorage
@@ -114,4 +113,17 @@ public class Storage : IStorage
         var key = storageAccount.GetKeys().Value.Keys.FirstOrDefault();
         return (resourceId.Name, key?.Value);
     }
+
+    public IEnumerable<string> GetAccounts(StorageType storageType)
+    {
+        switch (storageType)
+        {
+            case StorageType.Corpus:
+                return CorpusAccounts();
+            case StorageType.Config:
+                return new[] { GetFuncStorage() };
+            default:
+                throw new NotImplementedException();
+        }
+    }
 }

src/ApiService/ApiService/onefuzzlib/TaskOperations.cs

@@ -1,12 +1,18 @@
 using ApiService.OneFuzzLib.Orm;
-using System;
-using System.Linq;
 
 namespace Microsoft.OneFuzz.Service;
 
 public interface ITaskOperations : IOrm<Task>
 {
     Async.Task<Task?> GetByTaskId(Guid taskId);
+
+    Async.Task<Task?> GetByJobIdAndTaskId(Guid jobId, Guid taskId);
+
+
+    IAsyncEnumerable<Task> SearchStates(Guid? jobId = null, IEnumerable<TaskState>? states = null);
+
+    IEnumerable<string>? GetInputContainerQueues(TaskConfig config);
+
 }
 
 public class TaskOperations : Orm<Task>, ITaskOperations
@@ -25,4 +31,37 @@ public class TaskOperations : Orm<Task>, ITaskOperations
         return await data.FirstOrDefaultAsync();
     }
 
+    public async Async.Task<Task?> GetByJobIdAndTaskId(Guid jobId, Guid taskId)
+    {
+        var data = QueryAsync(filter: $"PartitionKey eq '{jobId}' and RowKey eq '{taskId}'");
+
+        return await data.FirstOrDefaultAsync();
+    }
+    public IAsyncEnumerable<Task> SearchStates(Guid? jobId = null, IEnumerable<TaskState>? states = null)
+    {
+        var queryString = String.Empty;
+        if (jobId != null)
+        {
+            queryString += $"PartitionKey eq '{jobId}'";
+        }
+
+        if (states != null)
+        {
+            if (jobId != null)
+            {
+                queryString += " and ";
+            }
+
+            var statesString = string.Join(",", states);
+            queryString += $"state in ({statesString})";
+        }
+
+        return QueryAsync(filter: queryString);
+    }
+
+    public IEnumerable<string>? GetInputContainerQueues(TaskConfig config)
+    {
+        throw new NotImplementedException();
+    }
+
 }

src/ApiService/ApiService/onefuzzlib/Utils.cs

@@ -1,6 +1,3 @@
-
-using System;
-
 namespace Microsoft.OneFuzz.Service;
 
 public static class ObjectExtention

src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs

@@ -1,6 +1,4 @@
 using ApiService.OneFuzzLib.Orm;
-using System;
-using System.Collections.Generic;
 
 namespace Microsoft.OneFuzz.Service;
 

src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs

@@ -1,8 +1,4 @@
-using System;
+namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System.Collections.Generic;
-using System.Linq;
-
-namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 
 public class CaseConverter
 {

src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs

@@ -1,7 +1,4 @@
-using System;
+using System.Diagnostics;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.Linq;
 using System.Reflection;
 using System.Runtime.CompilerServices;
 using System.Text.Encodings.Web;

src/ApiService/ApiService/onefuzzlib/orm/EntityConverter.cs

@@ -1,13 +1,10 @@
 using Azure.Data.Tables;
-using System;
 using System.Reflection;
-using System.Linq;
 using System.Linq.Expressions;
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using System.Collections.Concurrent;
 using Azure;
-using System.Collections.Generic;
 
 namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 

src/ApiService/ApiService/onefuzzlib/orm/Orm.cs

@@ -1,9 +1,6 @@
 using Azure.Data.Tables;
 using Microsoft.OneFuzz.Service;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
-using System;
-using System.Collections.Generic;
-using System.Linq;
 using System.Threading.Tasks;
 
 namespace ApiService.OneFuzzLib.Orm

src/ApiService/ApiService/packages.lock.json

@@ -107,6 +107,16 @@
           "System.Text.Json": "4.7.2"
         }
       },
+      "Azure.Storage.Blobs": {
+        "type": "Direct",
+        "requested": "[12.11.0, )",
+        "resolved": "12.11.0",
+        "contentHash": "50eRjIhY7Q1JN7kT2MSawDKCcwSb7uRZUkz00P/BLjSg47gm2hxUYsnJPyvzCHntYMbOWzrvaVQTwYwXabaR5Q==",
+        "dependencies": {
+          "Azure.Storage.Common": "12.10.0",
+          "System.Text.Json": "4.7.2"
+        }
+      },
       "Azure.Storage.Queues": {
         "type": "Direct",
         "requested": "[12.9.0, )",

src/ApiService/Tests/OrmModelsTest.cs

@@ -249,6 +249,41 @@ namespace Tests
             );
 
         }
+
+        public static Gen<Report> Report()
+        {
+            return Arb.Generate<Tuple<string, BlobRef, List<string>, Guid, int>>().Select(
+                arg =>
+                    new Report(
+                        InputUrl: arg.Item1,
+                        InputBlob: arg.Item2,
+                        Executable: arg.Item1,
+                        CrashType: arg.Item1,
+                        CrashSite: arg.Item1,
+                        CallStack: arg.Item3,
+                        CallStackSha256: arg.Item1,
+                        InputSha256: arg.Item1,
+                        AsanLog: arg.Item1,
+                        TaskId: arg.Item4,
+                        JobId: arg.Item4,
+                        ScarinessScore: arg.Item5,
+                        ScarinessDescription: arg.Item1,
+                        MinimizedStack: arg.Item3,
+                        MinimizedStackSha256: arg.Item1,
+                        MinimizedStackFunctionNames: arg.Item3,
+                        MinimizedStackFunctionNamesSha256: arg.Item1,
+                        MinimizedStackFunctionLines: arg.Item3,
+                        MinimizedStackFunctionLinesSha256: arg.Item1
+                    )
+            );
+        }
+
+        public static Gen<Container> Container()
+        {
+            return Arb.Generate<Tuple<NonNull<string>>>().Select(
+                arg => new Container(string.Join("", arg.Item1.Get.Where(c => char.IsLetterOrDigit(c) || c == '-'))!)
+            );
+        }
     }
 
     public class OrmArb
@@ -327,6 +362,16 @@ namespace Tests
         {
             return Arb.From(OrmGenerators.WebhookMessage());
         }
+
+        public static Arbitrary<Report> Report()
+        {
+            return Arb.From(OrmGenerators.Report());
+        }
+
+        public static Arbitrary<Container> Container()
+        {
+            return Arb.From(OrmGenerators.Container());
+        }
     }