Migrating timer_daily (#1803)

* Migrating timer_daily

* isabling the TimerDaily to prevent ollision with the existing

* adding missing keys

* Adding Report

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -1,4 +1,6 @@
-namespace Microsoft.OneFuzz.Service;
+using System.Collections.Concurrent;
+
+namespace Microsoft.OneFuzz.Service;
 public enum ErrorCode
 {
     INVALID_REQUEST = 450,
@@ -27,9 +29,9 @@ public enum ErrorCode
     INVALID_CONFIGURATION = 473,
 }
 
-public enum VmState
+public enum VmState
 {
-    Init,
+    Init,
     ExtensionsLaunched,
     ExtensionsFailed,
     VmAllocationFailed,
@@ -107,4 +109,75 @@ public enum TaskDebugFlag
 {
     KeepNodeOnFailure,
     KeepNodeOnCompletion,
+}
+
+public enum ScalesetState
+{
+    Init,
+    Setup,
+    Resize,
+    Running,
+    Shutdown,
+    Halt,
+    CreationFailed
+}
+
+public static class ScalesetStateHelper
+{
+
+    static ConcurrentDictionary<string, ScalesetState[]> _states = new ConcurrentDictionary<string, ScalesetState[]>();
+
+    /// set of states that indicate the scaleset can be updated
+    public static ScalesetState[] CanUpdate()
+    {
+        return
+        _states.GetOrAdd("CanUpdate", k => new[]{
+            ScalesetState.Running,
+            ScalesetState.Resize
+        });
+    }
+
+    /// set of states that indicate work is needed during eventing
+    public static ScalesetState[] NeedsWork()
+    {
+        return
+        _states.GetOrAdd("CanUpdate", k => new[]{
+            ScalesetState.Init,
+            ScalesetState.Setup,
+            ScalesetState.Resize,
+            ScalesetState.Shutdown,
+            ScalesetState.Halt,
+        });
+    }
+
+    /// set of states that indicate if it's available for work
+    public static ScalesetState[] Available()
+    {
+        return
+        _states.GetOrAdd("CanUpdate", k =>
+        {
+            return
+                new[]{
+                ScalesetState.Resize,
+                ScalesetState.Running,
+            };
+        });
+    }
+
+    /// set of states that indicate scaleset is resizing
+    public static ScalesetState[] Resizing()
+    {
+        return
+        _states.GetOrAdd("CanDelete", k =>
+        {
+            return
+                new[]{
+                ScalesetState.Halt,
+                ScalesetState.Init,
+                ScalesetState.Setup,
+            };
+        });
+    }
+
+
 }

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -373,3 +373,61 @@ public record InstanceConfig
         }
     }
 }
+
+
+public record ScalesetNodeState(
+    Guid MachineId,
+    string InstanceId,
+    NodeState? State
+
+);
+
+
+public record Scaleset(
+    [PartitionKey] PoolName PoolName,
+    [RowKey] Guid ScalesetId,
+    ScalesetState State,
+    Authentication? Auth,
+    string VmSku,
+    string Image,
+    Region Region,
+    int Size,
+    bool SpotInstance,
+    bool EphemeralOsDisks,
+    bool NeedsConfigUpdate,
+    List<ScalesetNodeState> Nodes,
+    Guid? ClientId,
+    Guid? ClientObjectId,
+    Dictionary<string, string> Tags
+
+) : EntityBase();
+
+
+public record BlobRef(
+    string Account,
+    Container Container,
+    string Name
+);
+
+
+public record Report(
+    string? InputURL,
+    BlobRef? InputBlob,
+    string? Executable,
+    string CrashType,
+    string CrashSite,
+    List<string> CallStack,
+    string CallStackSha256,
+    string InputSha256,
+    string? AsanLog,
+    Guid TaskID,
+    Guid JobID,
+    int? ScarinessScore,
+    string? ScarinessDescription,
+    List<string> MinimizedStack,
+    string? MinimizedStackSha256,
+    List<string> MinimizedStackFunctionNames,
+    string? MinimizedStackFunctionNamesSha256,
+    List<string> MinimizedStackFunctionLines,
+    string? MinimizedStackFunctionLinesSha256
+);

src/ApiService/ApiService/Program.cs

@@ -5,7 +5,6 @@ using System;
 using System.Collections.Generic;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.DependencyInjection;
-using ApiService.OneFuzzLib;
 using Microsoft.Azure.Functions.Worker.Middleware;
 using Microsoft.Azure.Functions.Worker;
 
@@ -22,7 +21,7 @@ public class Program
             {
                 //TODO
                 //if correlation ID is available in HTTP request
-                //if correlation ID is available in Queue message 
+                //if correlation ID is available in Queue message
                 //log.ReplaceCorrelationId(Guid from request)
 
                 log.ReplaceCorrelationId(Guid.NewGuid());
@@ -76,6 +75,7 @@ public class Program
             .AddScoped<IStorage, Storage>()
             .AddScoped<IProxyOperations, ProxyOperations>()
             .AddScoped<IConfigOperations, ConfigOperations>()
+            .AddScoped<IScalesetOperations, ScalesetOperations>()
 
         //TODO: move out expensive resources into separate class, and add those as Singleton
         // ArmClient, Table Client(s), Queue Client(s), HttpClient, etc.

src/ApiService/ApiService/TimerDaily.cs

@@ -0,0 +1,41 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Extensions.Logging;
+
+namespace Microsoft.OneFuzz.Service;
+
+
+public class TimerDaily
+{
+    private readonly ILogger _logger;
+
+    private readonly IScalesetOperations _scalesets;
+
+    private readonly IWebhookMessageLogOperations _webhookMessageLogs;
+
+    public TimerDaily(ILoggerFactory loggerFactory, IScalesetOperations scalesets, IWebhookMessageLogOperations webhookMessageLogs)
+    {
+        _logger = loggerFactory.CreateLogger<QueueTaskHearbeat>();
+        _scalesets = scalesets;
+        _webhookMessageLogs = webhookMessageLogs;
+    }
+
+    //[Function("TimerDaily")]
+    public async Async.Task Run([TimerTrigger("1.00:00:00")] TimerInfo myTimer)
+    {
+        var scalesets = _scalesets.Search();
+        await foreach (var scaleset in scalesets)
+        {
+            _logger.LogInformation($"updating scaleset configs: {scaleset.ScalesetId}");
+            // todo: do it in batches
+            await _scalesets.Replace(scaleset with { NeedsConfigUpdate = true });
+        }
+
+
+        var expiredWebhookLogs = _webhookMessageLogs.SearchExpired();
+        await foreach (var logEntry in expiredWebhookLogs)
+        {
+            _logger.LogInformation($"stopping expired webhook message log: {logEntry.WebhookId}:{logEntry.EventId}");
+            await _webhookMessageLogs.Delete(logEntry);
+        }
+    }
+}

src/ApiService/ApiService/onefuzzlib/Events.cs

@@ -1,5 +1,4 @@
-using ApiService.OneFuzzLib;
-using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
+using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 using System;
 using System.Collections.Generic;
 using System.Text;

src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs

@@ -0,0 +1,25 @@
+using ApiService.OneFuzzLib.Orm;
+using System.Collections.Generic;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IScalesetOperations : IOrm<Scaleset>
+{
+    IAsyncEnumerable<Scaleset> Search();
+}
+
+public class ScalesetOperations : Orm<Scaleset>, IScalesetOperations
+{
+
+    public ScalesetOperations(IStorage storage)
+        : base(storage)
+    {
+
+    }
+
+    public IAsyncEnumerable<Scaleset> Search()
+    {
+        return QueryAsync();
+    }
+
+}

src/ApiService/ApiService/onefuzzlib/WebhookOperations.cs

@@ -1,19 +1,20 @@
 using ApiService.OneFuzzLib.Orm;
-using Microsoft.OneFuzz.Service;
 using System;
 using System.Collections.Generic;
 
-namespace ApiService.OneFuzzLib;
+namespace Microsoft.OneFuzz.Service;
 
 
 public interface IWebhookMessageLogOperations : IOrm<WebhookMessageLog>
 {
-
+    IAsyncEnumerable<WebhookMessageLog> SearchExpired();
 }
 
 
 public class WebhookMessageLogOperations : Orm<WebhookMessageLog>, IWebhookMessageLogOperations
 {
+    const int EXPIRE_DAYS = 7;
+
     record WebhookMessageQueueObj(
         Guid WebhookId,
         Guid EventId
@@ -58,6 +59,14 @@ public class WebhookMessageLogOperations : Orm<WebhookMessageLog>, IWebhookMessa
     {
         throw new NotImplementedException();
     }
+
+    public IAsyncEnumerable<WebhookMessageLog> SearchExpired()
+    {
+        var expireTime = (DateTimeOffset.UtcNow - TimeSpan.FromDays(EXPIRE_DAYS)).ToString("o");
+
+        var timeFilter = $"Timestamp lt datetime'{expireTime}'";
+        return QueryAsync(filter: timeFilter);
+    }
 }
 
 

src/ApiService/ApiService/onefuzzlib/orm/Orm.cs

@@ -16,6 +16,7 @@ namespace ApiService.OneFuzzLib.Orm
 
         Task<T> GetEntityAsync(string partitionKey, string rowKey);
         Task<ResultOk<(int, string)>> Insert(T entity);
+        Task<ResultOk<(int, string)>> Delete(T entity);
     }
 
     public class Orm<T> : IOrm<T> where T : EntityBase
@@ -108,5 +109,20 @@ namespace ApiService.OneFuzzLib.Orm
             await tableClient.CreateTableIfNotExistsAsync(table);
             return tableClient.GetTableClient(table);
         }
+
+        public async Task<ResultOk<(int, string)>> Delete(T entity)
+        {
+            var tableClient = await GetTableClient(typeof(T).Name);
+            var tableEntity = _entityConverter.ToTableEntity(entity);
+            var response = await tableClient.DeleteEntityAsync(tableEntity.PartitionKey, tableEntity.RowKey);
+            if (response.IsError)
+            {
+                return ResultOk<(int, string)>.Error((response.Status, response.ReasonPhrase));
+            }
+            else
+            {
+                return ResultOk<(int, string)>.Ok();
+            }
+        }
     }
 }