Refactor timer_repro (#1839)

* Checkpoint

* lint

* PR comments and warnings

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -257,3 +257,45 @@ public static class TaskStateHelper
     }
 }
 
+public enum PoolState
+{
+    Init,
+    Running,
+    Shutdown,
+    Halt
+}
+
+public static class PoolStateHelper
+{
+    static ConcurrentDictionary<string, PoolState[]> _states = new ConcurrentDictionary<string, PoolState[]>();
+    public static PoolState[] NeedsWork()
+    {
+        return
+        _states.GetOrAdd("NeedsWork", k =>
+        {
+            return
+                 new[]{
+                    PoolState.Init,
+                    PoolState.Shutdown,
+                    PoolState.Halt
+                 };
+        });
+    }
+
+    public static PoolState[] Available()
+    {
+        return
+        _states.GetOrAdd("Available", k =>
+        {
+            return
+                 new[]{
+                    PoolState.Running
+                 };
+        });
+    }
+}
+
+public enum Architecture
+{
+    x86_64
+}

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -206,9 +206,9 @@ public record TaskVm(
     Region Region,
     string Sku,
     string Image,
-    int Count,
+    bool? RebootAfterSetup,
-    bool SpotInstance,
+    int Count = 1,
-    bool? RebootAfterSetup
+    bool SpotInstance = false
 );
 
 public record TaskPool(
@@ -497,6 +497,62 @@ public record TeamsTemplate();
 
 public record GithubIssuesTemplate();
 
+public record Repro(
+    DateTimeOffset Timestamp,
+    Guid VmId,
+    Guid TaskId,
+    ReproConfig Config,
+    VmState State,
+    Authentication? Auth,
+    Os Os,
+    Error? Error,
+    string? Ip,
+    DateTime? EndTime,
+    UserInfo? UserInfo
+) : StatefulEntityBase<VmState>(State);
+
+public record ReproConfig(
+    Container Container,
+    string Path,
+    // TODO: Make this >1 and < 7*24 (more than one hour, less than seven days)
+    int Duration
+);
+
+public record Pool(
+    DateTimeOffset Timestamp,
+    PoolName Name,
+    Guid PoolId,
+    Os Os,
+    bool Managed,
+    // Skipping AutoScaleConfig because it's not used anymore
+    Architecture Architecture,
+    PoolState State,
+    Guid? ClientId,
+    List<Node>? Nodes,
+    AgentConfig? Config,
+    List<WorkSetSummary>? WorkQueue,
+    List<ScalesetSummary>? ScalesetSummary
+) : StatefulEntityBase<PoolState>(State);
+
+
+// TODO
+public record AgentConfig();
+public record WorkSetSummary();
+public record ScalesetSummary();
+
+public record Vm(
+    string Name,
+    Region Region,
+    string Sku,
+    string Image,
+    Authentication Auth,
+    Nsg? Nsg,
+    IDictionary<string, string>? Tags
+)
+{
+    public string Name { get; } = Name.Length > 40 ? throw new ArgumentOutOfRangeException("VM name too long") : Name;
+};
+
 
 public record SecretAddress(Uri Url);
 

src/ApiService/ApiService/Program.cs

@@ -82,6 +82,11 @@ public class Program
             .AddScoped<IReports, Reports>()
             .AddScoped<INotificationOperations, NotificationOperations>()
             .AddScoped<IUserCredentials, UserCredentials>()
+            .AddScoped<IReproOperations, ReproOperations>()
+            .AddScoped<IPoolOperations, PoolOperations>()
+            .AddScoped<IIpOperations, IpOperations>()
+            .AddScoped<IDiskOperations, DiskOperations>()
+            .AddScoped<IVmOperations, VmOperations>()
             .AddScoped<ISecretsOperations, SecretsOperations>()
             .AddScoped<IJobOperations, JobOperations>()
 

src/ApiService/ApiService/TimerRepro.cs

@@ -0,0 +1,29 @@
+using Microsoft.Azure.Functions.Worker;
+
+namespace Microsoft.OneFuzz.Service;
+
+public class TimerRepro
+{
+    private readonly ILogTracer _log;
+
+    private readonly IStorage _storage;
+
+    private readonly IReproOperations _reproOperations;
+
+    public TimerRepro(ILogTracer log, IStorage storage, IReproOperations reproOperations)
+    {
+        _log = log;
+        _storage = storage;
+        _reproOperations = reproOperations;
+    }
+
+    public async Async.Task Run([TimerTrigger("00:00:30")] TimerInfo myTimer)
+    {
+        var expired = _reproOperations.SearchExpired();
+        await foreach (var repro in expired)
+        {
+            _log.Info($"stopping repro: {repro?.VmId}");
+        }
+    }
+
+}

src/ApiService/ApiService/onefuzzlib/Creds.cs

@@ -19,6 +19,8 @@ public interface ICreds
     public ArmClient ArmClient { get; }
 
     public ResourceGroupResource GetResourceGroupResource();
+
+    public string GetBaseRegion();
 }
 
 public class Creds : ICreds
@@ -69,4 +71,9 @@ public class Creds : ICreds
         var resourceId = GetResourceGroupResourceIdentifier();
         return ArmClient.GetResourceGroupResource(resourceId);
     }
+
+    public string GetBaseRegion()
+    {
+        return ArmClient.GetResourceGroupResource(GetResourceGroupResourceIdentifier()).Data.Location.Name;
+    }
 }

src/ApiService/ApiService/onefuzzlib/DiskOperations.cs

@@ -0,0 +1,35 @@
+using System.Threading.Tasks;
+using Azure.ResourceManager.Compute;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IDiskOperations
+{
+    DiskCollection ListDisks(string resourceGroup);
+
+    Async.Task<bool> DeleteDisk(string resourceGroup, string name);
+}
+
+public class DiskOperations : IDiskOperations
+{
+    private ILogTracer _logTracer;
+
+    private ICreds _creds;
+
+    public DiskOperations(ILogTracer log, ICreds creds)
+    {
+        _logTracer = log;
+        _creds = creds;
+    }
+
+    public Task<bool> DeleteDisk(string resourceGroup, string name)
+    {
+        throw new NotImplementedException();
+    }
+
+    public DiskCollection ListDisks(string resourceGroup)
+    {
+        _logTracer.Info($"listing disks {resourceGroup}");
+        return _creds.GetResourceGroupResource().GetDisks();
+    }
+}

src/ApiService/ApiService/onefuzzlib/IpOperations.cs

@@ -0,0 +1,49 @@
+using Azure.ResourceManager.Network;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IIpOperations
+{
+    public Async.Task<NetworkInterfaceResource> GetPublicNic(string resourceGroup, string name);
+
+    public Async.Task<PublicIPAddressResource> GetIp(string resourceGroup, string name);
+
+    public Async.Task DeleteNic(string resourceGroup, string name);
+
+    public Async.Task DeleteIp(string resourceGroup, string name);
+}
+
+public class IpOperations : IIpOperations
+{
+    private ILogTracer _logTracer;
+
+    private ICreds _creds;
+
+    public IpOperations(ILogTracer log, ICreds creds)
+    {
+        _logTracer = log;
+        _creds = creds;
+    }
+
+    public async Async.Task<NetworkInterfaceResource> GetPublicNic(string resourceGroup, string name)
+    {
+        _logTracer.Info($"getting nic: {resourceGroup} {name}");
+        return await _creds.GetResourceGroupResource().GetNetworkInterfaceAsync(name);
+    }
+
+    public async Async.Task<PublicIPAddressResource> GetIp(string resourceGroup, string name)
+    {
+        _logTracer.Info($"getting ip {resourceGroup}:{name}");
+        return await _creds.GetResourceGroupResource().GetPublicIPAddressAsync(name);
+    }
+
+    public System.Threading.Tasks.Task DeleteNic(string resourceGroup, string name)
+    {
+        throw new NotImplementedException();
+    }
+
+    public System.Threading.Tasks.Task DeleteIp(string resourceGroup, string name)
+    {
+        throw new NotImplementedException();
+    }
+}

src/ApiService/ApiService/onefuzzlib/Nsg.cs

@@ -11,6 +11,8 @@ namespace Microsoft.OneFuzz.Service
         IAsyncEnumerable<NetworkSecurityGroupResource> ListNsgs();
         bool OkToDelete(HashSet<string> active_regions, string nsg_region, string nsg_name);
         Async.Task<bool> StartDeleteNsg(string name);
+
+        Async.Task DissociateNic(NetworkInterfaceResource nic);
     }
 
 
@@ -52,6 +54,11 @@ namespace Microsoft.OneFuzz.Service
             return null;
         }
 
+        public System.Threading.Tasks.Task DissociateNic(NetworkInterfaceResource nic)
+        {
+            throw new NotImplementedException();
+        }
+
         public async Async.Task<NetworkSecurityGroupResource?> GetNsg(string name)
         {
             var response = await _creds.GetResourceGroupResource().GetNetworkSecurityGroupAsync(name);
@@ -84,4 +91,4 @@ namespace Microsoft.OneFuzz.Service
             return true;
         }
     }
-}
+}

src/ApiService/ApiService/onefuzzlib/PoolOperations.cs

@@ -0,0 +1,39 @@
+using ApiService.OneFuzzLib.Orm;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IPoolOperations
+{
+    public Async.Task<Result<Pool, Error>> GetByName(string poolName);
+
+}
+
+public class PoolOperations : StatefulOrm<Pool, PoolState>, IPoolOperations
+{
+    private IConfigOperations _configOperations;
+    private ITaskOperations _taskOperations;
+
+    public PoolOperations(IStorage storage, ILogTracer log, IServiceConfig config, IConfigOperations configOperations, ITaskOperations taskOperations)
+        : base(storage, log, config)
+    {
+        _configOperations = configOperations;
+        _taskOperations = taskOperations;
+    }
+
+    public async Async.Task<Result<Pool, Error>> GetByName(string poolName)
+    {
+        var pools = QueryAsync(filter: $"name eq '{poolName}'");
+
+        if (pools == null)
+        {
+            return new Result<Pool, Error>(new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find pool" }));
+        }
+
+        if (await pools.CountAsync() != 1)
+        {
+            return new Result<Pool, Error>(new Error(ErrorCode.INVALID_REQUEST, new[] { "error identifying pool" }));
+        }
+
+        return new Result<Pool, Error>(await pools.SingleAsync());
+    }
+}

src/ApiService/ApiService/onefuzzlib/ReproOperations.cs

@@ -0,0 +1,103 @@
+using ApiService.OneFuzzLib.Orm;
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IReproOperations
+{
+    public IAsyncEnumerable<Repro?> SearchExpired();
+}
+
+public class ReproOperations : StatefulOrm<Repro, VmState>, IReproOperations
+{
+    private static readonly Dictionary<Os, string> DEFAULT_OS = new Dictionary<Os, string>
+    {
+        {Os.Linux, "Canonical:UbuntuServer:18.04-LTS:latest"},
+        {Os.Windows, "MicrosoftWindowsDesktop:Windows-10:20h2-pro:latest"}
+    };
+
+    const string DEFAULT_SKU = "Standard_DS1_v2";
+
+    private IConfigOperations _configOperations;
+    private ITaskOperations _taskOperations;
+
+    private IVmOperations _vmOperations;
+
+    private ICreds _creds;
+
+    public ReproOperations(IStorage storage, ILogTracer log, IServiceConfig config, IConfigOperations configOperations, ITaskOperations taskOperations, ICreds creds, IVmOperations vmOperations)
+        : base(storage, log, config)
+    {
+        _configOperations = configOperations;
+        _taskOperations = taskOperations;
+        _creds = creds;
+        _vmOperations = vmOperations;
+    }
+
+    public IAsyncEnumerable<Repro?> SearchExpired()
+    {
+        return QueryAsync(filter: $"end_time lt datetime'{DateTime.UtcNow.ToString("o")}'");
+    }
+
+    public async Async.Task<Vm> GetVm(Repro repro, InstanceConfig config)
+    {
+        var tags = config.VmTags;
+        var task = await _taskOperations.GetByTaskId(repro.TaskId);
+        if (task == null)
+        {
+            throw new Exception($"previous existing task missing: {repro.TaskId}");
+        }
+
+        var vmConfig = await _taskOperations.GetReproVmConfig(task);
+        if (vmConfig == null)
+        {
+            if (!DEFAULT_OS.ContainsKey(task.Os))
+            {
+                throw new NotImplementedException($"unsupport OS for repro {task.Os}");
+            }
+
+            vmConfig = new TaskVm(
+                _creds.GetBaseRegion(),
+                DEFAULT_SKU,
+                DEFAULT_OS[task.Os],
+                null
+            );
+        }
+
+        if (repro.Auth == null)
+        {
+            throw new Exception("missing auth");
+        }
+
+        return new Vm(
+            repro.VmId.ToString(),
+            vmConfig.Region,
+            vmConfig.Sku,
+            vmConfig.Image,
+            repro.Auth,
+            null,
+            tags
+        );
+    }
+
+    public async System.Threading.Tasks.Task Stopping(Repro repro)
+    {
+        var config = await _configOperations.Fetch();
+        var vm = await GetVm(repro, config);
+        if (!await _vmOperations.IsDeleted(vm))
+        {
+            _logTracer.Info($"vm stopping: {repro.VmId}");
+            await _vmOperations.Delete(vm);
+            await Replace(repro);
+        }
+        else
+        {
+            await Stopped(repro);
+        }
+    }
+
+    public async Async.Task Stopped(Repro repro)
+    {
+        _logTracer.Info($"vm stopped: {repro.VmId}");
+        await Delete(repro);
+    }
+}

src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs

@@ -5,6 +5,9 @@ namespace Microsoft.OneFuzz.Service;
 public interface IScalesetOperations : IOrm<Scaleset>
 {
     IAsyncEnumerable<Scaleset> Search();
+
+    public IAsyncEnumerable<Scaleset?> SearchByPool(string poolName);
+
 }
 
 public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState>, IScalesetOperations
@@ -21,4 +24,9 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState>, IScalese
         return QueryAsync();
     }
 
+    public IAsyncEnumerable<Scaleset> SearchByPool(string poolName)
+    {
+        return QueryAsync(filter: $"pool_name eq '{poolName}'");
+    }
+
 }

src/ApiService/ApiService/onefuzzlib/TaskOperations.cs

@@ -13,15 +13,20 @@ public interface ITaskOperations : IStatefulOrm<Task, TaskState>
 
     IEnumerable<string>? GetInputContainerQueues(TaskConfig config);
 
+    Async.Task<TaskVm?> GetReproVmConfig(Task task);
 }
 
 public class TaskOperations : StatefulOrm<Task, TaskState>, ITaskOperations
 {
 
-    public TaskOperations(IStorage storage, ILogTracer log, IServiceConfig config)
+    private IPoolOperations _poolOperations;
+    private IScalesetOperations _scalesetOperations;
+
+    public TaskOperations(IStorage storage, ILogTracer log, IServiceConfig config, IPoolOperations poolOperations, IScalesetOperations scalesetOperations)
         : base(storage, log, config)
     {
-
+        _poolOperations = poolOperations;
+        _scalesetOperations = scalesetOperations;
     }
 
     public async Async.Task<Task?> GetByTaskId(Guid taskId)
@@ -64,4 +69,35 @@ public class TaskOperations : StatefulOrm<Task, TaskState>, ITaskOperations
         throw new NotImplementedException();
     }
 
+    public async Async.Task<TaskVm?> GetReproVmConfig(Task task)
+    {
+        if (task.Config.Vm != null)
+        {
+            return task.Config.Vm;
+        }
+
+        if (task.Config.Pool == null)
+        {
+            throw new Exception($"either pool or vm must be specified: {task.TaskId}");
+        }
+
+        var pool = await _poolOperations.GetByName(task.Config.Pool.PoolName);
+
+        if (!pool.IsOk)
+        {
+            _logTracer.Info($"unable to find pool from task: {task.TaskId}");
+            return null;
+        }
+
+        var scaleset = await _scalesetOperations.SearchByPool(task.Config.Pool.PoolName).FirstOrDefaultAsync();
+
+        if (scaleset == null)
+        {
+            _logTracer.Warning($"no scalesets are defined for task: {task.JobId}:{task.TaskId}");
+            return null;
+        }
+
+        return new TaskVm(scaleset.Region, scaleset.VmSku, scaleset.Image, null);
+    }
+
 }

src/ApiService/ApiService/onefuzzlib/VmOperations.cs

@@ -0,0 +1,133 @@
+using Azure.ResourceManager.Compute;
+
+
+namespace Microsoft.OneFuzz.Service;
+
+public interface IVmOperations
+{
+    Async.Task<bool> IsDeleted(Vm vm);
+
+    Async.Task<bool> HasComponents(string name);
+
+    Async.Task<VirtualMachineResource?> GetVm(string name);
+
+    Async.Task<bool> Delete(Vm vm);
+
+}
+
+public class VmOperations : IVmOperations
+{
+    private ILogTracer _logTracer;
+
+    private ICreds _creds;
+
+    private IIpOperations _ipOperations;
+
+    private IDiskOperations _diskOperations;
+
+    public VmOperations(ILogTracer log, ICreds creds, IIpOperations ipOperations, IDiskOperations diskOperations)
+    {
+        _logTracer = log;
+        _creds = creds;
+        _ipOperations = ipOperations;
+        _diskOperations = diskOperations;
+    }
+    public async Async.Task<bool> IsDeleted(Vm vm)
+    {
+        return !(await HasComponents(vm.Name));
+    }
+
+    public async Async.Task<bool> HasComponents(string name)
+    {
+        var resourceGroup = _creds.GetBaseResourceGroup();
+        if (await GetVm(name) != null)
+        {
+            return true;
+        }
+
+        if (await _ipOperations.GetPublicNic(resourceGroup, name) != null)
+        {
+            return true;
+        }
+
+        if (await _ipOperations.GetIp(resourceGroup, name) != null)
+        {
+            return true;
+        }
+
+        var disks = await _diskOperations.ListDisks(resourceGroup)
+            .ToAsyncEnumerable()
+            .Where(disk => disk.Data.Name.StartsWith(name))
+            .AnyAsync();
+
+        if (disks)
+        {
+            return true;
+        }
+
+        return false;
+    }
+
+    public async Async.Task<VirtualMachineResource?> GetVm(string name)
+    {
+        return await _creds.GetResourceGroupResource().GetVirtualMachineAsync(name);
+    }
+
+    public async Async.Task<bool> Delete(Vm vm)
+    {
+        return await DeleteVmComponents(vm.Name, vm.Nsg);
+    }
+
+    public async Async.Task<bool> DeleteVmComponents(string name, Nsg? nsg)
+    {
+        var resourceGroup = _creds.GetBaseResourceGroup();
+        _logTracer.Info($"deleting vm components {resourceGroup}:{name}");
+        if (GetVm(name) != null)
+        {
+            _logTracer.Info($"deleting vm {resourceGroup}:{name}");
+            DeleteVm(name);
+            return false;
+        }
+
+        var nic = await _ipOperations.GetPublicNic(resourceGroup, name);
+        if (nic != null)
+        {
+            _logTracer.Info($"deleting nic {resourceGroup}:{name}");
+            if (nic.Data.NetworkSecurityGroup != null && nsg != null)
+            {
+                await nsg.DissociateNic(nic);
+                return false;
+            }
+            await _ipOperations.DeleteNic(resourceGroup, name);
+            return false;
+        }
+
+        if (await _ipOperations.GetIp(resourceGroup, name) != null)
+        {
+            _logTracer.Info($"deleting ip {resourceGroup}:{name}");
+            await _ipOperations.DeleteIp(resourceGroup, name);
+            return false;
+        }
+
+        var disks = _diskOperations.ListDisks(resourceGroup)
+            .ToAsyncEnumerable()
+            .Where(disk => disk.Data.Name.StartsWith(name));
+
+        if (await disks.AnyAsync())
+        {
+            await foreach (var disk in disks)
+            {
+                _logTracer.Info($"deleting disk {resourceGroup}:{disk?.Data.Name}");
+                await _diskOperations.DeleteDisk(resourceGroup, disk?.Data.Name!);
+            }
+            return false;
+        }
+
+        return true;
+    }
+
+    public void DeleteVm(string name)
+    {
+        throw new NotImplementedException();
+    }
+}