Refactor timer repro part2 (#1852)

* Implement the rest of the function

* Initial implementation

* Pulling new formatting

* format

* Fixes after testing

* PR comments

src/ApiService/.editorconfig

@@ -130,4 +130,4 @@ csharp_preserve_single_line_blocks = true
 
 
 [*.{cs}]
-dotnet_diagnostic.IDE0005.severity = warning
+dotnet_diagnostic.IDE0005.severity = warning

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -30,7 +30,7 @@ public enum ErrorCode {
 
 public enum VmState {
     Init,
-    ExtensionsLaunched,
+    ExtensionsLaunch,
     ExtensionsFailed,
     VmAllocationFailed,
     Running,
@@ -192,7 +192,7 @@ public static class VmStateHelper {
             return
                 new[]{
                 VmState.Init,
-                VmState.ExtensionsLaunched,
+                VmState.ExtensionsLaunch,
                 VmState.Stopping
             };
         });
@@ -204,7 +204,7 @@ public static class VmStateHelper {
             return
                 new[]{
                 VmState.Init,
-                VmState.ExtensionsLaunched,
+                VmState.ExtensionsLaunch,
                 VmState.ExtensionsFailed,
                 VmState.VmAllocationFailed,
                 VmState.Running,

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -471,8 +471,8 @@ public record TeamsTemplate();
 public record GithubIssuesTemplate();
 
 public record Repro(
-    DateTimeOffset Timestamp,
+    [PartitionKey] Guid VmId,
-    Guid VmId,
+    [RowKey] Guid _,
     Guid TaskId,
     ReproConfig Config,
     VmState State,
@@ -589,6 +589,7 @@ public record Job(
     public UserInfo? UserInfo { get; set; }
 }
 
+public record Nsg(string Name, Region Region);
 
 public record WorkUnit(
     Guid JobId,

src/ApiService/ApiService/Program.cs

@@ -84,6 +84,7 @@ public class Program {
             .AddScoped<IVmOperations, VmOperations>()
             .AddScoped<ISecretsOperations, SecretsOperations>()
             .AddScoped<IJobOperations, JobOperations>()
+            .AddScoped<INsgOperations, NsgOperations>()
             .AddScoped<IScheduler, Scheduler>()
             .AddScoped<IConfig, Config>()
             .AddScoped<ILogAnalytics, LogAnalytics>()

src/ApiService/ApiService/TimerProxy.cs

@@ -10,7 +10,7 @@ public partial class TimerProxy {
 
     private readonly IScalesetOperations _scalesetOperations;
 
-    private readonly INsg _nsg;
+    private readonly INsgOperations _nsg;
 
     private readonly ICreds _creds;
 
@@ -18,7 +18,7 @@ public partial class TimerProxy {
 
     private readonly ISubnet _subnet;
 
-    public TimerProxy(ILogTracer logTracer, IProxyOperations proxies, IScalesetOperations scalesets, INsg nsg, ICreds creds, IConfigOperations configOperations, ISubnet subnet) {
+    public TimerProxy(ILogTracer logTracer, IProxyOperations proxies, IScalesetOperations scalesets, INsgOperations nsg, ICreds creds, IConfigOperations configOperations, ISubnet subnet) {
         _logger = logTracer;
         _proxYOperations = proxies;
         _scalesetOperations = scalesets;

src/ApiService/ApiService/TimerRepro.cs

@@ -15,10 +15,23 @@ public class TimerRepro {
         _reproOperations = reproOperations;
     }
 
+    // [Function("TimerRepro")]
     public async Async.Task Run([TimerTrigger("00:00:30")] TimerInfo myTimer) {
         var expired = _reproOperations.SearchExpired();
         await foreach (var repro in expired) {
-            _log.Info($"stopping repro: {repro?.VmId}");
+            _log.Info($"stopping repro: {repro.VmId}");
+            await _reproOperations.Stopping(repro);
+        }
+
+        var expiredVmIds = expired.Select(repro => repro?.VmId);
+
+        await foreach (var repro in _reproOperations.SearchStates(VmStateHelper.NeedsWork())) {
+            if (await expiredVmIds.ContainsAsync(repro.VmId)) {
+                // this VM already got processed during the expired phase
+                continue;
+            }
+            _log.Info($"update repro: {repro.VmId}");
+            await _reproOperations.ProcessStateUpdates(repro);
         }
     }
 

src/ApiService/ApiService/onefuzzlib/DiskOperations.cs

@@ -1,4 +1,5 @@
 using System.Threading.Tasks;
+using Azure;
 using Azure.ResourceManager.Compute;
 
 namespace Microsoft.OneFuzz.Service;
@@ -19,8 +20,19 @@ public class DiskOperations : IDiskOperations {
         _creds = creds;
     }
 
-    public Task<bool> DeleteDisk(string resourceGroup, string name) {
+    public async Task<bool> DeleteDisk(string resourceGroup, string name) {
-        throw new NotImplementedException();
+        try {
+            _logTracer.Info($"deleting disks {resourceGroup} : {name}");
+            var disk = await _creds.GetResourceGroupResource().GetDiskAsync(name);
+            if (disk != null) {
+                await disk.Value.DeleteAsync(WaitUntil.Started);
+                return true;
+            }
+        } catch (Exception e) {
+            _logTracer.Error($"unable to delete disk: {name} {e.Message}");
+            _logTracer.Exception(e);
+        }
+        return false;
     }
 
     public DiskCollection ListDisks(string resourceGroup) {

src/ApiService/ApiService/onefuzzlib/IpOperations.cs

@@ -1,4 +1,5 @@
-using Azure.ResourceManager.Network;
+using Azure;
+using Azure.ResourceManager.Network;
 
 namespace Microsoft.OneFuzz.Service;
 
@@ -32,11 +33,13 @@ public class IpOperations : IIpOperations {
         return await _creds.GetResourceGroupResource().GetPublicIPAddressAsync(name);
     }
 
-    public Async.Task DeleteNic(string resourceGroup, string name) {
+    public async System.Threading.Tasks.Task DeleteNic(string resourceGroup, string name) {
-        throw new NotImplementedException();
+        _logTracer.Info($"deleting nic {resourceGroup}:{name}");
+        await _creds.GetResourceGroupResource().GetNetworkInterfaceAsync(name).Result.Value.DeleteAsync(WaitUntil.Started);
     }
 
-    public Async.Task DeleteIp(string resourceGroup, string name) {
+    public async System.Threading.Tasks.Task DeleteIp(string resourceGroup, string name) {
-        throw new NotImplementedException();
+        _logTracer.Info($"deleting ip {resourceGroup}:{name}");
+        await _creds.GetResourceGroupResource().GetPublicIPAddressAsync(name).Result.Value.DeleteAsync(WaitUntil.Started);
     }
 }

src/ApiService/ApiService/onefuzzlib/NsgOperations.cs

@@ -3,24 +3,24 @@ using Azure.ResourceManager.Network;
 
 
 namespace Microsoft.OneFuzz.Service {
-    public interface INsg {
+    public interface INsgOperations {
         Async.Task<NetworkSecurityGroupResource?> GetNsg(string name);
         public Async.Task<Error?> AssociateSubnet(string name, VirtualNetworkResource vnet, SubnetResource subnet);
         IAsyncEnumerable<NetworkSecurityGroupResource> ListNsgs();
         bool OkToDelete(HashSet<string> active_regions, string nsg_region, string nsg_name);
         Async.Task<bool> StartDeleteNsg(string name);
 
-        Async.Task DissociateNic(NetworkInterfaceResource nic);
+        Async.Task<OneFuzzResultVoid> DissociateNic(Nsg nsg, NetworkInterfaceResource nic);
     }
 
 
-    public class Nsg : INsg {
+    public class NsgOperations : INsgOperations {
 
         private readonly ICreds _creds;
         private readonly ILogTracer _logTracer;
 
 
-        public Nsg(ICreds creds, ILogTracer logTracer) {
+        public NsgOperations(ICreds creds, ILogTracer logTracer) {
             _creds = creds;
             _logTracer = logTracer;
         }
@@ -46,8 +46,54 @@ namespace Microsoft.OneFuzz.Service {
             return null;
         }
 
-        public Async.Task DissociateNic(NetworkInterfaceResource nic) {
+        public async Async.Task<OneFuzzResultVoid> DissociateNic(Nsg nsg, NetworkInterfaceResource nic) {
-            throw new NotImplementedException();
+            if (nic.Data.NetworkSecurityGroup == null) {
+                return OneFuzzResultVoid.Ok();
+            }
+
+            var azureNsg = await GetNsg(nsg.Name);
+            if (azureNsg == null) {
+                return OneFuzzResultVoid.Error(
+                    ErrorCode.UNABLE_TO_FIND,
+                    new[] { $"cannot update nsg rules. nsg {nsg.Name} not found" }
+                );
+            }
+            if (azureNsg.Data.Id != nic.Data.NetworkSecurityGroup.Id) {
+                return OneFuzzResultVoid.Error(
+                    ErrorCode.UNABLE_TO_UPDATE,
+                    new[] {
+                        "network interface is not associated with this nsg.",
+                        $"nsg: {azureNsg.Id}, nic: {nic.Data.Name}, nic.nsg: {nic.Data.NetworkSecurityGroup.Id}"
+                    }
+                );
+            }
+
+            _logTracer.Info($"dissociating nic {nic.Data.Name} with nsg: {_creds.GetBaseResourceGroup()} {nsg.Name}");
+            nic.Data.NetworkSecurityGroup = null;
+            try {
+                await _creds.GetResourceGroupResource()
+                    .GetNetworkInterfaces()
+                    .CreateOrUpdateAsync(WaitUntil.Started, nic.Data.Name, nic.Data);
+            } catch (Exception e) {
+                if (IsConcurrentRequestError(e.Message)) {
+                    /*
+                    logging.debug(
+                        "dissociate nsg with nic had conflicts with ",
+                        "concurrent request, ignoring %s",
+                        err,
+                    )
+                    */
+                    return OneFuzzResultVoid.Ok();
+                }
+                return OneFuzzResultVoid.Error(
+                    ErrorCode.UNABLE_TO_UPDATE,
+                    new[] {
+                        $"Unable to dissociate nsg {nsg.Name} with nic {nic.Data.Name} due to {e.Message} {e.StackTrace}"
+                    }
+                );
+            }
+
+            return OneFuzzResultVoid.Ok();
         }
 
         public async Async.Task<NetworkSecurityGroupResource?> GetNsg(string name) {
@@ -76,5 +122,9 @@ namespace Microsoft.OneFuzz.Service {
             await nsg.Value.DeleteAsync(WaitUntil.Completed);
             return true;
         }
+
+        private static bool IsConcurrentRequestError(string err) {
+            return err.Contains("The request failed due to conflict with a concurrent request");
+        }
     }
 }

src/ApiService/ApiService/onefuzzlib/ReproOperations.cs

@@ -3,7 +3,11 @@
 namespace Microsoft.OneFuzz.Service;
 
 public interface IReproOperations : IStatefulOrm<Repro, VmState> {
-    public IAsyncEnumerable<Repro?> SearchExpired();
+    public IAsyncEnumerable<Repro> SearchExpired();
+
+    public System.Threading.Tasks.Task Stopping(Repro repro);
+
+    public IAsyncEnumerable<Repro> SearchStates(IEnumerable<VmState>? States);
 }
 
 public class ReproOperations : StatefulOrm<Repro, VmState>, IReproOperations {
@@ -30,7 +34,7 @@ public class ReproOperations : StatefulOrm<Repro, VmState>, IReproOperations {
         _vmOperations = vmOperations;
     }
 
-    public IAsyncEnumerable<Repro?> SearchExpired() {
+    public IAsyncEnumerable<Repro> SearchExpired() {
         return QueryAsync(filter: $"end_time lt datetime'{DateTime.UtcNow.ToString("o")}'");
     }
 
@@ -86,4 +90,15 @@ public class ReproOperations : StatefulOrm<Repro, VmState>, IReproOperations {
         _logTracer.Info($"vm stopped: {repro.VmId}");
         await Delete(repro);
     }
+
+    public IAsyncEnumerable<Repro> SearchStates(IEnumerable<VmState>? states) {
+        string? queryString = null;
+        if (states != null) {
+            queryString = string.Join(
+                " or ",
+                states.Select(s => $"state eq '{s}'")
+            );
+        }
+        return QueryAsync(queryString);
+    }
 }

src/ApiService/ApiService/onefuzzlib/TaskOperations.cs

@@ -58,8 +58,10 @@ public class TaskOperations : StatefulOrm<Task, TaskState>, ITaskOperations {
                 queryString += " and ";
             }
 
-            var statesString = string.Join(",", states);
+            queryString += "(" + string.Join(
-            queryString += $"state in ({statesString})";
+                " or ",
+                states.Select(s => $"state eq '{s}'")
+            ) + ")";
         }
 
         return QueryAsync(filter: queryString);
@@ -69,7 +71,6 @@ public class TaskOperations : StatefulOrm<Task, TaskState>, ITaskOperations {
         throw new NotImplementedException();
     }
 
-
     public IAsyncEnumerable<Task> SearchExpired() {
         var timeFilter = $"end_time lt Datetime'{DateTimeOffset.UtcNow.ToString("o") }'";
         return QueryAsync(filter: timeFilter);

src/ApiService/ApiService/onefuzzlib/VmOperations.cs

@@ -1,4 +1,5 @@
-using Azure.ResourceManager.Compute;
+using Azure;
+using Azure.ResourceManager.Compute;
 
 
 namespace Microsoft.OneFuzz.Service;
@@ -23,11 +24,14 @@ public class VmOperations : IVmOperations {
 
     private IDiskOperations _diskOperations;
 
-    public VmOperations(ILogTracer log, ICreds creds, IIpOperations ipOperations, IDiskOperations diskOperations) {
+    private INsgOperations _nsgOperations;
+
+    public VmOperations(ILogTracer log, ICreds creds, IIpOperations ipOperations, IDiskOperations diskOperations, INsgOperations nsgOperations) {
         _logTracer = log;
         _creds = creds;
         _ipOperations = ipOperations;
         _diskOperations = diskOperations;
+        _nsgOperations = nsgOperations;
     }
     public async Async.Task<bool> IsDeleted(Vm vm) {
         return !(await HasComponents(vm.Name));
@@ -72,7 +76,7 @@ public class VmOperations : IVmOperations {
         _logTracer.Info($"deleting vm components {resourceGroup}:{name}");
         if (GetVm(name) != null) {
             _logTracer.Info($"deleting vm {resourceGroup}:{name}");
-            DeleteVm(name);
+            await DeleteVm(name);
             return false;
         }
 
@@ -80,7 +84,7 @@ public class VmOperations : IVmOperations {
         if (nic != null) {
             _logTracer.Info($"deleting nic {resourceGroup}:{name}");
             if (nic.Data.NetworkSecurityGroup != null && nsg != null) {
-                await nsg.DissociateNic(nic);
+                await _nsgOperations.DissociateNic((Nsg)nsg, nic);
                 return false;
             }
             await _ipOperations.DeleteNic(resourceGroup, name);
@@ -108,7 +112,10 @@ public class VmOperations : IVmOperations {
         return true;
     }
 
-    public void DeleteVm(string name) {
+    public async System.Threading.Tasks.Task DeleteVm(string name) {
-        throw new NotImplementedException();
+        _logTracer.Info($"deleting vm: {_creds.GetBaseResourceGroup()} {name}");
+        await _creds.GetResourceGroupResource()
+            .GetVirtualMachineAsync(name).Result.Value
+            .DeleteAsync(WaitUntil.Started);
     }
 }