diff --git a/src/ApiService/.editorconfig b/src/ApiService/.editorconfig index 3ba00364b..741c0ab90 100644 --- a/src/ApiService/.editorconfig +++ b/src/ApiService/.editorconfig @@ -130,4 +130,4 @@ csharp_preserve_single_line_blocks = true [*.{cs}] -dotnet_diagnostic.IDE0005.severity = warning \ No newline at end of file +dotnet_diagnostic.IDE0005.severity = warning diff --git a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs index 3d2566f85..8b0969b99 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Enums.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Enums.cs @@ -30,7 +30,7 @@ public enum ErrorCode { public enum VmState { Init, - ExtensionsLaunched, + ExtensionsLaunch, ExtensionsFailed, VmAllocationFailed, Running, @@ -192,7 +192,7 @@ public static class VmStateHelper { return new[]{ VmState.Init, - VmState.ExtensionsLaunched, + VmState.ExtensionsLaunch, VmState.Stopping }; }); @@ -204,7 +204,7 @@ public static class VmStateHelper { return new[]{ VmState.Init, - VmState.ExtensionsLaunched, + VmState.ExtensionsLaunch, VmState.ExtensionsFailed, VmState.VmAllocationFailed, VmState.Running, diff --git a/src/ApiService/ApiService/OneFuzzTypes/Model.cs b/src/ApiService/ApiService/OneFuzzTypes/Model.cs index e074c552b..d98c277b8 100644 --- a/src/ApiService/ApiService/OneFuzzTypes/Model.cs +++ b/src/ApiService/ApiService/OneFuzzTypes/Model.cs @@ -471,8 +471,8 @@ public record TeamsTemplate(); public record GithubIssuesTemplate(); public record Repro( - DateTimeOffset Timestamp, - Guid VmId, + [PartitionKey] Guid VmId, + [RowKey] Guid _, Guid TaskId, ReproConfig Config, VmState State, @@ -589,6 +589,7 @@ public record Job( public UserInfo? UserInfo { get; set; } } +public record Nsg(string Name, Region Region); public record WorkUnit( Guid JobId, diff --git a/src/ApiService/ApiService/Program.cs b/src/ApiService/ApiService/Program.cs index 0cae4f43e..464e5d5ed 100644 --- a/src/ApiService/ApiService/Program.cs +++ b/src/ApiService/ApiService/Program.cs @@ -84,6 +84,7 @@ public class Program { .AddScoped() .AddScoped() .AddScoped() + .AddScoped() .AddScoped() .AddScoped() .AddScoped() diff --git a/src/ApiService/ApiService/TimerProxy.cs b/src/ApiService/ApiService/TimerProxy.cs index cc2668491..59d6c3187 100644 --- a/src/ApiService/ApiService/TimerProxy.cs +++ b/src/ApiService/ApiService/TimerProxy.cs @@ -10,7 +10,7 @@ public partial class TimerProxy { private readonly IScalesetOperations _scalesetOperations; - private readonly INsg _nsg; + private readonly INsgOperations _nsg; private readonly ICreds _creds; @@ -18,7 +18,7 @@ public partial class TimerProxy { private readonly ISubnet _subnet; - public TimerProxy(ILogTracer logTracer, IProxyOperations proxies, IScalesetOperations scalesets, INsg nsg, ICreds creds, IConfigOperations configOperations, ISubnet subnet) { + public TimerProxy(ILogTracer logTracer, IProxyOperations proxies, IScalesetOperations scalesets, INsgOperations nsg, ICreds creds, IConfigOperations configOperations, ISubnet subnet) { _logger = logTracer; _proxYOperations = proxies; _scalesetOperations = scalesets; diff --git a/src/ApiService/ApiService/TimerRepro.cs b/src/ApiService/ApiService/TimerRepro.cs index 648e85d7d..f45d24085 100644 --- a/src/ApiService/ApiService/TimerRepro.cs +++ b/src/ApiService/ApiService/TimerRepro.cs @@ -15,10 +15,23 @@ public class TimerRepro { _reproOperations = reproOperations; } + // [Function("TimerRepro")] public async Async.Task Run([TimerTrigger("00:00:30")] TimerInfo myTimer) { var expired = _reproOperations.SearchExpired(); await foreach (var repro in expired) { - _log.Info($"stopping repro: {repro?.VmId}"); + _log.Info($"stopping repro: {repro.VmId}"); + await _reproOperations.Stopping(repro); + } + + var expiredVmIds = expired.Select(repro => repro?.VmId); + + await foreach (var repro in _reproOperations.SearchStates(VmStateHelper.NeedsWork())) { + if (await expiredVmIds.ContainsAsync(repro.VmId)) { + // this VM already got processed during the expired phase + continue; + } + _log.Info($"update repro: {repro.VmId}"); + await _reproOperations.ProcessStateUpdates(repro); } } diff --git a/src/ApiService/ApiService/onefuzzlib/DiskOperations.cs b/src/ApiService/ApiService/onefuzzlib/DiskOperations.cs index 3e14f4ad1..6225bfeca 100644 --- a/src/ApiService/ApiService/onefuzzlib/DiskOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/DiskOperations.cs @@ -1,4 +1,5 @@ using System.Threading.Tasks; +using Azure; using Azure.ResourceManager.Compute; namespace Microsoft.OneFuzz.Service; @@ -19,8 +20,19 @@ public class DiskOperations : IDiskOperations { _creds = creds; } - public Task DeleteDisk(string resourceGroup, string name) { - throw new NotImplementedException(); + public async Task DeleteDisk(string resourceGroup, string name) { + try { + _logTracer.Info($"deleting disks {resourceGroup} : {name}"); + var disk = await _creds.GetResourceGroupResource().GetDiskAsync(name); + if (disk != null) { + await disk.Value.DeleteAsync(WaitUntil.Started); + return true; + } + } catch (Exception e) { + _logTracer.Error($"unable to delete disk: {name} {e.Message}"); + _logTracer.Exception(e); + } + return false; } public DiskCollection ListDisks(string resourceGroup) { diff --git a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs index a48034359..04d992d25 100644 --- a/src/ApiService/ApiService/onefuzzlib/IpOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/IpOperations.cs @@ -1,4 +1,5 @@ -using Azure.ResourceManager.Network; +using Azure; +using Azure.ResourceManager.Network; namespace Microsoft.OneFuzz.Service; @@ -32,11 +33,13 @@ public class IpOperations : IIpOperations { return await _creds.GetResourceGroupResource().GetPublicIPAddressAsync(name); } - public Async.Task DeleteNic(string resourceGroup, string name) { - throw new NotImplementedException(); + public async System.Threading.Tasks.Task DeleteNic(string resourceGroup, string name) { + _logTracer.Info($"deleting nic {resourceGroup}:{name}"); + await _creds.GetResourceGroupResource().GetNetworkInterfaceAsync(name).Result.Value.DeleteAsync(WaitUntil.Started); } - public Async.Task DeleteIp(string resourceGroup, string name) { - throw new NotImplementedException(); + public async System.Threading.Tasks.Task DeleteIp(string resourceGroup, string name) { + _logTracer.Info($"deleting ip {resourceGroup}:{name}"); + await _creds.GetResourceGroupResource().GetPublicIPAddressAsync(name).Result.Value.DeleteAsync(WaitUntil.Started); } } diff --git a/src/ApiService/ApiService/onefuzzlib/Nsg.cs b/src/ApiService/ApiService/onefuzzlib/NsgOperations.cs similarity index 54% rename from src/ApiService/ApiService/onefuzzlib/Nsg.cs rename to src/ApiService/ApiService/onefuzzlib/NsgOperations.cs index bb55b5687..cf9efb227 100644 --- a/src/ApiService/ApiService/onefuzzlib/Nsg.cs +++ b/src/ApiService/ApiService/onefuzzlib/NsgOperations.cs @@ -3,24 +3,24 @@ using Azure.ResourceManager.Network; namespace Microsoft.OneFuzz.Service { - public interface INsg { + public interface INsgOperations { Async.Task GetNsg(string name); public Async.Task AssociateSubnet(string name, VirtualNetworkResource vnet, SubnetResource subnet); IAsyncEnumerable ListNsgs(); bool OkToDelete(HashSet active_regions, string nsg_region, string nsg_name); Async.Task StartDeleteNsg(string name); - Async.Task DissociateNic(NetworkInterfaceResource nic); + Async.Task DissociateNic(Nsg nsg, NetworkInterfaceResource nic); } - public class Nsg : INsg { + public class NsgOperations : INsgOperations { private readonly ICreds _creds; private readonly ILogTracer _logTracer; - public Nsg(ICreds creds, ILogTracer logTracer) { + public NsgOperations(ICreds creds, ILogTracer logTracer) { _creds = creds; _logTracer = logTracer; } @@ -46,8 +46,54 @@ namespace Microsoft.OneFuzz.Service { return null; } - public Async.Task DissociateNic(NetworkInterfaceResource nic) { - throw new NotImplementedException(); + public async Async.Task DissociateNic(Nsg nsg, NetworkInterfaceResource nic) { + if (nic.Data.NetworkSecurityGroup == null) { + return OneFuzzResultVoid.Ok(); + } + + var azureNsg = await GetNsg(nsg.Name); + if (azureNsg == null) { + return OneFuzzResultVoid.Error( + ErrorCode.UNABLE_TO_FIND, + new[] { $"cannot update nsg rules. nsg {nsg.Name} not found" } + ); + } + if (azureNsg.Data.Id != nic.Data.NetworkSecurityGroup.Id) { + return OneFuzzResultVoid.Error( + ErrorCode.UNABLE_TO_UPDATE, + new[] { + "network interface is not associated with this nsg.", + $"nsg: {azureNsg.Id}, nic: {nic.Data.Name}, nic.nsg: {nic.Data.NetworkSecurityGroup.Id}" + } + ); + } + + _logTracer.Info($"dissociating nic {nic.Data.Name} with nsg: {_creds.GetBaseResourceGroup()} {nsg.Name}"); + nic.Data.NetworkSecurityGroup = null; + try { + await _creds.GetResourceGroupResource() + .GetNetworkInterfaces() + .CreateOrUpdateAsync(WaitUntil.Started, nic.Data.Name, nic.Data); + } catch (Exception e) { + if (IsConcurrentRequestError(e.Message)) { + /* + logging.debug( + "dissociate nsg with nic had conflicts with ", + "concurrent request, ignoring %s", + err, + ) + */ + return OneFuzzResultVoid.Ok(); + } + return OneFuzzResultVoid.Error( + ErrorCode.UNABLE_TO_UPDATE, + new[] { + $"Unable to dissociate nsg {nsg.Name} with nic {nic.Data.Name} due to {e.Message} {e.StackTrace}" + } + ); + } + + return OneFuzzResultVoid.Ok(); } public async Async.Task GetNsg(string name) { @@ -76,5 +122,9 @@ namespace Microsoft.OneFuzz.Service { await nsg.Value.DeleteAsync(WaitUntil.Completed); return true; } + + private static bool IsConcurrentRequestError(string err) { + return err.Contains("The request failed due to conflict with a concurrent request"); + } } } diff --git a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs index f5c28a637..43faf70dc 100644 --- a/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/ReproOperations.cs @@ -3,7 +3,11 @@ namespace Microsoft.OneFuzz.Service; public interface IReproOperations : IStatefulOrm { - public IAsyncEnumerable SearchExpired(); + public IAsyncEnumerable SearchExpired(); + + public System.Threading.Tasks.Task Stopping(Repro repro); + + public IAsyncEnumerable SearchStates(IEnumerable? States); } public class ReproOperations : StatefulOrm, IReproOperations { @@ -30,7 +34,7 @@ public class ReproOperations : StatefulOrm, IReproOperations { _vmOperations = vmOperations; } - public IAsyncEnumerable SearchExpired() { + public IAsyncEnumerable SearchExpired() { return QueryAsync(filter: $"end_time lt datetime'{DateTime.UtcNow.ToString("o")}'"); } @@ -86,4 +90,15 @@ public class ReproOperations : StatefulOrm, IReproOperations { _logTracer.Info($"vm stopped: {repro.VmId}"); await Delete(repro); } + + public IAsyncEnumerable SearchStates(IEnumerable? states) { + string? queryString = null; + if (states != null) { + queryString = string.Join( + " or ", + states.Select(s => $"state eq '{s}'") + ); + } + return QueryAsync(queryString); + } } diff --git a/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs b/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs index 5a9caf366..aed2bf4c8 100644 --- a/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/TaskOperations.cs @@ -58,8 +58,10 @@ public class TaskOperations : StatefulOrm, ITaskOperations { queryString += " and "; } - var statesString = string.Join(",", states); - queryString += $"state in ({statesString})"; + queryString += "(" + string.Join( + " or ", + states.Select(s => $"state eq '{s}'") + ) + ")"; } return QueryAsync(filter: queryString); @@ -69,7 +71,6 @@ public class TaskOperations : StatefulOrm, ITaskOperations { throw new NotImplementedException(); } - public IAsyncEnumerable SearchExpired() { var timeFilter = $"end_time lt Datetime'{DateTimeOffset.UtcNow.ToString("o") }'"; return QueryAsync(filter: timeFilter); diff --git a/src/ApiService/ApiService/onefuzzlib/VmOperations.cs b/src/ApiService/ApiService/onefuzzlib/VmOperations.cs index 44101b620..39a443728 100644 --- a/src/ApiService/ApiService/onefuzzlib/VmOperations.cs +++ b/src/ApiService/ApiService/onefuzzlib/VmOperations.cs @@ -1,4 +1,5 @@ -using Azure.ResourceManager.Compute; +using Azure; +using Azure.ResourceManager.Compute; namespace Microsoft.OneFuzz.Service; @@ -23,11 +24,14 @@ public class VmOperations : IVmOperations { private IDiskOperations _diskOperations; - public VmOperations(ILogTracer log, ICreds creds, IIpOperations ipOperations, IDiskOperations diskOperations) { + private INsgOperations _nsgOperations; + + public VmOperations(ILogTracer log, ICreds creds, IIpOperations ipOperations, IDiskOperations diskOperations, INsgOperations nsgOperations) { _logTracer = log; _creds = creds; _ipOperations = ipOperations; _diskOperations = diskOperations; + _nsgOperations = nsgOperations; } public async Async.Task IsDeleted(Vm vm) { return !(await HasComponents(vm.Name)); @@ -72,7 +76,7 @@ public class VmOperations : IVmOperations { _logTracer.Info($"deleting vm components {resourceGroup}:{name}"); if (GetVm(name) != null) { _logTracer.Info($"deleting vm {resourceGroup}:{name}"); - DeleteVm(name); + await DeleteVm(name); return false; } @@ -80,7 +84,7 @@ public class VmOperations : IVmOperations { if (nic != null) { _logTracer.Info($"deleting nic {resourceGroup}:{name}"); if (nic.Data.NetworkSecurityGroup != null && nsg != null) { - await nsg.DissociateNic(nic); + await _nsgOperations.DissociateNic((Nsg)nsg, nic); return false; } await _ipOperations.DeleteNic(resourceGroup, name); @@ -108,7 +112,10 @@ public class VmOperations : IVmOperations { return true; } - public void DeleteVm(string name) { - throw new NotImplementedException(); + public async System.Threading.Tasks.Task DeleteVm(string name) { + _logTracer.Info($"deleting vm: {_creds.GetBaseResourceGroup()} {name}"); + await _creds.GetResourceGroupResource() + .GetVirtualMachineAsync(name).Result.Value + .DeleteAsync(WaitUntil.Started); } }