mirror of
https://github.com/microsoft/onefuzz.git
synced 2025-06-19 21:13:46 +00:00
Fix logic of marking task as failed (#3083)
* Fix logic of markaing task as failed - Do not mark task as failed if it is already in the shutting down state - accumulate errors when setting task error to understand the context - refactor the Error record * fix tests * format * Fix build * Update src/ApiService/ApiService/onefuzzlib/ImageReference.cs Co-authored-by: George Pollard <porges@porg.es> * Update src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com> --------- Co-authored-by: George Pollard <porges@porg.es> Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>
This commit is contained in:
@ -35,11 +35,7 @@ public class AgentCanSchedule {
|
||||
_log.Warning($"Unable to find {canScheduleRequest.MachineId:Tag:MachineId}");
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
ErrorCode.UNABLE_TO_FIND,
|
||||
new string[] {
|
||||
"unable to find node"
|
||||
}),
|
||||
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
|
||||
canScheduleRequest.MachineId.ToString());
|
||||
}
|
||||
|
||||
|
@ -35,7 +35,7 @@ public class AgentEvents {
|
||||
NodeStateUpdate updateEvent => await OnStateUpdate(envelope.MachineId, updateEvent),
|
||||
WorkerEvent workerEvent => await OnWorkerEvent(envelope.MachineId, workerEvent),
|
||||
NodeEvent nodeEvent => await OnNodeEvent(envelope.MachineId, nodeEvent),
|
||||
_ => new Error(ErrorCode.INVALID_REQUEST, new string[] { $"invalid node event: {envelope.Event.GetType().Name}" }),
|
||||
_ => Error.Create(ErrorCode.INVALID_REQUEST, $"invalid node event: {envelope.Event.GetType().Name}"),
|
||||
};
|
||||
|
||||
if (error is Error e) {
|
||||
@ -114,17 +114,17 @@ public class AgentEvents {
|
||||
} else if (ev.State == NodeState.SettingUp) {
|
||||
if (ev.Data is NodeSettingUpEventData settingUpData) {
|
||||
if (!settingUpData.Tasks.Any()) {
|
||||
return new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] {
|
||||
$"setup without tasks. machine_id: {machineId}",
|
||||
});
|
||||
return Error.Create(ErrorCode.INVALID_REQUEST,
|
||||
$"setup without tasks. machine_id: {machineId}"
|
||||
);
|
||||
}
|
||||
|
||||
foreach (var taskId in settingUpData.Tasks) {
|
||||
var task = await _context.TaskOperations.GetByTaskId(taskId);
|
||||
if (task is null) {
|
||||
return new Error(
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { $"unable to find task: {taskId}" });
|
||||
$"unable to find task: {taskId}");
|
||||
}
|
||||
|
||||
_log.Info($"node starting task. {machineId:Tag:MachineId} {task.JobId:Tag:JobId} {task.TaskId:Tag:TaskId}");
|
||||
@ -154,7 +154,7 @@ public class AgentEvents {
|
||||
if (ev.Data is NodeDoneEventData doneData) {
|
||||
if (doneData.Error is not null) {
|
||||
var errorText = EntityConverter.ToJsonString(doneData);
|
||||
error = new Error(ErrorCode.TASK_FAILED, Errors: new string[] { errorText });
|
||||
error = Error.Create(ErrorCode.TASK_FAILED, errorText);
|
||||
_log.Error($"node 'done' {machineId:Tag:MachineId} - {errorText:Tag:Error}");
|
||||
}
|
||||
}
|
||||
@ -178,9 +178,9 @@ public class AgentEvents {
|
||||
return await OnWorkerEventRunning(machineId, ev.Running);
|
||||
}
|
||||
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { "WorkerEvent should have either 'done' or 'running' set" });
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
"WorkerEvent should have either 'done' or 'running' set");
|
||||
}
|
||||
|
||||
private async Async.Task<Error?> OnWorkerEventRunning(Guid machineId, WorkerRunningEvent running) {
|
||||
@ -189,15 +189,11 @@ public class AgentEvents {
|
||||
_context.NodeOperations.GetByMachineId(machineId));
|
||||
|
||||
if (task is null) {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { $"unable to find task: {running.TaskId}" });
|
||||
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find task: {running.TaskId}");
|
||||
}
|
||||
|
||||
if (node is null) {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { $"unable to find node: {machineId}" });
|
||||
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find node: {machineId}");
|
||||
}
|
||||
|
||||
if (!node.State.ReadyForReset()) {
|
||||
@ -240,15 +236,11 @@ public class AgentEvents {
|
||||
_context.NodeOperations.GetByMachineId(machineId));
|
||||
|
||||
if (task is null) {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { $"unable to find task: {done.TaskId}" });
|
||||
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find task: {done.TaskId}");
|
||||
}
|
||||
|
||||
if (node is null) {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { $"unable to find node: {machineId}" });
|
||||
return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find node: {machineId}");
|
||||
}
|
||||
|
||||
// trim stdout/stderr if too long
|
||||
@ -272,13 +264,12 @@ public class AgentEvents {
|
||||
} else {
|
||||
await _context.TaskOperations.MarkFailed(
|
||||
task,
|
||||
new Error(
|
||||
Code: ErrorCode.TASK_FAILED,
|
||||
Errors: new string[] {
|
||||
Error.Create(
|
||||
ErrorCode.TASK_FAILED,
|
||||
$"task failed. exit_status:{done.ExitStatus}",
|
||||
done.Stdout,
|
||||
done.Stderr,
|
||||
}));
|
||||
done.Stderr
|
||||
));
|
||||
|
||||
// keep node if any keep options are set
|
||||
if ((task.Config.Debug?.Contains(TaskDebugFlag.KeepNodeOnFailure) == true)
|
||||
|
@ -40,9 +40,9 @@ public class AgentRegistration {
|
||||
if (machineId == Guid.Empty) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] { "'machine_id' query parameter must be provided" }),
|
||||
"'machine_id' query parameter must be provided"),
|
||||
"agent registration");
|
||||
}
|
||||
|
||||
@ -50,9 +50,9 @@ public class AgentRegistration {
|
||||
if (agentNode is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] { $"unable to find a registration associated with machine_id '{machineId}'" }),
|
||||
$"unable to find a registration associated with machine_id '{machineId}'"),
|
||||
"agent registration");
|
||||
}
|
||||
|
||||
@ -60,9 +60,9 @@ public class AgentRegistration {
|
||||
if (!pool.IsOk) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] { "unable to find a pool associated with the provided machine_id" }),
|
||||
"unable to find a pool associated with the provided machine_id"),
|
||||
"agent registration");
|
||||
}
|
||||
|
||||
@ -101,18 +101,16 @@ public class AgentRegistration {
|
||||
if (machineId == Guid.Empty) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] { "'machine_id' query parameter must be provided" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST, "'machine_id' query parameter must be provided"),
|
||||
"agent registration");
|
||||
}
|
||||
|
||||
if (poolName is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] { "'pool_name' query parameter must be provided" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST, "'pool_name' query parameter must be provided"),
|
||||
"agent registration");
|
||||
}
|
||||
|
||||
@ -124,9 +122,8 @@ public class AgentRegistration {
|
||||
if (!poolResult.IsOk) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new[] { $"unable to find pool '{poolName}'" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST, $"unable to find pool '{poolName}'"),
|
||||
"agent registration");
|
||||
}
|
||||
|
||||
@ -140,9 +137,9 @@ public class AgentRegistration {
|
||||
if (os != null && pool.Os != os) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new[] { $"OS mismatch: pool '{poolName}' is configured for '{pool.Os}', but agent is running '{os}'" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
$"OS mismatch: pool '{poolName}' is configured for '{pool.Os}', but agent is running '{os}'"),
|
||||
"agent registration");
|
||||
}
|
||||
|
||||
|
@ -39,9 +39,9 @@ public class ContainersFunction {
|
||||
if (container is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new[] { "invalid container" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
"invalid container"),
|
||||
context: get.Name.String);
|
||||
}
|
||||
|
||||
@ -101,9 +101,9 @@ public class ContainersFunction {
|
||||
if (sas is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new[] { "invalid container" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
"invalid container"),
|
||||
context: post.Name.String);
|
||||
}
|
||||
|
||||
|
@ -25,9 +25,9 @@ public class Download {
|
||||
if (queryContainer is null || !Container.TryParse(queryContainer, out var container)) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] { "'container' query parameter must be provided and valid" }),
|
||||
"'container' query parameter must be provided and valid"),
|
||||
"download");
|
||||
}
|
||||
|
||||
@ -35,9 +35,9 @@ public class Download {
|
||||
if (filename is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] { "'filename' query parameter must be provided" }),
|
||||
"'filename' query parameter must be provided"),
|
||||
"download");
|
||||
}
|
||||
|
||||
|
@ -59,9 +59,7 @@ public class Jobs {
|
||||
if (containerSas is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_CREATE_CONTAINER,
|
||||
Errors: new string[] { "unable to create logs container " }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_CREATE_CONTAINER, "unable to create logs container "),
|
||||
"logs");
|
||||
}
|
||||
|
||||
@ -73,9 +71,9 @@ public class Jobs {
|
||||
_logTracer.WithTag("HttpRequest", "POST").WithHttpStatus(r.ErrorV).Error($"failed to insert job {job.JobId:Tag:JobId}");
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_CREATE,
|
||||
Errors: new string[] { "unable to create job " }
|
||||
Error.Create(
|
||||
ErrorCode.UNABLE_TO_CREATE,
|
||||
"unable to create job"
|
||||
),
|
||||
"job");
|
||||
}
|
||||
@ -95,9 +93,9 @@ public class Jobs {
|
||||
if (job is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_JOB,
|
||||
Errors: new string[] { "no such job" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_JOB,
|
||||
"no such job"),
|
||||
context: jobId.ToString());
|
||||
}
|
||||
|
||||
@ -124,9 +122,7 @@ public class Jobs {
|
||||
if (job is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_JOB,
|
||||
Errors: new string[] { "no such job" }),
|
||||
Error.Create(ErrorCode.INVALID_JOB, "no such job"),
|
||||
context: jobId.ToString());
|
||||
}
|
||||
|
||||
|
@ -72,7 +72,7 @@ public class JinjaToScriban {
|
||||
_log.Info($"Migrated notification: {notification.NotificationId} to jinja");
|
||||
} else {
|
||||
failedNotificationIds.Add(notification.NotificationId);
|
||||
_log.Error(new Error(ErrorCode.UNABLE_TO_UPDATE, new[] { r.ErrorV.Reason, r.ErrorV.Status.ToString() }));
|
||||
_log.Error(Error.Create(ErrorCode.UNABLE_TO_UPDATE, r.ErrorV.Reason, r.ErrorV.Status.ToString()));
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
failedNotificationIds.Add(notification.NotificationId);
|
||||
|
@ -38,9 +38,7 @@ public class Node {
|
||||
if (node is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_FIND,
|
||||
Errors: new string[] { "unable to find node" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
|
||||
context: machineId.ToString());
|
||||
}
|
||||
|
||||
@ -94,9 +92,7 @@ public class Node {
|
||||
if (node is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_FIND,
|
||||
Errors: new string[] { "unable to find node" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
|
||||
context: patch.MachineId.ToString());
|
||||
}
|
||||
|
||||
@ -130,9 +126,7 @@ public class Node {
|
||||
if (node is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_FIND,
|
||||
Errors: new string[] { "unable to find node" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
|
||||
context: post.MachineId.ToString());
|
||||
}
|
||||
|
||||
@ -166,9 +160,7 @@ public class Node {
|
||||
if (node is null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_FIND,
|
||||
new string[] { "unable to find node" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
|
||||
context: delete.MachineId.ToString());
|
||||
}
|
||||
|
||||
|
@ -30,7 +30,7 @@ public class NodeAddSshKey {
|
||||
if (node == null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(ErrorCode.UNABLE_TO_FIND, new[] { "unable to find node" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
|
||||
$"{request.OkV.MachineId}");
|
||||
}
|
||||
|
||||
|
@ -61,18 +61,18 @@ public class Notifications {
|
||||
var entries = await _context.NotificationOperations.SearchByPartitionKeys(new[] { $"{request.OkV.NotificationId}" }).ToListAsync();
|
||||
|
||||
if (entries.Count == 0) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find notification" }), context: "notification delete");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find notification"), context: "notification delete");
|
||||
}
|
||||
|
||||
if (entries.Count > 1) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "error identifying Notification" }), context: "notification delete");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "error identifying Notification"), context: "notification delete");
|
||||
}
|
||||
|
||||
var result = await _context.NotificationOperations.Delete(entries[0]);
|
||||
|
||||
if (!result.IsOk) {
|
||||
var (status, error) = result.ErrorV;
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.UNABLE_TO_UPDATE, new[] { error }), "notification delete");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.UNABLE_TO_UPDATE, error), "notification delete");
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
|
@ -63,9 +63,7 @@ public class Pool {
|
||||
if (pool.IsOk) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { "pool with that name already exists" }),
|
||||
Error.Create(ErrorCode.INVALID_REQUEST, "pool with that name already exists"),
|
||||
"PoolCreate");
|
||||
}
|
||||
var newPool = await _context.PoolOperations.Create(name: create.Name, os: create.Os, architecture: create.Arch, managed: create.Managed, objectId: create.ObjectId);
|
||||
@ -89,9 +87,7 @@ public class Pool {
|
||||
if (!pool.IsOk) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { "pool with that name does not exist" }),
|
||||
Error.Create(ErrorCode.INVALID_REQUEST, "pool with that name does not exist"),
|
||||
"PoolUpdate");
|
||||
}
|
||||
|
||||
@ -100,9 +96,7 @@ public class Pool {
|
||||
if (updatePool.IsOk) {
|
||||
return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(updated), true));
|
||||
} else {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new string[] { updatePool.ErrorV.Reason }), "PoolUpdate");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, updatePool.ErrorV.Reason), "PoolUpdate");
|
||||
}
|
||||
|
||||
|
||||
|
@ -63,7 +63,7 @@ public class Proxy {
|
||||
machineId: machineId, dstPort: dstPort).ToListAsync();
|
||||
|
||||
if (!forwards.Any()) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "no forwards for scaleset and node" }), "debug_proxy get");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "no forwards for scaleset and node"), "debug_proxy get");
|
||||
}
|
||||
|
||||
var response = req.CreateResponse();
|
||||
@ -78,7 +78,7 @@ public class Proxy {
|
||||
await r.WriteAsJsonAsync(new ProxyList(proxies));
|
||||
return r;
|
||||
default:
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "ProxyGet must provide all or none of the following: scaleset_id, machine_id, dst_port" }), "debug_proxy get");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "ProxyGet must provide all or none of the following: scaleset_id, machine_id, dst_port"), "debug_proxy get");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -35,7 +35,7 @@ public class ReproVmss {
|
||||
var vm = await _context.ReproOperations.SearchByPartitionKeys(new[] { $"{request.OkV.VmId}" }).FirstOrDefaultAsync();
|
||||
|
||||
if (vm == null) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "no such VM" }), $"{request.OkV.VmId}");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "no such VM"), $"{request.OkV.VmId}");
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
@ -98,7 +98,7 @@ public class ReproVmss {
|
||||
if (request.OkV.VmId == null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(ErrorCode.INVALID_REQUEST, new[] { "missing vm_id" }),
|
||||
Error.Create(ErrorCode.INVALID_REQUEST, "missing vm_id"),
|
||||
context: "repro delete");
|
||||
}
|
||||
|
||||
@ -107,7 +107,7 @@ public class ReproVmss {
|
||||
if (vm == null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(ErrorCode.INVALID_REQUEST, new[] { "no such vm" }),
|
||||
Error.Create(ErrorCode.INVALID_REQUEST, "no such vm"),
|
||||
context: "repro delete");
|
||||
}
|
||||
|
||||
|
@ -70,9 +70,7 @@ public class Scaleset {
|
||||
if (!pool.Managed) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_CREATE,
|
||||
Errors: new string[] { "scalesets can only be added to managed pools " }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_CREATE, "scalesets can only be added to managed pools "),
|
||||
context: "ScalesetCreate");
|
||||
}
|
||||
|
||||
@ -96,9 +94,7 @@ public class Scaleset {
|
||||
if (!validRegions.Contains(create.Region)) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_CREATE,
|
||||
Errors: new string[] { "invalid region" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_CREATE, "invalid region"),
|
||||
context: "ScalesetCreate");
|
||||
}
|
||||
|
||||
@ -109,9 +105,7 @@ public class Scaleset {
|
||||
if (!availableSkus.Contains(create.VmSku)) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_CREATE,
|
||||
Errors: new string[] { $"The specified VM SKU '{create.VmSku}' is not available in the location ${region}" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_CREATE, $"The specified VM SKU '{create.VmSku}' is not available in the location ${region}"),
|
||||
context: "ScalesetCreate");
|
||||
}
|
||||
|
||||
@ -142,9 +136,9 @@ public class Scaleset {
|
||||
_log.WithHttpStatus(inserted.ErrorV).Error($"failed to insert new scaleset {scaleset.ScalesetId:Tag:ScalesetId}");
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.UNABLE_TO_CREATE,
|
||||
new string[] { $"unable to insert scaleset: {inserted.ErrorV}" }
|
||||
Error.Create(
|
||||
ErrorCode.UNABLE_TO_CREATE,
|
||||
$"unable to insert scaleset: {inserted.ErrorV}"
|
||||
),
|
||||
context: "ScalesetCreate");
|
||||
}
|
||||
@ -191,9 +185,9 @@ public class Scaleset {
|
||||
if (!scaleset.State.CanUpdate()) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: new[] { $"scaleset must be in one of the following states to update: {string.Join(", ", ScalesetStateHelper.CanUpdateStates)}" }),
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
$"scaleset must be in one of the following states to update: {string.Join(", ", ScalesetStateHelper.CanUpdateStates)}"),
|
||||
"ScalesetUpdate");
|
||||
}
|
||||
|
||||
|
@ -37,9 +37,9 @@ public class Tasks {
|
||||
if (task == null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new[] { "unable to find task" }),
|
||||
"unable to find task"),
|
||||
"task get");
|
||||
}
|
||||
|
||||
@ -101,7 +101,7 @@ public class Tasks {
|
||||
if (!checkConfig.IsOk) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(ErrorCode.INVALID_REQUEST, new[] { checkConfig.ErrorV.Error }),
|
||||
Error.Create(ErrorCode.INVALID_REQUEST, checkConfig.ErrorV.Error),
|
||||
"task create");
|
||||
}
|
||||
|
||||
@ -115,14 +115,14 @@ public class Tasks {
|
||||
if (job == null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find job" }),
|
||||
Error.Create(ErrorCode.INVALID_REQUEST, "unable to find job"),
|
||||
cfg.JobId.ToString());
|
||||
}
|
||||
|
||||
if (job.State != JobState.Enabled && job.State != JobState.Init) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(ErrorCode.UNABLE_TO_ADD_TASK_TO_JOB, new[] { $"unable to add a job in state {job.State}" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_ADD_TASK_TO_JOB, $"unable to add a job in state {job.State}"),
|
||||
cfg.JobId.ToString());
|
||||
}
|
||||
|
||||
@ -133,7 +133,7 @@ public class Tasks {
|
||||
if (prereq == null) {
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find task " }),
|
||||
Error.Create(ErrorCode.INVALID_REQUEST, "unable to find task "),
|
||||
"task create prerequisite");
|
||||
}
|
||||
}
|
||||
@ -165,8 +165,8 @@ public class Tasks {
|
||||
|
||||
var task = await _context.TaskOperations.GetByTaskId(request.OkV.TaskId);
|
||||
if (task == null) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find task"
|
||||
}), "task delete");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find task"
|
||||
), "task delete");
|
||||
|
||||
}
|
||||
|
||||
|
@ -36,7 +36,7 @@ public class WebhookLogs {
|
||||
var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
|
||||
|
||||
if (webhook == null) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook log");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook log");
|
||||
}
|
||||
|
||||
_log.Info($"getting webhook logs: {request.OkV.WebhookId:Tag:WebhookId}");
|
||||
|
@ -36,7 +36,7 @@ public class WebhookPing {
|
||||
var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
|
||||
|
||||
if (webhook == null) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook ping");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook ping");
|
||||
}
|
||||
|
||||
_log.Info($"pinging webhook : {request.OkV.WebhookId:Tag:WebhookId}");
|
||||
|
@ -40,7 +40,7 @@ public class Webhooks {
|
||||
var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId.Value);
|
||||
|
||||
if (webhook == null) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook get");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook get");
|
||||
}
|
||||
|
||||
var response = req.CreateResponse(HttpStatusCode.OK);
|
||||
@ -73,7 +73,7 @@ public class Webhooks {
|
||||
var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
|
||||
|
||||
if (webhook == null) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook update");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook update");
|
||||
}
|
||||
|
||||
var updated = webhook with {
|
||||
@ -134,7 +134,7 @@ public class Webhooks {
|
||||
var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
|
||||
|
||||
if (webhook == null) {
|
||||
return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook delete");
|
||||
return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook delete");
|
||||
}
|
||||
|
||||
var r = await _context.WebhookOperations.Delete(webhook);
|
||||
|
@ -166,7 +166,10 @@ public record Proxy
|
||||
bool Outdated
|
||||
) : StatefulEntityBase<VmState>(State);
|
||||
|
||||
public record Error(ErrorCode Code, string[]? Errors = null) {
|
||||
public record Error(ErrorCode Code, List<string>? Errors) {
|
||||
public static Error Create(ErrorCode code, params string[] errors) {
|
||||
return new Error(code, errors.ToList());
|
||||
}
|
||||
public sealed override string ToString() {
|
||||
var errorsString = Errors != null ? string.Join("", Errors) : string.Empty;
|
||||
return $"Error {{ Code = {Code}, Errors = {errorsString} }}";
|
||||
|
@ -49,7 +49,7 @@ namespace Microsoft.OneFuzz.Service {
|
||||
|
||||
private OneFuzzResult(T_Ok ok) => (OkV, ErrorV, IsOk) = (ok, null, true);
|
||||
|
||||
private OneFuzzResult(ErrorCode errorCode, string[] errors) => (OkV, ErrorV, IsOk) = (default, new Error(errorCode, errors), false);
|
||||
private OneFuzzResult(ErrorCode errorCode, string[] errors) => (OkV, ErrorV, IsOk) = (default, new Error(errorCode, errors.ToList()), false);
|
||||
|
||||
private OneFuzzResult(Error err) => (OkV, ErrorV, IsOk) = (default, err, false);
|
||||
|
||||
@ -72,7 +72,7 @@ namespace Microsoft.OneFuzz.Service {
|
||||
|
||||
public OneFuzzResultVoid() => (ErrorV, IsOk) = (null, true);
|
||||
|
||||
private OneFuzzResultVoid(ErrorCode errorCode, string[] errors) => (ErrorV, IsOk) = (new Error(errorCode, errors), false);
|
||||
private OneFuzzResultVoid(ErrorCode errorCode, string[] errors) => (ErrorV, IsOk) = (new Error(errorCode, errors.ToList()), false);
|
||||
|
||||
private OneFuzzResultVoid(Error err) => (ErrorV, IsOk) = (err, false);
|
||||
|
||||
|
@ -26,7 +26,7 @@ namespace ApiService.TestHooks {
|
||||
|
||||
if (config is null) {
|
||||
_log.Error($"Instance config is null");
|
||||
Error err = new(ErrorCode.INVALID_REQUEST, new[] { "Instance config is null" });
|
||||
Error err = Error.Create(ErrorCode.INVALID_REQUEST, "Instance config is null");
|
||||
var resp = req.CreateResponse(HttpStatusCode.InternalServerError);
|
||||
await resp.WriteAsJsonAsync(err);
|
||||
return resp;
|
||||
|
@ -10,7 +10,7 @@ using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
|
||||
#if DEBUG
|
||||
namespace ApiService.TestHooks {
|
||||
|
||||
sealed record MarkTasks(Node node, Error? error);
|
||||
sealed record MarkTasks(Node node, Error error);
|
||||
|
||||
public class NodeOperationsTestHooks {
|
||||
private readonly ILogTracer _log;
|
||||
|
@ -75,9 +75,9 @@ public class EndpointAuthorization : IEndpointAuthorization {
|
||||
|
||||
return await _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.UNAUTHORIZED,
|
||||
new string[] { reason ?? "Unrecognized agent" }
|
||||
reason ?? "Unrecognized agent"
|
||||
),
|
||||
"token verification",
|
||||
HttpStatusCode.Unauthorized
|
||||
@ -92,9 +92,9 @@ public class EndpointAuthorization : IEndpointAuthorization {
|
||||
|
||||
var config = await _context.ConfigOperations.Fetch();
|
||||
if (config is null) {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_CONFIGURATION,
|
||||
Errors: new string[] { "no instance configuration found " });
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_CONFIGURATION,
|
||||
"no instance configuration found ");
|
||||
}
|
||||
|
||||
return CheckRequireAdminsImpl(config, tokenResult.OkV.UserInfo);
|
||||
@ -117,9 +117,7 @@ public class EndpointAuthorization : IEndpointAuthorization {
|
||||
}
|
||||
|
||||
if (config.Admins is null) {
|
||||
return new Error(
|
||||
Code: ErrorCode.UNAUTHORIZED,
|
||||
Errors: new string[] { "pool modification disabled " });
|
||||
return Error.Create(ErrorCode.UNAUTHORIZED, "pool modification disabled ");
|
||||
}
|
||||
|
||||
if (userInfo.ObjectId is Guid objectId) {
|
||||
@ -127,13 +125,9 @@ public class EndpointAuthorization : IEndpointAuthorization {
|
||||
return OneFuzzResultVoid.Ok;
|
||||
}
|
||||
|
||||
return new Error(
|
||||
Code: ErrorCode.UNAUTHORIZED,
|
||||
Errors: new string[] { "not authorized to manage instance" });
|
||||
return Error.Create(ErrorCode.UNAUTHORIZED, "not authorized to manage instance");
|
||||
} else {
|
||||
return new Error(
|
||||
Code: ErrorCode.UNAUTHORIZED,
|
||||
Errors: new string[] { "user had no Object ID" });
|
||||
return Error.Create(ErrorCode.UNAUTHORIZED, "user had no Object ID");
|
||||
}
|
||||
}
|
||||
|
||||
@ -157,16 +151,12 @@ public class EndpointAuthorization : IEndpointAuthorization {
|
||||
var allowed = await membershipChecker.IsMember(rule.AllowedGroupsIds, memberId);
|
||||
if (!allowed) {
|
||||
_log.Error($"unauthorized access: {memberId:Tag:MemberId} is not authorized to access {path:Tag:Path}");
|
||||
return new Error(
|
||||
Code: ErrorCode.UNAUTHORIZED,
|
||||
Errors: new string[] { "not approved to use this endpoint" });
|
||||
return Error.Create(ErrorCode.UNAUTHORIZED, "not approved to use this endpoint");
|
||||
} else {
|
||||
return OneFuzzResultVoid.Ok;
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
return new Error(
|
||||
Code: ErrorCode.UNAUTHORIZED,
|
||||
Errors: new string[] { "unable to interact with graph", ex.Message });
|
||||
return Error.Create(ErrorCode.UNAUTHORIZED, "unable to interact with graph", ex.Message);
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -22,7 +22,7 @@ public abstract record ImageReference {
|
||||
public static ImageReference MustParse(string image) {
|
||||
var result = TryParse(image);
|
||||
if (!result.IsOk) {
|
||||
var msg = string.Join(", ", result.ErrorV.Errors ?? Array.Empty<string>());
|
||||
var msg = result.ErrorV.Errors != null ? string.Join(", ", result.ErrorV.Errors) : string.Empty;
|
||||
throw new ArgumentException(msg, nameof(image));
|
||||
}
|
||||
|
||||
@ -46,18 +46,17 @@ public abstract record ImageReference {
|
||||
} else if (identifier.ResourceType == ImageResource.ResourceType) {
|
||||
result = new Image(identifier);
|
||||
} else {
|
||||
return new Error(
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_IMAGE,
|
||||
new[] { $"Unknown image resource type: {identifier.ResourceType}" });
|
||||
$"Unknown image resource type: {identifier.ResourceType}");
|
||||
}
|
||||
} catch (FormatException) {
|
||||
// not an ARM identifier, try to parse a marketplace image:
|
||||
var imageParts = image.Split(":");
|
||||
// The python code would throw if more than 4 parts are found in the split
|
||||
if (imageParts.Length != 4) {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_IMAGE,
|
||||
new[] { $"Expected 4 ':' separated parts in '{image}'" });
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_IMAGE, $"Expected 4 ':' separated parts in '{image}'");
|
||||
}
|
||||
|
||||
result = new Marketplace(
|
||||
@ -112,10 +111,10 @@ public abstract record ImageReference {
|
||||
if (resource.Value.Data.OSType is OperatingSystemTypes os) {
|
||||
return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
|
||||
} else {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
|
||||
}
|
||||
} catch (Exception ex) when (ex is RequestFailedException) {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -129,10 +128,10 @@ public abstract record ImageReference {
|
||||
if (resource.Value.Data.OSType is OperatingSystemTypes os) {
|
||||
return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
|
||||
} else {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
|
||||
}
|
||||
} catch (Exception ex) when (ex is RequestFailedException) {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -145,10 +144,10 @@ public abstract record ImageReference {
|
||||
if (resource.Value.Data.OSType is OperatingSystemTypes os) {
|
||||
return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
|
||||
} else {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
|
||||
}
|
||||
} catch (Exception ex) when (ex is RequestFailedException) {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -162,10 +161,10 @@ public abstract record ImageReference {
|
||||
if (resource.Value.Data.OSType is OperatingSystemTypes os) {
|
||||
return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
|
||||
} else {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
|
||||
}
|
||||
} catch (Exception ex) when (ex is RequestFailedException) {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -178,7 +177,7 @@ public abstract record ImageReference {
|
||||
var os = resource.Value.Data.StorageProfile.OSDisk.OSType.ToString();
|
||||
return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
|
||||
} catch (Exception ex) when (ex is RequestFailedException) {
|
||||
return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
|
||||
return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
@ -80,7 +80,7 @@ public class JobOperations : StatefulOrm<Job, JobState, JobOperations>, IJobOper
|
||||
|
||||
await foreach (var job in QueryAsync(filter)) {
|
||||
await foreach (var task in _context.TaskOperations.QueryAsync(Query.PartitionKey(job.JobId.ToString()))) {
|
||||
await _context.TaskOperations.MarkFailed(task, new Error(ErrorCode.TASK_FAILED, new[] { "job never not start" }));
|
||||
await _context.TaskOperations.MarkFailed(task, Error.Create(ErrorCode.TASK_FAILED, "job never not start"));
|
||||
}
|
||||
_logTracer.Info($"stopping job that never started: {job.JobId:Tag:JobId}");
|
||||
|
||||
|
@ -48,7 +48,7 @@ public interface INodeOperations : IStatefulOrm<Node, NodeState> {
|
||||
|
||||
IAsyncEnumerable<Node> GetDeadNodes(Guid scaleSetId, TimeSpan expirationPeriod);
|
||||
|
||||
Async.Task MarkTasksStoppedEarly(Node node, Error? error = null);
|
||||
Async.Task MarkTasksStoppedEarly(Node node, Error? error);
|
||||
static readonly TimeSpan NODE_EXPIRATION_TIME = TimeSpan.FromHours(1.0);
|
||||
static readonly TimeSpan NODE_REIMAGE_TIME = TimeSpan.FromDays(6.0);
|
||||
|
||||
@ -213,18 +213,18 @@ public class NodeOperations : StatefulOrm<Node, NodeState, NodeOperations>, INod
|
||||
|
||||
var scaleset = scalesetResult.OkV;
|
||||
if (!scaleset.State.IsAvailable()) {
|
||||
return CanProcessNewWorkResponse.NotAllowed($"scaleset not available for work. Scaleset state '{scaleset.State}'"); ;
|
||||
return CanProcessNewWorkResponse.NotAllowed($"scaleset not available for work. Scaleset state '{scaleset.State}'");
|
||||
}
|
||||
}
|
||||
|
||||
var poolResult = await _context.PoolOperations.GetByName(node.PoolName);
|
||||
if (!poolResult.IsOk) {
|
||||
return CanProcessNewWorkResponse.NotAllowed("invalid pool"); ;
|
||||
return CanProcessNewWorkResponse.NotAllowed("invalid pool");
|
||||
}
|
||||
|
||||
var pool = poolResult.OkV;
|
||||
if (!PoolStateHelper.Available.Contains(pool.State)) {
|
||||
return CanProcessNewWorkResponse.NotAllowed("pool is not available for work"); ;
|
||||
return CanProcessNewWorkResponse.NotAllowed("pool is not available for work");
|
||||
}
|
||||
|
||||
return CanProcessNewWorkResponse.Allowed();
|
||||
@ -585,14 +585,19 @@ public class NodeOperations : StatefulOrm<Node, NodeState, NodeOperations>, INod
|
||||
}
|
||||
|
||||
|
||||
public async Async.Task MarkTasksStoppedEarly(Node node, Error? error = null) {
|
||||
if (error is null) {
|
||||
error = new Error(ErrorCode.TASK_FAILED, new[] { $"node reimaged during task execution. machine_id: {node.MachineId}" });
|
||||
}
|
||||
|
||||
public async Async.Task MarkTasksStoppedEarly(Node node, Error? error) {
|
||||
await foreach (var entry in _context.NodeTasksOperations.GetByMachineId(node.MachineId)) {
|
||||
var task = await _context.TaskOperations.GetByTaskId(entry.TaskId);
|
||||
if (task is not null) {
|
||||
if (task is not null && !TaskStateHelper.ShuttingDown(task.State)) {
|
||||
var message = $"Node {node.MachineId} stopping while the task state is '{task.State}'";
|
||||
if (error is not null) {
|
||||
if (error.Errors == null) {
|
||||
error = error with { Errors = new List<string>() };
|
||||
}
|
||||
error.Errors.Add(message);
|
||||
} else {
|
||||
error = Error.Create(ErrorCode.TASK_FAILED, message);
|
||||
}
|
||||
await _context.TaskOperations.MarkFailed(task, error);
|
||||
}
|
||||
if (!node.DebugKeepNode) {
|
||||
@ -605,7 +610,7 @@ public class NodeOperations : StatefulOrm<Node, NodeState, NodeOperations>, INod
|
||||
}
|
||||
|
||||
public new async Async.Task Delete(Node node) {
|
||||
await MarkTasksStoppedEarly(node);
|
||||
await MarkTasksStoppedEarly(node, Error.Create(ErrorCode.INVALID_NODE, "node is being deleted"));
|
||||
await _context.NodeTasksOperations.ClearByMachineId(node.MachineId);
|
||||
await _context.NodeMessageOperations.ClearMessages(node.MachineId);
|
||||
var r = await base.Delete(node);
|
||||
|
@ -37,15 +37,14 @@ namespace Microsoft.OneFuzz.Service {
|
||||
public async Async.Task<OneFuzzResult<bool>> AssociateSubnet(string name, VirtualNetworkResource vnet, SubnetResource subnet) {
|
||||
var nsg = await GetNsg(name);
|
||||
if (nsg == null) {
|
||||
return OneFuzzResult<bool>.Error(new Error(ErrorCode.UNABLE_TO_FIND,
|
||||
new[] { $"cannot associate subnet. nsg {name} not found" }));
|
||||
return OneFuzzResult<bool>.Error(Error.Create(ErrorCode.UNABLE_TO_FIND,
|
||||
$"cannot associate subnet. nsg {name} not found"));
|
||||
}
|
||||
|
||||
if (nsg.Data.Location != vnet.Data.Location) {
|
||||
return OneFuzzResult<bool>.Error(new Error(ErrorCode.UNABLE_TO_UPDATE,
|
||||
new[] {
|
||||
return OneFuzzResult<bool>.Error(Error.Create(ErrorCode.UNABLE_TO_UPDATE,
|
||||
$"subnet and nsg have to be in the same region. nsg {nsg.Data.Name} {nsg.Data.Location}, subnet: {subnet.Data.Name} {subnet.Data}"
|
||||
}));
|
||||
));
|
||||
}
|
||||
|
||||
if (subnet.Data.NetworkSecurityGroup != null && subnet.Data.NetworkSecurityGroup.Id == nsg.Id) {
|
||||
|
@ -212,8 +212,8 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
|
||||
}
|
||||
|
||||
private async System.Threading.Tasks.Task<Proxy> SetProvisionFailed(Proxy proxy, VirtualMachineInstanceView? instanceView) {
|
||||
var errors = GetErrors(proxy, instanceView).ToArray();
|
||||
return await SetFailed(proxy, new Error(ErrorCode.PROXY_FAILED, errors));
|
||||
var errors = GetErrors(proxy, instanceView);
|
||||
return await SetFailed(proxy, new Error(ErrorCode.PROXY_FAILED, errors.ToList()));
|
||||
}
|
||||
|
||||
private async Task<Proxy> SetFailed(Proxy proxy, Error error) {
|
||||
@ -259,7 +259,7 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
|
||||
var vm = GetVm(proxy, config);
|
||||
var vmData = await _context.VmOperations.GetVm(vm.Name);
|
||||
if (vmData is null) {
|
||||
return await SetFailed(proxy, new Error(ErrorCode.PROXY_FAILED, new[] { "azure not able to find vm" }));
|
||||
return await SetFailed(proxy, Error.Create(ErrorCode.PROXY_FAILED, "azure not able to find vm"));
|
||||
}
|
||||
|
||||
if (vmData.ProvisioningState == "Failed") {
|
||||
|
@ -180,9 +180,9 @@ public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IRe
|
||||
if (vmData == null) {
|
||||
return await _context.ReproOperations.SetError(
|
||||
repro,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.VM_CREATE_FAILED,
|
||||
new string[] { "failed before launching extensions" }));
|
||||
"failed before launching extensions"));
|
||||
}
|
||||
if (vmData.ProvisioningState == "Failed") {
|
||||
var failedVmData = await _context.VmOperations.GetVmWithInstanceView(vm.Name);
|
||||
@ -232,7 +232,7 @@ public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IRe
|
||||
repro,
|
||||
new Error(
|
||||
ErrorCode.VM_CREATE_FAILED,
|
||||
errors));
|
||||
errors.ToList()));
|
||||
}
|
||||
|
||||
public async Task<OneFuzzResultVoid> BuildReproScript(Repro repro) {
|
||||
|
@ -98,9 +98,9 @@ public class RequestHandling : IRequestHandling {
|
||||
}
|
||||
|
||||
if (errors.Any()) {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: errors.ToArray());
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
errors.ToArray());
|
||||
}
|
||||
}
|
||||
|
||||
@ -109,9 +109,9 @@ public class RequestHandling : IRequestHandling {
|
||||
if (Validator.TryValidateObject(t, validationContext, validationResults, validateAllProperties: true)) {
|
||||
return OneFuzzResult.Ok(t);
|
||||
} else {
|
||||
return new Error(
|
||||
Code: ErrorCode.INVALID_REQUEST,
|
||||
Errors: validationResults.Select(vr => vr.ToString()).ToArray());
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
validationResults.Select(vr => vr.ToString()).ToArray());
|
||||
}
|
||||
} else {
|
||||
return OneFuzzResult<T>.Error(
|
||||
@ -154,13 +154,12 @@ public class RequestHandling : IRequestHandling {
|
||||
}
|
||||
}
|
||||
|
||||
return new Error(
|
||||
return Error.Create(
|
||||
ErrorCode.INVALID_REQUEST,
|
||||
new string[] {
|
||||
exception.Message,
|
||||
exception.Source ?? string.Empty,
|
||||
exception.StackTrace ?? string.Empty
|
||||
}
|
||||
|
||||
);
|
||||
}
|
||||
|
||||
|
@ -291,7 +291,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
|
||||
|
||||
if (scaleset.Auth is null) {
|
||||
_logTracer.Error($"Scaleset Auth is missing for scaleset {scaleset.ScalesetId:Tag:ScalesetId}");
|
||||
return await SetFailed(scaleset, new Error(ErrorCode.UNABLE_TO_CREATE, new[] { "missing required auth" }));
|
||||
return await SetFailed(scaleset, Error.Create(ErrorCode.UNABLE_TO_CREATE, "missing required auth"));
|
||||
}
|
||||
|
||||
var vmss = await _context.VmssOperations.GetVmss(scaleset.ScalesetId);
|
||||
@ -455,7 +455,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
|
||||
return await SetFailed(scaleset, imageOsResult.ErrorV);
|
||||
} else if (imageOsResult.OkV != pool.Os) {
|
||||
_logTracer.Error($"got invalid OS: {imageOsResult.OkV:Tag:ActualOs} for scaleset: {scaleset.ScalesetId:Tag:ScalesetId} expected OS {pool.Os:Tag:ExpectedOs}");
|
||||
return await SetFailed(scaleset, new Error(ErrorCode.INVALID_REQUEST, new[] { $"invalid os (got: {imageOsResult.OkV} needed: {pool.Os})" }));
|
||||
return await SetFailed(scaleset, Error.Create(ErrorCode.INVALID_REQUEST, $"invalid os (got: {imageOsResult.OkV} needed: {pool.Os})"));
|
||||
} else {
|
||||
return await SetState(scaleset, ScalesetState.Setup);
|
||||
}
|
||||
@ -605,7 +605,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
|
||||
|
||||
_log.Info($"{errorMessage} {deadNode.MachineId:Tag:MachineId} {deadNode.ScalesetId:Tag:ScalesetId}");
|
||||
|
||||
var error = new Error(ErrorCode.TASK_FAILED, new[] { $"{errorMessage} scaleset_id {deadNode.ScalesetId} last heartbeat:{deadNode.Heartbeat}" });
|
||||
var error = Error.Create(ErrorCode.TASK_FAILED, $"{errorMessage} scaleset_id {deadNode.ScalesetId} last heartbeat:{deadNode.Heartbeat}");
|
||||
await _context.NodeOperations.MarkTasksStoppedEarly(deadNode, error);
|
||||
toReimage[deadNode.MachineId] = await _context.NodeOperations.ToReimage(deadNode, true);
|
||||
}
|
||||
|
@ -106,7 +106,7 @@ public class TaskOperations : StatefulOrm<Task, TaskState, TaskOperations>, ITas
|
||||
}
|
||||
|
||||
if (!task.State.HasStarted()) {
|
||||
await MarkFailed(task, new Error(Code: ErrorCode.TASK_FAILED, Errors: new[] { "task never started" }));
|
||||
await MarkFailed(task, Error.Create(ErrorCode.TASK_FAILED, "task never started"));
|
||||
} else {
|
||||
_ = await SetState(task, TaskState.Stopping);
|
||||
}
|
||||
@ -133,7 +133,7 @@ public class TaskOperations : StatefulOrm<Task, TaskState, TaskOperations>, ITas
|
||||
foreach (var t in taskInJob) {
|
||||
if (t.Config.PrereqTasks != null) {
|
||||
if (t.Config.PrereqTasks.Contains(task.TaskId)) {
|
||||
await MarkFailed(t, new Error(ErrorCode.TASK_FAILED, new[] { $"prerequisite task failed. task_id:{t.TaskId}" }), taskInJob);
|
||||
await MarkFailed(t, Error.Create(ErrorCode.TASK_FAILED, $"prerequisite task failed. task_id:{t.TaskId}"), taskInJob);
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -264,7 +264,7 @@ public class TaskOperations : StatefulOrm<Task, TaskState, TaskOperations>, ITas
|
||||
|
||||
// if a prereq task fails, then mark this task as failed
|
||||
if (t == null) {
|
||||
await MarkFailed(task, new Error(ErrorCode.INVALID_REQUEST, Errors: new[] { "unable to find prereq task" }));
|
||||
await MarkFailed(task, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find prereq task"));
|
||||
return false;
|
||||
}
|
||||
|
||||
|
@ -235,7 +235,7 @@ public class VmssOperations : IVmssOperations {
|
||||
try {
|
||||
return OneFuzzResult.Ok(await GetInstanceIdForVmId(name, vmId));
|
||||
} catch {
|
||||
return new Error(ErrorCode.UNABLE_TO_FIND, new string[] { $"unable to find scaleset machine: {name}:{vmId}" });
|
||||
return Error.Create(ErrorCode.UNABLE_TO_FIND, $"unable to find scaleset machine: {name}:{vmId}");
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -189,7 +189,7 @@ public class JinjaTemplateAdapter {
|
||||
new List<TaskDebugFlag> { TaskDebugFlag.KeepNodeOnCompletion },
|
||||
true
|
||||
),
|
||||
new Error(ErrorCode.UNABLE_TO_FIND, new string[] { "some error message" }),
|
||||
Error.Create(ErrorCode.UNABLE_TO_FIND, "some error message"),
|
||||
new Authentication("password", "public key", "private key"),
|
||||
DateTimeOffset.UtcNow,
|
||||
DateTimeOffset.UtcNow,
|
||||
|
@ -17,7 +17,7 @@ public abstract class NotificationsBase {
|
||||
|
||||
public async Async.Task LogFailedNotification(Report report, Exception error, Guid notificationId) {
|
||||
_logTracer.Error($"notification failed: notification_id:{notificationId:Tag:NotificationId} job_id:{report.JobId:Tag:JobId} task_id:{report.TaskId:Tag:TaskId} err:{error.Message:Tag:Error}");
|
||||
Error? err = new Error(ErrorCode.NOTIFICATION_FAILURE, new string[] { $"{error}" });
|
||||
Error? err = Error.Create(ErrorCode.NOTIFICATION_FAILURE, $"{error}");
|
||||
await _context.Events.SendEvent(new EventNotificationFailed(
|
||||
NotificationId: notificationId,
|
||||
JobId: report.JobId,
|
||||
|
@ -36,9 +36,9 @@ sealed class TestEndpointAuthorization : EndpointAuthorization {
|
||||
|
||||
return _context.RequestHandling.NotOk(
|
||||
req,
|
||||
new Error(
|
||||
Error.Create(
|
||||
ErrorCode.UNAUTHORIZED,
|
||||
new string[] { "Unrecognized agent" }
|
||||
"Unrecognized agent"
|
||||
),
|
||||
"token verification",
|
||||
HttpStatusCode.Unauthorized
|
||||
|
Reference in New Issue
Block a user