Fix logic of marking task as failed (#3083)

* Fix logic of markaing task as failed
- Do not mark task as failed if it is already in the shutting down state
- accumulate errors when setting task error to understand the context
- refactor the Error record

* fix tests

* format

* Fix build

* Update src/ApiService/ApiService/onefuzzlib/ImageReference.cs

Co-authored-by: George Pollard <porges@porg.es>

* Update src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs

Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>

---------

Co-authored-by: George Pollard <porges@porg.es>
Co-authored-by: Teo Voinea <58236992+tevoinea@users.noreply.github.com>

src/ApiService/ApiService/Functions/AgentCanSchedule.cs

@@ -35,11 +35,7 @@ public class AgentCanSchedule {
             _log.Warning($"Unable to find {canScheduleRequest.MachineId:Tag:MachineId}");
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
-                    ErrorCode.UNABLE_TO_FIND,
-                    new string[] {
-                        "unable to find node"
-                    }),
                 canScheduleRequest.MachineId.ToString());
         }
 

src/ApiService/ApiService/Functions/AgentEvents.cs

@@ -35,7 +35,7 @@ public class AgentEvents {
             NodeStateUpdate updateEvent => await OnStateUpdate(envelope.MachineId, updateEvent),
             WorkerEvent workerEvent => await OnWorkerEvent(envelope.MachineId, workerEvent),
             NodeEvent nodeEvent => await OnNodeEvent(envelope.MachineId, nodeEvent),
-            _ => new Error(ErrorCode.INVALID_REQUEST, new string[] { $"invalid node event: {envelope.Event.GetType().Name}" }),
+            _ => Error.Create(ErrorCode.INVALID_REQUEST, $"invalid node event: {envelope.Event.GetType().Name}"),
         };
 
         if (error is Error e) {
@@ -114,17 +114,17 @@ public class AgentEvents {
         } else if (ev.State == NodeState.SettingUp) {
             if (ev.Data is NodeSettingUpEventData settingUpData) {
                 if (!settingUpData.Tasks.Any()) {
-                    return new Error(ErrorCode.INVALID_REQUEST, Errors: new string[] {
+                    return Error.Create(ErrorCode.INVALID_REQUEST,
-                        $"setup without tasks.  machine_id: {machineId}",
+                        $"setup without tasks.  machine_id: {machineId}"
-                    });
+                    );
                 }
 
                 foreach (var taskId in settingUpData.Tasks) {
                     var task = await _context.TaskOperations.GetByTaskId(taskId);
                     if (task is null) {
-                        return new Error(
+                        return Error.Create(
                             ErrorCode.INVALID_REQUEST,
-                            Errors: new string[] { $"unable to find task: {taskId}" });
+                            $"unable to find task: {taskId}");
                     }
 
                     _log.Info($"node starting task. {machineId:Tag:MachineId} {task.JobId:Tag:JobId} {task.TaskId:Tag:TaskId}");
@@ -154,7 +154,7 @@ public class AgentEvents {
             if (ev.Data is NodeDoneEventData doneData) {
                 if (doneData.Error is not null) {
                     var errorText = EntityConverter.ToJsonString(doneData);
-                    error = new Error(ErrorCode.TASK_FAILED, Errors: new string[] { errorText });
+                    error = Error.Create(ErrorCode.TASK_FAILED, errorText);
                     _log.Error($"node 'done' {machineId:Tag:MachineId} - {errorText:Tag:Error}");
                 }
             }
@@ -178,9 +178,9 @@ public class AgentEvents {
             return await OnWorkerEventRunning(machineId, ev.Running);
         }
 
-        return new Error(
+        return Error.Create(
-            Code: ErrorCode.INVALID_REQUEST,
+            ErrorCode.INVALID_REQUEST,
-            Errors: new string[] { "WorkerEvent should have either 'done' or 'running' set" });
+            "WorkerEvent should have either 'done' or 'running' set");
     }
 
     private async Async.Task<Error?> OnWorkerEventRunning(Guid machineId, WorkerRunningEvent running) {
@@ -189,15 +189,11 @@ public class AgentEvents {
             _context.NodeOperations.GetByMachineId(machineId));
 
         if (task is null) {
-            return new Error(
+            return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find task: {running.TaskId}");
-                Code: ErrorCode.INVALID_REQUEST,
-                Errors: new string[] { $"unable to find task: {running.TaskId}" });
         }
 
         if (node is null) {
-            return new Error(
+            return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find node: {machineId}");
-                Code: ErrorCode.INVALID_REQUEST,
-                Errors: new string[] { $"unable to find node: {machineId}" });
         }
 
         if (!node.State.ReadyForReset()) {
@@ -240,15 +236,11 @@ public class AgentEvents {
             _context.NodeOperations.GetByMachineId(machineId));
 
         if (task is null) {
-            return new Error(
+            return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find task: {done.TaskId}");
-                Code: ErrorCode.INVALID_REQUEST,
-                Errors: new string[] { $"unable to find task: {done.TaskId}" });
         }
 
         if (node is null) {
-            return new Error(
+            return Error.Create(ErrorCode.INVALID_REQUEST, $"unable to find node: {machineId}");
-                Code: ErrorCode.INVALID_REQUEST,
-                Errors: new string[] { $"unable to find node: {machineId}" });
         }
 
         // trim stdout/stderr if too long
@@ -272,13 +264,12 @@ public class AgentEvents {
         } else {
             await _context.TaskOperations.MarkFailed(
                 task,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.TASK_FAILED,
+                    ErrorCode.TASK_FAILED,
-                    Errors: new string[] {
+                    $"task failed. exit_status:{done.ExitStatus}",
-                        $"task failed. exit_status:{done.ExitStatus}",
+                    done.Stdout,
-                        done.Stdout,
+                    done.Stderr
-                        done.Stderr,
+                    ));
-                    }));
 
             // keep node if any keep options are set
             if ((task.Config.Debug?.Contains(TaskDebugFlag.KeepNodeOnFailure) == true)

src/ApiService/ApiService/Functions/AgentRegistration.cs

@@ -40,9 +40,9 @@ public class AgentRegistration {
         if (machineId == Guid.Empty) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
                     ErrorCode.INVALID_REQUEST,
-                    new string[] { "'machine_id' query parameter must be provided" }),
+                    "'machine_id' query parameter must be provided"),
                 "agent registration");
         }
 
@@ -50,9 +50,9 @@ public class AgentRegistration {
         if (agentNode is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
                     ErrorCode.INVALID_REQUEST,
-                    new string[] { $"unable to find a registration associated with machine_id '{machineId}'" }),
+                    $"unable to find a registration associated with machine_id '{machineId}'"),
                 "agent registration");
         }
 
@@ -60,9 +60,9 @@ public class AgentRegistration {
         if (!pool.IsOk) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
                     ErrorCode.INVALID_REQUEST,
-                    new string[] { "unable to find a pool associated with the provided machine_id" }),
+                    "unable to find a pool associated with the provided machine_id"),
                 "agent registration");
         }
 
@@ -101,18 +101,16 @@ public class AgentRegistration {
         if (machineId == Guid.Empty) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    ErrorCode.INVALID_REQUEST,
+                    ErrorCode.INVALID_REQUEST, "'machine_id' query parameter must be provided"),
-                    new string[] { "'machine_id' query parameter must be provided" }),
                 "agent registration");
         }
 
         if (poolName is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    ErrorCode.INVALID_REQUEST,
+                    ErrorCode.INVALID_REQUEST, "'pool_name' query parameter must be provided"),
-                    new string[] { "'pool_name' query parameter must be provided" }),
                 "agent registration");
         }
 
@@ -124,9 +122,8 @@ public class AgentRegistration {
         if (!poolResult.IsOk) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.INVALID_REQUEST,
+                    ErrorCode.INVALID_REQUEST, $"unable to find pool '{poolName}'"),
-                    Errors: new[] { $"unable to find pool '{poolName}'" }),
                 "agent registration");
         }
 
@@ -140,9 +137,9 @@ public class AgentRegistration {
         if (os != null && pool.Os != os) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.INVALID_REQUEST,
+                    ErrorCode.INVALID_REQUEST,
-                    Errors: new[] { $"OS mismatch: pool '{poolName}' is configured for '{pool.Os}', but agent is running '{os}'" }),
+                    $"OS mismatch: pool '{poolName}' is configured for '{pool.Os}', but agent is running '{os}'"),
                 "agent registration");
         }
 

src/ApiService/ApiService/Functions/Containers.cs

@@ -39,9 +39,9 @@ public class ContainersFunction {
             if (container is null) {
                 return await _context.RequestHandling.NotOk(
                     req,
-                    new Error(
+                    Error.Create(
-                        Code: ErrorCode.INVALID_REQUEST,
+                        ErrorCode.INVALID_REQUEST,
-                        Errors: new[] { "invalid container" }),
+                        "invalid container"),
                     context: get.Name.String);
             }
 
@@ -101,9 +101,9 @@ public class ContainersFunction {
         if (sas is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.INVALID_REQUEST,
+                    ErrorCode.INVALID_REQUEST,
-                    Errors: new[] { "invalid container" }),
+                    "invalid container"),
                 context: post.Name.String);
         }
 

src/ApiService/ApiService/Functions/Download.cs

@@ -25,9 +25,9 @@ public class Download {
         if (queryContainer is null || !Container.TryParse(queryContainer, out var container)) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
                     ErrorCode.INVALID_REQUEST,
-                    new string[] { "'container' query parameter must be provided and valid" }),
+                    "'container' query parameter must be provided and valid"),
                 "download");
         }
 
@@ -35,9 +35,9 @@ public class Download {
         if (filename is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
                     ErrorCode.INVALID_REQUEST,
-                    new string[] { "'filename' query parameter must be provided" }),
+                    "'filename' query parameter must be provided"),
                 "download");
         }
 

src/ApiService/ApiService/Functions/Jobs.cs

@@ -59,9 +59,7 @@ public class Jobs {
         if (containerSas is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.UNABLE_TO_CREATE_CONTAINER, "unable to create logs container "),
-                    Code: ErrorCode.UNABLE_TO_CREATE_CONTAINER,
-                    Errors: new string[] { "unable to create logs container " }),
                 "logs");
         }
 
@@ -73,9 +71,9 @@ public class Jobs {
             _logTracer.WithTag("HttpRequest", "POST").WithHttpStatus(r.ErrorV).Error($"failed to insert job {job.JobId:Tag:JobId}");
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.UNABLE_TO_CREATE,
+                ErrorCode.UNABLE_TO_CREATE,
-                    Errors: new string[] { "unable to create job " }
+                "unable to create job"
                 ),
                 "job");
         }
@@ -95,9 +93,9 @@ public class Jobs {
         if (job is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.INVALID_JOB,
+                    ErrorCode.INVALID_JOB,
-                    Errors: new string[] { "no such job" }),
+                    "no such job"),
                 context: jobId.ToString());
         }
 
@@ -124,9 +122,7 @@ public class Jobs {
             if (job is null) {
                 return await _context.RequestHandling.NotOk(
                     req,
-                    new Error(
+                    Error.Create(ErrorCode.INVALID_JOB, "no such job"),
-                        Code: ErrorCode.INVALID_JOB,
-                        Errors: new string[] { "no such job" }),
                     context: jobId.ToString());
             }
 

src/ApiService/ApiService/Functions/Migrations/JinjaToScriban.cs

@@ -72,7 +72,7 @@ public class JinjaToScriban {
                     _log.Info($"Migrated notification: {notification.NotificationId} to jinja");
                 } else {
                     failedNotificationIds.Add(notification.NotificationId);
-                    _log.Error(new Error(ErrorCode.UNABLE_TO_UPDATE, new[] { r.ErrorV.Reason, r.ErrorV.Status.ToString() }));
+                    _log.Error(Error.Create(ErrorCode.UNABLE_TO_UPDATE, r.ErrorV.Reason, r.ErrorV.Status.ToString()));
                 }
             } catch (Exception ex) {
                 failedNotificationIds.Add(notification.NotificationId);

src/ApiService/ApiService/Functions/Node.cs

@@ -38,9 +38,7 @@ public class Node {
             if (node is null) {
                 return await _context.RequestHandling.NotOk(
                     req,
-                    new Error(
+                    Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
-                        Code: ErrorCode.UNABLE_TO_FIND,
-                        Errors: new string[] { "unable to find node" }),
                     context: machineId.ToString());
             }
 
@@ -94,9 +92,7 @@ public class Node {
         if (node is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
-                    Code: ErrorCode.UNABLE_TO_FIND,
-                    Errors: new string[] { "unable to find node" }),
                 context: patch.MachineId.ToString());
         }
 
@@ -130,9 +126,7 @@ public class Node {
         if (node is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
-                    Code: ErrorCode.UNABLE_TO_FIND,
-                    Errors: new string[] { "unable to find node" }),
                 context: post.MachineId.ToString());
         }
 
@@ -166,9 +160,7 @@ public class Node {
         if (node is null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
-                    Code: ErrorCode.UNABLE_TO_FIND,
-                    new string[] { "unable to find node" }),
                 context: delete.MachineId.ToString());
         }
 

src/ApiService/ApiService/Functions/NodeAddSshKey.cs

@@ -30,7 +30,7 @@ public class NodeAddSshKey {
         if (node == null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(ErrorCode.UNABLE_TO_FIND, new[] { "unable to find node" }),
+                Error.Create(ErrorCode.UNABLE_TO_FIND, "unable to find node"),
                 $"{request.OkV.MachineId}");
         }
 

src/ApiService/ApiService/Functions/Notifications.cs

@@ -61,18 +61,18 @@ public class Notifications {
         var entries = await _context.NotificationOperations.SearchByPartitionKeys(new[] { $"{request.OkV.NotificationId}" }).ToListAsync();
 
         if (entries.Count == 0) {
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find notification" }), context: "notification delete");
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find notification"), context: "notification delete");
         }
 
         if (entries.Count > 1) {
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "error identifying Notification" }), context: "notification delete");
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "error identifying Notification"), context: "notification delete");
         }
 
         var result = await _context.NotificationOperations.Delete(entries[0]);
 
         if (!result.IsOk) {
             var (status, error) = result.ErrorV;
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.UNABLE_TO_UPDATE, new[] { error }), "notification delete");
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.UNABLE_TO_UPDATE, error), "notification delete");
         }
 
         var response = req.CreateResponse(HttpStatusCode.OK);

src/ApiService/ApiService/Functions/Pool.cs

@@ -63,9 +63,7 @@ public class Pool {
         if (pool.IsOk) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.INVALID_REQUEST, "pool with that name already exists"),
-                    Code: ErrorCode.INVALID_REQUEST,
-                    Errors: new string[] { "pool with that name already exists" }),
                 "PoolCreate");
         }
         var newPool = await _context.PoolOperations.Create(name: create.Name, os: create.Os, architecture: create.Arch, managed: create.Managed, objectId: create.ObjectId);
@@ -89,9 +87,7 @@ public class Pool {
         if (!pool.IsOk) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.INVALID_REQUEST, "pool with that name does not exist"),
-                    Code: ErrorCode.INVALID_REQUEST,
-                    Errors: new string[] { "pool with that name does not exist" }),
                 "PoolUpdate");
         }
 
@@ -100,9 +96,7 @@ public class Pool {
         if (updatePool.IsOk) {
             return await RequestHandling.Ok(req, await Populate(PoolToPoolResponse(updated), true));
         } else {
-            return await _context.RequestHandling.NotOk(req, new Error(
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, updatePool.ErrorV.Reason), "PoolUpdate");
-                Code: ErrorCode.INVALID_REQUEST,
-                Errors: new string[] { updatePool.ErrorV.Reason }), "PoolUpdate");
         }
 
 

src/ApiService/ApiService/Functions/Proxy.cs

@@ -63,7 +63,7 @@ public class Proxy {
                     machineId: machineId, dstPort: dstPort).ToListAsync();
 
                 if (!forwards.Any()) {
-                    return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "no forwards for scaleset and node" }), "debug_proxy get");
+                    return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "no forwards for scaleset and node"), "debug_proxy get");
                 }
 
                 var response = req.CreateResponse();
@@ -78,7 +78,7 @@ public class Proxy {
                 await r.WriteAsJsonAsync(new ProxyList(proxies));
                 return r;
             default:
-                return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "ProxyGet must provide all or none of the following: scaleset_id, machine_id, dst_port" }), "debug_proxy get");
+                return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "ProxyGet must provide all or none of the following: scaleset_id, machine_id, dst_port"), "debug_proxy get");
         }
     }
 

src/ApiService/ApiService/Functions/ReproVmss.cs

@@ -35,7 +35,7 @@ public class ReproVmss {
             var vm = await _context.ReproOperations.SearchByPartitionKeys(new[] { $"{request.OkV.VmId}" }).FirstOrDefaultAsync();
 
             if (vm == null) {
-                return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "no such VM" }), $"{request.OkV.VmId}");
+                return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "no such VM"), $"{request.OkV.VmId}");
             }
 
             var response = req.CreateResponse(HttpStatusCode.OK);
@@ -98,7 +98,7 @@ public class ReproVmss {
         if (request.OkV.VmId == null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(ErrorCode.INVALID_REQUEST, new[] { "missing vm_id" }),
+                Error.Create(ErrorCode.INVALID_REQUEST, "missing vm_id"),
                 context: "repro delete");
         }
 
@@ -107,7 +107,7 @@ public class ReproVmss {
         if (vm == null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(ErrorCode.INVALID_REQUEST, new[] { "no such vm" }),
+                Error.Create(ErrorCode.INVALID_REQUEST, "no such vm"),
                 context: "repro delete");
         }
 

src/ApiService/ApiService/Functions/Scaleset.cs

@@ -70,9 +70,7 @@ public class Scaleset {
         if (!pool.Managed) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.UNABLE_TO_CREATE, "scalesets can only be added to managed pools "),
-                    Code: ErrorCode.UNABLE_TO_CREATE,
-                    Errors: new string[] { "scalesets can only be added to managed pools " }),
                 context: "ScalesetCreate");
         }
 
@@ -96,9 +94,7 @@ public class Scaleset {
             if (!validRegions.Contains(create.Region)) {
                 return await _context.RequestHandling.NotOk(
                     req,
-                    new Error(
+                    Error.Create(ErrorCode.UNABLE_TO_CREATE, "invalid region"),
-                        Code: ErrorCode.UNABLE_TO_CREATE,
-                        Errors: new string[] { "invalid region" }),
                     context: "ScalesetCreate");
             }
 
@@ -109,9 +105,7 @@ public class Scaleset {
         if (!availableSkus.Contains(create.VmSku)) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(ErrorCode.UNABLE_TO_CREATE, $"The specified VM SKU '{create.VmSku}' is not available in the location ${region}"),
-                    Code: ErrorCode.UNABLE_TO_CREATE,
-                    Errors: new string[] { $"The specified VM SKU '{create.VmSku}' is not available in the location ${region}" }),
                 context: "ScalesetCreate");
         }
 
@@ -142,9 +136,9 @@ public class Scaleset {
             _log.WithHttpStatus(inserted.ErrorV).Error($"failed to insert new scaleset {scaleset.ScalesetId:Tag:ScalesetId}");
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.UNABLE_TO_CREATE,
+                    ErrorCode.UNABLE_TO_CREATE,
-                    new string[] { $"unable to insert scaleset: {inserted.ErrorV}" }
+                    $"unable to insert scaleset: {inserted.ErrorV}"
                 ),
                 context: "ScalesetCreate");
         }
@@ -191,9 +185,9 @@ public class Scaleset {
         if (!scaleset.State.CanUpdate()) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(
+                Error.Create(
-                    Code: ErrorCode.INVALID_REQUEST,
+                    ErrorCode.INVALID_REQUEST,
-                    Errors: new[] { $"scaleset must be in one of the following states to update: {string.Join(", ", ScalesetStateHelper.CanUpdateStates)}" }),
+                    $"scaleset must be in one of the following states to update: {string.Join(", ", ScalesetStateHelper.CanUpdateStates)}"),
                 "ScalesetUpdate");
         }
 

src/ApiService/ApiService/Functions/Tasks.cs

@@ -37,9 +37,9 @@ public class Tasks {
             if (task == null) {
                 return await _context.RequestHandling.NotOk(
                     req,
-                    new Error(
+                    Error.Create(
                         ErrorCode.INVALID_REQUEST,
-                        new[] { "unable to find task" }),
+                        "unable to find task"),
                     "task get");
             }
 
@@ -101,7 +101,7 @@ public class Tasks {
         if (!checkConfig.IsOk) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(ErrorCode.INVALID_REQUEST, new[] { checkConfig.ErrorV.Error }),
+                Error.Create(ErrorCode.INVALID_REQUEST, checkConfig.ErrorV.Error),
                 "task create");
         }
 
@@ -115,14 +115,14 @@ public class Tasks {
         if (job == null) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find job" }),
+                Error.Create(ErrorCode.INVALID_REQUEST, "unable to find job"),
                 cfg.JobId.ToString());
         }
 
         if (job.State != JobState.Enabled && job.State != JobState.Init) {
             return await _context.RequestHandling.NotOk(
                 req,
-                new Error(ErrorCode.UNABLE_TO_ADD_TASK_TO_JOB, new[] { $"unable to add a job in state {job.State}" }),
+                Error.Create(ErrorCode.UNABLE_TO_ADD_TASK_TO_JOB, $"unable to add a job in state {job.State}"),
                 cfg.JobId.ToString());
         }
 
@@ -133,7 +133,7 @@ public class Tasks {
                 if (prereq == null) {
                     return await _context.RequestHandling.NotOk(
                         req,
-                        new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find task " }),
+                        Error.Create(ErrorCode.INVALID_REQUEST, "unable to find task "),
                         "task create prerequisite");
                 }
             }
@@ -165,8 +165,8 @@ public class Tasks {
 
         var task = await _context.TaskOperations.GetByTaskId(request.OkV.TaskId);
         if (task == null) {
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find task"
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find task"
-            }), "task delete");
+            ), "task delete");
 
         }
 

src/ApiService/ApiService/Functions/WebhookLogs.cs

@@ -36,7 +36,7 @@ public class WebhookLogs {
         var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
 
         if (webhook == null) {
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook log");
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook log");
         }
 
         _log.Info($"getting webhook logs: {request.OkV.WebhookId:Tag:WebhookId}");

src/ApiService/ApiService/Functions/WebhookPing.cs

@@ -36,7 +36,7 @@ public class WebhookPing {
         var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
 
         if (webhook == null) {
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook ping");
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook ping");
         }
 
         _log.Info($"pinging webhook : {request.OkV.WebhookId:Tag:WebhookId}");

src/ApiService/ApiService/Functions/Webhooks.cs

@@ -40,7 +40,7 @@ public class Webhooks {
             var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId.Value);
 
             if (webhook == null) {
-                return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook get");
+                return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook get");
             }
 
             var response = req.CreateResponse(HttpStatusCode.OK);
@@ -73,7 +73,7 @@ public class Webhooks {
         var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
 
         if (webhook == null) {
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook update");
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook update");
         }
 
         var updated = webhook with {
@@ -134,7 +134,7 @@ public class Webhooks {
         var webhook = await _context.WebhookOperations.GetByWebhookId(request.OkV.WebhookId);
 
         if (webhook == null) {
-            return await _context.RequestHandling.NotOk(req, new Error(ErrorCode.INVALID_REQUEST, new[] { "unable to find webhook" }), "webhook delete");
+            return await _context.RequestHandling.NotOk(req, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find webhook"), "webhook delete");
         }
 
         var r = await _context.WebhookOperations.Delete(webhook);

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -166,7 +166,10 @@ public record Proxy
     bool Outdated
 ) : StatefulEntityBase<VmState>(State);
 
-public record Error(ErrorCode Code, string[]? Errors = null) {
+public record Error(ErrorCode Code, List<string>? Errors) {
+    public static Error Create(ErrorCode code, params string[] errors) {
+        return new Error(code, errors.ToList());
+    }
     public sealed override string ToString() {
         var errorsString = Errors != null ? string.Join("", Errors) : string.Empty;
         return $"Error {{ Code = {Code}, Errors = {errorsString} }}";

src/ApiService/ApiService/OneFuzzTypes/ReturnTypes.cs

@@ -49,7 +49,7 @@ namespace Microsoft.OneFuzz.Service {
 
         private OneFuzzResult(T_Ok ok) => (OkV, ErrorV, IsOk) = (ok, null, true);
 
-        private OneFuzzResult(ErrorCode errorCode, string[] errors) => (OkV, ErrorV, IsOk) = (default, new Error(errorCode, errors), false);
+        private OneFuzzResult(ErrorCode errorCode, string[] errors) => (OkV, ErrorV, IsOk) = (default, new Error(errorCode, errors.ToList()), false);
 
         private OneFuzzResult(Error err) => (OkV, ErrorV, IsOk) = (default, err, false);
 
@@ -72,7 +72,7 @@ namespace Microsoft.OneFuzz.Service {
 
         public OneFuzzResultVoid() => (ErrorV, IsOk) = (null, true);
 
-        private OneFuzzResultVoid(ErrorCode errorCode, string[] errors) => (ErrorV, IsOk) = (new Error(errorCode, errors), false);
+        private OneFuzzResultVoid(ErrorCode errorCode, string[] errors) => (ErrorV, IsOk) = (new Error(errorCode, errors.ToList()), false);
 
         private OneFuzzResultVoid(Error err) => (ErrorV, IsOk) = (err, false);
 

src/ApiService/ApiService/TestHooks/InstanceConfigTestHooks.cs

@@ -26,7 +26,7 @@ namespace ApiService.TestHooks {
 
             if (config is null) {
                 _log.Error($"Instance config is null");
-                Error err = new(ErrorCode.INVALID_REQUEST, new[] { "Instance config is null" });
+                Error err = Error.Create(ErrorCode.INVALID_REQUEST, "Instance config is null");
                 var resp = req.CreateResponse(HttpStatusCode.InternalServerError);
                 await resp.WriteAsJsonAsync(err);
                 return resp;

src/ApiService/ApiService/TestHooks/NodeOperationsTestHooks.cs

@@ -10,7 +10,7 @@ using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 #if DEBUG
 namespace ApiService.TestHooks {
 
-    sealed record MarkTasks(Node node, Error? error);
+    sealed record MarkTasks(Node node, Error error);
 
     public class NodeOperationsTestHooks {
         private readonly ILogTracer _log;

src/ApiService/ApiService/onefuzzlib/EndpointAuthorization.cs

@@ -75,9 +75,9 @@ public class EndpointAuthorization : IEndpointAuthorization {
 
         return await _context.RequestHandling.NotOk(
             req,
-            new Error(
+           Error.Create(
                 ErrorCode.UNAUTHORIZED,
-                new string[] { reason ?? "Unrecognized agent" }
+                reason ?? "Unrecognized agent"
             ),
             "token verification",
             HttpStatusCode.Unauthorized
@@ -92,9 +92,9 @@ public class EndpointAuthorization : IEndpointAuthorization {
 
         var config = await _context.ConfigOperations.Fetch();
         if (config is null) {
-            return new Error(
+            return Error.Create(
-                Code: ErrorCode.INVALID_CONFIGURATION,
+                ErrorCode.INVALID_CONFIGURATION,
-                Errors: new string[] { "no instance configuration found " });
+                "no instance configuration found ");
         }
 
         return CheckRequireAdminsImpl(config, tokenResult.OkV.UserInfo);
@@ -117,9 +117,7 @@ public class EndpointAuthorization : IEndpointAuthorization {
         }
 
         if (config.Admins is null) {
-            return new Error(
+            return Error.Create(ErrorCode.UNAUTHORIZED, "pool modification disabled ");
-                Code: ErrorCode.UNAUTHORIZED,
-                Errors: new string[] { "pool modification disabled " });
         }
 
         if (userInfo.ObjectId is Guid objectId) {
@@ -127,13 +125,9 @@ public class EndpointAuthorization : IEndpointAuthorization {
                 return OneFuzzResultVoid.Ok;
             }
 
-            return new Error(
+            return Error.Create(ErrorCode.UNAUTHORIZED, "not authorized to manage instance");
-                Code: ErrorCode.UNAUTHORIZED,
-                Errors: new string[] { "not authorized to manage instance" });
         } else {
-            return new Error(
+            return Error.Create(ErrorCode.UNAUTHORIZED, "user had no Object ID");
-                Code: ErrorCode.UNAUTHORIZED,
-                Errors: new string[] { "user had no Object ID" });
         }
     }
 
@@ -157,16 +151,12 @@ public class EndpointAuthorization : IEndpointAuthorization {
             var allowed = await membershipChecker.IsMember(rule.AllowedGroupsIds, memberId);
             if (!allowed) {
                 _log.Error($"unauthorized access: {memberId:Tag:MemberId} is not authorized to access {path:Tag:Path}");
-                return new Error(
+                return Error.Create(ErrorCode.UNAUTHORIZED, "not approved to use this endpoint");
-                    Code: ErrorCode.UNAUTHORIZED,
-                    Errors: new string[] { "not approved to use this endpoint" });
             } else {
                 return OneFuzzResultVoid.Ok;
             }
         } catch (Exception ex) {
-            return new Error(
+            return Error.Create(ErrorCode.UNAUTHORIZED, "unable to interact with graph", ex.Message);
-                Code: ErrorCode.UNAUTHORIZED,
-                Errors: new string[] { "unable to interact with graph", ex.Message });
         }
     }
 

src/ApiService/ApiService/onefuzzlib/ImageReference.cs

@@ -22,7 +22,7 @@ public abstract record ImageReference {
     public static ImageReference MustParse(string image) {
         var result = TryParse(image);
         if (!result.IsOk) {
-            var msg = string.Join(", ", result.ErrorV.Errors ?? Array.Empty<string>());
+            var msg = result.ErrorV.Errors != null ? string.Join(", ", result.ErrorV.Errors) : string.Empty;
             throw new ArgumentException(msg, nameof(image));
         }
 
@@ -46,18 +46,17 @@ public abstract record ImageReference {
             } else if (identifier.ResourceType == ImageResource.ResourceType) {
                 result = new Image(identifier);
             } else {
-                return new Error(
+                return Error.Create(
                     ErrorCode.INVALID_IMAGE,
-                    new[] { $"Unknown image resource type: {identifier.ResourceType}" });
+                    $"Unknown image resource type: {identifier.ResourceType}");
             }
         } catch (FormatException) {
             // not an ARM identifier, try to parse a marketplace image:
             var imageParts = image.Split(":");
             // The python code would throw if more than 4 parts are found in the split
             if (imageParts.Length != 4) {
-                return new Error(
+                return Error.Create(
-                    Code: ErrorCode.INVALID_IMAGE,
+                    ErrorCode.INVALID_IMAGE, $"Expected 4 ':' separated parts in '{image}'");
-                    new[] { $"Expected 4 ':' separated parts in '{image}'" });
             }
 
             result = new Marketplace(
@@ -112,10 +111,10 @@ public abstract record ImageReference {
                 if (resource.Value.Data.OSType is OperatingSystemTypes os) {
                     return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
                 } else {
-                    return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
+                    return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
                 }
             } catch (Exception ex) when (ex is RequestFailedException) {
-                return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
+                return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
             }
         }
     }
@@ -129,10 +128,10 @@ public abstract record ImageReference {
                 if (resource.Value.Data.OSType is OperatingSystemTypes os) {
                     return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
                 } else {
-                    return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
+                    return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
                 }
             } catch (Exception ex) when (ex is RequestFailedException) {
-                return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
+                return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
             }
         }
     }
@@ -145,10 +144,10 @@ public abstract record ImageReference {
                 if (resource.Value.Data.OSType is OperatingSystemTypes os) {
                     return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
                 } else {
-                    return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
+                    return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
                 }
             } catch (Exception ex) when (ex is RequestFailedException) {
-                return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
+                return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
             }
         }
     }
@@ -162,10 +161,10 @@ public abstract record ImageReference {
                 if (resource.Value.Data.OSType is OperatingSystemTypes os) {
                     return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
                 } else {
-                    return new Error(ErrorCode.INVALID_IMAGE, new[] { "Specified image had no OSType" });
+                    return Error.Create(ErrorCode.INVALID_IMAGE, "Specified image had no OSType");
                 }
             } catch (Exception ex) when (ex is RequestFailedException) {
-                return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
+                return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
             }
         }
     }
@@ -178,7 +177,7 @@ public abstract record ImageReference {
                 var os = resource.Value.Data.StorageProfile.OSDisk.OSType.ToString();
                 return OneFuzzResult.Ok(Enum.Parse<Os>(os.ToString(), ignoreCase: true));
             } catch (Exception ex) when (ex is RequestFailedException) {
-                return new Error(ErrorCode.INVALID_IMAGE, new[] { ex.ToString() });
+                return Error.Create(ErrorCode.INVALID_IMAGE, ex.ToString());
             }
         }
     }

src/ApiService/ApiService/onefuzzlib/JobOperations.cs

@@ -80,7 +80,7 @@ public class JobOperations : StatefulOrm<Job, JobState, JobOperations>, IJobOper
 
         await foreach (var job in QueryAsync(filter)) {
             await foreach (var task in _context.TaskOperations.QueryAsync(Query.PartitionKey(job.JobId.ToString()))) {
-                await _context.TaskOperations.MarkFailed(task, new Error(ErrorCode.TASK_FAILED, new[] { "job never not start" }));
+                await _context.TaskOperations.MarkFailed(task, Error.Create(ErrorCode.TASK_FAILED, "job never not start"));
             }
             _logTracer.Info($"stopping job that never started: {job.JobId:Tag:JobId}");
 

src/ApiService/ApiService/onefuzzlib/NodeOperations.cs

@@ -48,7 +48,7 @@ public interface INodeOperations : IStatefulOrm<Node, NodeState> {
 
     IAsyncEnumerable<Node> GetDeadNodes(Guid scaleSetId, TimeSpan expirationPeriod);
 
-    Async.Task MarkTasksStoppedEarly(Node node, Error? error = null);
+    Async.Task MarkTasksStoppedEarly(Node node, Error? error);
     static readonly TimeSpan NODE_EXPIRATION_TIME = TimeSpan.FromHours(1.0);
     static readonly TimeSpan NODE_REIMAGE_TIME = TimeSpan.FromDays(6.0);
 
@@ -213,18 +213,18 @@ public class NodeOperations : StatefulOrm<Node, NodeState, NodeOperations>, INod
 
             var scaleset = scalesetResult.OkV;
             if (!scaleset.State.IsAvailable()) {
-                return CanProcessNewWorkResponse.NotAllowed($"scaleset not available for work. Scaleset state '{scaleset.State}'"); ;
+                return CanProcessNewWorkResponse.NotAllowed($"scaleset not available for work. Scaleset state '{scaleset.State}'");
             }
         }
 
         var poolResult = await _context.PoolOperations.GetByName(node.PoolName);
         if (!poolResult.IsOk) {
-            return CanProcessNewWorkResponse.NotAllowed("invalid pool"); ;
+            return CanProcessNewWorkResponse.NotAllowed("invalid pool");
         }
 
         var pool = poolResult.OkV;
         if (!PoolStateHelper.Available.Contains(pool.State)) {
-            return CanProcessNewWorkResponse.NotAllowed("pool is not available for work"); ;
+            return CanProcessNewWorkResponse.NotAllowed("pool is not available for work");
         }
 
         return CanProcessNewWorkResponse.Allowed();
@@ -585,14 +585,19 @@ public class NodeOperations : StatefulOrm<Node, NodeState, NodeOperations>, INod
     }
 
 
-    public async Async.Task MarkTasksStoppedEarly(Node node, Error? error = null) {
+    public async Async.Task MarkTasksStoppedEarly(Node node, Error? error) {
-        if (error is null) {
-            error = new Error(ErrorCode.TASK_FAILED, new[] { $"node reimaged during task execution.  machine_id: {node.MachineId}" });
-        }
-
         await foreach (var entry in _context.NodeTasksOperations.GetByMachineId(node.MachineId)) {
             var task = await _context.TaskOperations.GetByTaskId(entry.TaskId);
-            if (task is not null) {
+            if (task is not null && !TaskStateHelper.ShuttingDown(task.State)) {
+                var message = $"Node {node.MachineId} stopping while the task state is '{task.State}'";
+                if (error is not null) {
+                    if (error.Errors == null) {
+                        error = error with { Errors = new List<string>() };
+                    }
+                    error.Errors.Add(message);
+                } else {
+                    error = Error.Create(ErrorCode.TASK_FAILED, message);
+                }
                 await _context.TaskOperations.MarkFailed(task, error);
             }
             if (!node.DebugKeepNode) {
@@ -605,7 +610,7 @@ public class NodeOperations : StatefulOrm<Node, NodeState, NodeOperations>, INod
     }
 
     public new async Async.Task Delete(Node node) {
-        await MarkTasksStoppedEarly(node);
+        await MarkTasksStoppedEarly(node, Error.Create(ErrorCode.INVALID_NODE, "node is being deleted"));
         await _context.NodeTasksOperations.ClearByMachineId(node.MachineId);
         await _context.NodeMessageOperations.ClearMessages(node.MachineId);
         var r = await base.Delete(node);

src/ApiService/ApiService/onefuzzlib/NsgOperations.cs

@@ -37,15 +37,14 @@ namespace Microsoft.OneFuzz.Service {
         public async Async.Task<OneFuzzResult<bool>> AssociateSubnet(string name, VirtualNetworkResource vnet, SubnetResource subnet) {
             var nsg = await GetNsg(name);
             if (nsg == null) {
-                return OneFuzzResult<bool>.Error(new Error(ErrorCode.UNABLE_TO_FIND,
+                return OneFuzzResult<bool>.Error(Error.Create(ErrorCode.UNABLE_TO_FIND,
-                    new[] { $"cannot associate subnet. nsg {name} not found" }));
+                    $"cannot associate subnet. nsg {name} not found"));
             }
 
             if (nsg.Data.Location != vnet.Data.Location) {
-                return OneFuzzResult<bool>.Error(new Error(ErrorCode.UNABLE_TO_UPDATE,
+                return OneFuzzResult<bool>.Error(Error.Create(ErrorCode.UNABLE_TO_UPDATE,
-                    new[] {
                         $"subnet and nsg have to be in the same region. nsg {nsg.Data.Name} {nsg.Data.Location}, subnet: {subnet.Data.Name} {subnet.Data}"
-                    }));
+                    ));
             }
 
             if (subnet.Data.NetworkSecurityGroup != null && subnet.Data.NetworkSecurityGroup.Id == nsg.Id) {

src/ApiService/ApiService/onefuzzlib/ProxyOperations.cs

@@ -212,8 +212,8 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
     }
 
     private async System.Threading.Tasks.Task<Proxy> SetProvisionFailed(Proxy proxy, VirtualMachineInstanceView? instanceView) {
-        var errors = GetErrors(proxy, instanceView).ToArray();
+        var errors = GetErrors(proxy, instanceView);
-        return await SetFailed(proxy, new Error(ErrorCode.PROXY_FAILED, errors));
+        return await SetFailed(proxy, new Error(ErrorCode.PROXY_FAILED, errors.ToList()));
     }
 
     private async Task<Proxy> SetFailed(Proxy proxy, Error error) {
@@ -259,7 +259,7 @@ public class ProxyOperations : StatefulOrm<Proxy, VmState, ProxyOperations>, IPr
         var vm = GetVm(proxy, config);
         var vmData = await _context.VmOperations.GetVm(vm.Name);
         if (vmData is null) {
-            return await SetFailed(proxy, new Error(ErrorCode.PROXY_FAILED, new[] { "azure not able to find vm" }));
+            return await SetFailed(proxy, Error.Create(ErrorCode.PROXY_FAILED, "azure not able to find vm"));
         }
 
         if (vmData.ProvisioningState == "Failed") {

src/ApiService/ApiService/onefuzzlib/ReproOperations.cs

@@ -180,9 +180,9 @@ public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IRe
         if (vmData == null) {
             return await _context.ReproOperations.SetError(
                 repro,
-                new Error(
+                Error.Create(
                     ErrorCode.VM_CREATE_FAILED,
-                    new string[] { "failed before launching extensions" }));
+                    "failed before launching extensions"));
         }
         if (vmData.ProvisioningState == "Failed") {
             var failedVmData = await _context.VmOperations.GetVmWithInstanceView(vm.Name);
@@ -232,7 +232,7 @@ public class ReproOperations : StatefulOrm<Repro, VmState, ReproOperations>, IRe
             repro,
             new Error(
                 ErrorCode.VM_CREATE_FAILED,
-                errors));
+                errors.ToList()));
     }
 
     public async Task<OneFuzzResultVoid> BuildReproScript(Repro repro) {

src/ApiService/ApiService/onefuzzlib/Request.cs

@@ -98,9 +98,9 @@ public class RequestHandling : IRequestHandling {
                     }
 
                     if (errors.Any()) {
-                        return new Error(
+                        return Error.Create(
-                            Code: ErrorCode.INVALID_REQUEST,
+                            ErrorCode.INVALID_REQUEST,
-                            Errors: errors.ToArray());
+                            errors.ToArray());
                     }
                 }
 
@@ -109,9 +109,9 @@ public class RequestHandling : IRequestHandling {
                 if (Validator.TryValidateObject(t, validationContext, validationResults, validateAllProperties: true)) {
                     return OneFuzzResult.Ok(t);
                 } else {
-                    return new Error(
+                    return Error.Create(
-                        Code: ErrorCode.INVALID_REQUEST,
+                        ErrorCode.INVALID_REQUEST,
-                        Errors: validationResults.Select(vr => vr.ToString()).ToArray());
+                        validationResults.Select(vr => vr.ToString()).ToArray());
                 }
             } else {
                 return OneFuzzResult<T>.Error(
@@ -154,13 +154,12 @@ public class RequestHandling : IRequestHandling {
             }
         }
 
-        return new Error(
+        return Error.Create(
             ErrorCode.INVALID_REQUEST,
-            new string[] {
                 exception.Message,
                 exception.Source ?? string.Empty,
                 exception.StackTrace ?? string.Empty
-            }
+
         );
     }
 

src/ApiService/ApiService/onefuzzlib/ScalesetOperations.cs

@@ -291,7 +291,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
 
         if (scaleset.Auth is null) {
             _logTracer.Error($"Scaleset Auth is missing for scaleset {scaleset.ScalesetId:Tag:ScalesetId}");
-            return await SetFailed(scaleset, new Error(ErrorCode.UNABLE_TO_CREATE, new[] { "missing required auth" }));
+            return await SetFailed(scaleset, Error.Create(ErrorCode.UNABLE_TO_CREATE, "missing required auth"));
         }
 
         var vmss = await _context.VmssOperations.GetVmss(scaleset.ScalesetId);
@@ -455,7 +455,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
                 return await SetFailed(scaleset, imageOsResult.ErrorV);
             } else if (imageOsResult.OkV != pool.Os) {
                 _logTracer.Error($"got invalid OS: {imageOsResult.OkV:Tag:ActualOs} for scaleset: {scaleset.ScalesetId:Tag:ScalesetId} expected OS {pool.Os:Tag:ExpectedOs}");
-                return await SetFailed(scaleset, new Error(ErrorCode.INVALID_REQUEST, new[] { $"invalid os (got: {imageOsResult.OkV} needed: {pool.Os})" }));
+                return await SetFailed(scaleset, Error.Create(ErrorCode.INVALID_REQUEST, $"invalid os (got: {imageOsResult.OkV} needed: {pool.Os})"));
             } else {
                 return await SetState(scaleset, ScalesetState.Setup);
             }
@@ -605,7 +605,7 @@ public class ScalesetOperations : StatefulOrm<Scaleset, ScalesetState, ScalesetO
 
             _log.Info($"{errorMessage} {deadNode.MachineId:Tag:MachineId} {deadNode.ScalesetId:Tag:ScalesetId}");
 
-            var error = new Error(ErrorCode.TASK_FAILED, new[] { $"{errorMessage} scaleset_id {deadNode.ScalesetId} last heartbeat:{deadNode.Heartbeat}" });
+            var error = Error.Create(ErrorCode.TASK_FAILED, $"{errorMessage} scaleset_id {deadNode.ScalesetId} last heartbeat:{deadNode.Heartbeat}");
             await _context.NodeOperations.MarkTasksStoppedEarly(deadNode, error);
             toReimage[deadNode.MachineId] = await _context.NodeOperations.ToReimage(deadNode, true);
         }

src/ApiService/ApiService/onefuzzlib/TaskOperations.cs

@@ -106,7 +106,7 @@ public class TaskOperations : StatefulOrm<Task, TaskState, TaskOperations>, ITas
         }
 
         if (!task.State.HasStarted()) {
-            await MarkFailed(task, new Error(Code: ErrorCode.TASK_FAILED, Errors: new[] { "task never started" }));
+            await MarkFailed(task, Error.Create(ErrorCode.TASK_FAILED, "task never started"));
         } else {
             _ = await SetState(task, TaskState.Stopping);
         }
@@ -133,7 +133,7 @@ public class TaskOperations : StatefulOrm<Task, TaskState, TaskOperations>, ITas
         foreach (var t in taskInJob) {
             if (t.Config.PrereqTasks != null) {
                 if (t.Config.PrereqTasks.Contains(task.TaskId)) {
-                    await MarkFailed(t, new Error(ErrorCode.TASK_FAILED, new[] { $"prerequisite task failed.  task_id:{t.TaskId}" }), taskInJob);
+                    await MarkFailed(t, Error.Create(ErrorCode.TASK_FAILED, $"prerequisite task failed.  task_id:{t.TaskId}"), taskInJob);
                 }
             }
         }
@@ -264,7 +264,7 @@ public class TaskOperations : StatefulOrm<Task, TaskState, TaskOperations>, ITas
 
                 // if a prereq task fails, then mark this task as failed
                 if (t == null) {
-                    await MarkFailed(task, new Error(ErrorCode.INVALID_REQUEST, Errors: new[] { "unable to find prereq task" }));
+                    await MarkFailed(task, Error.Create(ErrorCode.INVALID_REQUEST, "unable to find prereq task"));
                     return false;
                 }
 

src/ApiService/ApiService/onefuzzlib/VmssOperations.cs

@@ -235,7 +235,7 @@ public class VmssOperations : IVmssOperations {
         try {
             return OneFuzzResult.Ok(await GetInstanceIdForVmId(name, vmId));
         } catch {
-            return new Error(ErrorCode.UNABLE_TO_FIND, new string[] { $"unable to find scaleset machine: {name}:{vmId}" });
+            return Error.Create(ErrorCode.UNABLE_TO_FIND, $"unable to find scaleset machine: {name}:{vmId}");
         }
     }
 

src/ApiService/ApiService/onefuzzlib/notifications/JinjaTemplateAdapter.cs

@@ -189,7 +189,7 @@ public class JinjaTemplateAdapter {
                     new List<TaskDebugFlag> { TaskDebugFlag.KeepNodeOnCompletion },
                     true
                 ),
-                new Error(ErrorCode.UNABLE_TO_FIND, new string[] { "some error message" }),
+                Error.Create(ErrorCode.UNABLE_TO_FIND, "some error message"),
                 new Authentication("password", "public key", "private key"),
                 DateTimeOffset.UtcNow,
                 DateTimeOffset.UtcNow,

src/ApiService/ApiService/onefuzzlib/notifications/NotificationsBase.cs

@@ -17,7 +17,7 @@ public abstract class NotificationsBase {
 
     public async Async.Task LogFailedNotification(Report report, Exception error, Guid notificationId) {
         _logTracer.Error($"notification failed: notification_id:{notificationId:Tag:NotificationId} job_id:{report.JobId:Tag:JobId} task_id:{report.TaskId:Tag:TaskId} err:{error.Message:Tag:Error}");
-        Error? err = new Error(ErrorCode.NOTIFICATION_FAILURE, new string[] { $"{error}" });
+        Error? err = Error.Create(ErrorCode.NOTIFICATION_FAILURE, $"{error}");
         await _context.Events.SendEvent(new EventNotificationFailed(
             NotificationId: notificationId,
             JobId: report.JobId,

src/ApiService/IntegrationTests/Fakes/TestEndpointAuthorization.cs

@@ -36,9 +36,9 @@ sealed class TestEndpointAuthorization : EndpointAuthorization {
 
         return _context.RequestHandling.NotOk(
             req,
-            new Error(
+            Error.Create(
                 ErrorCode.UNAUTHORIZED,
-                new string[] { "Unrecognized agent" }
+                "Unrecognized agent"
             ),
             "token verification",
             HttpStatusCode.Unauthorized