Include a reason when a task has never started (#3148)

* Include a reason to mark a task as failed

* mark dependent task cancelled when the task is cancelled

* cleanup

* build fix

Author: chkeita

docs/webhook_events.md

@@ -1183,7 +1183,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 481,
                 482,
                 483,
-                484
+                484,
+                485
             ],
             "title": "ErrorCode"
         },
@@ -1829,7 +1830,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 481,
                 482,
                 483,
-                484
+                484,
+                485
             ],
             "title": "ErrorCode"
         }
@@ -2774,7 +2776,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 481,
                 482,
                 483,
-                484
+                484,
+                485
             ],
             "title": "ErrorCode"
         }
@@ -3501,7 +3504,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 481,
                 482,
                 483,
-                484
+                484,
+                485
             ],
             "title": "ErrorCode"
         },
@@ -5579,7 +5583,8 @@ If webhook is set to have Event Grid message format then the payload will look a
                 481,
                 482,
                 483,
-                484
+                484,
+                485
             ],
             "title": "ErrorCode"
         },

src/ApiService/ApiService/Functions/AgentEvents.cs

@@ -250,7 +250,7 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
 
         if (done.ExitStatus.Success) {
             _log.Info($"task done. {task.JobId:Tag:JobId}:{task.TaskId:Tag:TaskId} {done.ExitStatus:Tag:Status}");
-            await _context.TaskOperations.MarkStopping(task);
+            await _context.TaskOperations.MarkStopping(task, "task is done");
 
             // keep node if keep-on-completion is set
             if (task.Config.Debug?.Contains(TaskDebugFlag.KeepNodeOnCompletion) == true) {

src/ApiService/ApiService/Functions/Tasks.cs

@@ -170,7 +170,7 @@ private async Async.Task<HttpResponseData> Delete(HttpRequestData req) {
 
         }
 
-        await _context.TaskOperations.MarkStopping(task);
+        await _context.TaskOperations.MarkStopping(task, "task is deleted");
 
         var response = req.CreateResponse(HttpStatusCode.OK);
         await response.WriteAsJsonAsync(task);

src/ApiService/ApiService/Functions/TimerTasks.cs

@@ -26,7 +26,7 @@ public async Async.Task Run([TimerTrigger("00:00:15")] TimerInfo myTimer) {
         var expriredTasks = _taskOperations.SearchExpired();
         await foreach (var task in expriredTasks) {
             _logger.Info($"stopping expired task. job_id:{task.JobId:Tag:JobId} task_id:{task.TaskId:Tag:TaskId}");
-            await _taskOperations.MarkStopping(task);
+            await _taskOperations.MarkStopping(task, "task is expired");
         }
 
 

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -39,6 +39,7 @@ public enum ErrorCode {
     UNEXPECTED_DATA_SHAPE = 482,
     UNABLE_TO_SEND = 483,
     NODE_DELETED = 484,
+    TASK_CANCELLED = 485,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/onefuzzlib/JobOperations.cs

@@ -111,7 +111,7 @@ public async Async.Task<Job> Stopping(Job job) {
 
         if (notStopped.Any()) {
             foreach (var task in notStopped) {
-                await _context.TaskOperations.MarkStopping(task);
+                await _context.TaskOperations.MarkStopping(task, "job is stopping");
             }
         } else {
             job = job with { State = JobState.Stopped };

src/ApiService/ApiService/onefuzzlib/TaskOperations.cs

@@ -19,7 +19,7 @@ public interface ITaskOperations : IStatefulOrm<Task, TaskState> {
     Result<IEnumerable<Container>?, TaskConfigError> GetInputContainerQueues(TaskConfig config);
 
     IAsyncEnumerable<Task> SearchExpired();
-    Async.Task MarkStopping(Task task);
+    Async.Task MarkStopping(Task task, string reason);
     Async.Task MarkFailed(Task task, Error error, List<Task>? taskInJob = null);
 
     Async.Task<TaskVm?> GetReproVmConfig(Task task);
@@ -99,14 +99,14 @@ public IAsyncEnumerable<Task> SearchExpired() {
         return QueryAsync(filter: filter);
     }
 
-    public async Async.Task MarkStopping(Task task) {
+    public async Async.Task MarkStopping(Task task, string reason) {
         if (task.State.ShuttingDown()) {
             _logTracer.Verbose($"ignoring post - task stop calls to stop {task.JobId:Tag:JobId}:{task.TaskId:Tag:TaskId}");
             return;
         }
 
         if (!task.State.HasStarted()) {
-            await MarkFailed(task, Error.Create(ErrorCode.TASK_FAILED, "task never started"));
+            await MarkFailed(task, Error.Create(ErrorCode.TASK_CANCELLED, reason, "task never started"));
         } else {
             _ = await SetState(task, TaskState.Stopping);
         }
@@ -124,16 +124,16 @@ public async Async.Task MarkFailed(Task task, Error error, List<Task>? taskInJob
         _logTracer.Info($"task failed {task.JobId:Tag:JobId}:{task.TaskId:Tag:TaskId} - {error:Tag:Error}");
 
         task = await SetState(task with { Error = error }, TaskState.Stopping);
-        await MarkDependantsFailed(task, taskInJob);
-    }
-
-    private async Async.Task MarkDependantsFailed(Task task, List<Task>? taskInJob = null) {
         taskInJob ??= await SearchByPartitionKeys(new[] { $"{task.JobId}" }).ToListAsync();
 
+        var dependentError =
+            error.Code == ErrorCode.TASK_CANCELLED
+            ? Error.Create(ErrorCode.TASK_CANCELLED, $"prerequisite task is cancelled.")
+            : Error.Create(ErrorCode.TASK_FAILED, $"prerequisite task is failed.");
         foreach (var t in taskInJob) {
             if (t.Config.PrereqTasks != null) {
                 if (t.Config.PrereqTasks.Contains(task.TaskId)) {
-                    await MarkFailed(t, Error.Create(ErrorCode.TASK_FAILED, $"prerequisite task failed.  task_id:{t.TaskId}"), taskInJob);
+                    await MarkFailed(t, dependentError, taskInJob);
                 }
             }
         }

src/ApiService/IntegrationTests/AgentEventsTests.cs

@@ -123,7 +123,7 @@ await Context.InsertAll(
     }
 
     [Fact]
-    public async Async.Task WorkerDone_ForNonStartedTask_MarksTaskAsFailed() {
+    public async Async.Task WorkerDone_ForNonStartedTask_MarksTaskAsCancelled() {
         await Context.InsertAll(
             new Node(_poolName, _machineId, _poolId, _poolVersion),
             // task state is scheduled, not running
@@ -148,7 +148,7 @@ await Context.InsertAll(
 
         // should be failed - it never started running
         Assert.Equal(TaskState.Stopping, task.State);
-        Assert.Equal(ErrorCode.TASK_FAILED, task.Error?.Code);
+        Assert.Equal(ErrorCode.TASK_CANCELLED, task.Error?.Code);
     }
 
     [Fact]

src/pytypes/onefuzztypes/enums.py

@@ -291,6 +291,7 @@ class ErrorCode(Enum):
     UNEXPECTED_DATA_SHAPE = 482
     UNABLE_TO_SEND = 483
     NODE_DELETED = 484
+    TASK_CANCELLED = 485
     # NB: if you update this enum, also update Enums.cs