Merge branch 'main' into kanan/oom-logging

Author: kananb

src/ApiService/ApiService/Functions/Containers.cs

@@ -16,11 +16,12 @@ public ContainersFunction(ILogger<ContainersFunction> logger, IOnefuzzContext co
 
     [Function("Containers")]
     [Authorize(Allow.User)]
-    public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", "POST", "DELETE")] HttpRequestData req)
+    public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET", "POST", "PATCH", "DELETE")] HttpRequestData req)
         => req.Method switch {
             "GET" => Get(req),
             "POST" => Post(req),
             "DELETE" => Delete(req),
+            "PATCH" => Patch(req),
             _ => throw new NotSupportedException(),
         };
 
@@ -108,4 +109,21 @@ private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
                 SasUrl: sas,
                 Metadata: post.Metadata));
     }
+
+    private async Async.Task<HttpResponseData> Patch(HttpRequestData req) {
+        var request = await RequestHandling.ParseRequest<ContainerUpdate>(req);
+        if (!request.IsOk) {
+            return await _context.RequestHandling.NotOk(req, request.ErrorV, context: "container update");
+        }
+
+        var toUpdate = request.OkV;
+        _logger.LogInformation("updating {ContainerName}", toUpdate.Name);
+        var updated = await _context.Containers.CreateOrUpdateContainerTag(toUpdate.Name, StorageType.Corpus, toUpdate.Metadata.ToDictionary(x => x.Key, x => x.Value));
+
+        if (!updated.IsOk) {
+            return await _context.RequestHandling.NotOk(req, updated.ErrorV, "container update");
+        }
+
+        return await RequestHandling.Ok(req, new ContainerInfoBase(toUpdate.Name, toUpdate.Metadata));
+    }
 }

src/ApiService/ApiService/Functions/QueueFileChanges.cs

@@ -2,6 +2,8 @@
 using System.Text.Json.Nodes;
 using System.Threading.Tasks;
 using Azure.Core;
+using Azure.Storage.Blobs;
+using Azure.Storage.Blobs.Models;
 using Microsoft.Azure.Functions.Worker;
 using Microsoft.Extensions.Logging;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
@@ -60,7 +62,11 @@ public async Async.Task Run(
         try {
             var result = await FileAdded(storageAccount, fileChangeEvent);
             if (!result.IsOk) {
-                await RequeueMessage(msg, result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED ? TimeSpan.FromDays(1) : null);
+                if (result.ErrorV.Code == ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED) {
+                    await RequeueMessage(msg, TimeSpan.FromDays(1), incrementDequeueCount: false);
+                } else {
+                    await RequeueMessage(msg);
+                }
             }
         } catch (Exception e) {
             _log.LogError(e, "File Added failed");
@@ -83,21 +89,26 @@ private async Async.Task<OneFuzzResultVoid> FileAdded(ResourceIdentifier storage
 
         _log.LogInformation("file added : {Container} - {Path}", container.String, path);
 
+        var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
+        var containerClient = account.GetBlobContainerClient(container.String);
+        var containerProps = await containerClient.GetPropertiesAsync();
+
+        if (_context.NotificationOperations.ShouldPauseNotificationsForContainer(containerProps.Value.Metadata)) {
+            return Error.Create(ErrorCode.ADO_WORKITEM_PROCESSING_DISABLED, $"container {container} has a metadata tag set to pause notifications processing");
+        }
+
         var (_, result) = await (
-            ApplyRetentionPolicy(storageAccount, container, path),
+            ApplyRetentionPolicy(containerClient, containerProps, path),
             _notificationOperations.NewFiles(container, path));
 
         return result;
     }
 
-    private async Async.Task<bool> ApplyRetentionPolicy(ResourceIdentifier storageAccount, Container container, string path) {
+    private async Async.Task<bool> ApplyRetentionPolicy(BlobContainerClient containerClient, BlobContainerProperties containerProps, string path) {
         if (await _context.FeatureManagerSnapshot.IsEnabledAsync(FeatureFlagConstants.EnableContainerRetentionPolicies)) {
             // default retention period can be applied to the container
             // if one exists, we will set the expiry date on the newly-created blob, if it doesn't already have one
-            var account = await _storage.GetBlobServiceClientForAccount(storageAccount);
-            var containerClient = account.GetBlobContainerClient(container.String);
-            var containerProps = await containerClient.GetPropertiesAsync();
-            var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Value.Metadata);
+            var retentionPeriod = RetentionPolicyUtils.GetContainerRetentionPeriodFromMetadata(containerProps.Metadata);
             if (!retentionPeriod.IsOk) {
                 _log.LogError("invalid retention period: {Error}", retentionPeriod.ErrorV);
             } else if (retentionPeriod.OkV is TimeSpan period) {
@@ -116,7 +127,7 @@ private async Async.Task<bool> ApplyRetentionPolicy(ResourceIdentifier storageAc
         return false;
     }
 
-    private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null) {
+    private async Async.Task RequeueMessage(string msg, TimeSpan? visibilityTimeout = null, bool incrementDequeueCount = true) {
         var json = JsonNode.Parse(msg);
 
         // Messages that are 'manually' requeued by us as opposed to being requeued by the azure functions runtime
@@ -135,7 +146,9 @@ await _context.Queue.QueueObject(
                 StorageType.Config)
                 .IgnoreResult();
         } else {
-            json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1;
+            if (incrementDequeueCount) {
+                json!["data"]!["customDequeueCount"] = newCustomDequeueCount + 1;
+            }
             await _context.Queue.QueueObject(
                 QueueFileChangesQueueName,
                 json,

src/ApiService/ApiService/Functions/QueueJobResult.cs

@@ -31,7 +31,12 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo
 
         var job = await _jobs.Get(task.JobId);
         if (job == null) {
-            _log.LogWarning("invalid {JobId}", task.JobId);
+            _log.LogWarning("invalid message {JobId}", task.JobId);
+            return;
+        }
+
+        if (jr.CreatedAt == null) {
+            _log.LogWarning("invalid message, no created_at field {JobId}", task.JobId);
             return;
         }
 
@@ -52,7 +57,7 @@ public async Async.Task Run([QueueTrigger("job-result", Connection = "AzureWebJo
             return;
         }
 
-        var jobResult = await _context.JobResultOperations.CreateOrUpdate(job.JobId, jobResultType, value);
+        var jobResult = await _context.JobResultOperations.CreateOrUpdate(job.JobId, jr.TaskId, jr.MachineId, jr.CreatedAt.Value, jr.Version, jobResultType, value);
         if (!jobResult.IsOk) {
             _log.LogError("failed to create or update with job result {JobId}", job.JobId);
         }

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -53,6 +53,9 @@ public enum ErrorCode {
     INVALID_RETENTION_PERIOD = 497,
     INVALID_CLI_VERSION = 498,
     TRANSIENT_NOTIFICATION_FAILURE = 499,
+
+    FAILED_CONTAINER_PROPERTIES_ACCESS = 500,
+    FAILED_SAVING_CONTAINER_METADATA = 501,
     // NB: if you update this enum, also update enums.py
 }
 

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -34,19 +34,6 @@ public enum HeartbeatType {
     TaskAlive,
 }
 
-[SkipRename]
-public enum JobResultType {
-    NewCrashingInput,
-    NoReproCrashingInput,
-    NewReport,
-    NewUniqueReport,
-    NewRegressionReport,
-    NewCoverage,
-    NewCrashDump,
-    CoverageData,
-    RuntimeStats,
-}
-
 public record HeartbeatData(HeartbeatType Type);
 
 public record TaskHeartbeatEntry(
@@ -55,12 +42,14 @@ public record TaskHeartbeatEntry(
     [property: Required] Guid MachineId,
     HeartbeatData[] Data);
 
-public record JobResultData(JobResultType Type);
+public record JobResultData(string Type);
 
 public record TaskJobResultEntry(
     Guid TaskId,
     Guid? JobId,
     Guid MachineId,
+    DateTime? CreatedAt,
+    double Version,
     JobResultData Data,
     Dictionary<string, double> Value
     );
@@ -921,26 +910,24 @@ public record SecretAddress<T>(Uri Url) : ISecret<T> {
 public record SecretData<T>(ISecret<T> Secret) {
 }
 
+[SkipRename]
+public enum JobResultType {
+    CoverageData,
+    RuntimeStats,
+}
+
 public record JobResult(
-    [PartitionKey][RowKey] Guid JobId,
+    [PartitionKey] Guid JobId,
+    [RowKey] string TaskIdMachineIdMetric,
+    Guid TaskId,
+    Guid MachineId,
+    DateTime CreatedAt,
     string Project,
     string Name,
-    double NewCrashingInput = 0,
-    double NoReproCrashingInput = 0,
-    double NewReport = 0,
-    double NewUniqueReport = 0,
-    double NewRegressionReport = 0,
-    double NewCrashDump = 0,
-    double InstructionsCovered = 0,
-    double TotalInstructions = 0,
-    double CoverageRate = 0,
-    double IterationCount = 0
-) : EntityBase() {
-    public JobResult(Guid JobId, string Project, string Name) : this(
-        JobId: JobId,
-        Project: Project,
-        Name: Name, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0) { }
-}
+    string Type,
+    double Version,
+    Dictionary<string, double> MetricValue
+) : EntityBase();
 
 public record JobConfig(
     string Project,

src/ApiService/ApiService/OneFuzzTypes/Requests.cs

@@ -128,6 +128,11 @@ public record ContainerDelete(
     IDictionary<string, string>? Metadata = null
 ) : BaseRequest;
 
+public record ContainerUpdate(
+    [property: Required] Container Name,
+    [property: Required] IDictionary<string, string> Metadata
+) : BaseRequest;
+
 public record NotificationCreate(
     [property: Required] Container Container,
     [property: Required] bool ReplaceExisting,

src/ApiService/ApiService/host.json

@@ -1,5 +1,6 @@
 {
     "version": "2.0",
+    "functionTimeout": "12:00:00",
     "logging": {
         "applicationInsights": {
             "samplingSettings": {

src/ApiService/ApiService/onefuzzlib/Containers.cs

@@ -1,5 +1,6 @@
 using System.IO;
 using System.IO.Compression;
+using System.Text.Json;
 using System.Threading.Tasks;
 using ApiService.OneFuzzLib.Orm;
 using Azure;
@@ -41,6 +42,8 @@ public interface IContainers {
     public Async.Task<OneFuzzResultVoid> DownloadAsZip(Container container, StorageType storageType, Stream stream, string? prefix = null);
 
     public Async.Task DeleteAllExpiredBlobs();
+
+    public Async.Task<OneFuzzResultVoid> CreateOrUpdateContainerTag(Container container, StorageType storageType, Dictionary<string, string> tags);
 }
 
 public class Containers : Orm<ContainerInformation>, IContainers {
@@ -448,4 +451,29 @@ private async Async.Task DeleteExpiredBlobsForAccount(ResourceIdentifier storage
             }
         }
     }
+
+    public async Task<OneFuzzResultVoid> CreateOrUpdateContainerTag(Container container, StorageType storageType, Dictionary<string, string> tags) {
+        var client = await FindContainer(container, storageType);
+        if (client is null || !await client.ExistsAsync()) {
+            return Error.Create(ErrorCode.INVALID_CONTAINER, $"Could not find container {container} in {storageType}");
+        }
+
+        var metadataRequest = await client.GetPropertiesAsync();
+        if (metadataRequest is null || metadataRequest.GetRawResponse().IsError) {
+            return Error.Create(ErrorCode.FAILED_CONTAINER_PROPERTIES_ACCESS, $"Could not access container properties for container: {container} in {storageType}");
+        }
+
+        var metadata = metadataRequest.Value.Metadata ?? new Dictionary<string, string>();
+
+        foreach (var kvp in tags) {
+            metadata[kvp.Key] = kvp.Value;
+        }
+
+        var saveMetadataRequest = await client.SetMetadataAsync(metadata);
+        if (saveMetadataRequest is null || saveMetadataRequest.GetRawResponse().IsError) {
+            return Error.Create(ErrorCode.FAILED_SAVING_CONTAINER_METADATA, $"Could not save metadata to container: {container} in {storageType}. Metadata: {JsonSerializer.Serialize(metadata)}");
+        }
+
+        return OneFuzzResultVoid.Ok;
+    }
 }