Fork Sync: Update from parent repository

.github/workflows/ci.yml

@@ -18,7 +18,7 @@ env:
   CARGO_TERM_COLOR: always
   SCCACHE_DIR: ${{github.workspace}}/sccache/
   SCCACHE_CACHE_SIZE: 1G
-  ACTIONS_CACHE_KEY_DATE: 2022-11-21-02
+  ACTIONS_CACHE_KEY_DATE: 2023-04-19
   CI: true
 
 jobs:
@@ -36,6 +36,11 @@ jobs:
               "1ES.Pool=onefuzz-ci",
               "1ES.ImageOverride=github-runner-image-ubuntu-20.04",
             ]
+          - [
+              self-hosted,
+              "1ES.Pool=onefuzz-ci-arm64",
+              "1ES.ImageOverride=github-runner-image-ubuntu-20.04-arm64",
+            ]
     runs-on: "${{ matrix.os }}"
     steps:
       - uses: actions/checkout@v3
@@ -470,8 +475,10 @@ jobs:
           cp artifacts/agent-MINGW*/onefuzz-task.exe src/deployment/tools/win64/
           cp artifacts/agent-MINGW*/onefuzz_agent.pdb src/deployment/tools/win64/
           cp artifacts/agent-MINGW*/onefuzz_task.pdb src/deployment/tools/win64/
-          cp artifacts/agent-Linux/onefuzz-agent src/deployment/tools/linux/
-          cp artifacts/agent-Linux/onefuzz-task src/deployment/tools/linux/
+          cp artifacts/agent-Linux-x86_64/onefuzz-agent src/deployment/tools/linux/
+          cp artifacts/agent-Linux-x86_64/onefuzz-task src/deployment/tools/linux/
+          # cp artifacts/agent-Linux-aarch64/onefuzz-agent src/deployment/tools/linux-aarch64/
+          # cp artifacts/agent-Linux-aarch64/onefuzz-task src/deployment/tools/linux-aarch64/
 
           cp artifacts/proxy/onefuzz-proxy-manager src/deployment/tools/linux/
           cp artifacts/service/api-service.zip src/deployment

docs/webhook_events.md

@@ -238,6 +238,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Onefuzz Version",
                     "type": "string"
                 },
+                "report_url": {
+                    "title": "Report Url",
+                    "type": "string"
+                },
                 "scariness_description": {
                     "title": "Scariness Description",
                     "type": "string"
@@ -2158,6 +2162,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Onefuzz Version",
                     "type": "string"
                 },
+                "report_url": {
+                    "title": "Report Url",
+                    "type": "string"
+                },
                 "scariness_description": {
                     "title": "Scariness Description",
                     "type": "string"
@@ -6578,6 +6586,10 @@ If webhook is set to have Event Grid message format then the payload will look a
                     "title": "Onefuzz Version",
                     "type": "string"
                 },
+                "report_url": {
+                    "title": "Report Url",
+                    "type": "string"
+                },
                 "scariness_description": {
                     "title": "Scariness Description",
                     "type": "string"

src/ApiService/ApiService/ApiService.csproj

@@ -27,7 +27,7 @@
     <PackageReference Include="Microsoft.Azure.Functions.Worker" Version="1.10.0" />
     <PackageReference Include="Azure.Data.Tables" Version="12.8.0" />
     <PackageReference Include="Azure.ResourceManager.Compute" Version="1.0.0-beta.8" />
-    <PackageReference Include="Azure.Identity" Version="1.6.0" />
+    <PackageReference Include="Azure.Identity" Version="1.8.2" />
     <PackageReference Include="Azure.Messaging.EventGrid" Version="4.10.0" />
     <PackageReference Include="Azure.ResourceManager" Version="1.3.1" />
     <PackageReference Include="Azure.ResourceManager.Network" Version="1.0.0" />

src/ApiService/ApiService/Functions/Notifications.cs

@@ -22,7 +22,7 @@ private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
             return await _context.RequestHandling.NotOk(req, request.ErrorV, "notification search");
         }
 
-        var entries = request.OkV switch { { Container: null, NotificationId: null } => _context.NotificationOperations.SearchAll(), { Container: var c, NotificationId: null } => _context.NotificationOperations.SearchByRowKeys(c.Select(x => x.String)), { Container: var _, NotificationId: var n } => new[] { await _context.NotificationOperations.GetNotification(n.Value) }.ToAsyncEnumerable(),
+        var entries = request.OkV switch { { Container: null, NotificationId: null } => _context.NotificationOperations.SearchAll(), { Container: var c, NotificationId: null } => _context.NotificationOperations.SearchByRowKeys(c.Select(x => x.String)), { Container: var _, NotificationId: var n } => new[] { await _context.NotificationOperations.GetNotification(n.Value) }.ToAsyncEnumerable()
         };
 
         var response = req.CreateResponse(HttpStatusCode.OK);

src/ApiService/ApiService/Functions/NotificationsTest.cs

@@ -0,0 +1,41 @@
+using System.Net;
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+
+namespace Microsoft.OneFuzz.Service.Functions;
+
+public class NotificationsTest {
+    private readonly ILogTracer _log;
+    private readonly IEndpointAuthorization _auth;
+    private readonly IOnefuzzContext _context;
+
+    public NotificationsTest(ILogTracer log, IEndpointAuthorization auth, IOnefuzzContext context) {
+        _log = log;
+        _auth = auth;
+        _context = context;
+    }
+
+    private async Async.Task<HttpResponseData> Post(HttpRequestData req) {
+        _log.WithTag("HttpRequest", "GET").Info($"Notification test");
+        var request = await RequestHandling.ParseRequest<NotificationTest>(req);
+        if (!request.IsOk) {
+            return await _context.RequestHandling.NotOk(req, request.ErrorV, "notification search");
+        }
+
+        var notificationTest = request.OkV;
+        var result = await _context.NotificationOperations.TriggerNotification(notificationTest.Notification.Container, notificationTest.Notification,
+            notificationTest.Report, isLastRetryAttempt: true);
+        var response = req.CreateResponse(HttpStatusCode.OK);
+        await response.WriteAsJsonAsync(new NotificationTestResponse(result.IsOk, result.ErrorV?.ToString()));
+        return response;
+    }
+
+
+    [Function("NotificationsTest")]
+    public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymous, "POST", Route = "notifications/test")] HttpRequestData req) {
+        return _auth.CallIfUser(req, r => r.Method switch {
+            "POST" => Post(r),
+            _ => throw new InvalidOperationException("Unsupported HTTP method"),
+        });
+    }
+}

src/ApiService/ApiService/OneFuzzTypes/Model.cs

@@ -527,6 +527,10 @@ public RegressionReport Truncate(int maxLength) {
     }
 }
 
+public record UnknownReportType(
+    Uri? ReportUrl
+) : IReport;
+
 [JsonConverter(typeof(NotificationTemplateConverter))]
 #pragma warning disable CA1715
 public interface NotificationTemplate {

src/ApiService/ApiService/OneFuzzTypes/Requests.cs

@@ -129,6 +129,12 @@ public record NotificationSearch(
     Guid? NotificationId
 ) : BaseRequest;
 
+
+public record NotificationTest(
+    [property: Required] Report Report,
+    [property: Required] Notification Notification
+) : BaseRequest;
+
 public record NotificationGet(
     [property: Required] Guid NotificationId
 ) : BaseRequest;

src/ApiService/ApiService/OneFuzzTypes/Responses.cs

@@ -205,3 +205,8 @@ List<Guid> FailedNotificationIds
 public record JinjaToScribanMigrationDryRunResponse(
     List<Guid> NotificationIdsToUpdate
 ) : BaseResponse();
+
+public record NotificationTestResponse(
+    bool Success,
+    string? Error = null
+) : BaseResponse();

src/ApiService/ApiService/onefuzzlib/NotificationOperations.cs

@@ -10,6 +10,9 @@ public interface INotificationOperations : IOrm<Notification> {
     IAsyncEnumerable<(Task, IEnumerable<Container>)> GetQueueTasks();
     Async.Task<OneFuzzResult<Notification>> Create(Container container, NotificationTemplate config, bool replaceExisting);
     Async.Task<Notification?> GetNotification(Guid notifificationId);
+
+    System.Threading.Tasks.Task<OneFuzzResultVoid> TriggerNotification(Container container,
+        Notification notification, IReport? reportOrRegression, bool isLastRetryAttempt = false);
 }
 
 public class NotificationOperations : Orm<Notification>, INotificationOperations {
@@ -30,22 +33,7 @@ public async Async.Task NewFiles(Container container, string filename, bool isLa
                 }
 
                 done.Add(notification.Config);
-
-                if (notification.Config is TeamsTemplate teamsTemplate) {
-                    await _context.Teams.NotifyTeams(teamsTemplate, container, filename, reportOrRegression!, notification.NotificationId);
-                }
-
-                if (reportOrRegression == null) {
-                    continue;
-                }
-
-                if (notification.Config is AdoTemplate adoTemplate) {
-                    await _context.Ado.NotifyAdo(adoTemplate, container, filename, reportOrRegression, isLastRetryAttempt, notification.NotificationId);
-                }
-
-                if (notification.Config is GithubIssuesTemplate githubIssuesTemplate) {
-                    await _context.GithubIssues.GithubIssue(githubIssuesTemplate, container, filename, reportOrRegression, notification.NotificationId);
-                }
+                _ = await TriggerNotification(container, notification, reportOrRegression, isLastRetryAttempt);
             }
         }
 
@@ -74,6 +62,25 @@ public async Async.Task NewFiles(Container container, string filename, bool isLa
         }
     }
 
+    public async System.Threading.Tasks.Task<OneFuzzResultVoid> TriggerNotification(Container container,
+        Notification notification, IReport? reportOrRegression, bool isLastRetryAttempt = false) {
+        switch (notification.Config) {
+            case TeamsTemplate teamsTemplate:
+                await _context.Teams.NotifyTeams(teamsTemplate, container, reportOrRegression!,
+                    notification.NotificationId);
+                break;
+            case AdoTemplate adoTemplate when reportOrRegression is not null:
+                return await _context.Ado.NotifyAdo(adoTemplate, container, reportOrRegression, isLastRetryAttempt,
+                    notification.NotificationId);
+            case GithubIssuesTemplate githubIssuesTemplate when reportOrRegression is not null:
+                await _context.GithubIssues.GithubIssue(githubIssuesTemplate, container, reportOrRegression,
+                    notification.NotificationId);
+                break;
+        }
+
+        return OneFuzzResultVoid.Ok;
+    }
+
     public IAsyncEnumerable<Notification> GetNotifications(Container container) {
         return SearchByRowKeys(new[] { container.String });
     }

src/ApiService/ApiService/onefuzzlib/Reports.cs

@@ -35,7 +35,17 @@ public Reports(ILogTracer log, IContainers containers) {
             return null;
         }
 
-        var blob = await _containers.GetBlob(container, fileName, StorageType.Corpus);
+        var containerClient = await _containers.FindContainer(container, StorageType.Corpus);
+        if (containerClient == null) {
+            if (expectReports) {
+                _log.Error($"get_report invalid container: {filePath:Tag:FilePath}");
+            }
+            return null;
+        }
+
+        Uri reportUrl = containerClient.GetBlobClient(fileName).Uri;
+
+        var blob = (await containerClient.GetBlobClient(fileName).DownloadContentAsync()).Value.Content;
 
         if (blob == null) {
             if (expectReports) {
@@ -44,11 +54,9 @@ public Reports(ILogTracer log, IContainers containers) {
             return null;
         }
 
-        var reportUrl = await _containers.GetFileUrl(container, fileName, StorageType.Corpus);
-
         var reportOrRegression = ParseReportOrRegression(blob.ToString(), reportUrl);
 
-        if (reportOrRegression == null && expectReports) {
+        if (reportOrRegression is UnknownReportType && expectReports) {
             _log.Error($"unable to parse report ({filePath:Tag:FilePath}) as a report or regression");
         }
 
@@ -64,7 +72,7 @@ public Reports(ILogTracer log, IContainers containers) {
         }
     }
 
-    public static IReport? ParseReportOrRegression(string content, Uri? reportUrl) {
+    public static IReport ParseReportOrRegression(string content, Uri reportUrl) {
         var regressionReport = TryDeserialize<RegressionReport>(content);
         if (regressionReport is { CrashTestResult: { } }) {
             return regressionReport with { ReportUrl = reportUrl };
@@ -73,12 +81,17 @@ public Reports(ILogTracer log, IContainers containers) {
         if (report is { CrashType: { } }) {
             return report with { ReportUrl = reportUrl };
         }
-        return null;
+        return new UnknownReportType(reportUrl);
     }
 }
 
 public interface IReport {
     Uri? ReportUrl {
         init;
+        get;
+    }
+    public string FileName() {
+        var segments = (this.ReportUrl ?? throw new ArgumentException()).Segments.Skip(2);
+        return string.Concat(segments);
     }
 };

src/ApiService/ApiService/onefuzzlib/Secrets.cs

@@ -56,13 +56,11 @@ public virtual async Task<SecretData<T>> SaveToKeyvault<T>(SecretData<T> secretD
     }
 
     public async Task<string?> GetSecretStringValue<T>(SecretData<T> data) {
-
-        if (data.Secret is SecretAddress<T> secretAddress) {
-            var secret = await GetSecret(secretAddress.Url);
-            return secret.Value;
-        } else {
-            return data.Secret.ToString();
-        }
+        return (data.Secret) switch {
+            SecretAddress<T> secretAddress => (await GetSecret(secretAddress.Url)).Value,
+            SecretValue<T> sValue => sValue.Value?.ToString(),
+            _ => data.Secret.ToString(),
+        };
     }
 
     public Uri GetKeyvaultAddress() {