Merge branch 'main' of https://github.com/microsoft/onefuzz into tevoinea/MoreTruncations

Author: tevoinea

.gitignore

@@ -19,4 +19,8 @@ lcov.info
 /src/ApiService/ApiService/Properties/PublishProfiles/*
 /src/ApiService/ApiService/Properties/ServiceDependencies/*
 
-.vs
+.vs
+
+**/__azurite_db_*.json
+**/__blobstorage__
+**/__queuestorage__

src/ApiService/ApiService/ApiService.csproj

@@ -28,7 +28,7 @@
     <PackageReference Include="Azure.Data.Tables" Version="12.8.0" />
     <PackageReference Include="Azure.ResourceManager.Compute" Version="1.0.0-beta.8" />
     <PackageReference Include="Azure.Identity" Version="1.8.2" />
-    <PackageReference Include="Azure.Messaging.EventGrid" Version="4.10.0" />
+    <PackageReference Include="Azure.Messaging.EventGrid" Version="4.15.0" />
     <PackageReference Include="Azure.ResourceManager" Version="1.3.1" />
     <PackageReference Include="Azure.ResourceManager.Network" Version="1.0.0" />
     <PackageReference Include="Azure.ResourceManager.Resources" Version="1.3.0" />

src/ApiService/ApiService/Functions/Events.cs

@@ -0,0 +1,39 @@
+using Microsoft.Azure.Functions.Worker;
+using Microsoft.Azure.Functions.Worker.Http;
+
+namespace Microsoft.OneFuzz.Service.Functions;
+
+public class EventsFunction {
+    private readonly ILogTracer _log;
+    private readonly IEndpointAuthorization _auth;
+    private readonly IOnefuzzContext _context;
+
+    public EventsFunction(ILogTracer log, IEndpointAuthorization auth, IOnefuzzContext context) {
+        _auth = auth;
+        _context = context;
+        _log = log;
+    }
+
+    [Function("Events")]
+    public Async.Task<HttpResponseData> Run([HttpTrigger(AuthorizationLevel.Anonymous, "GET")] HttpRequestData req)
+        => _auth.CallIfUser(req, r => r.Method switch {
+            "GET" => Get(r),
+            _ => throw new NotSupportedException(),
+        });
+
+    private async Async.Task<HttpResponseData> Get(HttpRequestData req) {
+        var request = await RequestHandling.ParseRequest<EventsGet>(req);
+        if (!request.IsOk) {
+            return await _context.RequestHandling.NotOk(req, request.ErrorV, "events get");
+        }
+
+        var eventsGet = request.OkV;
+
+        var requestedEvent = await _context.Events.GetDownloadableEvent(eventsGet.EventId);
+        if (!requestedEvent.IsOk) {
+            return await _context.RequestHandling.NotOk(req, requestedEvent.ErrorV, "events get");
+        }
+
+        return await RequestHandling.Ok(req, new EventGetResponse(requestedEvent.OkV));
+    }
+}

src/ApiService/ApiService/OneFuzzTypes/Enums.cs

@@ -36,6 +36,8 @@ public enum ErrorCode {
     ADO_VALIDATION_INVALID_FIELDS = 479,
     GITHUB_VALIDATION_INVALID_PAT = 480,
     GITHUB_VALIDATION_INVALID_REPOSITORY = 481,
+    UNEXPECTED_DATA_SHAPE = 482,
+    UNABLE_TO_SEND = 483,
 }
 
 public enum VmState {

src/ApiService/ApiService/OneFuzzTypes/Events.cs

@@ -50,30 +50,28 @@ public enum EventType {
 }
 
 public abstract record BaseEvent() {
-
-    private static readonly IReadOnlyDictionary<Type, EventType> typeToEvent;
-    private static readonly IReadOnlyDictionary<EventType, Type> eventToType;
+    private static readonly IReadOnlyDictionary<Type, EventType> _typeToEvent;
+    private static readonly IReadOnlyDictionary<EventType, Type> _eventToType;
 
     static BaseEvent() {
-
-        EventType ExtractEventType(Type type) {
+        static EventType ExtractEventType(Type type) {
             var attr = type.GetCustomAttribute<EventTypeAttribute>();
             if (attr is null) {
                 throw new InvalidOperationException($"Type {type} is missing {nameof(EventTypeAttribute)}");
             }
             return attr.EventType;
         }
 
-        typeToEvent =
+        _typeToEvent =
             typeof(BaseEvent).Assembly.GetTypes()
             .Where(t => t.IsSubclassOf(typeof(BaseEvent)))
             .ToDictionary(x => x, ExtractEventType);
 
-        eventToType = typeToEvent.ToDictionary(x => x.Value, x => x.Key);
+        _eventToType = _typeToEvent.ToDictionary(x => x.Value, x => x.Key);
 
         // check that all event types are accounted for
         var allEventTypes = Enum.GetValues<EventType>();
-        var missingEventTypes = allEventTypes.Except(eventToType.Keys).ToList();
+        var missingEventTypes = allEventTypes.Except(_eventToType.Keys).ToList();
         if (missingEventTypes.Any()) {
             throw new InvalidOperationException($"Missing event types: {string.Join(", ", missingEventTypes)}");
         }
@@ -82,15 +80,15 @@ EventType ExtractEventType(Type type) {
 
     public EventType GetEventType() {
         var type = this.GetType();
-        if (typeToEvent.TryGetValue(type, out var eventType)) {
+        if (_typeToEvent.TryGetValue(type, out var eventType)) {
             return eventType;
         }
 
         throw new NotSupportedException($"Unknown event type: {type.GetType()}");
     }
 
     public static Type GetTypeInfo(EventType eventType) {
-        if (eventToType.TryGetValue(eventType, out var type)) {
+        if (_eventToType.TryGetValue(eventType, out var type)) {
             return type;
         }
 
@@ -350,14 +348,24 @@ public record EventNotificationFailed(
     Error? Error
 ) : BaseEvent();
 
+public record DownloadableEventMessage : EventMessage {
+    public Uri SasUrl { get; init; }
+
+    public DownloadableEventMessage(Guid EventId, EventType EventType, BaseEvent Event, Guid InstanceId, string InstanceName, DateTime CreatedAt, Uri SasUrl)
+        : base(EventId, EventType, Event, InstanceId, InstanceName, CreatedAt) {
+        this.SasUrl = SasUrl;
+    }
+}
 public record EventMessage(
     Guid EventId,
     EventType EventType,
     [property: TypeDiscrimnatorAttribute("EventType", typeof(EventTypeProvider))]
     [property: JsonConverter(typeof(BaseEventConverter))]
     BaseEvent Event,
     Guid InstanceId,
-    String InstanceName
+    String InstanceName,
+    DateTime CreatedAt,
+    String Version = "1.0"
 );
 
 public class BaseEventConverter : JsonConverter<BaseEvent> {

src/ApiService/ApiService/OneFuzzTypes/Requests.cs

@@ -328,3 +328,7 @@ public record TemplateValidationPost(
 public record JinjaToScribanMigrationPost(
     bool DryRun = false
 ) : BaseRequest;
+
+public record EventsGet(
+    [property: Required] Guid EventId
+) : BaseRequest;

src/ApiService/ApiService/OneFuzzTypes/Responses.cs

@@ -206,6 +206,10 @@ public record JinjaToScribanMigrationDryRunResponse(
     List<Guid> NotificationIdsToUpdate
 ) : BaseResponse();
 
+public record EventGetResponse(
+    DownloadableEventMessage Event
+) : BaseResponse();
+
 public record NotificationTestResponse(
     bool Success,
     string? Error = null

src/ApiService/ApiService/OneFuzzTypes/Webhooks.cs

@@ -19,7 +19,9 @@ public record WebhookMessage(Guid EventId,
     BaseEvent Event,
     Guid InstanceId,
     String InstanceName,
-    Guid WebhookId) : EventMessage(EventId, EventType, Event, InstanceId, InstanceName);
+    Guid WebhookId,
+    DateTime CreatedAt,
+    Uri SasUrl) : DownloadableEventMessage(EventId, EventType, Event, InstanceId, InstanceName, CreatedAt, SasUrl);
 
 
 public record WebhookMessageEventGrid(
@@ -30,7 +32,9 @@ public record WebhookMessageEventGrid(
     Guid Id,
     [property: TypeDiscrimnatorAttribute("EventType", typeof(EventTypeProvider))]
     [property: JsonConverter(typeof(BaseEventConverter))]
-    BaseEvent Data);
+    BaseEvent Data,
+    Uri SasUrl,
+    String Version = "1.0");
 
 public record WebhookMessageLog(
     [RowKey] Guid EventId,

src/ApiService/ApiService/TestHooks/EventsTestHooks.cs

@@ -25,7 +25,7 @@ public async Task<HttpResponseData> LogEvent([HttpTrigger(AuthorizationLevel.Ano
             _log.Info($"Log event");
 
             var s = await req.ReadAsStringAsync();
-            var msg = JsonSerializer.Deserialize<EventMessage>(s!, EntityConverter.GetJsonSerializerOptions());
+            var msg = JsonSerializer.Deserialize<DownloadableEventMessage>(s!, EntityConverter.GetJsonSerializerOptions());
             _events.LogEvent(msg!.Event);
             var resp = req.CreateResponse(HttpStatusCode.OK);
             return resp;
@@ -37,7 +37,7 @@ public async Task<HttpResponseData> SendEvent([HttpTrigger(AuthorizationLevel.An
             _log.Info($"Send event");
 
             var s = await req.ReadAsStringAsync();
-            var msg = JsonSerializer.Deserialize<EventMessage>(s!, EntityConverter.GetJsonSerializerOptions());
+            var msg = JsonSerializer.Deserialize<DownloadableEventMessage>(s!, EntityConverter.GetJsonSerializerOptions());
             await _events.SendEvent(msg!.Event);
             var resp = req.CreateResponse(HttpStatusCode.OK);
             return resp;

src/ApiService/ApiService/onefuzzlib/Events.cs

@@ -1,6 +1,8 @@
 using System.Text.Json;
 using System.Text.Json.Serialization;
 using Azure;
+using System.Threading.Tasks;
+using Azure.Storage.Sas;
 using Microsoft.OneFuzz.Service.OneFuzzLib.Orm;
 
 namespace Microsoft.OneFuzz.Service {
@@ -9,16 +11,20 @@ namespace Microsoft.OneFuzz.Service {
     public record SignalREvent
     (
         string Target,
-        List<EventMessage> arguments
+        List<DownloadableEventMessage> arguments
     );
 
 
     public interface IEvents {
         Async.Task SendEvent(BaseEvent anEvent);
 
-        Async.Task QueueSignalrEvent(EventMessage message);
+        Async.Task QueueSignalrEvent(DownloadableEventMessage message);
 
         void LogEvent(BaseEvent anEvent);
+
+        Async.Task<OneFuzzResult<EventMessage>> GetEvent(Guid eventId);
+        Async.Task<OneFuzzResult<DownloadableEventMessage>> GetDownloadableEvent(Guid eventId);
+        Async.Task<DownloadableEventMessage> MakeDownloadable(EventMessage eventMessage);
     }
 
     public class Events : IEvents {
@@ -29,63 +35,119 @@ public class Events : IEvents {
         private readonly ICreds _creds;
         private readonly JsonSerializerOptions _options;
 
-        public Events(IQueue queue, IWebhookOperations webhook, ILogTracer log, IContainers containers, ICreds creds) {
-            _queue = queue;
-            _webhook = webhook;
+        public Events(ILogTracer log, IOnefuzzContext context) {
+            _queue = context.Queue;
+            _webhook = context.WebhookOperations;
             _log = log;
-            _containers = containers;
-            _creds = creds;
+            _containers = context.Containers;
+            _creds = context.Creds;
             _options = new JsonSerializerOptions(EntityConverter.GetJsonSerializerOptions()) {
                 DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
             };
             _options.Converters.Add(new RemoveUserInfo());
         }
 
-        public async Async.Task QueueSignalrEvent(EventMessage message) {
+        public virtual async Async.Task QueueSignalrEvent(DownloadableEventMessage message) {
             var tags = new (string, string)[] {
                 ("event_type", message.EventType.ToString()),
                 ("event_id", message.EventId.ToString())
             };
             try {
-                var ev = new SignalREvent("events", new List<EventMessage>() { message });
+                var ev = new SignalREvent("events", new List<DownloadableEventMessage>() { message });
                 await _queue.SendMessage("signalr-events", JsonSerializer.Serialize(ev, _options), StorageType.Config);
             } catch (RequestFailedException ex) {
                 if (ex.Message.Contains("The request body is too large") && message.Event is ITruncatable<BaseEvent> truncatableEvent) {
                     _log.WithTags(tags).Warning($"The EventMessage was too large for Azure Queue. Truncating event data and trying again.");
                     message = message with {
                         Event = truncatableEvent.Truncate(1000)
                     };
-                    var ev = new SignalREvent("events", new List<EventMessage>() { message });
+                    var ev = new SignalREvent("events", new List<DownloadableEventMessage>() { message });
                     await _queue.SendMessage("signalr-events", JsonSerializer.Serialize(ev, _options), StorageType.Config);
                 } else {
                     // Not handled
                     throw ex;
                 }
             }
-
         }
 
         public async Async.Task SendEvent(BaseEvent anEvent) {
             var eventType = anEvent.GetEventType();
 
             var instanceId = await _containers.GetInstanceId();
-
+            var creationDate = DateTime.UtcNow;
             var eventMessage = new EventMessage(
                 Guid.NewGuid(),
                 eventType,
                 anEvent,
                 instanceId,
-                _creds.GetInstanceName()
+                _creds.GetInstanceName(),
+                creationDate
             );
-            await QueueSignalrEvent(eventMessage);
-            await _webhook.SendEvent(eventMessage);
+
+            var downloadableEventMessage = await MakeDownloadable(eventMessage);
+
+            await QueueSignalrEvent(downloadableEventMessage);
+            await _webhook.SendEvent(downloadableEventMessage);
             LogEvent(anEvent);
         }
 
-        public void LogEvent(BaseEvent anEvent) {
+        public virtual void LogEvent(BaseEvent anEvent) {
             var serializedEvent = JsonSerializer.Serialize(anEvent, anEvent.GetType(), _options);
             _log.Info($"sending event: {anEvent.GetEventType():Tag:EventType} - {serializedEvent}");
         }
+
+        public async Async.Task<OneFuzzResult<EventMessage>> GetEvent(Guid eventId) {
+            var blob = await _containers.GetBlob(WellKnownContainers.Events, eventId.ToString(), StorageType.Corpus);
+            if (blob == null) {
+                return OneFuzzResult<EventMessage>.Error(ErrorCode.UNABLE_TO_FIND, $"Could not find container for event with id {eventId}");
+            }
+
+            var eventMessage = JsonSerializer.Deserialize<EventMessage>(blob, _options);
+            if (eventMessage == null) {
+                return OneFuzzResult<EventMessage>.Error(ErrorCode.UNEXPECTED_DATA_SHAPE, $"Could not deserialize event with id {eventId}");
+            }
+
+            return OneFuzzResult<EventMessage>.Ok(eventMessage);
+        }
+
+        public async Async.Task<OneFuzzResult<DownloadableEventMessage>> GetDownloadableEvent(Guid eventId) {
+            var eventMessageResult = await GetEvent(eventId);
+            if (!eventMessageResult.IsOk) {
+                return eventMessageResult.ErrorV;
+            }
+
+            var sasUrl = await _containers.GetFileSasUrl(WellKnownContainers.Events, eventId.ToString(), StorageType.Corpus, BlobSasPermissions.Read);
+            if (sasUrl == null) {
+                return OneFuzzResult<DownloadableEventMessage>.Error(ErrorCode.UNABLE_TO_FIND, $"Could not find container for event with id {eventId}");
+            }
+
+            var eventMessage = eventMessageResult.OkV!;
+
+            return OneFuzzResult<DownloadableEventMessage>.Ok(new DownloadableEventMessage(
+                eventMessage.EventId,
+                eventMessage.EventType,
+                eventMessage.Event,
+                eventMessage.InstanceId,
+                eventMessage.InstanceName,
+                eventMessage.CreatedAt,
+                sasUrl
+            ));
+        }
+
+        public async Task<DownloadableEventMessage> MakeDownloadable(EventMessage eventMessage) {
+            await _containers.SaveBlob(WellKnownContainers.Events, eventMessage.EventId.ToString(), JsonSerializer.Serialize(eventMessage, _options), StorageType.Corpus);
+            var sasUrl = await _containers.GetFileSasUrl(WellKnownContainers.Events, eventMessage.EventId.ToString(), StorageType.Corpus, BlobSasPermissions.Read);
+
+            return new DownloadableEventMessage(
+                eventMessage.EventId,
+                eventMessage.EventType,
+                eventMessage.Event,
+                eventMessage.InstanceId,
+                eventMessage.InstanceName,
+                eventMessage.CreatedAt,
+                sasUrl
+            );
+        }
     }