[release/8.0-preview5] Refactor console logs to render on server and use virtualization (#3178)

* Refactor console logs to render on server and use virtualization

* Fix tests

* Dashboard line record

* PR feedback

* PR feedback

* Fix console scrolling issue

---------

Co-authored-by: James Newton-King <james@newtonking.com>

Author: github-actions[bot]

playground/Stress/Stress.ApiService/Program.cs

@@ -1,6 +1,8 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Diagnostics;
+using System.Threading.Channels;
 using Stress.ApiService;
 
 var builder = WebApplication.CreateBuilder(args);
@@ -25,4 +27,48 @@
     return "Big trace created";
 });
 
+app.MapGet("/many-logs", (ILoggerFactory loggerFactory, CancellationToken cancellationToken) =>
+{
+    var channel = Channel.CreateUnbounded<string>();
+    var logger = loggerFactory.CreateLogger("ManyLogs");
+
+    cancellationToken.Register(() =>
+    {
+        logger.LogInformation("Writing logs canceled.");
+    });
+
+    // Write logs for 1 minute.
+    _ = Task.Run(async () =>
+    {
+        var stopwatch = Stopwatch.StartNew();
+        var logCount = 0;
+        while (stopwatch.Elapsed < TimeSpan.FromMinutes(1))
+        {
+            cancellationToken.ThrowIfCancellationRequested();
+
+            logCount++;
+            logger.LogInformation("This is log message {LogCount}.", logCount);
+
+            if (logCount % 100 == 0)
+            {
+                channel.Writer.TryWrite($"Logged {logCount} messages.");
+            }
+
+            await Task.Delay(5, cancellationToken);
+        }
+
+        channel.Writer.Complete();
+    }, cancellationToken);
+
+    return WriteOutput();
+
+    async IAsyncEnumerable<string> WriteOutput()
+    {
+        await foreach (var message in channel.Reader.ReadAllAsync(cancellationToken))
+        {
+            yield return message;
+        }
+    }
+});
+
 app.Run();

src/Aspire.Dashboard/Components/Controls/LogViewer.razor

@@ -6,7 +6,27 @@
 @inject IJSRuntime JS
 @implements IAsyncDisposable
 
-<div class="log-overflow">
+<div class="log-overflow continuous-scroll-overflow">
     <div class="log-container" id="logContainer">
+        <Virtualize Items="_logEntries" ItemSize="20" OverscanCount="100">
+            <div class="line-row-container">
+                <div class="line-row">
+                    <span class="line-area" role="log">
+                        <span class="line-number">@context.LineNumber</span>
+                        <span class="content">
+                            @if (context.Timestamp is { } timestamp)
+                            {
+                                <span class="timestamp">@GetDisplayTimestamp(timestamp)</span>
+                            }
+                            @if (context.Type == LogEntryType.Error)
+                            {
+                                <fluent-badge appearance="accent">stderr</fluent-badge>
+                            }
+                            @((MarkupString)(context.Content ?? string.Empty))
+                        </span>
+                    </span>
+                </div>
+            </div>
+        </Virtualize>
     </div>
 </div>

src/Aspire.Dashboard/Components/Controls/LogViewer.razor.cs

@@ -1,7 +1,10 @@
 // Licensed to the .NET Foundation under one or more agreements.
 // The .NET Foundation licenses this file to you under the MIT license.
 
+using System.Diagnostics;
+using System.Globalization;
 using Aspire.Dashboard.ConsoleLogs;
+using Aspire.Dashboard.Extensions;
 using Aspire.Dashboard.Model;
 using Aspire.Dashboard.Utils;
 using Microsoft.AspNetCore.Components;
@@ -14,30 +17,35 @@ namespace Aspire.Dashboard.Components;
 /// </summary>
 public sealed partial class LogViewer
 {
-    private readonly TaskCompletionSource _whenDomReady = new();
     private readonly CancellationSeries _cancellationSeries = new();
-    private IJSObjectReference? _jsModule;
+    private bool _convertTimestampsFromUtc;
+    private bool _applicationChanged;
 
     [Inject]
     public required BrowserTimeProvider TimeProvider { get; init; }
 
     protected override async Task OnAfterRenderAsync(bool firstRender)
     {
+        if (_applicationChanged)
+        {
+            await JS.InvokeVoidAsync("resetContinuousScrollPosition");
+            _applicationChanged = false;
+        }
         if (firstRender)
         {
-            _jsModule ??= await JS.InvokeAsync<IJSObjectReference>("import", "/Components/Controls/LogViewer.razor.js");
-
-            _whenDomReady.TrySetResult();
+            await JS.InvokeVoidAsync("initializeContinuousScroll");
         }
     }
 
-    internal async Task SetLogSourceAsync(IAsyncEnumerable<IReadOnlyList<(string Content, bool IsErrorMessage)>> batches, bool convertTimestampsFromUtc)
+    private readonly List<LogEntry> _logEntries = new();
+    private int? _baseLineNumber;
+
+    internal async Task SetLogSourceAsync(IAsyncEnumerable<IReadOnlyList<ResourceLogLine>> batches, bool convertTimestampsFromUtc)
     {
-        var cancellationToken = await _cancellationSeries.NextAsync();
-        var logParser = new LogParser(TimeProvider, convertTimestampsFromUtc);
+        _convertTimestampsFromUtc = convertTimestampsFromUtc;
 
-        // Ensure we are able to write to the DOM.
-        await _whenDomReady.Task;
+        var cancellationToken = await _cancellationSeries.NextAsync();
+        var logParser = new LogParser();
 
         await foreach (var batch in batches.WithCancellation(cancellationToken))
         {
@@ -46,33 +54,105 @@ internal async Task SetLogSourceAsync(IAsyncEnumerable<IReadOnlyList<(string Con
                 continue;
             }
 
-            List<LogEntry> entries = new(batch.Count);
-
-            foreach (var (content, isErrorOutput) in batch)
+            foreach (var (lineNumber, content, isErrorOutput) in batch)
             {
-                entries.Add(logParser.CreateLogEntry(content, isErrorOutput));
+                // Keep track of the base line number to ensure that we can calculate the line number of each log entry.
+                // This becomes important when the total number of log entries exceeds the limit and is truncated.
+                if (_baseLineNumber is null)
+                {
+                    _baseLineNumber = lineNumber;
+                }
+
+                InsertSorted(_logEntries, logParser.CreateLogEntry(content, isErrorOutput));
             }
 
-            await _jsModule!.InvokeVoidAsync("addLogEntries", cancellationToken, entries);
+            StateHasChanged();
         }
     }
 
-    internal async Task ClearLogsAsync(CancellationToken cancellationToken = default)
+    private void InsertSorted(List<LogEntry> logEntries, LogEntry logEntry)
     {
-        await _cancellationSeries.ClearAsync();
+        if (logEntry.ParentId != null)
+        {
+            // If we have a parent id, then we know we're on a non-timestamped line that is part
+            // of a multi-line log entry. We need to find the prior line from that entry
+            for (var rowIndex = logEntries.Count - 1; rowIndex >= 0; rowIndex--)
+            {
+                var current = logEntries[rowIndex];
 
-        if (_jsModule is not null)
+                if (current.Id == logEntry.ParentId && logEntry.LineIndex - 1 == current.LineIndex)
+                {
+                    InsertLogEntry(logEntries, rowIndex + 1, logEntry);
+                    return;
+                }
+            }
+        }
+        else if (logEntry.Timestamp != null)
         {
-            await _jsModule.InvokeVoidAsync("clearLogs", cancellationToken);
+            // Otherwise, if we have a timestamped line, we just need to find the prior line.
+            // Since the rows are always in order, as soon as we see a timestamp
+            // that is less than the one we're adding, we can insert it immediately after that
+            for (var rowIndex = logEntries.Count - 1; rowIndex >= 0; rowIndex--)
+            {
+                var current = logEntries[rowIndex];
+                var currentTimestamp = current.Timestamp ?? current.ParentTimestamp;
+
+                if (currentTimestamp != null && currentTimestamp < logEntry.Timestamp)
+                {
+                    InsertLogEntry(logEntries, rowIndex + 1, logEntry);
+                    return;
+                }
+            }
+        }
+
+        // If we didn't find a place to insert then append it to the end. This happens with the first entry, but
+        // could also happen if the logs don't have recognized timestamps.
+        InsertLogEntry(logEntries, logEntries.Count, logEntry);
+
+        void InsertLogEntry(List<LogEntry> logEntries, int index, LogEntry logEntry)
+        {
+            // Set the line number of the log entry.
+            if (index == 0)
+            {
+                Debug.Assert(_baseLineNumber != null, "Should be set before this method is run.");
+                logEntry.LineNumber = _baseLineNumber.Value;
+            }
+            else
+            {
+                logEntry.LineNumber = logEntries[index - 1].LineNumber + 1;
+            }
+
+            logEntries.Insert(index, logEntry);
+
+            // If a log entry isn't inserted at the end then update the line numbers of all subsequent entries.
+            for (var i = index + 1; i < logEntries.Count; i++)
+            {
+                logEntries[i].LineNumber++;
+            }
         }
     }
 
-    public async ValueTask DisposeAsync()
+    private string GetDisplayTimestamp(DateTimeOffset timestamp)
     {
-        _whenDomReady.TrySetCanceled();
+        if (_convertTimestampsFromUtc)
+        {
+            timestamp = TimeProvider.ToLocal(timestamp);
+        }
+
+        return timestamp.ToString(KnownFormats.ConsoleLogsTimestampFormat, CultureInfo.InvariantCulture);
+    }
 
+    internal async Task ClearLogsAsync()
+    {
         await _cancellationSeries.ClearAsync();
 
-        await JSInteropHelpers.SafeDisposeAsync(_jsModule);
+        _applicationChanged = true;
+        _logEntries.Clear();
+        StateHasChanged();
+    }
+
+    public async ValueTask DisposeAsync()
+    {
+        await _cancellationSeries.ClearAsync();
     }
 }

src/Aspire.Dashboard/Components/Controls/LogViewer.razor.css

@@ -34,21 +34,19 @@
     background: var(--console-background-color);
     color: var(--console-font-color);
     font-family: 'Cascadia Mono', Consolas, monospace;
-    font-size: 12px;
+    font-size: 13px;
     margin: 16px 0 0 0;
     padding-bottom: 24px;
     line-height: 20px;
     overflow: visible;
     display: flex;
     flex-direction: column;
     width: 100%;
-    counter-reset: line-number 0;
 }
 
 ::deep .line-row-container {
     width: 100%;
     overflow: hidden;
-    counter-increment: line-number 1;
 }
 
 ::deep .line-row {
@@ -79,10 +77,8 @@
     align-self: flex-start;
     flex-shrink: 0;
     color: var(--line-number-color);
-}
-
-::deep .line-number::before {
-    content: counter(line-number);
+    user-select: none;
+    cursor: default;
 }
 
 ::deep .content {