Respect SETTINGS_MAX_HEADER_LIST_SIZE on HTTP/2 and HTTP/3 (#79281)

Author: MihaZupan

src/libraries/Common/tests/System/Net/Http/Http3LoopbackConnection.cs

@@ -14,7 +14,7 @@
 
 namespace System.Net.Test.Common
 {
-    internal sealed class Http3LoopbackConnection : GenericLoopbackConnection
+    public sealed class Http3LoopbackConnection : GenericLoopbackConnection
     {
         public const long H3_NO_ERROR = 0x100;
         public const long H3_GENERAL_PROTOCOL_ERROR = 0x101;
@@ -188,11 +188,11 @@ public async Task<Http3LoopbackStream> AcceptRequestStreamAsync()
             return (controlStream, requestStream);
         }
 
-        public async Task EstablishControlStreamAsync()
+        public async Task EstablishControlStreamAsync(SettingsEntry[] settingsEntries)
         {
             _outboundControlStream = await OpenUnidirectionalStreamAsync();
             await _outboundControlStream.SendUnidirectionalStreamTypeAsync(Http3LoopbackStream.ControlStream);
-            await _outboundControlStream.SendSettingsFrameAsync();
+            await _outboundControlStream.SendSettingsFrameAsync(settingsEntries);
         }
 
         public override async Task<byte[]> ReadRequestBodyAsync()

src/libraries/Common/tests/System/Net/Http/Http3LoopbackServer.cs

@@ -66,12 +66,12 @@ public override void Dispose()
             _cert.Dispose();
         }
 
-        private async Task<Http3LoopbackConnection> EstablishHttp3ConnectionAsync()
+        private async Task<Http3LoopbackConnection> EstablishHttp3ConnectionAsync(params SettingsEntry[] settingsEntries)
         {
             QuicConnection con = await _listener.AcceptConnectionAsync().ConfigureAwait(false);
             Http3LoopbackConnection connection = new Http3LoopbackConnection(con);
 
-            await connection.EstablishControlStreamAsync();
+            await connection.EstablishControlStreamAsync(settingsEntries);
             return connection;
         }
 
@@ -80,6 +80,11 @@ public override async Task<GenericLoopbackConnection> EstablishGenericConnection
             return await EstablishHttp3ConnectionAsync();
         }
 
+        public Task<Http3LoopbackConnection> EstablishConnectionAsync(params SettingsEntry[] settingsEntries)
+        {
+            return EstablishHttp3ConnectionAsync(settingsEntries);
+        }
+
         public override async Task AcceptConnectionAsync(Func<GenericLoopbackConnection, Task> funcAsync)
         {
             await using Http3LoopbackConnection con = await EstablishHttp3ConnectionAsync().ConfigureAwait(false);

src/libraries/Common/tests/System/Net/Http/Http3LoopbackStream.cs

@@ -14,8 +14,7 @@
 
 namespace System.Net.Test.Common
 {
-
-    internal sealed class Http3LoopbackStream : IAsyncDisposable
+    public sealed class Http3LoopbackStream : IAsyncDisposable
     {
         private const int MaximumVarIntBytes = 8;
         private const long VarIntMax = (1L << 62) - 1;
@@ -58,18 +57,16 @@ public async Task SendUnidirectionalStreamTypeAsync(long streamType)
             await _stream.WriteAsync(buffer.AsMemory(0, bytesWritten)).ConfigureAwait(false);
         }
 
-        public async Task SendSettingsFrameAsync(ICollection<(long settingId, long settingValue)> settings = null)
+        public async Task SendSettingsFrameAsync(SettingsEntry[] settingsEntries)
         {
-            settings ??= Array.Empty<(long settingId, long settingValue)>();
-
-            var buffer = new byte[settings.Count * MaximumVarIntBytes * 2];
+            var buffer = new byte[settingsEntries.Length * MaximumVarIntBytes * 2];
 
             int bytesWritten = 0;
 
-            foreach ((long settingId, long settingValue) in settings)
+            foreach (SettingsEntry setting in settingsEntries)
             {
-                bytesWritten += EncodeHttpInteger(settingId, buffer.AsSpan(bytesWritten));
-                bytesWritten += EncodeHttpInteger(settingValue, buffer.AsSpan(bytesWritten));
+                bytesWritten += EncodeHttpInteger((int)setting.SettingId, buffer.AsSpan(bytesWritten));
+                bytesWritten += EncodeHttpInteger(setting.Value, buffer.AsSpan(bytesWritten));
             }
 
             await SendFrameAsync(SettingsFrame, buffer.AsMemory(0, bytesWritten)).ConfigureAwait(false);

src/libraries/System.Net.Http/src/Resources/Strings.resx

@@ -357,6 +357,9 @@
   <data name="net_http_buffer_insufficient_length" xml:space="preserve">
     <value>The buffer was not long enough.</value>
   </data>
+  <data name="net_http_request_headers_exceeded_length" xml:space="preserve">
+    <value>The HTTP request headers length exceeded the server limit of {0} bytes.</value>
+  </data>
   <data name="net_http_response_headers_exceeded_length" xml:space="preserve">
     <value>The HTTP response headers length exceeded the set limit of {0} bytes.</value>
   </data>

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http2Connection.cs

@@ -56,6 +56,10 @@ internal sealed partial class Http2Connection : HttpConnectionBase
         private readonly Channel<WriteQueueEntry> _writeChannel;
         private bool _lastPendingWriterShouldFlush;
 
+        // Server-advertised SETTINGS_MAX_HEADER_LIST_SIZE
+        // https://www.rfc-editor.org/rfc/rfc9113.html#section-6.5.2-2.12.1
+        private uint _maxHeaderListSize = uint.MaxValue; // Defaults to infinite
+
         // This flag indicates that the connection is shutting down and cannot accept new requests, because of one of the following conditions:
         // (1) We received a GOAWAY frame from the server
         // (2) We have exhaustead StreamIds (i.e. _nextStream == MaxStreamId)
@@ -162,6 +166,14 @@ public Http2Connection(HttpConnectionPool pool, Stream stream)
             _nextPingRequestTimestamp = Environment.TickCount64 + _keepAlivePingDelay;
             _keepAlivePingPolicy = _pool.Settings._keepAlivePingPolicy;
 
+            uint maxHeaderListSize = _pool._lastSeenHttp2MaxHeaderListSize;
+            if (maxHeaderListSize > 0)
+            {
+                // Previous connections to the same host advertised a limit.
+                // Use this as an initial value before we receive the SETTINGS frame.
+                _maxHeaderListSize = maxHeaderListSize;
+            }
+
             if (HttpTelemetry.Log.IsEnabled())
             {
                 HttpTelemetry.Log.Http20ConnectionEstablished();
@@ -822,6 +834,8 @@ private void ProcessSettingsFrame(FrameHeader frameHeader, bool initialFrame = f
                     uint settingValue = BinaryPrimitives.ReadUInt32BigEndian(settings);
                     settings = settings.Slice(4);
 
+                    if (NetEventSource.Log.IsEnabled()) Trace($"Applying setting {(SettingId)settingId}={settingValue}");
+
                     switch ((SettingId)settingId)
                     {
                         case SettingId.MaxConcurrentStreams:
@@ -861,6 +875,11 @@ private void ProcessSettingsFrame(FrameHeader frameHeader, bool initialFrame = f
                             }
                             break;
 
+                        case SettingId.MaxHeaderListSize:
+                            _maxHeaderListSize = settingValue;
+                            _pool._lastSeenHttp2MaxHeaderListSize = _maxHeaderListSize;
+                            break;
+
                         default:
                             // All others are ignored because we don't care about them.
                             // Note, per RFC, unknown settings IDs should be ignored.
@@ -1379,14 +1398,18 @@ private void WriteBytes(ReadOnlySpan<byte> bytes, ref ArrayBuffer headerBuffer)
             headerBuffer.Commit(bytes.Length);
         }
 
-        private void WriteHeaderCollection(HttpRequestMessage request, HttpHeaders headers, ref ArrayBuffer headerBuffer)
+        private int WriteHeaderCollection(HttpRequestMessage request, HttpHeaders headers, ref ArrayBuffer headerBuffer)
         {
             if (NetEventSource.Log.IsEnabled()) Trace("");
 
             HeaderEncodingSelector<HttpRequestMessage>? encodingSelector = _pool.Settings._requestHeaderEncodingSelector;
 
             ref string[]? tmpHeaderValuesArray = ref t_headerValues;
-            foreach (HeaderEntry header in headers.GetEntries())
+
+            ReadOnlySpan<HeaderEntry> entries = headers.GetEntries();
+            int headerListSize = entries.Length * HeaderField.RfcOverhead;
+
+            foreach (HeaderEntry header in entries)
             {
                 int headerValuesCount = HttpHeaders.GetStoreValuesIntoStringArray(header.Key, header.Value, ref tmpHeaderValuesArray);
                 Debug.Assert(headerValuesCount > 0, "No values for header??");
@@ -1402,6 +1425,10 @@ private void WriteHeaderCollection(HttpRequestMessage request, HttpHeaders heade
                     // The Connection, Upgrade and ProxyConnection headers are also not supported in HTTP2.
                     if (knownHeader != KnownHeaders.Host && knownHeader != KnownHeaders.Connection && knownHeader != KnownHeaders.Upgrade && knownHeader != KnownHeaders.ProxyConnection)
                     {
+                        // The length of the encoded name may be shorter than the actual name.
+                        // Ensure that headerListSize is always >= of the actual size.
+                        headerListSize += knownHeader.Name.Length;
+
                         if (knownHeader == KnownHeaders.TE)
                         {
                             // HTTP/2 allows only 'trailers' TE header. rfc7540 8.1.2.2
@@ -1442,6 +1469,8 @@ private void WriteHeaderCollection(HttpRequestMessage request, HttpHeaders heade
                     WriteLiteralHeader(header.Key.Name, headerValues, valueEncoding, ref headerBuffer);
                 }
             }
+
+            return headerListSize;
         }
 
         private void WriteHeaders(HttpRequestMessage request, ref ArrayBuffer headerBuffer)
@@ -1472,9 +1501,9 @@ private void WriteHeaders(HttpRequestMessage request, ref ArrayBuffer headerBuff
 
             WriteIndexedHeader(_pool.IsSecure ? H2StaticTable.SchemeHttps : H2StaticTable.SchemeHttp, ref headerBuffer);
 
-            if (request.HasHeaders && request.Headers.Host != null)
+            if (request.HasHeaders && request.Headers.Host is string host)
             {
-                WriteIndexedHeader(H2StaticTable.Authority, request.Headers.Host, ref headerBuffer);
+                WriteIndexedHeader(H2StaticTable.Authority, host, ref headerBuffer);
             }
             else
             {
@@ -1492,16 +1521,19 @@ private void WriteHeaders(HttpRequestMessage request, ref ArrayBuffer headerBuff
                 WriteIndexedHeader(H2StaticTable.PathSlash, pathAndQuery, ref headerBuffer);
             }
 
+            int headerListSize = 3 * HeaderField.RfcOverhead; // Method, Authority, Path
+
             if (request.HasHeaders)
             {
                 if (request.Headers.Protocol != null)
                 {
                     WriteBytes(ProtocolLiteralHeaderBytes, ref headerBuffer);
                     Encoding? protocolEncoding = _pool.Settings._requestHeaderEncodingSelector?.Invoke(":protocol", request);
                     WriteLiteralHeaderValue(request.Headers.Protocol, protocolEncoding, ref headerBuffer);
+                    headerListSize += HeaderField.RfcOverhead;
                 }
 
-                WriteHeaderCollection(request, request.Headers, ref headerBuffer);
+                headerListSize += WriteHeaderCollection(request, request.Headers, ref headerBuffer);
             }
 
             // Determine cookies to send.
@@ -1511,9 +1543,9 @@ private void WriteHeaders(HttpRequestMessage request, ref ArrayBuffer headerBuff
                 if (cookiesFromContainer != string.Empty)
                 {
                     WriteBytes(KnownHeaders.Cookie.Http2EncodedName, ref headerBuffer);
-
                     Encoding? cookieEncoding = _pool.Settings._requestHeaderEncodingSelector?.Invoke(KnownHeaders.Cookie.Name, request);
                     WriteLiteralHeaderValue(cookiesFromContainer, cookieEncoding, ref headerBuffer);
+                    headerListSize += HttpKnownHeaderNames.Cookie.Length + HeaderField.RfcOverhead;
                 }
             }
 
@@ -1525,11 +1557,24 @@ private void WriteHeaders(HttpRequestMessage request, ref ArrayBuffer headerBuff
                 {
                     WriteBytes(KnownHeaders.ContentLength.Http2EncodedName, ref headerBuffer);
                     WriteLiteralHeaderValue("0", valueEncoding: null, ref headerBuffer);
+                    headerListSize += HttpKnownHeaderNames.ContentLength.Length + HeaderField.RfcOverhead;
                 }
             }
             else
             {
-                WriteHeaderCollection(request, request.Content.Headers, ref headerBuffer);
+                headerListSize += WriteHeaderCollection(request, request.Content.Headers, ref headerBuffer);
+            }
+
+            // The headerListSize is an approximation of the total header length.
+            // This is acceptable as long as the value is always >= the actual length.
+            // We must avoid ever sending more than the server allowed.
+            // This approach must be revisted if we ever support the dynamic table or compression when sending requests.
+            headerListSize += headerBuffer.ActiveLength;
+
+            uint maxHeaderListSize = _maxHeaderListSize;
+            if ((uint)headerListSize > maxHeaderListSize)
+            {
+                throw new HttpRequestException(SR.Format(SR.net_http_request_headers_exceeded_length, maxHeaderListSize));
             }
         }
 
@@ -1602,10 +1647,10 @@ private async ValueTask<Http2Stream> SendHeadersAsync(HttpRequestMessage request
                 // streams are created and started in order.
                 await PerformWriteAsync(totalSize, (thisRef: this, http2Stream, headerBytes, endStream: (request.Content == null && !request.IsExtendedConnectRequest), mustFlush), static (s, writeBuffer) =>
                 {
-                    if (NetEventSource.Log.IsEnabled()) s.thisRef.Trace(s.http2Stream.StreamId, $"Started writing. Total header bytes={s.headerBytes.Length}");
-
                     s.thisRef.AddStream(s.http2Stream);
 
+                    if (NetEventSource.Log.IsEnabled()) s.thisRef.Trace(s.http2Stream.StreamId, $"Started writing. Total header bytes={s.headerBytes.Length}");
+
                     Span<byte> span = writeBuffer.Span;
 
                     // Copy the HEADERS frame.

src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/Http3Connection.cs

@@ -36,8 +36,9 @@ internal sealed class Http3Connection : HttpConnectionBase
         // Our control stream.
         private QuicStream? _clientControl;
 
-        // Current SETTINGS from the server.
-        private int _maximumHeadersLength = int.MaxValue; // TODO: this is not yet observed by Http3Stream when buffering headers.
+        // Server-advertised SETTINGS_MAX_FIELD_SECTION_SIZE
+        // https://www.rfc-editor.org/rfc/rfc9114.html#section-7.2.4.1-2.2.1
+        private uint _maxHeaderListSize = uint.MaxValue; // Defaults to infinite
 
         // Once the server's streams are received, these are set to 1. Further receipt of these streams results in a connection error.
         private int _haveServerControlStream;
@@ -53,7 +54,7 @@ internal sealed class Http3Connection : HttpConnectionBase
 
         public HttpAuthority Authority => _authority;
         public HttpConnectionPool Pool => _pool;
-        public int MaximumRequestHeadersLength => _maximumHeadersLength;
+        public uint MaxHeaderListSize => _maxHeaderListSize;
         public byte[]? AltUsedEncodedHeaderBytes => _altUsedEncodedHeader;
         public Exception? AbortException => Volatile.Read(ref _abortException);
         private object SyncObj => _activeRequests;
@@ -84,6 +85,13 @@ public Http3Connection(HttpConnectionPool pool, HttpAuthority? origin, HttpAutho
                 _altUsedEncodedHeader = QPack.QPackEncoder.EncodeLiteralHeaderFieldWithoutNameReferenceToArray(KnownHeaders.AltUsed.Name, altUsedValue);
             }
 
+            uint maxHeaderListSize = _pool._lastSeenHttp3MaxHeaderListSize;
+            if (maxHeaderListSize > 0)
+            {
+                // Previous connections to the same host advertised a limit.
+                // Use this as an initial value before we receive the SETTINGS frame.
+                _maxHeaderListSize = maxHeaderListSize;
+            }
 
             if (HttpTelemetry.Log.IsEnabled())
             {
@@ -725,10 +733,13 @@ async ValueTask ProcessSettingsFrameAsync(long settingsPayloadLength)
 
                     buffer.Discard(bytesRead);
 
+                    if (NetEventSource.Log.IsEnabled()) Trace($"Applying setting {(Http3SettingType)settingId}={settingValue}");
+
                     switch ((Http3SettingType)settingId)
                     {
                         case Http3SettingType.MaxHeaderListSize:
-                            _maximumHeadersLength = (int)Math.Min(settingValue, int.MaxValue);
+                            _maxHeaderListSize = (uint)Math.Min((ulong)settingValue, uint.MaxValue);
+                            _pool._lastSeenHttp3MaxHeaderListSize = _maxHeaderListSize;
                             break;
                         case Http3SettingType.ReservedHttp2EnablePush:
                         case Http3SettingType.ReservedHttp2MaxConcurrentStreams: