E2E tests for different header encodings

docs/docfx/articles/proxyhttpclientconfig.md

@@ -28,7 +28,8 @@ HTTP client configuration is based on [ProxyHttpClientOptions](xref:Microsoft.Re
         "Store": "<string>",
         "Location": "<string>",
         "AllowInvalid": "<bool>"
-    }
+    },
+    "RequestHeaderEncoding": "<encoding-name>"
 }
 ```
 
@@ -75,6 +76,32 @@ Configuration settings:
 }
 
 ```
+- RequestHeaderEncoding - enables other than ASCII encoding for outgoing request headers. Setting this value will leverage [`SocketsHttpHandler.RequestHeaderEncodingSelector`](https://docs.microsoft.com/en-us/dotnet/api/system.net.http.socketshttphandler.requestheaderencodingselector?view=net-5.0) and use the selected encoding for all headers. If you need more granular approach, please use custom `IProxyHttpClientFactory`. The value is then parsed by [`Encoding.GetEncoding`](https://docs.microsoft.com/en-us/dotnet/api/system.text.encoding.getencoding?view=net-5.0#System_Text_Encoding_GetEncoding_System_String_), use values like: "utf-8", "iso-8859-1", etc. **This setting is only available for .NET 5.0.**
+```JSON
+"RequestHeaderEncoding": "utf-8"
+```
+If you're using an encoding other than "ascii" (or "utf-8" for Kestrel) you also need to set your server to accept requests with such headers. For example, use [`KestrelServerOptions.RequestHeaderEncodingSelector`](https://docs.microsoft.com/en-us/dotnet/api/Microsoft.AspNetCore.Server.Kestrel.Core.KestrelServerOptions.RequestHeaderEncodingSelector?view=aspnetcore-5.0) to set up Kestrel to accept Latin1 ("iso-8859-1") headers:
+```C#
+private static IHostBuilder CreateHostBuilder(string[] args) =>
+    Host.CreateDefaultBuilder(args)
+        .ConfigureWebHostDefaults(webBuilder =>
+        {
+            webBuilder.UseStartup<Startup>();
+        })
+        .ConfigureWebHost(webBuilder => webBuilder.UseKestrel(kestrel =>
+        {
+            kestrel.RequestHeaderEncodingSelector = _ => Encoding.Latin1;
+        }));
+```
+
+For .NET Core 3.1, Latin1 ("iso-8859-1") is the only non-ASCII header encoding that can be accepted and only via application wide switch:
+```C#
+AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.AllowLatin1Headers", true);
+AppContext.SetSwitch("Microsoft.AspNetCore.Server.Kestrel.Latin1RequestHeaders", true);
+```
+
+At the moment, there is no solution for changing encoding for response headers, only ASCII is accepted.
+
 
 ### HttpRequest
 HTTP request configuration is based on [ProxyHttpRequestOptions](xref:Microsoft.ReverseProxy.Abstractions.ProxyHttpRequestOptions) and represented by the following configuration schema.
@@ -256,6 +283,12 @@ public class CustomProxyHttpClientFactory : IProxyHttpClientFactory
         {
             handler.SslOptions.RemoteCertificateValidationCallback = (sender, cert, chain, errors) => cert.Subject == "dev.mydomain";
         }
+#if NET
+        if (newClientOptions.RequestHeaderEncoding != null)
+        {
+            handler.RequestHeaderEncodingSelector = (_, _) => newClientOptions.RequestHeaderEncoding;
+        }
+#endif
 
         return new HttpMessageInvoker(handler, disposeHandler: true);
     }

src/ReverseProxy/Abstractions/ClusterDiscovery/Contract/ProxyHttpClientOptions.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Security.Authentication;
 using System.Security.Cryptography.X509Certificates;
+using System.Text;
 
 namespace Microsoft.ReverseProxy.Abstractions
 {
@@ -50,6 +51,11 @@ public sealed record ProxyHttpClientOptions
         //  is reached on all existing connections.
         /// </summary>
         public bool? EnableMultipleHttp2Connections { get; init; }
+
+        /// <summary>
+        /// Enables non-ASCII header encoding for outgoing requests.
+        /// </summary>
+        public Encoding RequestHeaderEncoding { get; init; }
 #endif
 
         /// <inheritdoc />
@@ -61,13 +67,15 @@ public bool Equals(ProxyHttpClientOptions other)
             }
 
             return SslProtocols == other.SslProtocols
-                && CertEquals(ClientCertificate, other.ClientCertificate)
-                && DangerousAcceptAnyServerCertificate == other.DangerousAcceptAnyServerCertificate
-                && MaxConnectionsPerServer == other.MaxConnectionsPerServer
+                   && CertEquals(ClientCertificate, other.ClientCertificate)
+                   && DangerousAcceptAnyServerCertificate == other.DangerousAcceptAnyServerCertificate
+                   && MaxConnectionsPerServer == other.MaxConnectionsPerServer
 #if NET
-                && EnableMultipleHttp2Connections == other.EnableMultipleHttp2Connections
+                   && EnableMultipleHttp2Connections == other.EnableMultipleHttp2Connections
+                   // Comparing by reference is fine here since Encoding.GetEncoding returns the same instance for each encoding.
+                   && RequestHeaderEncoding == other.RequestHeaderEncoding
 #endif
-                && ActivityContextHeaders == other.ActivityContextHeaders;
+                   && ActivityContextHeaders == other.ActivityContextHeaders;
         }
 
         private static bool CertEquals(X509Certificate2 certificate1, X509Certificate2 certificate2)
@@ -94,6 +102,7 @@ public override int GetHashCode()
                 MaxConnectionsPerServer,
 #if NET
                 EnableMultipleHttp2Connections,
+                RequestHeaderEncoding,
 #endif
                 ActivityContextHeaders);
         }

src/ReverseProxy/Configuration/ConfigurationConfigProvider.cs

@@ -8,6 +8,7 @@
 using System.Net.Http;
 using System.Security.Authentication;
 using System.Security.Cryptography.X509Certificates;
+using System.Text;
 using System.Threading;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.Logging;
@@ -327,6 +328,7 @@ private ProxyHttpClientOptions CreateProxyHttpClientOptions(IConfigurationSectio
                 MaxConnectionsPerServer = section.ReadInt32(nameof(ProxyHttpClientOptions.MaxConnectionsPerServer)),
 #if NET
                 EnableMultipleHttp2Connections = section.ReadBool(nameof(ProxyHttpClientOptions.EnableMultipleHttp2Connections)),
+                RequestHeaderEncoding = section[nameof(ProxyHttpClientOptions.RequestHeaderEncoding)] is string encoding ? Encoding.GetEncoding(encoding) : null,
 #endif
                 ActivityContextHeaders = section.ReadEnum<ActivityContextHeaders>(nameof(ProxyHttpClientOptions.ActivityContextHeaders))
             };

src/ReverseProxy/Service/Proxy/HttpProxy.cs

@@ -172,9 +172,20 @@ public async Task ProxyAsync(
                     throw new InvalidOperationException("Proxying the Client request body to the Destination server hasn't started. This is a coding defect.");
                 }
 
-                // :: Step 5: Copy response status line Client ◄-- Proxy ◄-- Destination
-                // :: Step 6: Copy response headers Client ◄-- Proxy ◄-- Destination
-                await CopyResponseStatusAndHeadersAsync(destinationResponse, context, transformer);
+                try
+                {
+                    // :: Step 5: Copy response status line Client ◄-- Proxy ◄-- Destination
+                    // :: Step 6: Copy response headers Client ◄-- Proxy ◄-- Destination
+                    await CopyResponseStatusAndHeadersAsync(destinationResponse, context, transformer);
+                }
+                catch (Exception ex)
+                {
+                    ReportProxyError(context, ProxyError.ResponseHeaders, ex);
+                    // Clear the response since status code, reason and some headers might have already been copied and we want clean 502 response.
+                    context.Response.Clear();
+                    context.Response.StatusCode = StatusCodes.Status502BadGateway;
+                    return;
+                }
 
                 // :: Step 7-A: Check for a 101 upgrade response, this takes care of WebSockets as well as any other upgradeable protocol.
                 if (destinationResponse.StatusCode == HttpStatusCode.SwitchingProtocols)
@@ -285,7 +296,7 @@ public async Task ProxyAsync(
             // Allow someone to custom build the request uri, otherwise provide a default for them.
             var request = context.Request;
             destinationRequest.RequestUri ??= RequestUtilities.MakeDestinationAddress(destinationPrefix, request.Path, request.QueryString);
-            
+
             Log.Proxying(_logger, destinationRequest.RequestUri);
 
             // TODO: What if they replace the HttpContent object? That would mess with our tracking and error handling.
@@ -689,6 +700,7 @@ private static string GetMessage(ProxyError error)
                     ProxyError.ResponseBodyCanceled => "Copying the response body was canceled.",
                     ProxyError.ResponseBodyClient => "The client reported an error when copying the response body.",
                     ProxyError.ResponseBodyDestination => "The destination reported an error when copying the response body.",
+                    ProxyError.ResponseHeaders => "The destination returned a response that cannot be proxied back to the client.",
                     ProxyError.UpgradeRequestCanceled => "Copying the upgraded request body was canceled.",
                     ProxyError.UpgradeRequestClient => "The client reported an error when copying the upgraded request body.",
                     ProxyError.UpgradeRequestDestination => "The destination reported an error when copying the upgraded request body.",

src/ReverseProxy/Service/Proxy/Infrastructure/ProxyHttpClientFactory.cs

@@ -70,6 +70,10 @@ public HttpMessageInvoker CreateClient(ProxyHttpClientContext context)
             {
                 handler.EnableMultipleHttp2Connections = newClientOptions.EnableMultipleHttp2Connections.Value;
             }
+            if (newClientOptions.RequestHeaderEncoding != null)
+            {
+                handler.RequestHeaderEncodingSelector = (_, _) => newClientOptions.RequestHeaderEncoding;
+            }
 #endif
 
             Log.ProxyClientCreated(_logger, context.ClusterId);

src/ReverseProxy/Service/Proxy/ProxyError.cs

@@ -43,6 +43,11 @@ public enum ProxyError : int
         /// </summary>
         RequestBodyDestination,
 
+        /// <summary>
+        /// Failed to copy response headers.
+        /// </summary>
+        ResponseHeaders,
+
         /// <summary>
         /// Canceled while copying the response body.
         /// </summary>

test/ReverseProxy.FunctionalTests/Common/TestEnvironment.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Collections.Generic;
 using System.Net;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
@@ -25,13 +26,14 @@ public class TestEnvironment
         private readonly Action<IApplicationBuilder> _configureProxyApp;
         private readonly HttpProtocols _proxyProtocol;
         private readonly bool _useHttpsOnDestination;
+        private readonly Encoding _headerEncoding;
 
         public string ClusterId { get; set; } = "cluster1";
 
         public TestEnvironment(
             RequestDelegate destinationGetDelegate,
             Action<IReverseProxyBuilder> configureProxy, Action<IApplicationBuilder> configureProxyApp,
-            HttpProtocols proxyProtocol = HttpProtocols.Http1AndHttp2, bool useHttpsOnDestination = false)
+            HttpProtocols proxyProtocol = HttpProtocols.Http1AndHttp2, bool useHttpsOnDestination = false, Encoding headerEncoding = null)
             : this(
                   destinationServices => { },
                   destinationApp =>
@@ -41,95 +43,113 @@ public TestEnvironment(
                   configureProxy,
                   configureProxyApp,
                   proxyProtocol,
-                  useHttpsOnDestination)
+                  useHttpsOnDestination,
+                  headerEncoding)
         { }
 
         public TestEnvironment(
             Action<IServiceCollection> configureDestinationServices, Action<IApplicationBuilder> configureDestinationApp,
             Action<IReverseProxyBuilder> configureProxy, Action<IApplicationBuilder> configureProxyApp,
-            HttpProtocols proxyProtocol = HttpProtocols.Http1AndHttp2, bool useHttpsOnDestination = false)
+            HttpProtocols proxyProtocol = HttpProtocols.Http1AndHttp2, bool useHttpsOnDestination = false, Encoding headerEncoding = null)
         {
             _configureDestinationServices = configureDestinationServices;
             _configureDestinationApp = configureDestinationApp;
             _configureProxy = configureProxy;
             _configureProxyApp = configureProxyApp;
             _proxyProtocol = proxyProtocol;
             _useHttpsOnDestination = useHttpsOnDestination;
+            _headerEncoding = headerEncoding;
         }
 
         public async Task Invoke(Func<string, Task> clientFunc, CancellationToken cancellationToken = default)
         {
-            using var destination = CreateHost(HttpProtocols.Http1AndHttp2, _useHttpsOnDestination, _configureDestinationServices, _configureDestinationApp);
+            using var destination = CreateHost(HttpProtocols.Http1AndHttp2, _useHttpsOnDestination, _headerEncoding, _configureDestinationServices, _configureDestinationApp);
             await destination.StartAsync(cancellationToken);
 
-            using var proxy = CreateHost(_proxyProtocol, false,
+            using var proxy = CreateProxy(_proxyProtocol, _useHttpsOnDestination, _headerEncoding, ClusterId, destination.GetAddress(), _configureProxy, _configureProxyApp);
+            await proxy.StartAsync(cancellationToken);
+
+            try
+            {
+                await clientFunc(proxy.GetAddress());
+            }
+            finally
+            {
+                await proxy.StopAsync(cancellationToken);
+                await destination.StopAsync(cancellationToken);
+            }
+        }
+
+        public static IHost CreateProxy(HttpProtocols protocols, bool useHttps, Encoding requestHeaderEncoding, string clusterId, string destinationAddress,
+            Action<IReverseProxyBuilder> configureProxy, Action<IApplicationBuilder> configureProxyApp)
+        {
+            return CreateHost(protocols, false, requestHeaderEncoding,
                 services =>
                 {
                     var proxyRoute = new ProxyRoute
                     {
                         RouteId = "route1",
-                        ClusterId = ClusterId,
+                        ClusterId = clusterId,
                         Match = new ProxyMatch { Path = "/{**catchall}" }
                     };
 
                     var cluster = new Cluster
                     {
-                        Id = ClusterId,
+                        Id = clusterId,
                         Destinations = new Dictionary<string, Destination>(StringComparer.OrdinalIgnoreCase)
                         {
-                            { "destination1",  new Destination() { Address = destination.GetAddress() } }
+                            { "destination1",  new Destination() { Address = destinationAddress } }
                         },
                         HttpClient = new ProxyHttpClientOptions
                         {
-                            DangerousAcceptAnyServerCertificate = _useHttpsOnDestination
+                            DangerousAcceptAnyServerCertificate = useHttps,
+#if NET
+                            RequestHeaderEncoding = requestHeaderEncoding,
+#endif
                         }
                     };
 
                     var proxyBuilder = services.AddReverseProxy().LoadFromMemory(new[] { proxyRoute }, new[] { cluster });
-                    _configureProxy(proxyBuilder);
+                    configureProxy(proxyBuilder);
                 },
                 app =>
                 {
-                    _configureProxyApp(app);
+                    configureProxyApp(app);
                     app.UseRouting();
                     app.UseEndpoints(builder =>
                     {
                         builder.MapReverseProxy();
                     });
                 });
-            await proxy.StartAsync(cancellationToken);
-
-            try
-            {
-                await clientFunc(proxy.GetAddress());
-            }
-            finally
-            {
-                await proxy.StopAsync(cancellationToken);
-                await destination.StopAsync(cancellationToken);
-            }
         }
 
-        private static IHost CreateHost(HttpProtocols protocols, bool useHttps, Action<IServiceCollection> configureServices, Action<IApplicationBuilder> configureApp)
+        private static IHost CreateHost(HttpProtocols protocols, bool useHttps, Encoding requestHeaderEncoding,
+            Action<IServiceCollection> configureServices, Action<IApplicationBuilder> configureApp)
         {
             return new HostBuilder()
-               .ConfigureWebHost(webHostBuilder =>
-               {
-                   webHostBuilder
-                       .ConfigureServices(configureServices)
-                       .UseKestrel(kestrel =>
-                       {
-                           kestrel.Listen(IPAddress.Loopback, 0, listenOptions =>
-                           {
-                               listenOptions.Protocols = protocols;
-                               if (useHttps)
-                               {
-                                   listenOptions.UseHttps(TestResources.GetTestCertificate());
-                               }
-                           });
-                       })
-                       .Configure(configureApp);
-               }).Build();
+                .ConfigureWebHost(webHostBuilder =>
+                {
+                    webHostBuilder
+                        .ConfigureServices(configureServices)
+                        .UseKestrel(kestrel =>
+                        {
+#if NET
+                            if (requestHeaderEncoding != null)
+                            {
+                                kestrel.RequestHeaderEncodingSelector = _ => requestHeaderEncoding;
+                            }
+#endif
+                            kestrel.Listen(IPAddress.Loopback, 0, listenOptions =>
+                            {
+                                listenOptions.Protocols = protocols;
+                                if (useHttps)
+                                {
+                                    listenOptions.UseHttps(TestResources.GetTestCertificate());
+                                }
+                            });
+                        })
+                        .Configure(configureApp);
+                }).Build();
         }
     }
 }