Support configuring the MaxRequestBodySize for the upload endpoint (#863

)

## Motivation and Context (Why the change? What's the scenario?)

We have some important documents that need to be imported to Kermel
Memory that are larger than the current ASP.NET Core default maximum
request body size limit of 30 MB (~28.6 MiB, see 
aspnet/Announcements#267).

It is not currently possible to set the `MaxRequestBodySize` from
configuration making it impossible to override this limit when running
Kernel Memory from the Docker container image.

## High level description (Approach, Design)

Add a new `MaxUploadSizeMb` property to the `ServiceConfig` class
that defaults to `null`.

---------

Co-authored-by: Devis Lucato <devis@microsoft.com>

service/Core/Configuration/ServiceConfig.cs

@@ -1,5 +1,6 @@
 // Copyright (c) Microsoft. All rights reserved.
 
+using System;
 using System.Collections.Generic;
 
 namespace Microsoft.KernelMemory.Configuration;
@@ -27,4 +28,20 @@ public class ServiceConfig
     /// List of handlers to enable
     /// </summary>
     public Dictionary<string, HandlerConfig> Handlers { get; set; } = new();
+
+    /// <summary>
+    /// The maximum allowed size in megabytes for a request body posted to the upload endpoint.
+    /// If not set the solution defaults to 30,000,000 bytes (~28.6 MB) (ASP.NET default).
+    /// </summary>
+    public long? MaxUploadSizeMb { get; set; } = null;
+
+    public long? GetMaxUploadSizeInBytes()
+    {
+        if (this.MaxUploadSizeMb.HasValue)
+        {
+            return Math.Min(10, this.MaxUploadSizeMb.Value) * 1024 * 1024;
+        }
+
+        return null;
+    }
 }

service/Service.AspNetCore/WebAPIEndpoints.cs

@@ -7,6 +7,7 @@
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Http.HttpResults;
 using Microsoft.AspNetCore.Mvc;
 using Microsoft.AspNetCore.Routing;
@@ -22,9 +23,10 @@ public static class WebAPIEndpoints
     public static IEndpointRouteBuilder AddKernelMemoryEndpoints(
         this IEndpointRouteBuilder builder,
         string apiPrefix = "/",
+        KernelMemoryConfig? kmConfig = null,
         IEndpointFilter? authFilter = null)
     {
-        builder.AddPostUploadEndpoint(apiPrefix, authFilter);
+        builder.AddPostUploadEndpoint(apiPrefix, authFilter, kmConfig?.Service.GetMaxUploadSizeInBytes());
         builder.AddGetIndexesEndpoint(apiPrefix, authFilter);
         builder.AddDeleteIndexesEndpoint(apiPrefix, authFilter);
         builder.AddDeleteDocumentsEndpoint(apiPrefix, authFilter);
@@ -37,7 +39,10 @@ public static IEndpointRouteBuilder AddKernelMemoryEndpoints(
     }
 
     public static void AddPostUploadEndpoint(
-        this IEndpointRouteBuilder builder, string apiPrefix = "/", IEndpointFilter? authFilter = null)
+        this IEndpointRouteBuilder builder,
+        string apiPrefix = "/",
+        IEndpointFilter? authFilter = null,
+        long? maxUploadSizeInBytes = null)
     {
         RouteGroupBuilder group = builder.MapGroup(apiPrefix);
 
@@ -49,6 +54,12 @@ public static void AddPostUploadEndpoint(
                 IContextProvider contextProvider,
                 CancellationToken cancellationToken) =>
             {
+                if (maxUploadSizeInBytes.HasValue && request.HttpContext.Features.Get<IHttpMaxRequestBodySizeFeature>() is { } feature)
+                {
+                    log.LogTrace("Max upload request body size set to {0} bytes", maxUploadSizeInBytes.Value);
+                    feature.MaxRequestBodySize = maxUploadSizeInBytes;
+                }
+
                 log.LogTrace("New upload HTTP request, content length {0}", request.ContentLength);
 
                 // Note: .NET doesn't yet support binding multipart forms including data and files

service/Service.AspNetCore/WebApplicationBuilderExtensions.cs

@@ -19,20 +19,25 @@ public static partial class WebApplicationBuilderExtensions
     /// <param name="appBuilder">Hosting application builder</param>
     /// <param name="configureMemoryBuilder">Optional configuration steps for the memory builder</param>
     /// <param name="configureMemory">Optional configuration steps for the memory instance</param>
+    /// <param name="configureServices">Optional configuration for the internal dependencies</param>
     public static WebApplicationBuilder AddKernelMemory(
         this WebApplicationBuilder appBuilder,
         Action<IKernelMemoryBuilder>? configureMemoryBuilder = null,
-        Action<IKernelMemory>? configureMemory = null)
+        Action<IKernelMemory>? configureMemory = null,
+        Action<IServiceCollection>? configureServices = null)
     {
         // Prepare memory builder, sharing the service collection used by the hosting service
         var memoryBuilder = new KernelMemoryBuilder(appBuilder.Services);
 
+        // Optional services configuration provided by the user
+        configureServices?.Invoke(appBuilder.Services);
+
         // Optional configuration provided by the user
         configureMemoryBuilder?.Invoke(memoryBuilder);
 
         var memory = memoryBuilder.Build();
 
-        // Optional configuration provided by the user
+        // Optional memory configuration provided by the user
         configureMemory?.Invoke(memory);
 
         appBuilder.Services.AddSingleton<IKernelMemory>(memory);

service/Service/Program.cs

@@ -5,7 +5,9 @@
 using System.Linq;
 using Microsoft.AspNetCore.Builder;
 using Microsoft.AspNetCore.Http;
+using Microsoft.AspNetCore.Http.Features;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.AspNetCore.Server.Kestrel.Core;
 using Microsoft.Extensions.Configuration;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Logging;
@@ -97,6 +99,19 @@ public static void Main(string[] args)
                 syncHandlersCount = AddHandlersToServerlessMemory(config, memory);
 
                 memoryType = ((memory is MemoryServerless) ? "Sync - " : "Async - ") + memory.GetType().FullName;
+            },
+            services =>
+            {
+                long? maxSize = config.Service.GetMaxUploadSizeInBytes();
+                if (!maxSize.HasValue) { return; }
+
+                services.Configure<IISServerOptions>(x => { x.MaxRequestBodySize = maxSize.Value; });
+                services.Configure<KestrelServerOptions>(x => { x.Limits.MaxRequestBodySize = maxSize.Value; });
+                services.Configure<FormOptions>(x =>
+                {
+                    x.MultipartBodyLengthLimit = maxSize.Value;
+                    x.ValueLengthLimit = int.MaxValue;
+                });
             });
 
         // CORS
@@ -138,7 +153,7 @@ public static void Main(string[] args)
                 .Produces<ProblemDetails>(StatusCodes.Status403Forbidden);
 
             // Add HTTP endpoints using minimal API (https://learn.microsoft.com/aspnet/core/fundamentals/minimal-apis)
-            app.AddKernelMemoryEndpoints("/", authFilter);
+            app.AddKernelMemoryEndpoints("/", config, authFilter);
 
             // Health probe
             app.MapGet("/health", () => Results.Ok("Service is running."))

service/Service/appsettings.json

@@ -44,6 +44,9 @@
       "RunWebService": true,
       // Whether to expose OpenAPI swagger UI at http://127.0.0.1:9001/swagger/index.html
       "OpenApiEnabled": false,
+      // The maximum allowed size in MB for the payload posted to the upload endpoint
+      // If not set the solution defaults to 30,000,000 bytes (~28.6 MB)
+      "MaxUploadSizeMb": null,
       // Whether to run the asynchronous pipeline handlers
       // Use these booleans to deploy the web service and the handlers on same/different VMs
       "RunHandlers": true,