Wait for password-check capacity during bursts

docs/password-authentication.md

@@ -0,0 +1,9 @@
+# Password authentication
+
+Interactive login and Basic password authentication share admission state. Each account allows five failed checks per quarter-hour window; each client IP allows fifteen. Checks underway reserve capacity before validating credentials. Successful checks return their capacity, and account recovery clears completed account failures.
+
+When capacity is occupied by checks underway, requests wait for up to ten seconds and retry before returning the existing login-denied response. Completed failures still deny admission. Cancellation releases any partial reservation. Pending reservations from a stopped host remain charged until the window ends.
+
+Configure Redis for deployments with multiple API hosts so admission state is shared. In-memory caching coordinates only callers within the same process. The installed Foundatio 13.0.4 in-memory increment primitive loses updates under contention; admission currently uses conditional cache operations instead.
+
+Clients behind a shared IP reuse capacity after successful checks. Fifteen actual failures from that IP still affect its other clients until the next window. The bounded wait addresses concurrent successful bursts; it does not change that failure policy or guarantee admission under sustained load.

src/Exceptionless.Core/Services/AuthService.cs

@@ -11,6 +11,7 @@ public sealed class AuthService
     private const int UserFailureLimit = 5;
     private const int IpAddressFailureLimit = 15;
     private static readonly TimeSpan AttemptWindow = TimeSpan.FromMinutes(15);
+    private static readonly TimeSpan CapacityWaitTimeout = TimeSpan.FromSeconds(10);
     private readonly ScopedCacheClient _cache;
     private readonly TimeProvider _timeProvider;
 
@@ -29,36 +30,63 @@ public AuthService(ICacheClient cacheClient, TimeProvider timeProvider)
             ArgumentException.ThrowIfNullOrWhiteSpace(ipAddress);
         cancellationToken.ThrowIfCancellationRequested();
 
-        var expiresUtc = GetWindowExpiration();
-        string[] userKeys = GetKeys($"user:{emailAddress.Trim().ToLowerInvariant()}", UserFailureLimit, expiresUtc);
-        var failures = await _cache.GetAllAsync<string>(userKeys);
-        var observedFailures = failures.Where(pair => pair.Value.HasValue && pair.Value.Value.StartsWith("failed:", StringComparison.Ordinal))
-            .Select(pair => new KeyValuePair<string, string>(pair.Key, pair.Value.Value)).ToArray();
+        long started = _timeProvider.GetTimestamp();
         string reservation = $"pending:{Guid.NewGuid():N}";
-        var keys = new List<string>(2);
-        try
+        while (true)
         {
-            string? userKey = await ReserveAsync(userKeys, reservation, expiresUtc);
-            if (userKey is null)
+            cancellationToken.ThrowIfCancellationRequested();
+            if (_timeProvider.GetElapsedTime(started) >= CapacityWaitTimeout)
                 return null;
-            keys.Add(userKey);
-            if (ipAddress is not null)
+            var expiresUtc = GetWindowExpiration();
+            string[] userKeys = GetKeys($"user:{emailAddress.Trim().ToLowerInvariant()}", UserFailureLimit, expiresUtc);
+            var userState = await _cache.GetAllAsync<string>(userKeys);
+            var keys = new List<string>(2);
+            try
             {
-                string? ipKey = await ReserveAsync(GetKeys($"ip:{ipAddress}", IpAddressFailureLimit, expiresUtc), reservation, expiresUtc);
-                if (ipKey is null)
+                string? userKey = await ReserveAsync(userKeys, userState, reservation, expiresUtc);
+                if (userKey is null && HasReachedFailureLimit(userKeys, userState))
+                    return null;
+                if (userKey is not null)
                 {
+                    keys.Add(userKey);
+                    string? ipKey = null;
+                    if (ipAddress is not null)
+                    {
+                        string[] ipKeys = GetKeys($"ip:{ipAddress}", IpAddressFailureLimit, expiresUtc);
+                        var ipState = await _cache.GetAllAsync<string>(ipKeys);
+                        ipKey = await ReserveAsync(ipKeys, ipState, reservation, expiresUtc);
+                        if (ipKey is null && HasReachedFailureLimit(ipKeys, ipState))
+                        {
+                            await ReleaseAsync(keys, reservation);
+                            return null;
+                        }
+                    }
+                    if (ipAddress is null || ipKey is not null)
+                    {
+                        if (ipKey is not null)
+                            keys.Add(ipKey);
+                        cancellationToken.ThrowIfCancellationRequested();
+                        var observedFailures = userState.Where(pair => IsFailure(pair.Value))
+                            .Select(pair => new KeyValuePair<string, string>(pair.Key, pair.Value.Value)).ToArray();
+                        return new LoginAttempt(this, expiresUtc, keys.ToArray(), reservation, observedFailures);
+                    }
+                    // Release the account slot while waiting on shared IP capacity.
                     await ReleaseAsync(keys, reservation);
-                    return null;
                 }
-                keys.Add(ipKey);
             }
-            cancellationToken.ThrowIfCancellationRequested();
-            return new LoginAttempt(this, expiresUtc, keys.ToArray(), reservation, observedFailures);
-        }
-        catch
-        {
-            await ReleaseAsync(keys, reservation);
-            throw;
+            catch
+            {
+                await ReleaseAsync(keys, reservation);
+                throw;
+            }
+
+            var remaining = CapacityWaitTimeout - _timeProvider.GetElapsedTime(started);
+            if (remaining <= TimeSpan.Zero)
+                return null;
+            // Pending checks may succeed and return capacity. Spread retries across
+            // hosts; completed failures still deny admission without waiting.
+            var delay = TimeSpan.FromMilliseconds(Random.Shared.Next(75, 126));
+            await Task.Delay(delay < remaining ? delay : remaining, _timeProvider, cancellationToken);
         }
     }
 
@@ -89,14 +117,20 @@ await RemoveFailuresAsync(failures.Where(pair => pair.Value.HasValue && pair.Val
             .Select(pair => new KeyValuePair<string, string>(pair.Key, pair.Value.Value)));
     }
 
-    private async Task<string?> ReserveAsync(string[] keys, string reservation, DateTime expiresUtc)
+    private async Task<string?> ReserveAsync(string[] keys, IDictionary<string, CacheValue<string>> state, string reservation, DateTime expiresUtc)
     {
         foreach (string key in keys)
-            if (await _cache.AddAsync(key, reservation, expiresUtc))
+            if ((!state.TryGetValue(key, out var value) || !value.HasValue) && await _cache.AddAsync(key, reservation, expiresUtc))
                 return key;
         return null;
     }
 
+    private static bool HasReachedFailureLimit(string[] keys, IDictionary<string, CacheValue<string>> state)
+        => keys.All(key => state.TryGetValue(key, out var value) && IsFailure(value));
+
+    private static bool IsFailure(CacheValue<string> value)
+        => value.HasValue && value.Value.StartsWith("failed:", StringComparison.Ordinal);
+
     private Task ReleaseAsync(IEnumerable<string> keys, string reservation)
         => Task.WhenAll(keys.Select(key => _cache.RemoveIfEqualAsync(key, reservation)));
 

tests/Exceptionless.Tests/Services/AuthServiceTests.cs

@@ -11,27 +11,101 @@ public async Task TryBeginLoginAsync_ConcurrentInstances_BoundsChecksBeforeFailu
     {
         var first = GetService<AuthService>();
         var second = new AuthService(GetService<ICacheClient>(), TimeProvider);
-        var attempts = await Task.WhenAll(Enumerable.Range(0, 100).Select(i =>
-            (i % 2 == 0 ? first : second).TryBeginLoginAsync(" User@exceptionless.test ", null, TestCancellationToken)));
+        var requests = Enumerable.Range(0, 100).Select(i =>
+            (i % 2 == 0 ? first : second).TryBeginLoginAsync(" User@exceptionless.test ", null, TestCancellationToken)).ToArray();
+        var admitted = requests.Where(task => task.IsCompletedSuccessfully).Select(task => task.Result).ToArray();
+        Assert.Equal(5, admitted.Length);
+        Assert.All(admitted, Assert.NotNull);
+        await Task.WhenAll(admitted.Select(a => first.RecordLoginFailureAsync(a!)));
+        var attempts = await Task.WhenAll(requests);
         Assert.Equal(5, attempts.Count(a => a is not null));
-        await Task.WhenAll(attempts.Where(a => a is not null).Select(a => first.RecordLoginFailureAsync(a!)));
         Assert.Null(await second.TryBeginLoginAsync("user@exceptionless.test", null, TestCancellationToken));
     }
 
     [Fact]
     public async Task TryBeginLoginAsync_ConcurrentUsers_BoundsChecksAtSharedIpAddress()
     {
         var service = GetService<AuthService>();
-        var attempts = await Task.WhenAll(Enumerable.Range(0, 100).Select(i =>
-            service.TryBeginLoginAsync($"user{i}@exceptionless.test", "192.0.2.1", TestCancellationToken)));
+        var requests = Enumerable.Range(0, 100).Select(i =>
+            service.TryBeginLoginAsync($"user{i}@exceptionless.test", "192.0.2.1", TestCancellationToken)).ToArray();
+        var admitted = requests.Where(task => task.IsCompletedSuccessfully).Select(task => task.Result).ToArray();
+        Assert.Equal(15, admitted.Length);
+        Assert.All(admitted, Assert.NotNull);
+        await Task.WhenAll(admitted.Select(a => service.RecordLoginFailureAsync(a!)));
+        var attempts = await Task.WhenAll(requests);
         Assert.Equal(15, attempts.Count(a => a is not null));
-        await Task.WhenAll(attempts.Where(a => a is not null).Select(a => service.RecordLoginFailureAsync(a!)));
         Assert.Null(await service.TryBeginLoginAsync("other@exceptionless.test", "192.0.2.1", TestCancellationToken));
         // Rejected IP admission must release the partially reserved account slot.
         await using var allowed = await service.TryBeginLoginAsync("user99@exceptionless.test", "192.0.2.2", TestCancellationToken);
         Assert.NotNull(allowed);
     }
 
+    [Theory]
+    [InlineData(false, 5)]
+    [InlineData(true, 15)]
+    public async Task TryBeginLoginAsync_ValidBurstAcrossInstances_WaitsForCapacity(bool distinctUsers, int capacity)
+    {
+        var first = GetService<AuthService>();
+        var second = new AuthService(GetService<ICacheClient>(), TimeProvider);
+        var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
+        int checks = 0;
+        int pending = 0;
+        async Task AuthenticateAsync(int index)
+        {
+            var service = index % 2 == 0 ? first : second;
+            string email = distinctUsers ? $"user{index}@example.test" : "user@example.test";
+            await using var attempt = await service.TryBeginLoginAsync(email, "192.0.2.1", TestCancellationToken);
+            Assert.NotNull(attempt);
+            Assert.InRange(Interlocked.Increment(ref pending), 1, capacity);
+            Interlocked.Increment(ref checks);
+            await release.Task;
+            Interlocked.Decrement(ref pending);
+            await service.RecordLoginSuccessAsync(attempt);
+        }
+        var requests = Enumerable.Range(0, 100).Select(AuthenticateAsync).ToArray();
+        try
+        {
+            Assert.Equal(capacity, Volatile.Read(ref checks));
+            Assert.All(requests, task => Assert.False(task.IsCompleted));
+        }
+        finally
+        {
+            release.TrySetResult();
+        }
+        await Task.WhenAll(requests);
+        Assert.Equal(100, checks);
+        Assert.Equal(0, pending);
+    }
+
+    [Theory]
+    [InlineData(false)]
+    [InlineData(true)]
+    public async Task TryBeginLoginAsync_PendingIpCapacity_CancellationOrTimeoutReleasesAccountSlot(bool timeout)
+    {
+        TimeProvider.SetUtcNow(new DateTimeOffset(2026, 1, 1, 12, 1, 0, TimeSpan.Zero));
+        var service = GetService<AuthService>();
+        var held = await Task.WhenAll(Enumerable.Range(0, 15).Select(index =>
+            service.TryBeginLoginAsync($"other{index}@example.test", "192.0.2.1", TestCancellationToken)));
+        Assert.All(held, Assert.NotNull);
+        using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken);
+        var waiting = service.TryBeginLoginAsync("waiting@example.test", "192.0.2.1", cancellation.Token);
+        Assert.False(waiting.IsCompleted);
+        if (timeout)
+        {
+            TimeProvider.Advance(TimeSpan.FromSeconds(10));
+            Assert.Null(await waiting);
+        }
+        else
+        {
+            await cancellation.CancelAsync();
+            await Assert.ThrowsAnyAsync<OperationCanceledException>(() => waiting);
+        }
+        var allowed = await Task.WhenAll(Enumerable.Range(0, 5).Select(_ =>
+            service.TryBeginLoginAsync("waiting@example.test", "192.0.2.2", TestCancellationToken)));
+        Assert.All(allowed, Assert.NotNull);
+        await Task.WhenAll(held.Concat(allowed).Select(attempt => attempt!.DisposeAsync().AsTask()));
+    }
+
     [Fact]
     public async Task RecordLoginSuccessAsync_ValidRequests_DoNotConsumeFailureQuota()
     {
@@ -100,8 +174,13 @@ public async Task ClearUserLoginAttemptsAsync_PreservesIpFailuresAndChecksUnderw
             await FailAsync(service);
         var pending = await BeginAsync(service);
         await service.ClearUserLoginAttemptsAsync(" User@exceptionless.test ");
-        var remaining = await Task.WhenAll(Enumerable.Range(0, 5).Select(_ => service.TryBeginLoginAsync("user@exceptionless.test", null, TestCancellationToken)));
-        Assert.Equal(4, remaining.Count(a => a is not null));
+        var remaining = await Task.WhenAll(Enumerable.Range(0, 4).Select(_ => service.TryBeginLoginAsync("user@exceptionless.test", null, TestCancellationToken)));
+        Assert.All(remaining, Assert.NotNull);
+        using var waitingCancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken);
+        var waiting = service.TryBeginLoginAsync("user@exceptionless.test", null, waitingCancellation.Token);
+        Assert.False(waiting.IsCompleted);
+        await waitingCancellation.CancelAsync();
+        await Assert.ThrowsAnyAsync<OperationCanceledException>(() => waiting);
         await pending.DisposeAsync();
         await Task.WhenAll(remaining.Where(a => a is not null).Select(a => a!.DisposeAsync().AsTask()));
         for (int i = 0; i < 11; i++)
@@ -117,7 +196,11 @@ public async Task TryBeginLoginAsync_QuarterHour_ExpiresFailuresAndAbandonedRese
         var old = await BeginAsync(service);
         for (int i = 0; i < 4; i++)
             await FailAsync(service);
-        Assert.Null(await service.TryBeginLoginAsync("user@exceptionless.test", null, TestCancellationToken));
+        using var cancellation = CancellationTokenSource.CreateLinkedTokenSource(TestCancellationToken);
+        var waiting = service.TryBeginLoginAsync("user@exceptionless.test", null, cancellation.Token);
+        Assert.False(waiting.IsCompleted);
+        await cancellation.CancelAsync();
+        await Assert.ThrowsAnyAsync<OperationCanceledException>(() => waiting);
         TimeProvider.Advance(TimeSpan.FromMinutes(1));
         for (int i = 0; i < 5; i++)
             await FailAsync(service);

tests/http/users.http

@@ -20,6 +20,7 @@ Content-Type: application/json
 @token = {{login.response.body.$.token}}
 
 ### Password authentication shares the login attempt window
+# Requests wait up to ten seconds when password checks are underway; completed failures retain their limit.
 # Password changes replace the current user session; use the returned token afterward.
 # Password sign-in checks the current stored credentials, account status, and email.
 GET {{apiUrl}}/users/me