OIDC: Limit number of dynamic tenants

Introduce a new configuration option `quarkus.oidc.dynamic-tenant-limit` as an opt-in to limit the number of active dynamic OIDC tenants. The default is no dynamic tenant limit.

This change adds the field `volatile long TenantConfigContext.lastUsed`, and removes the field `TenantConfigContext.ready` to keep the heap footprint the same. The `lastUsed` field is initially set to "now" and updated when the dynamic tenant is being accessed.

When a new dynamic tenant is about to be added to the dynamic tenants map, eviction runs, if there are more dynamic tenants than configured via `dynami-tenant-limit`. The eviction algo is built to iterate over the dynamic tenants only once - it may need to iterate more often, if one of the eviction candidates has been accessed in the mean time. There is no linked-list structure to form an LRU list/queue, as that likely causes more runtime overhead (pointer updates, synchronization) than the cost of iterating over the list once in a while. The assumption is that the map of dynamic tenants has not a lot of "churn".

extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/DefaultTenantConfigResolver.java

@@ -153,7 +153,7 @@ public Uni<? extends TenantConfigContext> get() {
     private Uni<TenantConfigContext> initializeStaticTenantIfContextNotReady(TenantConfigContext tenantContext) {
         requireNonNull(tenantContext, "tenantContext must never be null");
 
-        if (!tenantContext.ready) {
+        if (!tenantContext.isReady()) {
             return tenantConfigBean.getOrCreateTenantContext(tenantContext.oidcConfig, false);
         }
 

extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/OidcConfig.java

@@ -3,6 +3,7 @@
 import java.time.Duration;
 import java.util.Map;
 import java.util.Optional;
+import java.util.OptionalInt;
 
 import io.quarkus.oidc.OidcTenantConfig;
 import io.quarkus.runtime.annotations.ConfigDocMapKey;
@@ -43,6 +44,13 @@ public class OidcConfig {
     @ConfigItem(defaultValue = "false")
     public boolean resolveTenantsWithIssuer;
 
+    /**
+     * If configured, limit the number of active dynamic OIDC tenant contexts to this value.
+     * If not set, the number of active dynamic OIDC tenant contexts is unlimited.
+     */
+    @ConfigItem
+    public OptionalInt dynamicTenantLimit;
+
     /**
      * Default TokenIntrospection and UserInfo cache configuration.
      */

extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/OidcRecorder.java

@@ -16,6 +16,7 @@
 import java.util.function.BiFunction;
 import java.util.function.Consumer;
 import java.util.function.Function;
+import java.util.function.LongSupplier;
 import java.util.function.Supplier;
 
 import jakarta.enterprise.inject.CreationException;
@@ -123,7 +124,13 @@ public TenantConfigBean setup(OidcConfig config, Vertx vertxValue, TlsConfigurat
                     createStaticTenantContext(vertxValue, tenant.getValue(), false, tenant.getKey(), defaultTlsConfiguration));
         }
 
-        return new TenantConfigBean(staticTenantsConfig, defaultTenantContext,
+        return new TenantConfigBean(staticTenantsConfig, defaultTenantContext, new LongSupplier() {
+            @Override
+            public long getAsLong() {
+                return System.nanoTime();
+            }
+        },
+                Math.max(0, config.dynamicTenantLimit.orElse(0)),
                 new TenantConfigBean.TenantContextFactory() {
                     @Override
                     public Uni<TenantConfigContext> create(OidcTenantConfig oidcConfig, boolean dynamicTenant,
@@ -157,14 +164,14 @@ public TenantConfigContext apply(Throwable t) {
                                         + " Access to resources protected by this tenant may fail"
                                         + " if OIDC server will not become available",
                                         tenantId, t.getMessage());
-                                return new TenantConfigContext(null, oidcConfig, false);
+                                return TenantConfigContext.notReadyContext(oidcConfig);
                             }
                             logTenantConfigContextFailure(t, tenantId);
                             if (t instanceof ConfigurationException
                                     && oidcConfig.authServerUrl.isEmpty()
                                     && LaunchMode.DEVELOPMENT == LaunchMode.current()) {
                                 // Let it start if it is a DEV mode and auth-server-url has not been configured yet
-                                return new TenantConfigContext(null, oidcConfig, false);
+                                return TenantConfigContext.notReadyContext(oidcConfig);
                             }
                             // fail in all other cases
                             throw new OIDCException(t);
@@ -175,7 +182,7 @@ public TenantConfigContext apply(Throwable t) {
             LOG.warnf("Tenant '%s': OIDC server is not available after a %d seconds timeout, an attempt to connect will be made"
                     + " during the first request. Access to resources protected by this tenant may fail if OIDC server"
                     + " will not become available", tenantId, oidcConfig.getConnectionTimeout().getSeconds());
-            return new TenantConfigContext(null, oidcConfig, false);
+            return TenantConfigContext.notReadyContext(oidcConfig);
         }
     }
 
@@ -197,7 +204,8 @@ private Uni<TenantConfigContext> createTenantContext(Vertx vertx, OidcTenantConf
 
         if (!oidcConfig.tenantEnabled) {
             LOG.debugf("'%s' tenant configuration is disabled", tenantId);
-            return Uni.createFrom().item(new TenantConfigContext(new OidcProvider(null, null, null, null), oidcConfig));
+            return Uni.createFrom()
+                    .item(TenantConfigContext.readyContext(new OidcProvider(null, null, null, null), oidcConfig));
         }
 
         if (oidcConfig.getAuthServerUrl().isEmpty()) {
@@ -224,7 +232,7 @@ private Uni<TenantConfigContext> createTenantContext(Vertx vertx, OidcTenantConf
                                 + " or named tenants are configured.");
                         oidcConfig.setTenantEnabled(false);
                         return Uni.createFrom()
-                                .item(new TenantConfigContext(new OidcProvider(null, null, null, null), oidcConfig));
+                                .item(TenantConfigContext.readyContext(new OidcProvider(null, null, null, null), oidcConfig));
                     }
                 }
                 throw new ConfigurationException(
@@ -350,7 +358,7 @@ private Uni<TenantConfigContext> createTenantContext(Vertx vertx, OidcTenantConf
                 .onItem().transform(new Function<OidcProvider, TenantConfigContext>() {
                     @Override
                     public TenantConfigContext apply(OidcProvider p) {
-                        return new TenantConfigContext(p, oidcConfig);
+                        return TenantConfigContext.readyContext(p, oidcConfig);
                     }
                 });
     }
@@ -380,7 +388,7 @@ private static TenantConfigContext createTenantContextFromPublicKey(OidcTenantCo
         LOG.debug("'public-key' property for the local token verification is set,"
                 + " no connection to the OIDC server will be created");
 
-        return new TenantConfigContext(
+        return TenantConfigContext.readyContext(
                 new OidcProvider(oidcConfig.publicKey.orElseThrow(), oidcConfig, readTokenDecryptionKey(oidcConfig)),
                 oidcConfig);
     }
@@ -391,7 +399,7 @@ private static TenantConfigContext createTenantContextToVerifyCertChain(OidcTena
                     "Currently only 'service' applications can be used to verify tokens with inlined certificate chains");
         }
 
-        return new TenantConfigContext(
+        return TenantConfigContext.readyContext(
                 new OidcProvider(null, oidcConfig, readTokenDecryptionKey(oidcConfig)), oidcConfig);
     }
 

extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/TenantConfigBean.java

@@ -4,7 +4,9 @@
 
 import java.util.Map;
 import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.function.Function;
+import java.util.function.LongSupplier;
 
 import jakarta.enterprise.context.spi.CreationalContext;
 
@@ -26,6 +28,10 @@ public class TenantConfigBean {
     private final Map<String, TenantConfigContext> dynamicTenantsConfig;
     private final TenantConfigContext defaultTenant;
     private final TenantContextFactory tenantContextFactory;
+    private final LongSupplier clock;
+    private final int dynamicTenantLimit;
+    // VisibleForTesting
+    final AtomicBoolean tenantEvictionRunning = new AtomicBoolean(false);
 
     @FunctionalInterface
     public interface TenantContextFactory {
@@ -35,9 +41,13 @@ public interface TenantContextFactory {
     TenantConfigBean(
             Map<String, TenantConfigContext> staticTenantsConfig,
             TenantConfigContext defaultTenant,
+            LongSupplier clock,
+            int dynamicTenantLimit,
             TenantContextFactory tenantContextFactory) {
         this.staticTenantsConfig = new ConcurrentHashMap<>(staticTenantsConfig);
         this.dynamicTenantsConfig = new ConcurrentHashMap<>();
+        this.clock = clock;
+        this.dynamicTenantLimit = dynamicTenantLimit;
         this.defaultTenant = defaultTenant;
         this.tenantContextFactory = tenantContextFactory;
     }
@@ -46,24 +56,35 @@ Uni<TenantConfigContext> getOrCreateTenantContext(OidcTenantConfig oidcConfig, b
         var tenantId = oidcConfig.getTenantId().orElseThrow();
         var tenants = dynamicTenant ? dynamicTenantsConfig : staticTenantsConfig;
         var tenant = tenants.get(tenantId);
-        if (tenant == null || !tenant.ready) {
+        if (tenant == null || !tenant.isReady()) {
             LOG.tracef("Creating %s tenant config for %s", dynamicTenant ? "dynamic" : "static", tenantId);
             if (dynamicTenant && oidcConfig.logout.backchannel.path.isPresent()) {
                 throw new ConfigurationException(
                         "BackChannel Logout is currently not supported for dynamic tenants (tenant ID: " + tenantId + ")");
             }
             Uni<TenantConfigContext> uniContext = tenantContextFactory.create(oidcConfig, dynamicTenant, tenantId);
             return uniContext.onItem().transform(
-                    new Function<TenantConfigContext, TenantConfigContext>() {
+                    new Function<>() {
                         @Override
                         public TenantConfigContext apply(TenantConfigContext t) {
                             LOG.debugf("Updating %s %s tenant config for %s", dynamicTenant ? "dynamic" : "static",
-                                    t.ready ? "ready" : "not-ready", tenantId);
-                            tenants.put(tenantId, t);
+                                    t.isReady() ? "ready" : "not-ready", tenantId);
+                            t.lastUsed = clock.getAsLong();
+                            TenantConfigContext previous = tenants.put(tenantId, t);
+                            if (previous != null) {
+                                // Concurrent calls to createTenantContext may race, better "destroy" the previous
+                                // provider, if there's one.
+                                destroyContext(previous);
+                            } else if (dynamicTenant) {
+                                enforceDynamicTenantLimit();
+                            }
                             return t;
                         }
                     });
         }
+        if (dynamicTenant) {
+            tenant.lastUsed = clock.getAsLong();
+        }
         LOG.tracef("Immediately returning ready %s tenant config for %s", dynamicTenant ? "dynamic" : "static", tenantId);
         return Uni.createFrom().item(tenant);
     }
@@ -94,7 +115,12 @@ public Map<String, TenantConfigContext> getStaticTenantsConfig() {
      * Returns a dynamic tenant's config context or {@code null}, if the tenant does not exist.
      */
     public TenantConfigContext getDynamicTenantConfigContext(String tenantId) {
-        return dynamicTenantsConfig.get(tenantId);
+        TenantConfigContext context = dynamicTenantsConfig.get(tenantId);
+        if (context == null) {
+            return null;
+        }
+        context.lastUsed = clock.getAsLong();
+        return context;
     }
 
     /**
@@ -119,24 +145,120 @@ public TenantConfigContext getDefaultTenant() {
         return defaultTenant;
     }
 
+    static void destroyContext(TenantConfigContext context) {
+        if (context != null && context.provider != null) {
+            context.provider.close();
+        }
+    }
+
     public static class Destroyer implements BeanDestroyer<TenantConfigBean> {
 
         @Override
         public void destroy(TenantConfigBean instance, CreationalContext<TenantConfigBean> creationalContext,
                 Map<String, Object> params) {
-            if (instance.defaultTenant != null && instance.defaultTenant.provider != null) {
-                instance.defaultTenant.provider.close();
-            }
+            destroyContext(instance.defaultTenant);
             for (var i : instance.staticTenantsConfig.values()) {
-                if (i.provider != null) {
-                    i.provider.close();
-                }
+                destroyContext(i);
             }
             for (var i : instance.dynamicTenantsConfig.values()) {
-                if (i.provider != null) {
-                    i.provider.close();
-                }
+                destroyContext(i);
             }
         }
     }
+
+    record EvictionCandidate(String tenantId, TenantConfigContext context, long lastUsed) {
+    }
+
+    /**
+     * Enforces the dynamic tenants limit, if configured.
+     *
+     * <p>
+     * Eviction runs at max on one thread at any time.
+     *
+     * <p>
+     * Iterate over all tenants at best only once, unless an eviction candidate was used during the eviction run.
+     */
+    private void enforceDynamicTenantLimit() {
+        int limit = dynamicTenantLimit;
+        if (limit == 0) {
+            // No dynamic tenant limit, nothing to do
+            return;
+        }
+        int toEvict = dynamicTenantsConfig.size() - limit;
+        if (toEvict <= 0) {
+            // Nothing to evict
+            return;
+        }
+        if (!tenantEvictionRunning.compareAndSet(false, true)) {
+            // Eviction running in another thread, don't start a concurrent one.
+            return;
+        }
+        try {
+            do {
+                EvictionCandidate[] candidates = new EvictionCandidate[toEvict];
+                int numCandidates = 0;
+                // Current max
+                long maxLastUsed = Long.MAX_VALUE;
+
+                // Collect the required number of tenants to evict by visiting each dynamic tenant
+                for (Map.Entry<String, TenantConfigContext> e : dynamicTenantsConfig.entrySet()) {
+                    TenantConfigContext c = e.getValue();
+                    long lastUsed = c.lastUsed;
+                    if (lastUsed >= maxLastUsed) {
+                        // Tenant is too young, skip
+                        continue;
+                    }
+
+                    // Found a candidate with a lastUsed less than the current oldest
+                    EvictionCandidate evictionCandidate = new EvictionCandidate(e.getKey(), c, lastUsed);
+                    if (numCandidates < toEvict) {
+                        // Collect until we hit the number of tenants to evict
+                        candidates[numCandidates++] = evictionCandidate;
+                        if (numCandidates == toEvict) {
+                            // Calculate the new max lastUsed from the list of eviction candidates
+                            maxLastUsed = evictionCandidatesMaxLastUsed(candidates);
+                        }
+                    } else {
+                        // Replace the current newest eviction candidate with the current candidate
+                        for (int i = 0; i < numCandidates; i++) {
+                            if (candidates[i].lastUsed == maxLastUsed) {
+                                candidates[i] = evictionCandidate;
+                                break;
+                            }
+                        }
+                        // Recalculate the max lastUsed
+                        maxLastUsed = evictionCandidatesMaxLastUsed(candidates);
+                    }
+                }
+
+                // Evict the tenants that haven't been used since eviction started
+                for (EvictionCandidate candidate : candidates) {
+                    // Only evict the tenant, if it hasn't been used since eviction started
+                    evictTenant(candidate);
+                }
+
+                toEvict = dynamicTenantsConfig.size() - limit;
+            } while (toEvict > 0);
+        } finally {
+            tenantEvictionRunning.set(false);
+        }
+    }
+
+    // VisibleForTesting
+    boolean evictTenant(EvictionCandidate candidate) {
+        if (candidate != null && candidate.lastUsed == candidate.context.lastUsed) {
+            dynamicTenantsConfig.remove(candidate.tenantId);
+            destroyContext(candidate.context);
+            return true;
+        }
+        return false;
+    }
+
+    private static long evictionCandidatesMaxLastUsed(EvictionCandidate[] candidates) {
+        long max = 0L;
+        for (EvictionCandidate candidate : candidates) {
+            max = Math.max(max, candidate.lastUsed);
+        }
+        return max;
+    }
 }

extensions/oidc/runtime/src/main/java/io/quarkus/oidc/runtime/TenantConfigContext.java

@@ -51,17 +51,23 @@ public class TenantConfigContext {
      */
     private final SecretKey internalIdTokenGeneratedKey;
 
-    final boolean ready;
+    volatile long lastUsed;
 
-    public TenantConfigContext(OidcProvider client, OidcTenantConfig config) {
-        this(client, config, true);
+    public static TenantConfigContext notReadyContext(OidcTenantConfig config) {
+        return new TenantConfigContext(null, config,
+                getRedirectFiltersMap(TenantFeatureFinder.find(config, OidcRedirectFilter.class)));
     }
 
-    public TenantConfigContext(OidcProvider provider, OidcTenantConfig config, boolean ready) {
+    public static TenantConfigContext readyContext(OidcProvider client, OidcTenantConfig config) {
+        return new TenantConfigContext(client, config,
+                getRedirectFiltersMap(TenantFeatureFinder.find(config, OidcRedirectFilter.class)));
+    }
+
+    TenantConfigContext(OidcProvider provider, OidcTenantConfig config,
+            Map<Redirect.Location, List<OidcRedirectFilter>> redirectFilters) {
         this.provider = provider;
         this.oidcConfig = config;
-        this.redirectFilters = getRedirectFiltersMap(TenantFeatureFinder.find(config, OidcRedirectFilter.class));
-        this.ready = ready;
+        this.redirectFilters = redirectFilters;
 
         boolean isService = OidcUtils.isServiceApp(config);
         stateSecretKey = !isService && provider != null && provider.client != null ? createStateSecretKey(config) : null;
@@ -175,6 +181,10 @@ private static SecretKey generateIdTokenSecretKey(OidcTenantConfig config, OidcP
         }
     }
 
+    public boolean isReady() {
+        return provider != null;
+    }
+
     public OidcTenantConfig getOidcTenantConfig() {
         return oidcConfig;
     }