KEYCLOAK-5656 Use standard infinispan remote-store

filipeferraz · Oct 16, 2017 · 056ba75 · 056ba75
1 parent b6ab285
commit 056ba75
Show file tree

Hide file tree

Showing 23 changed files with 213 additions and 475 deletions.
diff --git a/misc/CrossDataCenter.md b/misc/CrossDataCenter.md
@@ -116,64 +116,62 @@ Keycloak servers setup
  <cache-container name="keycloak" jndi-name="infinispan/Keycloak" module="org.keycloak.keycloak-model-infinispan">
 ``` 
 
-3.3) Add the `store` under `work` cache:
+3.3) Add the `remote-store` under `work` cache:
 
 ```xml
 <replicated-cache name="work" mode="SYNC">
-    <store class="org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">   
-        <property name="rawValues">true</property>	
+    <remote-store cache="work" remote-servers="remote-cache" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">
+        <property name="rawValues">true</property>
         <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
-        <property name="remoteCacheName">work</property> 
-        <property name="sessionCache">false</property>
-    </store>
+    </remote-store>
 </replicated-cache>
 ```
 
-3.5) Add the `store` like this under `sessions` cache:
+3.5) Add the `remote-store` like this under `sessions` cache:
 
 ```xml
 <distributed-cache name="sessions" mode="SYNC" owners="1">
-    <store class="org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">   
-        <property name="remoteCacheName">sessions</property> 
-        <property name="useConfigTemplateFromCache">work</property>
-        <property name="sessionCache">true</property>
-    </store>
+    <remote-store cache="sessions" remote-servers="remote-cache" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">   
+        <property name="rawValues">true</property>
+        <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
+    </remote-store>
 </distributed-cache>
 ```
 
-3.6) Same for `offlineSessions` and `loginFailures` caches (The only difference from `sessions` cache is, that `remoteCacheName` property value are different:
+3.6) Same for `offlineSessions`, `loginFailures`, and `actionTokens` caches (the only difference from `sessions` cache is that `cache` property value are different):
 
 ```xml
 <distributed-cache name="offlineSessions" mode="SYNC" owners="1">
-    <store class="org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">   
-        <property name="remoteCacheName">offlineSessions</property> 
-        <property name="useConfigTemplateFromCache">work</property>
-        <property name="sessionCache">true</property>
-    </store>
+    <remote-store cache="offlineSessions" remote-servers="remote-cache" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">
+        <property name="rawValues">true</property>
+        <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
+    </remote-store>
 </distributed-cache>
 
 <distributed-cache name="loginFailures" mode="SYNC" owners="1">
-    <store class="org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">   
-        <property name="remoteCacheName">loginFailures</property> 
-        <property name="useConfigTemplateFromCache">work</property>
-        <property name="sessionCache">true</property>
-    </store>
+    <remote-store cache="loginFailures" remote-servers="remote-cache" passivation="false" fetch-state="false" purge="false" preload="false" shared="true">
+        <property name="rawValues">true</property>
+        <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
+    </remote-store>
 </distributed-cache>
-```
-
-3.7) The configuration of `actionTokens` cache have different `remoteCacheName`, `sessionCache` and the `preload` attribute:
 
-```xml
 <distributed-cache name="actionTokens" mode="SYNC" owners="2">
     <eviction max-entries="-1" strategy="NONE"/>
     <expiration max-idle="-1" interval="300000"/>
-    <store class="org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder" passivation="false" fetch-state="false" purge="false" preload="true" shared="true">   
-        <property name="remoteCacheName">actionTokens</property> 
-        <property name="useConfigTemplateFromCache">work</property>
-        <property name="sessionCache">false</property>
-    </store>
+    <remote-store cache="actionTokens" remote-servers="remote-cache" passivation="false" fetch-state="false" purge="false" preload="true" shared="true">
+        <property name="rawValues">true</property>
+        <property name="marshaller">org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory</property>
+    </remote-store>
 </distributed-cache>
-```            
+```
+
+3.7) Add outbound socket binding for the remote store into `socket-binding-group` configuration:
+
+```xml
+<outbound-socket-binding name="remote-cache">
+    <remote-destination host="${remote.cache.host:localhost}" port="${remote.cache.port:11222}"/>
+</outbound-socket-binding>
+```
 
 3.8) The configuration of distributed cache `authenticationSessions` and other caches is left unchanged.
 

diff --git a/...rc/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProvider.java b/...rc/main/java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProvider.java
@@ -25,7 +25,7 @@
  */
 public class DefaultInfinispanConnectionProvider implements InfinispanConnectionProvider {
 
-    private EmbeddedCacheManager cacheManager;
+    private final EmbeddedCacheManager cacheManager;
     private final String siteName;
     private final String nodeName;
 

diff --git a/.../java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java b/.../java/org/keycloak/connections/infinispan/DefaultInfinispanConnectionProviderFactory.java
@@ -41,9 +41,9 @@
 import org.keycloak.cluster.infinispan.KeycloakHotRodMarshallerFactory;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.KeycloakSessionFactory;
-import org.keycloak.models.sessions.infinispan.remotestore.KeycloakRemoteStoreConfigurationBuilder;
 
 import javax.naming.InitialContext;
+import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder;
 
 /**
  * @author <a href="mailto:sthorger@redhat.com">Stian Thorgersen</a>
@@ -157,7 +157,7 @@ protected void initContainerManaged(String cacheContainerLookup) {
                 this.nodeName = generateNodeName();
             }
 
-            logger.debugv("Using container managed Infinispan cache container, lookup={1}", cacheContainerLookup);
+            logger.debugv("Using container managed Infinispan cache container, lookup={0}", cacheContainerLookup);
         } catch (Exception e) {
             throw new RuntimeException("Failed to retrieve cache container", e);
         }
@@ -354,8 +354,7 @@ private void configureRemoteCacheStore(ConfigurationBuilder builder, boolean asy
 
         builder.persistence()
                 .passivation(false)
-                .addStore(KeycloakRemoteStoreConfigurationBuilder.class)
-                    .sessionCache(sessionCache)
+                .addStore(RemoteStoreConfigurationBuilder.class)
                     .fetchPersistentState(false)
                     .ignoreModifications(false)
                     .purgeOnStartup(false)
@@ -382,8 +381,7 @@ private void configureRemoteActionTokenCacheStore(ConfigurationBuilder builder,
 
         builder.persistence()
                 .passivation(false)
-                .addStore(KeycloakRemoteStoreConfigurationBuilder.class)
-                    .sessionCache(false)
+                .addStore(RemoteStoreConfigurationBuilder.class)
                     .fetchPersistentState(false)
                     .ignoreModifications(false)
                     .purgeOnStartup(false)

diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/CacheDecorators.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/CacheDecorators.java
@@ -26,13 +26,31 @@
  */
 public class CacheDecorators {
 
+    /**
+     * Adds {@link Flag#CACHE_MODE_LOCAL} flag to the cache.
+     * @param cache
+     * @return Cache with the flag applied.
+     */
     public static <K, V> AdvancedCache<K, V> localCache(Cache<K, V> cache) {
         return cache.getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL);
     }
 
+    /**
+     * Adds {@link Flag#SKIP_CACHE_LOAD} and {@link Flag#SKIP_CACHE_STORE} flags to the cache.
+     * @param cache
+     * @return Cache with the flags applied.
+     */
     public static <K, V> AdvancedCache<K, V> skipCacheLoaders(Cache<K, V> cache) {
         return cache.getAdvancedCache().withFlags(Flag.SKIP_CACHE_LOAD, Flag.SKIP_CACHE_STORE);
     }
 
+    /**
+     * Adds {@link Flag#SKIP_CACHE_STORE} flag to the cache.
+     * @param cache
+     * @return Cache with the flags applied.
+     */
+    public static <K, V> AdvancedCache<K, V> skipCacheStore(Cache<K, V> cache) {
+        return cache.getAdvancedCache().withFlags(Flag.SKIP_CACHE_STORE);
+    }
 
 }
diff --git a/.../src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java b/.../src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProvider.java
@@ -303,8 +303,9 @@ public UserSessionModel getUserSessionWithPredicate(RealmModel realm, String id,
         RemoteCache remoteCache = InfinispanUtil.getRemoteCache(cache);
 
         if (remoteCache != null) {
-            UserSessionEntity remoteSessionEntity = (UserSessionEntity) remoteCache.get(id);
-            if (remoteSessionEntity != null) {
+            SessionEntityWrapper<UserSessionEntity> remoteSessionEntityWrapper = (SessionEntityWrapper<UserSessionEntity>) remoteCache.get(id);
+            if (remoteSessionEntityWrapper != null) {
+                UserSessionEntity remoteSessionEntity = remoteSessionEntityWrapper.getEntity();
                 log.debugf("getUserSessionWithPredicate(%s): remote cache contains session entity %s", id, remoteSessionEntity);
 
                 UserSessionModel remoteSessionAdapter = wrap(realm, remoteSessionEntity, offline);
@@ -399,7 +400,7 @@ private void removeExpiredUserSessions(RealmModel realm) {
 
         FuturesHelper futures = new FuturesHelper();
 
-        // Each cluster node cleanups just local sessions, which are those owned by himself (+ few more taking l1 cache into account)
+        // Each cluster node cleanups just local sessions, which are those owned by itself (+ few more taking l1 cache into account)
         Cache<String, SessionEntityWrapper<UserSessionEntity>> localCache = CacheDecorators.localCache(sessionCache);
 
         Cache<String, SessionEntityWrapper<UserSessionEntity>> localCacheStoreIgnore = CacheDecorators.skipCacheLoaders(localCache);

diff --git a/...in/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java b/...in/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java
@@ -39,6 +39,7 @@
 import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper;
 import org.keycloak.models.sessions.infinispan.entities.LoginFailureEntity;
 import org.keycloak.models.sessions.infinispan.entities.LoginFailureKey;
+import org.keycloak.models.sessions.infinispan.entities.SessionEntity;
 import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity;
 import org.keycloak.models.sessions.infinispan.events.AbstractUserSessionClusterListener;
 import org.keycloak.models.sessions.infinispan.events.ClientRemovedSessionEvent;
@@ -204,7 +205,7 @@ protected void checkRemoteCaches(KeycloakSession session) {
 
         InfinispanConnectionProvider ispn = session.getProvider(InfinispanConnectionProvider.class);
 
-        Cache sessionsCache = ispn.getCache(InfinispanConnectionProvider.SESSION_CACHE_NAME);
+        Cache<String, SessionEntityWrapper<UserSessionEntity>> sessionsCache = ispn.getCache(InfinispanConnectionProvider.SESSION_CACHE_NAME);
         boolean sessionsRemoteCache = checkRemoteCache(session, sessionsCache, (RealmModel realm) -> {
             return realm.getSsoSessionIdleTimeout() * 1000;
         });
@@ -214,7 +215,7 @@ protected void checkRemoteCaches(KeycloakSession session) {
         }
 
 
-        Cache offlineSessionsCache = ispn.getCache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME);
+        Cache<String, SessionEntityWrapper<UserSessionEntity>> offlineSessionsCache = ispn.getCache(InfinispanConnectionProvider.OFFLINE_SESSION_CACHE_NAME);
         boolean offlineSessionsRemoteCache = checkRemoteCache(session, offlineSessionsCache, (RealmModel realm) -> {
             return realm.getOfflineSessionIdleTimeout() * 1000;
         });
@@ -223,13 +224,13 @@ protected void checkRemoteCaches(KeycloakSession session) {
             offlineLastSessionRefreshStore = new LastSessionRefreshStoreFactory().createAndInit(session, offlineSessionsCache, true);
         }
 
-        Cache loginFailuresCache = ispn.getCache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME);
+        Cache<LoginFailureKey, SessionEntityWrapper<LoginFailureEntity>> loginFailuresCache = ispn.getCache(InfinispanConnectionProvider.LOGIN_FAILURE_CACHE_NAME);
         boolean loginFailuresRemoteCache = checkRemoteCache(session, loginFailuresCache, (RealmModel realm) -> {
             return realm.getMaxDeltaTimeSeconds() * 1000;
         });
     }
 
-    private boolean checkRemoteCache(KeycloakSession session, Cache ispnCache, RemoteCacheInvoker.MaxIdleTimeLoader maxIdleLoader) {
+    private <K, V extends SessionEntity> boolean checkRemoteCache(KeycloakSession session, Cache<K, SessionEntityWrapper<V>> ispnCache, RemoteCacheInvoker.MaxIdleTimeLoader maxIdleLoader) {
         Set<RemoteStore> remoteStores = InfinispanUtil.getRemoteStores(ispnCache);
 
         if (remoteStores.isEmpty()) {
@@ -238,7 +239,7 @@ private boolean checkRemoteCache(KeycloakSession session, Cache ispnCache, Remot
         } else {
             log.infof("Remote store configured for cache '%s'", ispnCache.getName());
 
-            RemoteCache remoteCache = remoteStores.iterator().next().getRemoteCache();
+            RemoteCache<K, SessionEntityWrapper<V>> remoteCache = (RemoteCache) remoteStores.iterator().next().getRemoteCache();
 
             remoteCacheInvoker.addRemoteCache(ispnCache.getName(), remoteCache, maxIdleLoader);
 

diff --git a/.../org/keycloak/models/sessions/infinispan/changes/InfinispanChangelogBasedTransaction.java b/.../org/keycloak/models/sessions/infinispan/changes/InfinispanChangelogBasedTransaction.java
@@ -27,6 +27,7 @@
 import org.keycloak.models.AbstractKeycloakTransaction;
 import org.keycloak.models.KeycloakSession;
 import org.keycloak.models.RealmModel;
+import org.keycloak.models.sessions.infinispan.CacheDecorators;
 import org.keycloak.models.sessions.infinispan.entities.SessionEntity;
 import org.keycloak.models.sessions.infinispan.remotestore.RemoteCacheInvoker;
 
@@ -172,17 +173,17 @@ private void runOperationInCluster(K key, MergedUpdate<V> task,  SessionEntityWr
         switch (operation) {
             case REMOVE:
                 // Just remove it
-                cache
+                CacheDecorators.skipCacheStore(cache)
                         .getAdvancedCache().withFlags(Flag.IGNORE_RETURN_VALUES)
                         .remove(key);
                 break;
             case ADD:
-                cache
+                CacheDecorators.skipCacheStore(cache)
                         .getAdvancedCache().withFlags(Flag.IGNORE_RETURN_VALUES)
                         .put(key, sessionWrapper, task.getLifespanMs(), TimeUnit.MILLISECONDS);
                 break;
             case ADD_IF_ABSENT:
-                SessionEntityWrapper<V> existing = cache.putIfAbsent(key, sessionWrapper);
+                SessionEntityWrapper<V> existing = CacheDecorators.skipCacheStore(cache).putIfAbsent(key, sessionWrapper);
                 if (existing != null) {
                     logger.debugf("Existing entity in cache for key: %s . Will update it", key);
 
@@ -210,7 +211,7 @@ private void replace(K key, MergedUpdate<V> task, SessionEntityWrapper<V> oldVer
             SessionEntityWrapper<V> newVersionEntity = generateNewVersionAndWrapEntity(session, oldVersionEntity.getLocalMetadata());
 
             // Atomic cluster-aware replace
-            replaced = cache.replace(key, oldVersionEntity, newVersionEntity);
+            replaced = CacheDecorators.skipCacheStore(cache).replace(key, oldVersionEntity, newVersionEntity);
 
             // Replace fail. Need to load latest entity from cache, apply updates again and try to replace in cache again
             if (!replaced) {