diff --git a/misc/CrossDataCenter.md b/misc/CrossDataCenter.md index a57a7b9b44f0..5d9812d2ffba 100644 --- a/misc/CrossDataCenter.md +++ b/misc/CrossDataCenter.md @@ -98,7 +98,9 @@ Infinispan Server setup - + + + diff --git a/misc/Testsuite.md b/misc/Testsuite.md index f7b4e8e38388..142326a95cc6 100644 --- a/misc/Testsuite.md +++ b/misc/Testsuite.md @@ -117,7 +117,7 @@ But additionally you can enable Kerberos authentication in LDAP provider with th * KeyTab: $KEYCLOAK_SOURCES/testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/http.keytab (Replace $KEYCLOAK_SOURCES with correct absolute path of your sources) Once you do this, you should also ensure that your Kerberos client configuration file is properly configured with KEYCLOAK.ORG domain. -See [../testsuite/integration-arquillian/src/test/resources/kerberos/test-krb5.conf](../testsuite/integration-arquillian/src/test/resources/kerberos/test-krb5.conf) for inspiration. The location of Kerberos configuration file +See [../testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/test-krb5.conf](../testsuite/integration-arquillian/tests/base/src/test/resources/kerberos/test-krb5.conf) for inspiration. The location of Kerberos configuration file is platform dependent (In linux it's file `/etc/krb5.conf` ) Then you need to configure your browser to allow SPNEGO/Kerberos login from `localhost` . diff --git a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java index 89140e38b7ff..c9022fc08484 100644 --- a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java +++ b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProvider.java @@ -22,6 +22,7 @@ import org.keycloak.cluster.ClusterListener; import org.keycloak.cluster.ClusterProvider; import org.keycloak.cluster.ExecutionResult; +import org.keycloak.common.util.Retry; import org.keycloak.common.util.Time; import java.util.concurrent.Callable; @@ -140,7 +141,7 @@ private LockEntry createLockEntry() { private boolean tryLock(String cacheKey, int taskTimeoutInSeconds) { LockEntry myLock = createLockEntry(); - LockEntry existingLock = (LockEntry) crossDCAwareCacheFactory.getCache().putIfAbsent(cacheKey, myLock, taskTimeoutInSeconds, TimeUnit.SECONDS); + LockEntry existingLock = InfinispanClusterProviderFactory.putIfAbsentWithRetries(crossDCAwareCacheFactory, cacheKey, myLock, taskTimeoutInSeconds); if (existingLock != null) { if (logger.isTraceEnabled()) { logger.tracef("Task %s in progress already by node %s. Ignoring task.", cacheKey, existingLock.getNode()); @@ -156,22 +157,15 @@ private boolean tryLock(String cacheKey, int taskTimeoutInSeconds) { private void removeFromCache(String cacheKey) { - // 3 attempts to send the message (it may fail if some node fails in the meantime) - int retry = 3; - while (true) { - try { - crossDCAwareCacheFactory.getCache().remove(cacheKey); - if (logger.isTraceEnabled()) { - logger.tracef("Task %s removed from the cache", cacheKey); - } - return; - } catch (RuntimeException e) { - retry--; - if (retry == 0) { - throw e; - } + // More attempts to send the message (it may fail if some node fails in the meantime) + Retry.executeWithBackoff((int iteration) -> { + + crossDCAwareCacheFactory.getCache().remove(cacheKey); + if (logger.isTraceEnabled()) { + logger.tracef("Task %s removed from the cache", cacheKey); } - } + + }, 10, 10); } } diff --git a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java index 54cfd042df84..4334c299150a 100644 --- a/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/cluster/infinispan/InfinispanClusterProviderFactory.java @@ -18,6 +18,7 @@ package org.keycloak.cluster.infinispan; import org.infinispan.Cache; +import org.infinispan.client.hotrod.exceptions.HotRodClientException; import org.infinispan.manager.EmbeddedCacheManager; import org.infinispan.notifications.Listener; import org.infinispan.notifications.cachemanagerlistener.annotation.ViewChanged; @@ -29,6 +30,7 @@ import org.keycloak.Config; import org.keycloak.cluster.ClusterProvider; import org.keycloak.cluster.ClusterProviderFactory; +import org.keycloak.common.util.Retry; import org.keycloak.common.util.Time; import org.keycloak.connections.infinispan.InfinispanConnectionProvider; import org.keycloak.models.KeycloakSession; @@ -42,6 +44,8 @@ import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; import java.util.function.Function; import java.util.function.Predicate; import java.util.stream.Collectors; @@ -111,7 +115,7 @@ protected int initClusterStartupTime(KeycloakSession session) { // clusterStartTime not yet initialized. Let's try to put our startupTime int serverStartTime = (int) (session.getKeycloakSessionFactory().getServerStartupTimestamp() / 1000); - existingClusterStartTime = (Integer) crossDCAwareCacheFactory.getCache().putIfAbsent(InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime); + existingClusterStartTime = putIfAbsentWithRetries(crossDCAwareCacheFactory, InfinispanClusterProvider.CLUSTER_STARTUP_TIME_KEY, serverStartTime, -1); if (existingClusterStartTime == null) { logger.debugf("Initialized cluster startup time to %s", Time.toDate(serverStartTime).toString()); return serverStartTime; @@ -123,6 +127,35 @@ protected int initClusterStartupTime(KeycloakSession session) { } + // Will retry few times for the case when backup site not available in cross-dc environment. + // The site might be taken offline automatically if "take-offline" properly configured + static V putIfAbsentWithRetries(CrossDCAwareCacheFactory crossDCAwareCacheFactory, String key, V value, int taskTimeoutInSeconds) { + AtomicReference resultRef = new AtomicReference<>(); + + Retry.executeWithBackoff((int iteration) -> { + + try { + V result; + if (taskTimeoutInSeconds > 0) { + result = (V) crossDCAwareCacheFactory.getCache().putIfAbsent(key, value); + } else { + result = (V) crossDCAwareCacheFactory.getCache().putIfAbsent(key, value, taskTimeoutInSeconds, TimeUnit.SECONDS); + } + resultRef.set(result); + + } catch (HotRodClientException re) { + logger.warnf(re, "Failed to write key '%s' and value '%s' in iteration '%d' . Retrying", key, value, iteration); + + // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation. + throw re; + } + + }, 10, 10); + + return resultRef.get(); + } + + @Override public void init(Config.Scope config) { diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java index 76a010eb3843..22b7382383f8 100755 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/InfinispanUserSessionProviderFactory.java @@ -276,7 +276,7 @@ private void loadSessionsFromRemoteCaches(KeycloakSession session) { private void loadSessionsFromRemoteCache(final KeycloakSessionFactory sessionFactory, String cacheName, final int sessionsPerSegment, final int maxErrors) { - log.debugf("Check pre-loading userSessions from remote cache '%s'", cacheName); + log.debugf("Check pre-loading sessions from remote cache '%s'", cacheName); KeycloakModelUtils.runJobInTransaction(sessionFactory, new KeycloakSessionTask() { @@ -293,7 +293,7 @@ public void run(KeycloakSession session) { }); - log.debugf("Pre-loading userSessions from remote cache '%s' finished", cacheName); + log.debugf("Pre-loading sessions from remote cache '%s' finished", cacheName); } diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java index 5379e4068b3f..7b60dcb39d5b 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/initializer/OfflinePersistentUserSessionLoader.java @@ -18,15 +18,18 @@ package org.keycloak.models.sessions.infinispan.initializer; import org.infinispan.Cache; +import org.infinispan.client.hotrod.exceptions.HotRodClientException; import org.infinispan.context.Flag; import org.jboss.logging.Logger; import org.keycloak.cluster.ClusterProvider; +import org.keycloak.common.util.Retry; import org.keycloak.models.KeycloakSession; import org.keycloak.models.UserSessionModel; import org.keycloak.models.session.UserSessionPersisterProvider; import java.io.Serializable; import java.util.List; +import java.util.concurrent.TimeUnit; /** * @author Marek Posolda @@ -101,10 +104,24 @@ public boolean isFinished(BaseCacheInitializer initializer) { public void afterAllSessionsLoaded(BaseCacheInitializer initializer) { Cache workCache = initializer.getWorkCache(); - // Cross-DC aware flag - workCache - .getAdvancedCache().withFlags(Flag.SKIP_REMOTE_LOOKUP) - .put(PERSISTENT_SESSIONS_LOADED, true); + // Will retry few times for the case when backup site not available in cross-dc environment. + // The site might be taken offline automatically if "take-offline" properly configured + Retry.executeWithBackoff((int iteration) -> { + + try { + // Cross-DC aware flag + workCache + .getAdvancedCache().withFlags(Flag.SKIP_REMOTE_LOOKUP) + .put(PERSISTENT_SESSIONS_LOADED, true); + + } catch (HotRodClientException re) { + log.warnf(re, "Failed to write flag PERSISTENT_SESSIONS_LOADED in iteration '%d' . Retrying", iteration); + + // Rethrow the exception. Retry will take care of handle the exception and eventually retry the operation. + throw re; + } + + }, 10, 10); // Just local-DC aware flag workCache diff --git a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java index 128a7e98a493..120156337a54 100644 --- a/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java +++ b/model/infinispan/src/main/java/org/keycloak/models/sessions/infinispan/remotestore/RemoteCacheSessionsLoader.java @@ -142,7 +142,8 @@ public boolean isFinished(BaseCacheInitializer initializer) { .getAdvancedCache().withFlags(Flag.SKIP_CACHE_LOAD, Flag.SKIP_CACHE_STORE) .get(OfflinePersistentUserSessionLoader.PERSISTENT_SESSIONS_LOADED_IN_CURRENT_DC); - if (cacheName.equals(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) && sessionsLoaded != null && sessionsLoaded) { + if ((cacheName.equals(InfinispanConnectionProvider.OFFLINE_USER_SESSION_CACHE_NAME) || (cacheName.equals(InfinispanConnectionProvider.OFFLINE_CLIENT_SESSION_CACHE_NAME))) + && sessionsLoaded != null && sessionsLoaded) { log.debugf("Sessions already loaded in current DC. Skip sessions loading from remote cache '%s'", cacheName); return true; } else { diff --git a/model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGOfflineBackupsTest.java b/model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGOfflineBackupsTest.java new file mode 100644 index 000000000000..19fe4f7b12ee --- /dev/null +++ b/model/infinispan/src/test/java/org/keycloak/cluster/infinispan/ConcurrencyJDGOfflineBackupsTest.java @@ -0,0 +1,114 @@ +/* + * Copyright 2017 Red Hat, Inc. and/or its affiliates + * and other contributors as indicated by the @author tags. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.keycloak.cluster.infinispan; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.infinispan.Cache; +import org.infinispan.client.hotrod.exceptions.HotRodClientException; +import org.infinispan.context.Flag; +import org.infinispan.manager.EmbeddedCacheManager; +import org.infinispan.persistence.remote.configuration.RemoteStoreConfigurationBuilder; +import org.jboss.logging.Logger; +import org.keycloak.common.util.Time; +import org.keycloak.connections.infinispan.InfinispanConnectionProvider; +import org.keycloak.models.sessions.infinispan.changes.SessionEntityWrapper; +import org.keycloak.models.sessions.infinispan.entities.UserSessionEntity; + +/** + * @author Marek Posolda + */ +public class ConcurrencyJDGOfflineBackupsTest { + + protected static final Logger logger = Logger.getLogger(ConcurrencyJDGOfflineBackupsTest.class); + + public static void main(String[] args) throws Exception { + + Cache> cache1 = createManager(1).getCache(InfinispanConnectionProvider.USER_SESSION_CACHE_NAME); + + try { + // Create initial item + UserSessionEntity session = new UserSessionEntity(); + session.setId("123"); + session.setRealmId("foo"); + session.setBrokerSessionId("!23123123"); + session.setBrokerUserId(null); + session.setUser("foo"); + session.setLoginUsername("foo"); + session.setIpAddress("123.44.143.178"); + session.setStarted(Time.currentTime()); + session.setLastSessionRefresh(Time.currentTime()); + +// AuthenticatedClientSessionEntity clientSession = new AuthenticatedClientSessionEntity(); +// clientSession.setAuthMethod("saml"); +// clientSession.setAction("something"); +// clientSession.setTimestamp(1234); +// clientSession.setProtocolMappers(new HashSet<>(Arrays.asList("mapper1", "mapper2"))); +// clientSession.setRoles(new HashSet<>(Arrays.asList("role1", "role2"))); +// session.getAuthenticatedClientSessions().put(CLIENT_1_UUID.toString(), clientSession.getId()); + + SessionEntityWrapper wrappedSession = new SessionEntityWrapper<>(session); + + // Some dummy testing of remoteStore behaviour + logger.info("Before put"); + + + AtomicInteger successCount = new AtomicInteger(0); + AtomicInteger errorsCount = new AtomicInteger(0); + for (int i=0 ; i<100 ; i++) { + try { + cache1 + .getAdvancedCache().withFlags(Flag.CACHE_MODE_LOCAL) // will still invoke remoteStore . Just doesn't propagate to cluster + .put("123", wrappedSession); + successCount.incrementAndGet(); + Thread.sleep(1000); + logger.infof("Success in the iteration: %d", i); + } catch (HotRodClientException hrce) { + logger.errorf("Failed to put the item in the iteration: %d ", i); + errorsCount.incrementAndGet(); + } + } + + logger.infof("SuccessCount: %d, ErrorsCount: %d", successCount.get(), errorsCount.get()); + +// logger.info("After put"); +// +// cache1.replace("123", wrappedSession); +// +// logger.info("After replace"); +// +// cache1.get("123"); +// +// logger.info("After cache1.get"); + +// cache2.get("123"); +// +// logger.info("After cache2.get"); + + } finally { + // Finish JVM + cache1.getCacheManager().stop(); + } + + } + + private static EmbeddedCacheManager createManager(int threadId) { + return new TestCacheManagerFactory().createManager(threadId, InfinispanConnectionProvider.USER_SESSION_CACHE_NAME, RemoteStoreConfigurationBuilder.class); + } + +} diff --git a/testsuite/integration-arquillian/HOW-TO-RUN.md b/testsuite/integration-arquillian/HOW-TO-RUN.md index 81815619eef0..23191b26f431 100644 --- a/testsuite/integration-arquillian/HOW-TO-RUN.md +++ b/testsuite/integration-arquillian/HOW-TO-RUN.md @@ -458,114 +458,115 @@ The cross DC requires setting a profile specifying used cache server by specifyi #### Run Cross-DC Tests from Maven -a) First compile the Infinispan/JDG test server via the following command: +a) Prepare the environment. Compile the infinispan server and eventually Keycloak on JBoss server. + +a1) If you want to use **Undertow** based Keycloak container, you just need to download and prepare the +Infinispan/JDG test server via the following command: `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -f testsuite/integration-arquillian -DskipTests clean install` -or +*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'* - `mvn -Pcache-server-jdg,auth-servers-crossdc-undertow -f testsuite/integration-arquillian -DskipTests clean install` +a2) If you want to use **JBoss-based** Keycloak backend containers instead of containers on Embedded Undertow, + you need to prepare both the Infinispan/JDG test server and the Keycloak server on Wildfly/EAP. Run following command: -b) Then in case you want to use **JBoss-based** Keycloak backend containers instead of containers on Embedded Undertow run following command: + `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -f testsuite/integration-arquillian -DskipTests clean install` - `mvn -Pauth-servers-crossdc-jboss,auth-server-wildfly -f testsuite/integration-arquillian -DskipTests clean install` +*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'* *note: 'auth-server-wildfly' can be replaced by 'auth-server-eap'* -By default JBoss-based containers use in-memory h2 database. It can be configured to use real DB, e.g. with following command: +By default JBoss-based containers use TCP-based h2 database. It can be configured to use real DB, e.g. with following command: - `mvn -Pauth-servers-crossdc-jboss,auth-server-wildfly,jpa -f testsuite/integration-arquillian -DskipTests clean install -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak` + `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,jpa -f testsuite/integration-arquillian -DskipTests clean install -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak` -c1) Then you can run the tests using the following command (adjust the test specification according to your needs) for Keycloak backend containers on **Undertow**: +b1) For **Undertow** Keycloak backend containers, you can run the tests using the following command (adjust the test specification according to your needs): `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install` -or +*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'* - `mvn -Pcache-server-jdg,auth-servers-crossdc-undertow -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install` +*note: It can be useful to add additional system property to enable logging:* + + `-Dkeycloak.infinispan.logging.level=debug` -c2) For **JBoss-based** Keycloak backend containers: +b2) For **JBoss-based** Keycloak backend containers, you can run the tests like this: `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install` -or - - `mvn -Pcache-server-jdg,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.* -pl testsuite/integration-arquillian/tests/base clean install` +*note: 'cache-server-infinispan' can be replaced by 'cache-server-jdg'* *note: 'auth-server-wildfly can be replaced by auth-server-eap'* -**note** -Previous commands can be "squashed" into one. E.g.: +**note**: +For **JBoss-based** Keycloak backend containers on real DB, the previous commands from (a2) and (b2) can be "squashed" into one. E.g.: `mvn -f testsuite/integration-arquillian clean install -Dtest=*.crossdc.* -Djdbc.mvn.groupId=org.mariadb.jdbc -Djdbc.mvn.artifactId=mariadb-java-client -Djdbc.mvn.version=2.0.3 -Dkeycloak.connectionsJpa.url=jdbc:mariadb://localhost:3306/keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsJpa.user=keycloak -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly,jpa clean install` -It can be useful to add additional system property to enable logging: - - -Dkeycloak.infinispan.logging.level=debug -**Tests from package "manual"** uses manual lifecycle for all servers, so needs to be executed manually. Also needs to be executed with real DB like MySQL. You can run them with: +#### Run "Manual" Cross-DC Tests from Maven + +Tests from package "manual" uses manual lifecycle for all servers, so needs to be executed manually. + +First prepare the environment and do the step (a) from previous paragraph. + +c1) For **Undertow** Keycloak backend containers, you can run the test using following command: + + `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.manual.* -Dmanual.mode=true -Drun.h2=true -Dkeycloak.connectionsJpa.url.crossdc="jdbc:h2:tcp://localhost:9092/mem:keycloak-dc-shared;DB_CLOSE_DELAY=-1" -pl testsuite/integration-arquillian/tests/base clean install` + +*note: As you can see, there is a need to run TCP-Based H2 for this test. In-memory H2 won't work due the data need +to persist the stop of all the Keycloak servers.* + +If you want to test with real DB like MySQL, you can run them with: - mvn -Pcache-server-infinispan -Dtest=*.crossdc.manual.* -Dmanual.mode=true \ - -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver \ - -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak \ - -pl testsuite/integration-arquillian/tests/base test + `mvn -Pcache-server-infinispan,auth-servers-crossdc-undertow -Dtest=*.crossdc.manual.* -Dmanual.mode=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -pl testsuite/integration-arquillian/tests/base clean install` +c2) For **JBoss-based** Keycloak backend containers, you can run the tests like this: + `mvn -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.manual.* -Dmanual.mode=true -pl testsuite/integration-arquillian/tests/base clean install` +*note: TCP-based H2 is used by default when running cross-dc tests on JBoss-based Keycloak container. +So no need to explicitly specify it like in (c1) for undertow.* #### Run Cross-DC Tests from Intellij IDEA -First we will manually download, configure and run infinispan server. Then we can run the tests from IDE against 1 server. It's more effective during -development as there is no need to restart infinispan server(s) among test runs. +First we will manually download, configure and run infinispan servers. Then we can run the tests from IDE against the servers. +It's more effective during development as there is no need to restart infinispan server(s) among test runs. -1) Download infinispan server 8.2.X from http://infinispan.org/download/ +1) Download infinispan server 8.2.X from http://infinispan.org/download/ and go through the steps +from the [../../misc/CrossDataCenter.md](../../misc/CrossDataCenter.md) and the `Infinispan Server Setup` part. -2) Edit `ISPN_SERVER_HOME/standalone/configuration/standalone.xml` and add these local-caches to the section under cache-container `local` : +Assume you have both Infinispan/JDG servers up and running. - - - - - - - - - - - - -3) Run the server through `./standalone.sh` - -4) Setup MySQL database or some other shared database. +**TODO:** Change this once CrossDataCenter.md is removed and converted to the proper docs. + +2) Setup MySQL database or some other shared database. -5) Ensure that org.wildfly.arquillian:wildfly-arquillian-container-managed is on the classpath when running test. On Intellij, it can be -done by going to: View -> Tool Windows -> Maven projects. Then check profile "cache-server-infinispan". The tests will use this profile when executed. +3) Ensure that `org.wildfly.arquillian:wildfly-arquillian-container-managed` is on the classpath when running test. On Intellij, it can be +done by going to: `View` -> `Tool Windows` -> `Maven projects`. Then check profile `cache-server-infinispan` and `auth-servers-crossdc-undertow`. +The tests will use this profile when executed. -6) Run the LoginCrossDCTest (or any other test) with those properties. In shortcut, it's using MySQL database, disabled L1 lifespan and +4) Run the LoginCrossDCTest (or any other test) with those properties. In shortcut, it's using MySQL database and connects to the remoteStore provided by infinispan server configured in previous steps: - -Dauth.server.crossdc=true -Dauth.server.undertow.crossdc=true -Dcache.server.lifecycle.skip=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak - -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak - -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsInfinispan.l1Lifespan=0 - -Dkeycloak.connectionsInfinispan.remoteStorePort=11222 -Dkeycloak.connectionsInfinispan.remoteStorePort.2=11222 -Dkeycloak.connectionsInfinispan.sessionsOwners=1 - -Dsession.cache.owners=1 -Dkeycloak.infinispan.logging.level=debug -Dresources + `-Dauth.server.crossdc=true -Dauth.server.undertow.crossdc=true -Dcache.server.lifecycle.skip=true -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsJpa.url.crossdc=jdbc:mysql://localhost/keycloak -Dkeycloak.connectionsJpa.driver.crossdc=com.mysql.jdbc.Driver -Dkeycloak.connectionsJpa.user=keycloak -Dkeycloak.connectionsJpa.password=keycloak -Dkeycloak.connectionsInfinispan.clustered=true -Dkeycloak.connectionsInfinispan.remoteStorePort=12232 -Dkeycloak.connectionsInfinispan.remoteStorePort.2=13232 -Dkeycloak.connectionsInfinispan.sessionsOwners=1 -Dsession.cache.owners=1 -Dkeycloak.infinispan.logging.level=debug -Dresources` -NOTE: Tests from package "manual" (eg. SessionsPreloadCrossDCTest) needs to be executed with managed containers. +**NOTE**: Tests from package `manual` (eg. SessionsPreloadCrossDCTest) needs to be executed with managed containers. So skip steps 1,2 and add property `-Dmanual.mode=true` and change "cache.server.lifecycle.skip" to false `-Dcache.server.lifecycle.skip=false` or remove it. -7) If you want to debug and test manually, the servers are running on these ports (Note that not all backend servers are running by default and some might be also unused by loadbalancer): +5) If you want to debug or test manually, the servers are running on these ports (Note that not all backend servers are running by default and some might be also unused by loadbalancer): + +* *Loadbalancer* -> "http://localhost:8180/auth" + +* *auth-server-undertow-cross-dc-0_1* -> "http://localhost:8101/auth" + +* *auth-server-undertow-cross-dc-0_2-manual* -> "http://localhost:8102/auth" + +* *auth-server-undertow-cross-dc-1_1* -> "http://localhost:8111/auth" - Loadbalancer -> "http://localhost:8180/auth" - auth-server-undertow-cross-dc-0_1 -> "http://localhost:8101/auth" - auth-server-undertow-cross-dc-0_2-manual -> "http://localhost:8102/auth" - auth-server-undertow-cross-dc-1_1 -> "http://localhost:8111/auth" - auth-server-undertow-cross-dc-1_2-manual -> "http://localhost:8112/auth" +* *auth-server-undertow-cross-dc-1_2-manual* -> "http://localhost:8112/auth" ## Run Docker Authentication test diff --git a/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl b/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl index e6c16da2e149..d3881a5c07f4 100644 --- a/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl +++ b/testsuite/integration-arquillian/servers/cache-server/jboss/common/add-keycloak-caches.xsl @@ -39,7 +39,9 @@ - + + + diff --git a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java index bc1243b47c9e..14a508d67f43 100644 --- a/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java +++ b/testsuite/integration-arquillian/tests/base/src/test/java/org/keycloak/testsuite/crossdc/manual/SessionsPreloadCrossDCTest.java @@ -52,8 +52,9 @@ public void beforeAbstractKeycloakTest() throws Exception { stopAllCacheServersAndAuthServers(); - // Start DC1 only + // Start DC1 and only the cache container from DC2. All Keycloak nodes on DC2 are stopped containerController.start(getCacheServer(DC.FIRST).getQualifier()); + containerController.start(getCacheServer(DC.SECOND).getQualifier()); startBackendNode(DC.FIRST, 0); enableLoadBalancerNode(DC.FIRST, 0); @@ -119,7 +120,6 @@ public void sessionsPreloadTest() throws Exception { List tokenResponses = createInitialSessions(false); // Start 2nd DC. - containerController.start(getCacheServer(DC.SECOND).getQualifier()); startBackendNode(DC.SECOND, 0); enableLoadBalancerNode(DC.SECOND, 0); @@ -130,7 +130,7 @@ public void sessionsPreloadTest() throws Exception { Assert.assertEquals(sessions01, sessionsBefore + SESSIONS_COUNT); Assert.assertEquals(sessions02, sessionsBefore + SESSIONS_COUNT); - // On DC2 sessions were preloaded from from remoteCache + // On DC2 sessions were preloaded from remoteCache Assert.assertTrue(getTestingClientForStartedNodeInDc(1).testing().cache(InfinispanConnectionProvider.WORK_CACHE_NAME).contains("distributed::remoteCacheLoad::sessions")); // Assert refreshing works @@ -157,13 +157,15 @@ public void offlineSessionsPreloadTest() throws Exception { // Stop Everything stopAllCacheServersAndAuthServers(); - // Start DC1. Sessions should be preloaded from DB + // Start cache containers on both DC1 and DC2 containerController.start(getCacheServer(DC.FIRST).getQualifier()); + containerController.start(getCacheServer(DC.SECOND).getQualifier()); + + // Start Keycloak on DC1. Sessions should be preloaded from DB startBackendNode(DC.FIRST, 0); enableLoadBalancerNode(DC.FIRST, 0); - // Start DC2. Sessions should be preloaded from remoteCache - containerController.start(getCacheServer(DC.SECOND).getQualifier()); + // Start Keycloak on DC2. Sessions should be preloaded from remoteCache startBackendNode(DC.SECOND, 0); enableLoadBalancerNode(DC.SECOND, 0); @@ -210,7 +212,6 @@ public void loginFailuresPreloadTest() throws Exception { } // Start 2nd DC. - containerController.start(getCacheServer(DC.SECOND).getQualifier()); startBackendNode(DC.SECOND, 0); enableLoadBalancerNode(DC.SECOND, 0); diff --git a/travis-run-tests.sh b/travis-run-tests.sh index bee06b3ab850..532eaf916815 100755 --- a/travis-run-tests.sh +++ b/travis-run-tests.sh @@ -82,5 +82,17 @@ if [ $1 == "crossdc" ]; then cd tests/base mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.**.* 2>&1 | java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer - exit ${PIPESTATUS[0]} + BASE_TESTS_STATUS=${PIPESTATUS[0]} + + mvn clean test -B -nsu -Pcache-server-infinispan,auth-servers-crossdc-jboss,auth-server-wildfly -Dtest=*.crossdc.manual.* -Dmanual.mode=true 2>&1 | + java -cp ../../../utils/target/classes org.keycloak.testsuite.LogTrimmer + MANUAL_TESTS_STATUS=${PIPESTATUS[0]} + + echo "BASE_TESTS_STATUS=$BASE_TESTS_STATUS, MANUAL_TESTS_STATUS=$MANUAL_TESTS_STATUS"; + if [ $BASE_TESTS_STATUS -eq 0 -a $MANUAL_TESTS_STATUS -eq 0 ]; then + exit 0; + else + exit 1; + fi; + fi