[KEYCLOAK-11330] - Quarkus clustering tests

vidhya03 · Jun 16, 2020 · a8bad5b · a8bad5b
1 parent 3d5e976
commit a8bad5b
Show file tree

Hide file tree

Showing 18 changed files with 412 additions and 84 deletions.
diff --git a/common/src/main/java/org/keycloak/common/util/Retry.java b/common/src/main/java/org/keycloak/common/util/Retry.java
@@ -131,7 +131,7 @@ public static <T> T call(Supplier<T> supplier, int attemptsCount, long intervalM
         while (true) {
             try {
                 return supplier.get(iteration);
-            } catch (RuntimeException | AssertionError e) {
+            } catch (Exception | AssertionError e) {
                 attemptsCount--;
                 iteration++;
                 if (attemptsCount > 0) {

diff --git a/testsuite/integration-arquillian/HOW-TO-RUN.md b/testsuite/integration-arquillian/HOW-TO-RUN.md
@@ -649,6 +649,46 @@ After you build the distribution, you run this command to setup servers and run
     -Dauth.server.log.check=false \
     -Dfrontend.console.output=true \
     -Dtest=org.keycloak.testsuite.cluster.**.*Test clean install
+
+### Cluster tests with Keycloak on Quarkus
+
+Run tests using the `auth-server-cluster-quarkus` profile:
+
+     mvn -f testsuite/integration-arquillian/tests/base/pom.xml clean install \
+     -Pauth-server-cluster-quarkus \
+     -Dsession.cache.owners=2  \
+     -Dtest=AuthenticationSessionFailoverClusterTest
+     
+---
+**NOTE**
+
+Right now, tests are using a H2 database.
+
+To run tests using a different database such as PostgreSQL, add the following properties into the `testsuite/integration-arquillian/servers/auth-server/quarkus/src/main/content/conf/keycloak.properties` configuration file:
+
+```
+# HA using PostgreSQL
+%ha.datasource.dialect=org.hibernate.dialect.PostgreSQL9Dialect
+%ha.datasource.driver = org.postgresql.xa.PGXADataSource
+%ha.datasource.url = jdbc:postgresql://localhost/keycloak
+%ha.datasource.username = keycloak
+%ha.datasource.password = password
+```
+
+The `ha` profile is automatically set when running clustering tests. 
+
+This is temporary and database configuration should be more integrated with the test suite once we review Quarkus configuration.
+
+---
+     
+#### Run cluster tests from IDE on Quarkus
+
+Activate the following profiles:
+
+* `quarkus`
+* `auth-server-cluster-quarkus`
+
+Then run any cluster test as usual.
 
 ### Cluster tests with Keycloak on embedded undertow
 
@@ -915,13 +955,22 @@ When running the test, add the following arguments to the command line:
 Java 11 requires some arguments to be passed to JVM. Those can be activated using `-Pjava11-auth-server` and
 `-Pjava11-app-server` profiles, respectively.
 
-### Running tests using Quarkus distribution
+## Running tests using Quarkus distribution
+
+### Before Everything
 
 Make sure you build the project using the `quarkus` profile as follows:
 
     mvn -Pdistribution,quarkus clean install
+
+### Running tests
 
-Then, just run tests using the `auth-server-quarkus` profile:
+Run tests using the `auth-server-quarkus` profile:
 
     mvn -f testsuite/integration-arquillian/tests/base/pom.xml clean install -Pauth-server-quarkus
+
+### Debug the Server
+
+Right now, the server runs in a separate process. To debug the server set `auth.server.debug` system property to `true`.
 
+To configure the debugger port, set the `auth.server.debug.port` system property with any valid port number. Default is `5005`. 
diff --git a/testsuite/integration-arquillian/servers/auth-server/quarkus/assembly.xml b/testsuite/integration-arquillian/servers/auth-server/quarkus/assembly.xml
@@ -42,5 +42,12 @@
             <fileMode>0755</fileMode>
         </fileSet>
     </fileSets>
-
+    <files>
+        <file>
+            <source>src/main/content/conf/cluster-${auth.server.quarkus.config}.xml</source>
+            <outputDirectory>/auth-server-quarkus/conf</outputDirectory>
+            <destName>cluster.xml</destName>
+            <filtered>true</filtered>
+        </file>
+    </files>
 </assembly>
diff --git a/testsuite/integration-arquillian/servers/auth-server/quarkus/pom.xml b/testsuite/integration-arquillian/servers/auth-server/quarkus/pom.xml
@@ -14,6 +14,10 @@
 
     <properties>
         <auth.server.home>${project.build.directory}/unpacked/keycloak.x-${project.version}</auth.server.home>
+        <session.cache.owners>2</session.cache.owners>
+        <offline.session.cache.owners>2</offline.session.cache.owners>
+        <login.failure.cache.owners>2</login.failure.cache.owners>
+        <auth.server.quarkus.config>local</auth.server.quarkus.config>
     </properties>
 
     <dependencies>
@@ -112,4 +116,13 @@
             </plugin>
         </plugins>
     </build>
+
+    <profiles>
+        <profile>
+            <id>auth-server-cluster-quarkus</id>
+            <properties>
+                <auth.server.quarkus.config>ha</auth.server.quarkus.config>
+            </properties>
+        </profile>
+    </profiles>
 </project>
diff --git a/...e/integration-arquillian/servers/auth-server/quarkus/src/main/content/conf/cluster-ha.xml b/...e/integration-arquillian/servers/auth-server/quarkus/src/main/content/conf/cluster-ha.xml
@@ -0,0 +1,62 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+  ~ Copyright 2019 Red Hat, Inc. and/or its affiliates
+  ~ and other contributors as indicated by the @author tags.
+  ~
+  ~ Licensed under the Apache License, Version 2.0 (the "License");
+  ~ you may not use this file except in compliance with the License.
+  ~ You may obtain a copy of the License at
+  ~
+  ~ http://www.apache.org/licenses/LICENSE-2.0
+  ~
+  ~ Unless required by applicable law or agreed to in writing, software
+  ~ distributed under the License is distributed on an "AS IS" BASIS,
+  ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  ~ See the License for the specific language governing permissions and
+  ~ limitations under the License.
+  -->
+
+<infinispan
+        xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+        xsi:schemaLocation="urn:infinispan:config:9.4 http://www.infinispan.org/schemas/infinispan-config-9.4.xsd"
+        xmlns="urn:infinispan:config:9.4">
+
+    <!-- Distributed Cache Container Configuration -->
+    <cache-container name="keycloak">
+        <transport lock-timeout="60000" node-name="${jboss.node.name}" />
+        <local-cache name="realms">
+            <memory>
+                <object size="10000"/>
+            </memory>
+        </local-cache>
+        <local-cache name="users">
+            <memory>
+                <object size="10000"/>
+            </memory>
+        </local-cache>
+        <distributed-cache name="sessions" owners="${session.cache.owners}"/>
+        <distributed-cache name="authenticationSessions" owners="${session.cache.owners}"/>
+        <distributed-cache name="offlineSessions" owners="${offline.session.cache.owners}"/>
+        <distributed-cache name="clientSessions" owners="${session.cache.owners}"/>
+        <distributed-cache name="offlineClientSessions" owners="${offline.session.cache.owners}"/>
+        <distributed-cache name="loginFailures" owners="${login.failure.cache.owners}"/>
+        <local-cache name="authorization">
+            <memory>
+                <object size="10000"/>
+            </memory>
+        </local-cache>
+        <replicated-cache name="work"/>
+        <local-cache name="keys">
+            <expiration max-idle="3600000"/>
+            <memory>
+                <object size="1000"/>
+            </memory>
+        </local-cache>
+        <distributed-cache name="actionTokens" owners="2">
+            <expiration max-idle="-1" interval="300000"/>
+            <memory>
+                <object size="-1"/>
+            </memory>
+        </distributed-cache>
+    </cache-container>
+</infinispan>
diff --git a/...quarkus/src/main/content/conf/cluster.xml → ...s/src/main/content/conf/cluster-local.xml b/...quarkus/src/main/content/conf/cluster.xml → ...s/src/main/content/conf/cluster-local.xml
@@ -21,46 +21,6 @@
         xsi:schemaLocation="urn:infinispan:config:9.4 http://www.infinispan.org/schemas/infinispan-config-9.4.xsd"
         xmlns="urn:infinispan:config:9.4">
 
-    <!-- Distributed Cache Container Configuration
-    <cache-container name="keycloak">
-        <transport lock-timeout="60000"/>
-        <local-cache name="realms">
-            <memory>
-                <object size="10000"/>
-            </memory>
-        </local-cache>
-        <local-cache name="users">
-            <memory>
-                <object size="10000"/>
-            </memory>
-        </local-cache>
-        <distributed-cache name="sessions" owners="1"/>
-        <distributed-cache name="authenticationSessions" owners="1"/>
-        <distributed-cache name="offlineSessions" owners="1"/>
-        <distributed-cache name="clientSessions" owners="1"/>
-        <distributed-cache name="offlineClientSessions" owners="1"/>
-        <distributed-cache name="loginFailures" owners="1"/>
-        <local-cache name="authorization">
-            <memory>
-                <object size="10000"/>
-            </memory>
-        </local-cache>
-        <replicated-cache name="work"/>
-        <local-cache name="keys">
-            <expiration max-idle="3600000"/>
-            <memory>
-                <object size="1000"/>
-            </memory>
-        </local-cache>
-        <distributed-cache name="actionTokens" owners="2">
-            <expiration max-idle="-1" interval="300000"/>
-            <memory>
-                <object size="-1"/>
-            </memory>
-        </distributed-cache>
-    </cache-container>
-    -->
-
     <!-- Local Cache Container Configuration -->
     <cache-container name="keycloak">
         <local-cache name="default">

diff --git a/...egration-arquillian/servers/auth-server/quarkus/src/main/content/conf/keycloak.properties b/...egration-arquillian/servers/auth-server/quarkus/src/main/content/conf/keycloak.properties
@@ -6,14 +6,17 @@ datasource.jdbc.transactions=xa
 # H2
 datasource.dialect=org.hibernate.dialect.H2Dialect
 datasource.driver=org.h2.jdbcx.JdbcDataSource
-datasource.url = jdbc:h2:file:${keycloak.home.dir}/data/keycloakdb;AUTO_SERVER=TRUE
+datasource.url = jdbc:h2:file:${keycloak.home.dir}/data/keycloakdb;AUTO_SERVER=TRUE;DB_CLOSE_DELAY=-1
 datasource.username = sa
 datasource.password = keycloak
 
 # SSL
 http.ssl.certificate.key-store-file=${keycloak.home.dir}/conf/keycloak.jks
 http.ssl.certificate.key-store-password=secret
 
+# Proxy
+http.proxy-address-forwarding=true
+
 # Truststore Provider
 truststore.file.file=${keycloak.home.dir}/conf/keycloak.truststore
 truststore.file.password=secret
diff --git a/.../main/java/org/keycloak/testsuite/arquillian/containers/KeycloakQuarkusConfiguration.java b/.../main/java/org/keycloak/testsuite/arquillian/containers/KeycloakQuarkusConfiguration.java
@@ -1,11 +1,16 @@
 package org.keycloak.testsuite.arquillian.containers;
 
+import com.fasterxml.jackson.core.type.TypeReference;
 import org.jboss.arquillian.container.spi.ConfigurationException;
 import org.jboss.arquillian.container.spi.client.container.ContainerConfiguration;
 import org.jboss.logging.Logger;
+import org.keycloak.util.JsonSerialization;
 
+import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.HashMap;
+import java.util.Map;
 
 /**
  * @author mhajas
@@ -20,6 +25,10 @@ public class KeycloakQuarkusConfiguration implements ContainerConfiguration {
     private int bindHttpsPort = Integer.valueOf(System.getProperty("auth.server.https.port", "8543"));
     private Path providersPath = Paths.get(System.getProperty("auth.server.home"));
     private int startupTimeoutInSeconds = 60;
+    private String route;
+    private String keycloakConfigPropertyOverrides;
+    private HashMap<String, Object> keycloakConfigPropertyOverridesMap;
+    private String profile;
 
     @Override
     public void validate() throws ConfigurationException {
@@ -32,6 +41,15 @@ public void validate() throws ConfigurationException {
         setBindHttpsPort(newHttpsPort);
 
         log.info("Keycloak will listen for http on port: " + newPort + " and for https on port: " + newHttpsPort);
+
+        if (this.keycloakConfigPropertyOverrides != null) {
+            try {
+                TypeReference<HashMap<String,Object>> typeRef = new TypeReference<HashMap<String,Object>>() {};
+                this.keycloakConfigPropertyOverridesMap = JsonSerialization.sysPropertiesAwareMapper.readValue(this.keycloakConfigPropertyOverrides, typeRef);
+            } catch (IOException ex) {
+                throw new ConfigurationException(ex);
+            }
+        }
     }
 
     public int getBindHttpPortOffset() {
@@ -81,4 +99,32 @@ public int getStartupTimeoutInSeconds() {
     public void setStartupTimeoutInSeconds(int startupTimeoutInSeconds) {
         this.startupTimeoutInSeconds = startupTimeoutInSeconds;
     }
+
+    public String getRoute() {
+        return route;
+    }
+
+    public void setRoute(String route) {
+        this.route = route;
+    }
+
+    public String getProfile() {
+        return profile;
+    }
+
+    public void setProfile(String profile) {
+        this.profile = profile;
+    }
+
+    public String getKeycloakConfigPropertyOverrides() {
+        return keycloakConfigPropertyOverrides;
+    }
+
+    public void setKeycloakConfigPropertyOverrides(String keycloakConfigPropertyOverrides) {
+        this.keycloakConfigPropertyOverrides = keycloakConfigPropertyOverrides;
+    }
+
+    public Map<String, Object> getKeycloakConfigPropertyOverridesMap() {
+        return keycloakConfigPropertyOverridesMap;
+    }
 }
diff --git a/...rg/keycloak/testsuite/arquillian/containers/KeycloakQuarkusServerDeployableContainer.java b/...rg/keycloak/testsuite/arquillian/containers/KeycloakQuarkusServerDeployableContainer.java
@@ -42,7 +42,7 @@ public class KeycloakQuarkusServerDeployableContainer implements DeployableConta
 
     private KeycloakQuarkusConfiguration configuration;
     private Process container;
-    private AtomicBoolean restart = new AtomicBoolean();
+    private static AtomicBoolean restart = new AtomicBoolean();
 
     @Inject
     private Instance<SuiteContext> suiteContext;
@@ -130,14 +130,20 @@ private String[] getProcessCommands() {
         commands.add("-Dquarkus.http.port=" + configuration.getBindHttpPort());
         commands.add("-Dquarkus.http.ssl-port=" + configuration.getBindHttpsPort());
 
+        if (configuration.getRoute() != null) {
+            commands.add("-Djboss.node.name=" + configuration.getRoute());
+        }
+
+        commands.add("-Dquarkus.profile=" + System.getProperty("auth.server.quarkus.config", "local"));
+
         return commands.toArray(new String[commands.size()]);
     }
 
     private void waitForReadiness() throws MalformedURLException, LifecycleException {
         SuiteContext suiteContext = this.suiteContext.get();
         //TODO: not sure if the best endpoint but it makes sure that everything is properly initialized. Once we have
         // support for MP Health this should change
-        URL contextRoot = new URL(suiteContext.getAuthServerInfo().getContextRoot() + "/auth/realms/master/");
+        URL contextRoot = new URL(getBaseUrl(suiteContext) + "/auth/realms/master/");
         HttpURLConnection connection;
         long startTime = System.currentTimeMillis();
 
@@ -150,7 +156,7 @@ private void waitForReadiness() throws MalformedURLException, LifecycleException
             try {
                 // wait before checking for opening a new connection
                 Thread.sleep(1000);
-                if ("https".equals(contextRoot.toURI().getScheme())) {
+                if ("https".equals(contextRoot.getProtocol())) {
                     HttpsURLConnection httpsConnection = (HttpsURLConnection) (connection = (HttpURLConnection) contextRoot.openConnection());
                     httpsConnection.setSSLSocketFactory(createInsecureSslSocketFactory());
                     httpsConnection.setHostnameVerifier(createInsecureHostnameVerifier());
@@ -171,7 +177,19 @@ private void waitForReadiness() throws MalformedURLException, LifecycleException
             }
         }
 
-        log.infof("Keycloak is ready at %s", this.suiteContext.get().getAuthServerInfo().getContextRoot());
+        log.infof("Keycloak is ready at %s", contextRoot);
+    }
+
+    private URL getBaseUrl(SuiteContext suiteContext) throws MalformedURLException {
+        URL baseUrl = suiteContext.getAuthServerInfo().getContextRoot();
+
+        // might be running behind a load balancer
+        if ("https".equals(baseUrl.getProtocol())) {
+            baseUrl = new URL(baseUrl.toString().replace(String.valueOf(baseUrl.getPort()), String.valueOf(configuration.getBindHttpsPort())));
+        } else {
+            baseUrl = new URL(baseUrl.toString().replace(String.valueOf(baseUrl.getPort()), String.valueOf(configuration.getBindHttpPort())));
+        }
+        return baseUrl;
     }
 
     private HostnameVerifier createInsecureHostnameVerifier() {