Skip to content

Commit

Permalink
KEYCLOAK-4407 Ability to restart arquillian containers from test
Browse files Browse the repository at this point in the history
Co-Authored-By: Hynek Mlnarik <hmlnarik@redhat.com>
KEYCLOAK-4407 Fix connection error if underlying container restarts (63b9da857a8174a0b5e65e70c47ef2e2842f4d4e)
  • Loading branch information
vramik authored and hmlnarik committed Jul 27, 2018
1 parent d885682 commit 38017d3
Show file tree
Hide file tree
Showing 28 changed files with 630 additions and 293 deletions.
3 changes: 2 additions & 1 deletion .travis.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ env:
- TESTS=server-group3
- TESTS=server-group4
- TESTS=old
- TESTS=crossdc
- TESTS=crossdc1
- TESTS=crossdc2

jdk:
- oraclejdk8
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -145,4 +145,11 @@ public <T> T proxy(Class<T> proxyClass, URI absoluteURI) {
public void close() {
client.close();
}

/**
* @return true if the underlying client is closed.
*/
public boolean isClosed() {
return client.isClosed();
}
}
7 changes: 6 additions & 1 deletion testsuite/integration-arquillian/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,12 @@
<app.server>undertow</app.server>

<!--component versions-->
<!--to update arquillian-core to 1.3.0.Final or higher see https://issues.jboss.org/browse/ARQ-2181 -->
<!--
to update arquillian-core to 1.3.0.Final or higher
- see https://issues.jboss.org/browse/ARQ-2181
- update org.keycloak.testsuite.arquillian.containers.KeycloakContainerTestExtension according to
current version of org.jboss.arquillian.container.test.impl.ContainerTestExtension
-->
<arquillian-core.version>1.2.1.Final</arquillian-core.version>
<!--the version of shrinkwrap_resolver should align with the version in arquillian-bom-->
<shrinkwrap-resolver.version>2.2.6</shrinkwrap-resolver.version>
Expand Down
2 changes: 2 additions & 0 deletions testsuite/integration-arquillian/tests/base/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -670,13 +670,15 @@
<id>auth-servers-crossdc-undertow</id>
<properties>
<skip.clean.second.cache>false</skip.clean.second.cache>
<exclude.crossdc>-</exclude.crossdc>
</properties>
</profile>
<profile>
<id>auth-servers-crossdc-jboss</id>
<properties>
<skip.clean.second.cache>false</skip.clean.second.cache>
<skip.copy.auth.crossdc.nodes>false</skip.copy.auth.crossdc.nodes>
<exclude.crossdc>-</exclude.crossdc>
</properties>
</profile>
<profile>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@ public class ProfileAssume {
}

public static void assumeFeatureEnabled(Profile.Feature feature) {
Assume.assumeTrue("Ignoring test as " + feature.name() + " is not enabled", isFeatureEnabled(feature));
Assume.assumeTrue("Ignoring test as feature " + feature.name() + " is not enabled", isFeatureEnabled(feature));
}

public static void assumePreview() {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.jboss.arquillian.core.api.Instance;
import org.jboss.arquillian.core.api.annotation.Inject;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.spi.event.suite.AfterClass;
import org.jboss.arquillian.test.spi.event.suite.BeforeClass;
import org.jboss.logging.Logger;
import org.keycloak.testsuite.arquillian.annotation.AppServerContainer;
Expand Down Expand Up @@ -63,9 +64,9 @@ public static List<String> getAppServerQualifiers(Class testClass) {
Class<?> annotatedClass = getNearestSuperclassWithAppServerAnnotation(testClass);

if (annotatedClass == null) return null; // no @AppServerContainer annotation --> no adapter test

AppServerContainer[] appServerContainers = annotatedClass.getAnnotationsByType(AppServerContainer.class);

List<String> appServerQualifiers = new ArrayList<>();
for (AppServerContainer appServerContainer : appServerContainers) {
appServerQualifiers.add(appServerContainer.value());
Expand All @@ -87,7 +88,7 @@ public static String getAppServerContextRoot(int clusterPortOffset) {

return String.format("%s://%s:%s", scheme, host, port + clusterPortOffset);
}

private static int parsePort(String property) {
try {
return Integer.parseInt(System.getProperty(property));
Expand Down Expand Up @@ -182,6 +183,19 @@ public void startAppServer(@Observes(precedence = -1) BeforeClass event) throws
}
}

public void stopAppServer(@Observes(precedence = 1) AfterClass event) {
if (testContext.getAppServerInfo() == null) {
return; // no adapter test
}

ContainerController controller = containerConrollerInstance.get();

if (controller.isStarted(testContext.getAppServerInfo().getQualifier())) {
log.info("Stopping app server: " + testContext.getAppServerInfo().getQualifier());
controller.stop(testContext.getAppServerInfo().getQualifier());
}
}

/**
* Workaround for WFARQ-44. It cannot be used 'cleanServerBaseDir' property.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,15 @@
*/
package org.keycloak.testsuite.arquillian;

import org.jboss.arquillian.container.spi.Container;
import org.jboss.arquillian.container.spi.ContainerRegistry;
import org.jboss.arquillian.container.spi.event.StartContainer;
import org.jboss.arquillian.container.spi.event.StartSuiteContainers;
import org.jboss.arquillian.container.spi.event.StopContainer;
import org.jboss.arquillian.container.test.api.ContainerController;
import org.jboss.arquillian.core.api.Event;
import org.jboss.arquillian.core.api.Instance;
import org.jboss.arquillian.core.api.InstanceProducer;
import org.jboss.arquillian.core.api.annotation.ApplicationScoped;
import org.jboss.arquillian.core.api.annotation.Inject;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.spi.annotation.ClassScoped;
Expand All @@ -50,7 +51,6 @@

import java.util.stream.Collectors;
import javax.ws.rs.NotFoundException;
import org.jboss.arquillian.core.api.annotation.ApplicationScoped;

/**
*
Expand All @@ -61,6 +61,8 @@ public class AuthServerTestEnricher {

protected static final Logger log = Logger.getLogger(AuthServerTestEnricher.class);

@Inject
private Instance<ContainerController> containerConroller;
@Inject
private Instance<ContainerRegistry> containerRegistry;

Expand Down Expand Up @@ -89,10 +91,6 @@ public class AuthServerTestEnricher {
public static final Boolean START_MIGRATION_CONTAINER = "auto".equals(System.getProperty("migration.mode")) ||
"manual".equals(System.getProperty("migration.mode"));

// In manual mode are all containers despite loadbalancers started in mode "manual" and nothing is managed through "suite".
// Useful for tests, which require restart servers etc.
public static final String MANUAL_MODE = "manual.mode";

@Inject
@SuiteScoped
private InstanceProducer<SuiteContext> suiteContextProducer;
Expand Down Expand Up @@ -123,21 +121,17 @@ public static String getAuthServerContextRoot(int clusterPortOffset) {
}

public static OnlineManagementClient getManagementClient() {
OnlineManagementClient managementClient;
try {
managementClient = ManagementClient.online(OnlineOptions
return ManagementClient.online(OnlineOptions
.standalone()
.hostAndPort(System.getProperty("auth.server.host", "localhost"), Integer.parseInt(System.getProperty("auth.server.management.port", "10090")))
.build()
);
} catch (IOException e) {
throw new RuntimeException(e);
}


return managementClient;
}

public void distinguishContainersInConsoleOutput(@Observes(precedence = 5) StartContainer event) {
log.info("************************" + event.getContainer().getName()
+ "*****************************************************************************");
Expand All @@ -148,21 +142,18 @@ public void initializeSuiteContext(@Observes(precedence = 2) BeforeSuite event)
.map(ContainerInfo::new)
.collect(Collectors.toSet());

// A way to specify that containers should be in mode "manual" rather then "suite"
checkManualMode(containers);

suiteContext = new SuiteContext(containers);

if (AUTH_SERVER_CROSS_DC) {
// if cross-dc mode enabled, load-balancer is the frontend of datacenter cluster
containers.stream()
.filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER + "-cross-dc"))
.forEach(c -> {
String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
suiteContext.addAuthServerInfo(Integer.valueOf(dcString), c);
});
.filter(c -> c.getQualifier().startsWith(AUTH_SERVER_BALANCER + "-cross-dc"))
.forEach(c -> {
String portOffsetString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("bindHttpPortOffset", "0");
String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
updateWithAuthServerInfo(c, Integer.valueOf(portOffsetString));
suiteContext.addAuthServerInfo(Integer.valueOf(dcString), c);
});

if (suiteContext.getDcAuthServerInfo().isEmpty()) {
throw new IllegalStateException("Not found frontend container (load balancer): " + AUTH_SERVER_BALANCER);
Expand All @@ -181,7 +172,7 @@ public void initializeSuiteContext(@Observes(precedence = 2) BeforeSuite event)
String dcString = c.getArquillianContainer().getContainerConfiguration().getContainerProperties().getOrDefault("dataCenter", "0");
suiteContext.addAuthServerBackendsInfo(Integer.valueOf(dcString), c);
});

containers.stream()
.filter(c -> c.getQualifier().startsWith("cache-server-cross-dc-"))
.sorted((a, b) -> a.getQualifier().compareTo(b.getQualifier()))
Expand Down Expand Up @@ -258,6 +249,7 @@ public void initializeSuiteContext(@Observes(precedence = 2) BeforeSuite event)
}

suiteContextProducer.set(suiteContext);
CacheServerTestEnricher.initializeSuiteContext(suiteContext);
log.info("\n\n" + suiteContext);
}

Expand Down Expand Up @@ -295,6 +287,12 @@ public void stopMigratedContainer(@Observes(precedence = 1) StartSuiteContainers
}
}

public void startAuthContainer(@Observes(precedence = 0) StartSuiteContainers event) {
//frontend-only (either load-balancer or auth-server)
log.debug("Starting auth server before suite");
startContainerEvent.fire(new StartContainer(suiteContext.getAuthServerInfo().getArquillianContainer()));
}

public void checkServerLogs(@Observes(precedence = -1) BeforeSuite event) throws IOException, InterruptedException {
boolean checkLog = Boolean.parseBoolean(System.getProperty("auth.server.log.check", "true"));
if (checkLog && suiteContext.getAuthServerInfo().isJBossBased()) {
Expand All @@ -316,6 +314,13 @@ public void initializeOAuthClient(@Observes(precedence = 3) BeforeClass event) {
}

public void afterClass(@Observes(precedence = 2) AfterClass event) {
//check if a test accidentally left the auth-server not running
ContainerController controller = containerConroller.get();
if (!controller.isStarted(suiteContext.getAuthServerInfo().getQualifier())) {
log.warn("Auth server wasn't running. Starting " + suiteContext.getAuthServerInfo().getQualifier());
controller.start(suiteContext.getAuthServerInfo().getQualifier());
}

TestContext testContext = testContextProducer.get();

Keycloak adminClient = testContext.getAdminClient();
Expand All @@ -335,32 +340,18 @@ public void afterClass(@Observes(precedence = 2) AfterClass event) {

public static void removeTestRealms(TestContext testContext, Keycloak adminClient) {
List<RealmRepresentation> testRealmReps = testContext.getTestRealmReps();
if (testRealmReps != null) {
if (testRealmReps != null && !testRealmReps.isEmpty()) {
log.info("removing test realms after test class");
StringBuilder realms = new StringBuilder();
for (RealmRepresentation testRealm : testRealmReps) {
String realmName = testRealm.getRealm();
log.info("removing realm: " + realmName);
try {
adminClient.realms().realm(realmName).remove();
adminClient.realms().realm(testRealm.getRealm()).remove();
realms.append(testRealm.getRealm()).append(", ");
} catch (NotFoundException e) {
// Ignore
}
}
}
}


private void checkManualMode(Set<ContainerInfo> containers) {
String manualMode = System.getProperty(MANUAL_MODE);

if (Boolean.parseBoolean(manualMode)) {

containers.stream()
.filter(containerInfo -> !containerInfo.getQualifier().contains("balancer"))
.forEach(containerInfo -> {
log.infof("Container '%s' will be in manual mode", containerInfo.getQualifier());
containerInfo.getArquillianContainer().getContainerConfiguration().setMode("manual");
});
log.info("removed realms: " + realms);
}
}

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
/*
* Copyright 2018 Red Hat, Inc. and/or its affiliates
* and other contributors as indicated by the @author tags.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.keycloak.testsuite.arquillian;

import java.io.File;
import java.io.IOException;

import org.apache.commons.io.FileUtils;
import org.jboss.arquillian.container.test.api.ContainerController;
import org.jboss.arquillian.core.api.Instance;
import org.jboss.arquillian.core.spi.Validate;
import org.jboss.arquillian.core.api.annotation.Inject;
import org.jboss.arquillian.core.api.annotation.Observes;
import org.jboss.arquillian.test.spi.event.suite.AfterClass;
import org.jboss.logging.Logger;

import org.keycloak.testsuite.crossdc.DC;

/**
*
* @author vramik
*/
public class CacheServerTestEnricher {

protected static final Logger log = Logger.getLogger(CacheServerTestEnricher.class);
private static SuiteContext suiteContext;

@Inject
private Instance<ContainerController> containerController;

static void initializeSuiteContext(SuiteContext suiteContext) {
Validate.notNull(suiteContext, "Suite context cannot be null.");
CacheServerTestEnricher.suiteContext = suiteContext;
}

public void afterClass(@Observes(precedence = 4) AfterClass event) {
if (!suiteContext.getCacheServersInfo().isEmpty()) {
stopCacheServer(suiteContext.getCacheServersInfo().get(DC.FIRST.ordinal()));
stopCacheServer(suiteContext.getCacheServersInfo().get(DC.SECOND.ordinal()));
}
}

private void stopCacheServer(ContainerInfo cacheServer) {
if (containerController.get().isStarted(cacheServer.getQualifier())) {
log.infof("Stopping %s", cacheServer.getQualifier());

containerController.get().stop(cacheServer.getQualifier());

// Workaround for possible arquillian bug. Needs to cleanup dir manually
String setupCleanServerBaseDir = getContainerProperty(cacheServer, "setupCleanServerBaseDir");
String cleanServerBaseDir = getContainerProperty(cacheServer, "cleanServerBaseDir");

if (Boolean.parseBoolean(setupCleanServerBaseDir)) {
log.infof("Going to clean directory: %s", cleanServerBaseDir);

File dir = new File(cleanServerBaseDir);
if (dir.exists()) {
try {
dir.renameTo(new File(dir.getParentFile(), dir.getName() + "--" + System.currentTimeMillis()));

File deploymentsDir = new File(dir, "deployments");
FileUtils.forceMkdir(deploymentsDir);
} catch (IOException ioe) {
throw new RuntimeException("Failed to clean directory: " + cleanServerBaseDir, ioe);
}
}
}

log.infof("Stopped %s", cacheServer.getQualifier());
}
}

private String getContainerProperty(ContainerInfo cacheServer, String propertyName) {
return cacheServer.getArquillianContainer().getContainerConfiguration().getContainerProperties().get(propertyName);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,9 @@ private InfinispanStatistics getInfinispanCacheStatistics(JmxInfinispanCacheStat

if (annotation.domain().isEmpty()) {
try {
Retry.execute(() -> value.reset(), 2, 150);
LOG.debug("Going to try reset InfinispanCacheStatistics (2 attempts, 150 ms interval)");
int execute = Retry.execute(() -> value.reset(), 2, 150);
LOG.debug("reset in " + execute + " attempts");
} catch (RuntimeException ex) {
if (annotation.dc() != DC.UNDEFINED && annotation.dcNodeIndex() != -1
&& suiteContext.get().getAuthServerBackendsInfo(annotation.dc().getDcIndex()).get(annotation.dcNodeIndex()).isStarted()) {
Expand Down
Loading

0 comments on commit 38017d3

Please sign in to comment.