forked from opensearch-project/OpenSearch
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Testclusters: improove timeout handling (#43440)
- Loading branch information
Showing
6 changed files
with
224 additions
and
87 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
59 changes: 59 additions & 0 deletions
59
...Src/src/main/java/org/elasticsearch/gradle/testclusters/TestClusterCleanupOnShutdown.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,59 @@ | ||
| package org.elasticsearch.gradle.testclusters; | ||
|
|
||
| import org.gradle.api.logging.Logger; | ||
| import org.gradle.api.logging.Logging; | ||
|
|
||
| import java.util.Collection; | ||
| import java.util.HashSet; | ||
| import java.util.Iterator; | ||
| import java.util.Set; | ||
|
|
||
| /** | ||
| * Keep an inventory of all running Clusters and stop them when interrupted | ||
| * | ||
| * This takes advantage of the fact that Gradle interrupts all the threads in the daemon when the build completes. | ||
| */ | ||
| public class TestClusterCleanupOnShutdown implements Runnable { | ||
|
|
||
| private final Logger logger = Logging.getLogger(TestClusterCleanupOnShutdown.class); | ||
|
|
||
| private Set<ElasticsearchCluster> clustersToWatch = new HashSet<>(); | ||
|
|
||
| public void watch(Collection<ElasticsearchCluster> cluster) { | ||
| synchronized (clustersToWatch) { | ||
| clustersToWatch.addAll(clustersToWatch); | ||
| } | ||
| } | ||
|
|
||
| public void unWatch(Collection<ElasticsearchCluster> cluster) { | ||
| synchronized (clustersToWatch) { | ||
| clustersToWatch.removeAll(clustersToWatch); | ||
| } | ||
| } | ||
|
|
||
| @Override | ||
| public void run() { | ||
| try { | ||
| while (true) { | ||
| Thread.sleep(Long.MAX_VALUE); | ||
| } | ||
| } catch (InterruptedException interrupted) { | ||
| synchronized (clustersToWatch) { | ||
| if (clustersToWatch.isEmpty()) { | ||
| return; | ||
| } | ||
| logger.info("Cleanup thread was interrupted, shutting down all clusters"); | ||
| Iterator<ElasticsearchCluster> iterator = clustersToWatch.iterator(); | ||
| while (iterator.hasNext()) { | ||
| ElasticsearchCluster cluster = iterator.next(); | ||
| iterator.remove(); | ||
| try { | ||
| cluster.stop(false); | ||
| } catch (Exception e) { | ||
| logger.warn("Could not shut down {}", cluster, e); | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } |
74 changes: 74 additions & 0 deletions
74
...Src/src/main/java/org/elasticsearch/gradle/testclusters/TestClustersCleanupExtension.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,74 @@ | ||
| package org.elasticsearch.gradle.testclusters; | ||
|
|
||
| import org.gradle.api.Project; | ||
| import org.gradle.api.logging.Logger; | ||
| import org.gradle.api.logging.Logging; | ||
|
|
||
| import java.util.concurrent.ExecutorService; | ||
| import java.util.concurrent.Executors; | ||
| import java.util.concurrent.TimeUnit; | ||
|
|
||
| /** | ||
| * This extensions was meant to be used internally by testclusters | ||
| * | ||
| * It holds synchronization primitives needed to implement the rate limiting. | ||
| * This is tricky because we can't use Gradle workers as there's no way to make sure that tests and their clusters are | ||
| * allocated atomically, so we could be in a situation where all workers are tests waiting for clusters to start up. | ||
| * | ||
| * Also auto configures cleanup of executors to make sure we don't leak threads in the daemon. | ||
| */ | ||
| public class TestClustersCleanupExtension { | ||
|
|
||
| private static final int EXECUTOR_SHUTDOWN_TIMEOUT = 1; | ||
| private static final TimeUnit EXECUTOR_SHUTDOWN_TIMEOUT_UNIT = TimeUnit.MINUTES; | ||
|
|
||
| private static final Logger logger = Logging.getLogger(TestClustersCleanupExtension.class); | ||
|
|
||
| private final ExecutorService executorService; | ||
| private final TestClusterCleanupOnShutdown cleanupThread; | ||
|
|
||
| public TestClustersCleanupExtension() { | ||
| executorService = Executors.newSingleThreadExecutor(); | ||
| cleanupThread = new TestClusterCleanupOnShutdown(); | ||
| executorService.submit(cleanupThread); | ||
| } | ||
|
|
||
|
|
||
| public static void createExtension(Project project) { | ||
| if (project.getRootProject().getExtensions().findByType(TestClustersCleanupExtension.class) != null) { | ||
| return; | ||
| } | ||
| // Configure the extension on the root project so we have a single instance per run | ||
| TestClustersCleanupExtension ext = project.getRootProject().getExtensions().create( | ||
| "__testclusters_rate_limit", | ||
| TestClustersCleanupExtension.class | ||
| ); | ||
| Thread shutdownHook = new Thread(ext.cleanupThread::run); | ||
| Runtime.getRuntime().addShutdownHook(shutdownHook); | ||
| project.getGradle().buildFinished(buildResult -> { | ||
| ext.executorService.shutdownNow(); | ||
| try { | ||
| if (ext.executorService.awaitTermination(EXECUTOR_SHUTDOWN_TIMEOUT, EXECUTOR_SHUTDOWN_TIMEOUT_UNIT) == false) { | ||
| throw new IllegalStateException( | ||
| "Failed to shut down executor service after " + | ||
| EXECUTOR_SHUTDOWN_TIMEOUT + " " + EXECUTOR_SHUTDOWN_TIMEOUT_UNIT | ||
| ); | ||
| } | ||
| } catch (InterruptedException e) { | ||
| Thread.currentThread().interrupt(); | ||
| } | ||
| try { | ||
| if (false == Runtime.getRuntime().removeShutdownHook(shutdownHook)) { | ||
| logger.warn("Trying to deregister shutdown hook when it was not registered."); | ||
| } | ||
| } catch (IllegalStateException ese) { | ||
| // Thrown when shutdown is in progress | ||
| logger.warn("Can't remove shutdown hook", ese); | ||
| } | ||
| }); | ||
| } | ||
|
|
||
| public TestClusterCleanupOnShutdown getCleanupThread() { | ||
| return cleanupThread; | ||
| } | ||
| } |
Oops, something went wrong.