Skip to content

Commit

Permalink
Testclusters: start using it for testing some plugins (#39693)
Browse files Browse the repository at this point in the history 
* enable testclusters for some plugins
  • Loading branch information
@alpar-t
alpar-t committed Mar 7, 2019
1 parent 7dcc191 commit 6c75a2f
Show file tree
Hide file tree
Showing 10 changed files with 229 additions and 83 deletions.
3 changes: 2 additions & 1 deletion build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,8 @@ allprojects {
description = "Elasticsearch subproject ${project.path}"
}

BuildPlugin.configureRepositories(project)

apply plugin: 'nebula.info-scm'
String licenseCommit
if (VersionProperties.elasticsearch.toString().endsWith('-SNAPSHOT')) {
Expand Down Expand Up @@ -227,7 +229,6 @@ allprojects {
"org.elasticsearch.client:transport:${version}": ':client:transport',
"org.elasticsearch.plugin:elasticsearch-scripting-painless-spi:${version}": ':modules:lang-painless:spi',
"org.elasticsearch.test:framework:${version}": ':test:framework',
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${version}": ':distribution:archives:integ-test-zip',
"org.elasticsearch.test:logger-usage:${version}": ':test:logger-usage',
"org.elasticsearch.xpack.test:feature-aware:${version}": ':x-pack:test:feature-aware',
// for transport client
Expand Down
35 changes: 27 additions & 8 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/plugin/PluginBuildPlugin.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import org.elasticsearch.gradle.BuildPlugin
import org.elasticsearch.gradle.NoticeTask
import org.elasticsearch.gradle.test.RestIntegTestTask
import org.elasticsearch.gradle.test.RunTask
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.gradle.api.Project
import org.gradle.api.publish.maven.MavenPublication
import org.gradle.api.publish.maven.plugins.MavenPublishPlugin
Expand Down Expand Up @@ -51,21 +52,36 @@ public class PluginBuildPlugin extends BuildPlugin {
String name = project.pluginProperties.extension.name
project.archivesBaseName = name

// set teh project description so it will be picked up by publishing
// set the project description so it will be picked up by publishing
project.description = project.pluginProperties.extension.description

configurePublishing(project)

project.integTestCluster.dependsOn(project.bundlePlugin)
project.tasks.run.dependsOn(project.bundlePlugin)
if (project.plugins.hasPlugin(TestClustersPlugin.class) == false) {
project.integTestCluster.dependsOn(project.tasks.bundlePlugin)
if (isModule) {
project.integTestCluster.module(project)
} else {
project.integTestCluster.plugin(project.path)
}
} else {
project.tasks.integTest.dependsOn(project.tasks.bundlePlugin)
if (isModule) {
throw new RuntimeException("Testclusters does not support modules yet");
} else {
project.testClusters.integTestCluster.plugin(
project.file(project.tasks.bundlePlugin.archiveFile)
)
}
}

project.tasks.run.dependsOn(project.tasks.bundlePlugin)
if (isModule) {
project.integTestCluster.module(project)
project.tasks.run.clusterConfig.module(project)
project.tasks.run.clusterConfig.distribution = System.getProperty(
'run.distribution', 'integ-test-zip'
)
} else {
project.integTestCluster.plugin(project.path)
project.tasks.run.clusterConfig.plugin(project.path)
}

Expand Down Expand Up @@ -136,7 +152,10 @@ public class PluginBuildPlugin extends BuildPlugin {
private static void createIntegTestTask(Project project) {
RestIntegTestTask integTest = project.tasks.create('integTest', RestIntegTestTask.class)
integTest.mustRunAfter(project.precommit, project.test)
project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
if (project.plugins.hasPlugin(TestClustersPlugin.class) == false) {
// only if not using test clusters
project.integTestCluster.distribution = System.getProperty('tests.distribution', 'integ-test-zip')
}
project.check.dependsOn(integTest)
}

Expand Down Expand Up @@ -214,7 +233,7 @@ public class PluginBuildPlugin extends BuildPlugin {
protected void addNoticeGeneration(Project project) {
File licenseFile = project.pluginProperties.extension.licenseFile
if (licenseFile != null) {
project.bundlePlugin.from(licenseFile.parentFile) {
project.tasks.bundlePlugin.from(licenseFile.parentFile) {
include(licenseFile.name)
rename { 'LICENSE.txt' }
}
Expand All @@ -223,7 +242,7 @@ public class PluginBuildPlugin extends BuildPlugin {
if (noticeFile != null) {
NoticeTask generateNotice = project.tasks.create('generateNotice', NoticeTask.class)
generateNotice.inputFile = noticeFile
project.bundlePlugin.from(generateNotice)
project.tasks.bundlePlugin.from(generateNotice)
}
}
}
17 changes: 14 additions & 3 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,10 +174,20 @@ class ClusterFormationTasks {

/** Adds a dependency on the given distribution */
static void configureDistributionDependency(Project project, String distro, Configuration configuration, String elasticsearchVersion) {
boolean internalBuild = project.hasProperty('bwcVersions')
if (distro.equals("integ-test-zip")) {
// short circuit integ test so it doesn't complicate the rest of the distribution setup below
project.dependencies.add(configuration.name,
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip")
if (internalBuild) {
project.dependencies.add(
configuration.name,
project.dependencies.project(path: ":distribution", configuration: 'integ-test-zip')
)
} else {
project.dependencies.add(
configuration.name,
"org.elasticsearch.distribution.integ-test-zip:elasticsearch:${elasticsearchVersion}@zip"
)
}
return
}
// TEMP HACK
Expand Down Expand Up @@ -209,8 +219,9 @@ class ClusterFormationTasks {
if (distro.equals("oss")) {
snapshotProject = "oss-" + snapshotProject
}
boolean internalBuild = project.hasProperty('bwcVersions')

BwcVersions.UnreleasedVersionInfo unreleasedInfo = null

if (project.hasProperty('bwcVersions')) {
// NOTE: leniency is needed for external plugin authors using build-tools. maybe build the version compat info into build-tools?
unreleasedInfo = project.bwcVersions.unreleasedInfo(version)
Expand Down
77 changes: 50 additions & 27 deletions buildSrc/src/main/groovy/org/elasticsearch/gradle/test/RestIntegTestTask.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ package org.elasticsearch.gradle.test

import com.carrotsearch.gradle.junit4.RandomizedTestingTask
import org.elasticsearch.gradle.VersionProperties
import org.elasticsearch.gradle.testclusters.ElasticsearchNode
import org.elasticsearch.gradle.testclusters.TestClustersPlugin
import org.gradle.api.DefaultTask
import org.gradle.api.Task
import org.gradle.api.execution.TaskExecutionAdapter
Expand Down Expand Up @@ -55,7 +57,19 @@ public class RestIntegTestTask extends DefaultTask {
super.dependsOn(runner)
clusterInit = project.tasks.create(name: "${name}Cluster#init", dependsOn: project.testClasses)
runner.dependsOn(clusterInit)
clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
boolean usesTestclusters = project.plugins.hasPlugin(TestClustersPlugin.class)
if (usesTestclusters == false) {
clusterConfig = project.extensions.create("${name}Cluster", ClusterConfiguration.class, project)
} else {
project.testClusters {
integTestCluster {
distribution = 'INTEG_TEST'
version = project.version
javaHome = project.file(project.ext.runtimeJavaHome)
}
}
runner.useCluster project.testClusters.integTestCluster
}

// override/add more for rest tests
runner.parallelism = '1'
Expand All @@ -66,31 +80,38 @@ public class RestIntegTestTask extends DefaultTask {
if (System.getProperty("tests.cluster") != null) {
throw new IllegalArgumentException("tests.rest.cluster and tests.cluster must both be null or non-null")
}
// we pass all nodes to the rest cluster to allow the clients to round-robin between them
// this is more realistic than just talking to a single node
runner.systemProperty('tests.rest.cluster', "${-> nodes.collect{it.httpUri()}.join(",")}")
runner.systemProperty('tests.config.dir', "${-> nodes[0].pathConf}")
// TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
// that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
// both as separate sysprops
runner.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")

// dump errors and warnings from cluster log on failure
TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() {
@Override
void afterExecute(Task task, TaskState state) {
if (state.failure != null) {
for (NodeInfo nodeInfo : nodes) {
printLogExcerpt(nodeInfo)
if (usesTestclusters == true) {
ElasticsearchNode node = project.testClusters.integTestCluster
runner.systemProperty('tests.rest.cluster', {node.allHttpSocketURI.join(",") })
runner.systemProperty('tests.config.dir', {node.getConfigDir()})
runner.systemProperty('tests.cluster', {node.transportPortURI})
} else {
// we pass all nodes to the rest cluster to allow the clients to round-robin between them
// this is more realistic than just talking to a single node
runner.systemProperty('tests.rest.cluster', "${-> nodes.collect { it.httpUri() }.join(",")}")
runner.systemProperty('tests.config.dir', "${-> nodes[0].pathConf}")
// TODO: our "client" qa tests currently use the rest-test plugin. instead they should have their own plugin
// that sets up the test cluster and passes this transport uri instead of http uri. Until then, we pass
// both as separate sysprops
runner.systemProperty('tests.cluster', "${-> nodes[0].transportUri()}")

// dump errors and warnings from cluster log on failure
TaskExecutionAdapter logDumpListener = new TaskExecutionAdapter() {
@Override
void afterExecute(Task task, TaskState state) {
if (state.failure != null) {
for (NodeInfo nodeInfo : nodes) {
printLogExcerpt(nodeInfo)
}
}
}
}
}
runner.doFirst {
project.gradle.addListener(logDumpListener)
}
runner.doLast {
project.gradle.removeListener(logDumpListener)
runner.doFirst {
project.gradle.addListener(logDumpListener)
}
runner.doLast {
project.gradle.removeListener(logDumpListener)
}
}
} else {
if (System.getProperty("tests.cluster") == null) {
Expand All @@ -113,11 +134,13 @@ public class RestIntegTestTask extends DefaultTask {
clusterInit.enabled = false
return // no need to add cluster formation tasks if the task won't run!
}
// only create the cluster if needed as otherwise an external cluster to use was specified
if (System.getProperty("tests.rest.cluster") == null) {
nodes = ClusterFormationTasks.setup(project, "${name}Cluster", runner, clusterConfig)
if (usesTestclusters == false) {
// only create the cluster if needed as otherwise an external cluster to use was specified
if (System.getProperty("tests.rest.cluster") == null) {
nodes = ClusterFormationTasks.setup(project, "${name}Cluster", runner, clusterConfig)
}
super.dependsOn(runner.finalizedBy)
}
super.dependsOn(runner.finalizedBy)
}
}

Expand Down
47 changes: 35 additions & 12 deletions buildSrc/src/main/java/org/elasticsearch/gradle/Distribution.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,18 +20,24 @@

public enum Distribution {

INTEG_TEST("integ-test"),
DEFAULT("elasticsearch"),
OSS("elasticsearch-oss");
INTEG_TEST("elasticsearch", "integ-test-zip"),
DEFAULT("elasticsearch", "elasticsearch"),
OSS("elasticsearch-oss", "elasticsearch-oss");

private final String fileName;
private final String artifactName;
private final String group;

Distribution(String name) {
this.fileName = name;
Distribution(String name, String group) {
this.artifactName = name;
this.group = group;
}

public String getArtifactName() {
return fileName;
return artifactName;
}

public String getGroup() {
return "org.elasticsearch.distribution." + group;
}

public String getFileExtension() {
Expand All @@ -46,10 +52,27 @@ public String getFileExtension() {
}

public String getClassifier() {
return OS.<String>conditional()
.onLinux(() -> "linux-x86_64")
.onWindows(() -> "windows-x86_64")
.onMac(() -> "darwin-x86_64")
.supply();
if (this.equals(INTEG_TEST)) {
return "";
} else {
return OS.<String>conditional()
.onLinux(() -> "linux-x86_64")
.onWindows(() -> "windows-x86_64")
.onMac(() -> "darwin-x86_64")
.supply();
}
}

public String getLiveConfiguration() {
if (this.equals(INTEG_TEST)) {
return "integ-test-zip";
} else {
return (this.equals(OSS) ? "oss-" : "") + OS.<String>conditional()
.onLinux(() -> "linux-tar")
.onWindows(() -> "windows-zip")
.onMac(() -> "darwin-tar")
.supply();
}
}

}
19 changes: 17 additions & 2 deletions buildSrc/src/main/java/org/elasticsearch/gradle/testclusters/ElasticsearchNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,10 @@ public void plugin(File plugin) {
plugin(plugin.toURI());
}

public Path getConfigDir() {
return configFile.getParent();
}

public void freeze() {
requireNonNull(distribution, "null distribution passed when configuring test cluster `" + this + "`");
requireNonNull(version, "null version passed when configuring test cluster `" + this + "`");
Expand Down Expand Up @@ -196,6 +200,7 @@ synchronized void start() {
logger.info("Starting `{}`", this);

Path distroArtifact = artifactsExtractDir
.resolve(distribution.getGroup())
.resolve(distribution.getArtifactName() + "-" + getVersion());

if (Files.exists(distroArtifact) == false) {
Expand All @@ -205,8 +210,8 @@ synchronized void start() {
throw new TestClustersException("Can not start " + this + ", is not a directory: " + distroArtifact);
}
services.sync(spec -> {
spec.from(distroArtifact.resolve("config").toFile());
spec.into(configFile.getParent());
spec.from(distroArtifact);
spec.into(workingDir);
});

try {
Expand Down Expand Up @@ -297,6 +302,16 @@ public String getTransportPortURI() {
return getTransportPortInternal().get(0);
}

public List<String> getAllHttpSocketURI() {
waitForAllConditions();
return getHttpPortInternal();
}

public List<String> getAllTransportPortURI() {
waitForAllConditions();
return getTransportPortInternal();
}

synchronized void stop(boolean tailLogs) {
if (esProcess == null && tailLogs) {
// This is a special case. If start() throws an exception the plugin will still call stop
Expand Down
Loading

0 comments on commit 6c75a2f

Please sign in to comment.