Skip to content

Commit

Permalink
Consolidate REST API specifications and publish under Apache 2.0 lice…
Browse files Browse the repository at this point in the history 
…nse (#70036)
  • Loading branch information
@mark-vieira
mark-vieira authored Mar 26, 2021
1 parent c9f7ed7 commit 6339691
Show file tree
Hide file tree
Showing 871 changed files with 4,022 additions and 453 deletions.
16 changes: 7 additions & 9 deletions buildSrc/src/integTest/groovy/org/elasticsearch/gradle/RestResourcesPluginFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
file("/build/restResources/yamlSpecs/rest-api-spec/api/" + api).exists()
}

def "restResources copies all core API (but not x-pack) by default for projects with copied tests"() {
def "restResources copies all API by default for projects with copied tests"() {
given:
internalBuild()
buildFile << """
Expand All @@ -71,7 +71,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
String apiXpack = "xpack.json"
String coreTest = "foo/10_basic.yml"
String xpackTest = "bar/10_basic.yml"
setupRestResources([apiCore1, apiCore2], [coreTest], [apiXpack], [xpackTest])
setupRestResources([apiCore1, apiCore2, apiXpack], [coreTest], [xpackTest])
// intentionally not adding tests to project, they will be copied over via the plugin
// this tests that the test copy happens before the api copy since the api copy will only trigger if there are tests in the project

Expand All @@ -83,7 +83,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
result.task(':copyYamlTestsTask').outcome == TaskOutcome.SUCCESS
file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiCore1).exists()
file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiCore2).exists()
file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiXpack).exists() == false //x-pack specs must be explicitly configured
file("/build/restResources/yamlSpecs/rest-api-spec/api/" + apiXpack).exists()
file("/build/restResources/yamlTests/rest-api-spec/test/" + coreTest).exists()
file("/build/restResources/yamlTests/rest-api-spec/test/" + xpackTest).exists()
}
Expand All @@ -97,16 +97,15 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
restResources {
restApi {
includeCore 'foo'
includeXpack 'xpackfoo'
include 'foo', 'xpackfoo'
}
}
"""
String apiFoo = "foo.json"
String apiXpackFoo = "xpackfoo.json"
String apiBar = "bar.json"
String apiXpackBar = "xpackbar.json"
setupRestResources([apiFoo, apiBar], [], [apiXpackFoo, apiXpackBar])
setupRestResources([apiFoo, apiBar, apiXpackFoo, apiXpackBar])
addRestTestsToProject(["10_basic.yml"])

when:
Expand All @@ -130,8 +129,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
restResources {
restApi {
includeCore '*'
includeXpack '*'
include '*'
}
restTests {
includeCore 'foo'
Expand All @@ -144,7 +142,7 @@ class RestResourcesPluginFuncTest extends AbstractRestResourcesFuncTest {
String apiXpack = "xpack.json"
String coreTest = "foo/10_basic.yml"
String xpackTest = "bar/10_basic.yml"
setupRestResources([apiCore1, apiCore2], [coreTest], [apiXpack], [xpackTest])
setupRestResources([apiCore1, apiCore2, apiXpack], [coreTest], [xpackTest])
// intentionally not adding tests to project, they will be copied over via the plugin
// this tests that the test copy happens before the api copy since the api copy will only trigger if there are tests in the project

Expand Down
15 changes: 5 additions & 10 deletions ...c/integTest/groovy/org/elasticsearch/gradle/fixtures/AbstractRestResourcesFuncTest.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,37 +11,32 @@ package org.elasticsearch.gradle.fixtures;

abstract class AbstractRestResourcesFuncTest extends AbstractGradleFuncTest {

void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackApis = [], List<String> xpackTests = []) {
void setupRestResources(List<String> apis, List<String> tests = [], List<String> xpackTests = []) {
addSubProject(":test:framework") << "apply plugin: 'elasticsearch.java'"
addSubProject(":distribution:archives:integ-test-zip") << "configurations { extracted }"
addSubProject(":rest-api-spec") << """
configurations { restSpecs\nrestTests }
artifacts {
restSpecs(new File(projectDir, "src/main/resources/rest-api-spec/api"))
restTests(new File(projectDir, "src/main/resources/rest-api-spec/test"))
restTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
}
"""
addSubProject(":x-pack:plugin") << """
configurations { restXpackSpecs\nrestXpackTests }
artifacts {
//The api and tests need to stay at src/test/... since some external tooling depends on that exact file path.
restXpackSpecs(new File(projectDir, "src/test/resources/rest-api-spec/api"))
restXpackTests(new File(projectDir, "src/test/resources/rest-api-spec/test"))
restXpackTests(new File(projectDir, "src/yamlRestTest/resources/rest-api-spec/test"))
}
"""

xpackApis.each { api ->
file("x-pack/plugin/src/test/resources/rest-api-spec/api/" + api) << ""
}
xpackTests.each { test ->
file("x-pack/plugin/src/test/resources/rest-api-spec/test/" + test) << ""
file("x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
}

apis.each { api ->
file("rest-api-spec/src/main/resources/rest-api-spec/api/" + api) << ""
}
tests.each { test ->
file("rest-api-spec/src/main/resources/rest-api-spec/test/" + test) << ""
file("rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/" + test) << ""
}
}

Expand Down
13 changes: 3 additions & 10 deletions ...src/main/java/org/elasticsearch/gradle/internal/rest/compat/YamlRestCompatTestPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -87,11 +87,9 @@ public void apply(Project project) {
Provider<CopyRestApiTask> copyCompatYamlSpecTask = project.getTasks()
.register("copyRestCompatApiTask", CopyRestApiTask.class, task -> {
task.dependsOn(bwcMinorConfig);
task.setCoreConfig(bwcMinorConfig);
task.setXpackConfig(bwcMinorConfig);
task.setConfig(bwcMinorConfig);
task.setAdditionalConfig(bwcMinorConfig);
task.getIncludeCore().set(extension.getRestApi().getIncludeCore());
task.getIncludeXpack().set(extension.getRestApi().getIncludeXpack());
task.getInclude().set(extension.getRestApi().getInclude());
task.getOutputResourceDir().set(project.getLayout().getBuildDirectory().dir(compatSpecsDir.toString()));
task.setSourceResourceDir(
yamlCompatTestSourceSet.getResources()
Expand All @@ -102,16 +100,11 @@ public void apply(Project project) {
.orElse(null)
);
task.setSkipHasRestTestCheck(true);
task.setCoreConfigToFileTree(
task.setConfigToFileTree(
config -> project.fileTree(
config.getSingleFile().toPath().resolve(RELATIVE_REST_API_RESOURCES).resolve(RELATIVE_API_PATH)
)
);
task.setXpackConfigToFileTree(
config -> project.fileTree(
config.getSingleFile().toPath().resolve(RELATIVE_REST_XPACK_RESOURCES).resolve(RELATIVE_API_PATH)
)
);
task.setAdditionalConfigToFileTree(
config -> project.fileTree(
getCompatProjectPath(project, config.getSingleFile().toPath()).resolve(RELATIVE_REST_PROJECT_RESOURCES)
Expand Down
95 changes: 26 additions & 69 deletions buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/CopyRestApiTask.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,22 +47,18 @@
public class CopyRestApiTask extends DefaultTask {
private static final String REST_API_PREFIX = "rest-api-spec/api";
private static final String REST_TEST_PREFIX = "rest-api-spec/test";
private final ListProperty<String> includeCore;
private final ListProperty<String> includeXpack;
private final ListProperty<String> include;
private final DirectoryProperty outputResourceDir;
private final DirectoryProperty additionalYamlTestsDir;

private File sourceResourceDir;
private boolean skipHasRestTestCheck;
private FileCollection coreConfig;
private FileCollection xpackConfig;
private FileCollection config;
private FileCollection additionalConfig;
private Function<FileCollection, FileTree> coreConfigToFileTree = FileCollection::getAsFileTree;
private Function<FileCollection, FileTree> xpackConfigToFileTree = FileCollection::getAsFileTree;
private Function<FileCollection, FileTree> configToFileTree = FileCollection::getAsFileTree;
private Function<FileCollection, FileTree> additionalConfigToFileTree = FileCollection::getAsFileTree;

private final PatternFilterable corePatternSet;
private final PatternFilterable xpackPatternSet;
private final PatternFilterable patternSet;
private final ProjectLayout projectLayout;
private final FileSystemOperations fileSystemOperations;
private final ArchiveOperations archiveOperations;
Expand All @@ -75,25 +71,18 @@ public CopyRestApiTask(
ArchiveOperations archiveOperations,
ObjectFactory objectFactory
) {
this.includeCore = objectFactory.listProperty(String.class);
this.includeXpack = objectFactory.listProperty(String.class);
this.include = objectFactory.listProperty(String.class);
this.outputResourceDir = objectFactory.directoryProperty();
this.additionalYamlTestsDir = objectFactory.directoryProperty();
this.corePatternSet = patternSetFactory.create();
this.xpackPatternSet = patternSetFactory.create();
this.patternSet = patternSetFactory.create();
this.projectLayout = projectLayout;
this.fileSystemOperations = fileSystemOperations;
this.archiveOperations = archiveOperations;
}

@Input
public ListProperty<String> getIncludeCore() {
return includeCore;
}

@Input
public ListProperty<String> getIncludeXpack() {
return includeXpack;
public ListProperty<String> getInclude() {
return include;
}

@Input
Expand All @@ -105,29 +94,22 @@ public boolean isSkipHasRestTestCheck() {
@InputFiles
public FileTree getInputDir() {
FileTree coreFileTree = null;
FileTree xpackFileTree = null;
if (includeXpack.get().isEmpty() == false) {
xpackPatternSet.setIncludes(includeXpack.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
xpackFileTree = xpackConfigToFileTree.apply(xpackConfig).matching(xpackPatternSet);
}
boolean projectHasYamlRestTests = skipHasRestTestCheck || projectHasYamlRestTests();
if (includeCore.get().isEmpty() == false || projectHasYamlRestTests) {
if (include.get().isEmpty() == false || projectHasYamlRestTests) {
if (BuildParams.isInternal()) {
corePatternSet.setIncludes(includeCore.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
coreFileTree = coreConfigToFileTree.apply(coreConfig).matching(corePatternSet); // directory on disk
patternSet.setIncludes(include.get().stream().map(prefix -> prefix + "*/**").collect(Collectors.toList()));
coreFileTree = configToFileTree.apply(config).matching(patternSet); // directory on disk
} else {
coreFileTree = coreConfig.getAsFileTree(); // jar file
coreFileTree = config.getAsFileTree(); // jar file
}
}

FileCollection fileCollection = additionalConfig == null
? projectLayout.files(coreFileTree, xpackFileTree)
: projectLayout.files(coreFileTree, xpackFileTree, additionalConfigToFileTree.apply(additionalConfig));
? coreFileTree
: projectLayout.files(coreFileTree, additionalConfigToFileTree.apply(additionalConfig));

// if project has rest tests or the includes are explicitly configured execute the task, else NO-SOURCE due to the null input
return projectHasYamlRestTests || includeCore.get().isEmpty() == false || includeXpack.get().isEmpty() == false
? fileCollection.getAsFileTree()
: null;
return projectHasYamlRestTests || include.get().isEmpty() == false ? fileCollection.getAsFileTree() : null;
}

@OutputDirectory
Expand All @@ -152,9 +134,9 @@ void copy() {
if (BuildParams.isInternal()) {
getLogger().debug("Rest specs for project [{}] will be copied to the test resources.", projectPath);
fileSystemOperations.copy(c -> {
c.from(coreConfigToFileTree.apply(coreConfig));
c.from(configToFileTree.apply(config));
c.into(restSpecOutputDir);
c.include(corePatternSet.getIncludes());
c.include(patternSet.getIncludes());
});
} else {
getLogger().debug(
Expand All @@ -163,26 +145,15 @@ void copy() {
VersionProperties.getElasticsearch()
);
fileSystemOperations.copy(c -> {
c.from(archiveOperations.zipTree(coreConfig.getSingleFile())); // jar file
c.from(archiveOperations.zipTree(config.getSingleFile())); // jar file
c.into(outputResourceDir);
if (includeCore.get().isEmpty()) {
if (include.get().isEmpty()) {
c.include(REST_API_PREFIX + "/**");
} else {
c.include(
includeCore.get().stream().map(prefix -> REST_API_PREFIX + "/" + prefix + "*/**").collect(Collectors.toList())
);
c.include(include.get().stream().map(prefix -> REST_API_PREFIX + "/" + prefix + "*/**").collect(Collectors.toList()));
}
});
}
// only copy x-pack specs if explicitly instructed
if (includeXpack.get().isEmpty() == false) {
getLogger().debug("X-pack rest specs for project [{}] will be copied to the test resources.", projectPath);
fileSystemOperations.copy(c -> {
c.from(xpackConfigToFileTree.apply(xpackConfig));
c.into(restSpecOutputDir);
c.include(xpackPatternSet.getIncludes());
});
}

// copy any additional config
if (additionalConfig != null) {
Expand Down Expand Up @@ -222,38 +193,24 @@ public void setSkipHasRestTestCheck(boolean skipHasRestTestCheck) {
this.skipHasRestTestCheck = skipHasRestTestCheck;
}

public void setCoreConfig(FileCollection coreConfig) {
this.coreConfig = coreConfig;
}

public void setXpackConfig(FileCollection xpackConfig) {
this.xpackConfig = xpackConfig;
public void setConfig(FileCollection config) {
this.config = config;
}

public void setAdditionalConfig(FileCollection additionalConfig) {
this.additionalConfig = additionalConfig;
}

public void setCoreConfigToFileTree(Function<FileCollection, FileTree> coreConfigToFileTree) {
this.coreConfigToFileTree = coreConfigToFileTree;
}

public void setXpackConfigToFileTree(Function<FileCollection, FileTree> xpackConfigToFileTree) {
this.xpackConfigToFileTree = xpackConfigToFileTree;
public void setConfigToFileTree(Function<FileCollection, FileTree> configToFileTree) {
this.configToFileTree = configToFileTree;
}

public void setAdditionalConfigToFileTree(Function<FileCollection, FileTree> additionalConfigToFileTree) {
this.additionalConfigToFileTree = additionalConfigToFileTree;
}

@Internal
public FileCollection getCoreConfig() {
return coreConfig;
public FileCollection getConfig() {
return config;
}

@Internal
public FileCollection getXpackConfig() {
return xpackConfig;
}

}
36 changes: 27 additions & 9 deletions buildSrc/src/main/java/org/elasticsearch/gradle/test/rest/RestResourcesExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,37 +19,58 @@
*/
public class RestResourcesExtension {

final RestResourcesSpec restApi;
final RestResourcesSpec restTests;
private final RestResourcesSpec restApi;
private final XpackRestResourcesSpec restTests;

@Inject
public RestResourcesExtension(ObjectFactory objects) {
restApi = new RestResourcesSpec(objects);
restTests = new RestResourcesSpec(objects);
restTests = new XpackRestResourcesSpec(objects);
}

void restApi(Action<? super RestResourcesSpec> spec) {
spec.execute(restApi);
}

void restTests(Action<? super RestResourcesSpec> spec) {
void restTests(Action<? super XpackRestResourcesSpec> spec) {
if (BuildParams.isInternal() == false) {
// TODO: Separate this out into an "internal" plugin so we don't even expose this API to external folks
throw new UnsupportedOperationException("Including tests is not supported from external builds.");
}
spec.execute(restTests);
}

public RestResourcesSpec getRestApi() {
return restApi;
}

public RestResourcesSpec getRestTests() {
public XpackRestResourcesSpec getRestTests() {
return restTests;
}

public static class RestResourcesSpec {

private final ListProperty<String> include;

RestResourcesSpec(ObjectFactory objects) {
include = objects.listProperty(String.class);
}

public void include(String... include) {
this.include.addAll(include);
}

public ListProperty<String> getInclude() {
return include;
}
}

public static class XpackRestResourcesSpec {

private final ListProperty<String> includeCore;
private final ListProperty<String> includeXpack;

RestResourcesSpec(ObjectFactory objects) {
XpackRestResourcesSpec(ObjectFactory objects) {
includeCore = objects.listProperty(String.class);
includeXpack = objects.listProperty(String.class);
}
Expand All @@ -59,9 +80,6 @@ public void includeCore(String... include) {
}

public void includeXpack(String... include) {
if (BuildParams.isInternal() == false) {
throw new IllegalStateException("Can not include x-pack rest resources from an external build.");
}
this.includeXpack.addAll(include);
}

Expand Down
Loading

0 comments on commit 6339691

Please sign in to comment.