Add a task to run forbiddenapis using cli (#32076)

* Add a task to run forbiddenapis using cli

Add a task that offers an equivalent check to the forbidden APIs plugin,
but runs it using the forbiddenAPIs CLI instead.

This isn't wired into precommit first, and doesn't work for projects
that require specific signatures, etc. It's meant to show how this can
be used. The next step is to make a custom task type and configure it
based on the project extension from the pugin and make some minor
adjustments to some build scripts as we can't  bee 100% compatible with
that at least due to how additional signatures are passed.

Notes:
- there's no `--target` for the CLI version so we have to pass in
specific bundled signature names
- the cli task already wires to `runtimeJavaHome`
- no equivalent for `failOnUnsupportedJava = false` but doesn't seem to
be a problem. Tested with Java 8 to 11
- there's no way to pass additional signatures as URL, these will have
to be on the file system, and can't be resources on the cp unless we
rely on how forbiddenapis is implemented and mimic them as boundled
signatures.
- the average of 3 runs is 4% slower using the CLI for :server.
( `./gradlew clean :server:forbiddenApis` vs `./gradlew clean
:server:forbiddenApisCli`)
- up-to-date checks don't work on the cli task yet, that will happen
with the custom task.

See also: #31715

Author: alpar-t

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -20,10 +20,14 @@ package org.elasticsearch.gradle.precommit
 
 import de.thetaphi.forbiddenapis.gradle.CheckForbiddenApis
 import de.thetaphi.forbiddenapis.gradle.ForbiddenApisPlugin
+import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.file.FileCollection
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.plugins.quality.Checkstyle
+import org.gradle.api.tasks.JavaExec
+import org.gradle.api.tasks.StopExecutionException
 
 /**
  * Validation tasks which should be run before committing. These run before tests.
@@ -40,7 +44,11 @@ class PrecommitTasks {
             project.tasks.create('licenseHeaders', LicenseHeadersTask.class),
             project.tasks.create('filepermissions', FilePermissionsTask.class),
             project.tasks.create('jarHell', JarHellTask.class),
-            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)]
+            project.tasks.create('thirdPartyAudit', ThirdPartyAuditTask.class)
+        ]
+
+        // Configure it but don't add it as a dependency yet
+        configureForbiddenApisCli(project)
 
         // tasks with just tests don't need dependency licenses, so this flag makes adding
         // the task optional
@@ -96,9 +104,58 @@ class PrecommitTasks {
         }
         Task forbiddenApis = project.tasks.findByName('forbiddenApis')
         forbiddenApis.group = "" // clear group, so this does not show up under verification tasks
+
         return forbiddenApis
     }
 
+    private static Task configureForbiddenApisCli(Project project) {
+        project.configurations.create("forbiddenApisCliJar")
+        project.dependencies {
+            forbiddenApisCliJar 'de.thetaphi:forbiddenapis:2.5'
+        }
+        Task forbiddenApisCli = project.tasks.create('forbiddenApisCli')
+
+        project.sourceSets.forEach { sourceSet ->
+            forbiddenApisCli.dependsOn(
+                project.tasks.create(sourceSet.getTaskName('forbiddenApisCli', null), JavaExec) {
+                    ExportElasticsearchBuildResourcesTask buildResources = project.tasks.getByName('buildResources')
+                    dependsOn(buildResources)
+                    classpath = project.files(
+                            project.configurations.forbiddenApisCliJar,
+                            sourceSet.compileClasspath,
+                            sourceSet.runtimeClasspath
+                    )
+                    main = 'de.thetaphi.forbiddenapis.cli.CliMain'
+                    executable = "${project.runtimeJavaHome}/bin/java"
+                    args "-b", 'jdk-unsafe-1.8'
+                    args "-b", 'jdk-deprecated-1.8'
+                    args "-b", 'jdk-non-portable'
+                    args "-b", 'jdk-system-out'
+                    args "-f", buildResources.copy("forbidden/jdk-signatures.txt")
+                    args "-f", buildResources.copy("forbidden/es-all-signatures.txt")
+                    args "--suppressannotation", '**.SuppressForbidden'
+                    if (sourceSet.name == 'test') {
+                        args "-f", buildResources.copy("forbidden/es-test-signatures.txt")
+                        args "-f", buildResources.copy("forbidden/http-signatures.txt")
+                    } else {
+                        args "-f", buildResources.copy("forbidden/es-server-signatures.txt")
+                    }
+                    dependsOn sourceSet.classesTaskName
+                    doFirst {
+                        // Forbidden APIs expects only existing dirs, and requires at least one
+                        FileCollection existingOutputs = sourceSet.output.classesDirs
+                                .filter { it.exists() }
+                        if (existingOutputs.isEmpty()) {
+                            throw new StopExecutionException("${sourceSet.name} has no outputs")
+                        }
+                        existingOutputs.forEach { args "-d", it }
+                    }
+                }
+            )
+        }
+        return forbiddenApisCli
+    }
+
     private static Task configureCheckstyle(Project project) {
         // Always copy the checkstyle configuration files to 'buildDir/checkstyle' since the resources could be located in a jar
         // file. If the resources are located in a jar, Gradle will fail when it tries to turn the URL into a file

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/StandaloneRestTestPlugin.groovy

@@ -22,15 +22,14 @@ package org.elasticsearch.gradle.test
 
 import com.carrotsearch.gradle.junit4.RandomizedTestingPlugin
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.ExportElasticsearchBuildResourcesTask
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.precommit.PrecommitTasks
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
-import org.gradle.api.Task
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.compile.JavaCompile
-
 /**
  * Configures the build to compile tests against Elasticsearch's test framework
  * and run REST tests. Use BuildPlugin if you want to build main code as well
@@ -48,6 +47,7 @@ public class StandaloneRestTestPlugin implements Plugin<Project> {
         project.pluginManager.apply(JavaBasePlugin)
         project.pluginManager.apply(RandomizedTestingPlugin)
 
+        project.getTasks().create("buildResources", ExportElasticsearchBuildResourcesTask)
         BuildPlugin.globalBuildInfo(project)
         BuildPlugin.configureRepositories(project)
 

buildSrc/src/main/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTask.java

@@ -21,7 +21,6 @@
 import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
 import org.gradle.api.file.DirectoryProperty;
-import org.gradle.api.file.RegularFileProperty;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.tasks.Classpath;
@@ -31,6 +30,7 @@
 import org.gradle.api.tasks.StopExecutionException;
 import org.gradle.api.tasks.TaskAction;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.InputStream;
 import java.nio.file.Files;
@@ -82,14 +82,14 @@ public void setOutputDir(DirectoryProperty outputDir) {
         this.outputDir = outputDir;
     }
 
-    public RegularFileProperty take(String resource) {
+    public File copy(String resource) {
         if (getState().getExecuted() || getState().getExecuting()) {
             throw new GradleException("buildResources can't be configured after the task ran. " +
                 "Make sure task is not used after configuration time"
             );
         }
         resources.add(resource);
-        return getProject().getLayout().fileProperty(outputDir.file(resource));
+        return outputDir.file(resource).get().getAsFile();
     }
 
     @TaskAction
@@ -101,12 +101,13 @@ public void doExport() {
             .forEach(resourcePath -> {
                 Path destination = outputDir.get().file(resourcePath).getAsFile().toPath();
                 try (InputStream is = getClass().getClassLoader().getResourceAsStream(resourcePath)) {
+                    Files.createDirectories(destination.getParent());
                     if (is == null) {
                         throw new GradleException("Can't export `" + resourcePath + "` from build-tools: not found");
                     }
                     Files.copy(is, destination);
                 } catch (IOException e) {
-                    throw new GradleException("Can't write resource `" + resourcePath + "` to " + destination);
+                    throw new GradleException("Can't write resource `" + resourcePath + "` to " + destination, e);
                 }
             });
     }

buildSrc/src/test/java/org/elasticsearch/gradle/ExportElasticsearchBuildResourcesTaskIT.java

@@ -23,6 +23,7 @@
 import org.gradle.testkit.runner.BuildResult;
 import org.gradle.testkit.runner.GradleRunner;
 
+
 public class ExportElasticsearchBuildResourcesTaskIT extends GradleIntegrationTestCase {
 
     public static final String PROJECT_NAME = "elasticsearch-build-resources";
@@ -59,6 +60,7 @@ public void testImplicitTaskDependencyCopy() {
             .withArguments("clean", "sampleCopyAll", "-s", "-i")
             .withPluginClasspath()
             .build();
+
         assertTaskSuccessfull(result, ":buildResources");
         assertTaskSuccessfull(result, ":sampleCopyAll");
         assertBuildFileExists(result, PROJECT_NAME, "sampleCopyAll/checkstyle.xml");

buildSrc/src/testKit/elasticsearch-build-resources/build.gradle

@@ -6,7 +6,7 @@ ext.licenseFile = file("$buildDir/dummy/license")
 ext.noticeFile = file("$buildDir/dummy/notice")
 
 buildResources {
-    take 'checkstyle.xml'
+    copy 'checkstyle.xml'
 }
 
 task sampleCopyAll(type: Sync) {
@@ -17,13 +17,13 @@ task sampleCopyAll(type: Sync) {
 
 task sample {
     // This does not work, task dependencies can't be providers
-    // dependsOn exportBuildResources.resource('minimumRuntimeVersion')
+    // dependsOn buildResources.resource('minimumRuntimeVersion')
     // Nor does this, despite https://github.com/gradle/gradle/issues/3811
-    // dependsOn exportBuildResources.outputDir
+    // dependsOn buildResources.outputDir
     // for now it's just
     dependsOn buildResources
     // we have to refference it at configuration time in order to be picked up
-    ext.checkstyle_suppressions = buildResources.take('checkstyle_suppressions.xml')
+    ext.checkstyle_suppressions = buildResources.copy('checkstyle_suppressions.xml')
     doLast {
         println "This task is using ${file(checkstyle_suppressions)}"
     }
@@ -33,6 +33,6 @@ task noConfigAfterExecution {
     dependsOn buildResources
     doLast {
         println "This should cause an error because we are refferencing " +
-                "${buildResources.take('checkstyle_suppressions.xml')} after the `buildResources` task has ran."
+                "${buildResources.copy('checkstyle_suppressions.xml')} after the `buildResources` task has ran."
     }
 }