[8.19] Update Gradle wrapper to 9.0.0 (#2438)

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/BuildPlugin.groovy

@@ -839,6 +839,8 @@ class BuildPlugin implements Plugin<Project> {
 
         if (!project.path.startsWith(":qa")) {
             TaskProvider<DependencyLicensesTask> dependencyLicenses = project.tasks.register('dependencyLicenses', DependencyLicensesTask.class) {
+                licensesDir = project.file("licenses")
+                outputMarker = new File(project.getBuildDir(), "dependencyLicense")
                 dependencies = project.configurations.runtimeClasspath
                 mapping from: /hadoop-.*/, to: 'hadoop'
                 mapping from: /hive-.*/, to: 'hive'

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/buildtools/AntFixture.groovy

@@ -23,6 +23,7 @@ import org.apache.tools.ant.taskdefs.condition.Os
 import org.gradle.api.GradleException
 import org.gradle.api.tasks.Internal
 import org.gradle.api.tasks.TaskProvider
+import groovy.ant.AntBuilder;
 
 /**
  * A fixture for integration tests which runs in a separate process launched by Ant.

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/buildtools/AntFixtureStop.groovy

@@ -25,7 +25,7 @@ import org.gradle.api.tasks.Internal
 
 import javax.inject.Inject
 
-class AntFixtureStop extends LoggedExec {
+abstract class AntFixtureStop extends LoggedExec {
 
     @Internal
     AntFixture fixture

buildSrc/src/main/groovy/org/elasticsearch/hadoop/gradle/buildtools/AntTask.groovy

@@ -30,6 +30,7 @@ import org.gradle.api.tasks.TaskAction
 
 import javax.inject.Inject
 import java.nio.charset.Charset
+import groovy.ant.AntBuilder
 
 /**
  * A task which will run ant commands.

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/DependencyLicensesTask.java

@@ -122,7 +122,7 @@ public abstract class DependencyLicensesTask extends DefaultTask {
     /**
      * The directory to find the license and sha files in.
      */
-    private File licensesDir = new File(getProject().getProjectDir(), "licenses");
+    private File licensesDir;
 
     /**
      * A map of patterns to prefix, used to find the LICENSE and NOTICE file.
@@ -134,6 +134,13 @@ public abstract class DependencyLicensesTask extends DefaultTask {
      */
     private Set<String> ignoreShas = new HashSet<>();
 
+    /**
+     * A marker output directory to allow this task being up-to-date.
+     * The check logic is exception driven so a failed tasks will not be defined
+     * by this output but when successful we can safely mark the task as up-to-date.
+     */
+    private File outputMarker;
+
     /**
      * Add a mapping from a regex pattern for the jar name, to a prefix to find
      * the LICENSE and NOTICE file for that jar.
@@ -257,12 +264,13 @@ public void checkDependencies() throws IOException, NoSuchAlgorithmException {
 
     }
 
-    // This is just a marker output folder to allow this task being up-to-date.
-    // The check logic is exception driven so a failed tasks will not be defined
-    // by this output but when successful we can safely mark the task as up-to-date.
     @OutputDirectory
     public File getOutputMarker() {
-        return new File(getProject().getBuildDir(), "dependencyLicense");
+        return outputMarker;
+    }
+
+    public void setOutputMarker(File outputMarker) {
+        this.outputMarker = outputMarker;
     }
 
     private void failIfAnyMissing(String item, Boolean exists, String type) {

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/LoggedExec.java

@@ -50,7 +50,7 @@
  * A wrapper around gradle's Exec task to capture output and log on error.
  */
 @SuppressWarnings("unchecked")
-public class LoggedExec extends Exec implements FileSystemOperationsAware {
+public abstract class LoggedExec extends Exec implements FileSystemOperationsAware {
 
     private static final Logger LOGGER = Logging.getLogger(LoggedExec.class);
     private Consumer<Logger> outputLogger;

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/Util.java

@@ -25,7 +25,7 @@
 import org.gradle.api.GradleException;
 import org.gradle.api.Project;
 import org.gradle.api.file.FileTree;
-import org.gradle.api.plugins.JavaPluginConvention;
+import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.util.PatternFilterable;
 
@@ -134,7 +134,7 @@ public static FileTree getJavaTestAndMainSourceResources(Project project, Action
      * @return An Optional that contains the Java test SourceSet if it exists.
      */
     public static Optional<SourceSet> getJavaTestSourceSet(Project project) {
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.TEST_SOURCE_SET_NAME));
     }
@@ -144,7 +144,7 @@ public static Optional<SourceSet> getJavaTestSourceSet(Project project) {
      * @return An Optional that contains the Java main SourceSet if it exists.
      */
     public static Optional<SourceSet> getJavaMainSourceSet(Project project) {
-        return project.getConvention().findPlugin(JavaPluginConvention.class) == null
+        return project.getExtensions().findByType(JavaPluginExtension.class) == null
             ? Optional.empty()
             : Optional.ofNullable(GradleUtils.getJavaSourceSets(project).findByName(SourceSet.MAIN_SOURCE_SET_NAME));
     }

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/buildtools/info/GlobalBuildInfoPlugin.java

@@ -27,6 +27,7 @@
 import org.gradle.api.logging.Logger;
 import org.gradle.api.logging.Logging;
 import org.gradle.api.provider.Provider;
+import org.gradle.process.ExecOperations;
 import org.gradle.api.provider.ProviderFactory;
 import org.gradle.internal.jvm.Jvm;
 import org.gradle.internal.jvm.inspection.JavaInstallationRegistry;
@@ -75,13 +76,16 @@ public class GlobalBuildInfoPlugin implements Plugin<Project> {
     private final JavaInstallationRegistry javaInstallationRegistry;
     private final JvmMetadataDetector metadataDetector;
     private final ProviderFactory providers;
+    private final ExecOperations execOperations;
 
     @Inject
     public GlobalBuildInfoPlugin(
+            ExecOperations execOperations,
         JavaInstallationRegistry javaInstallationRegistry,
         JvmMetadataDetector metadataDetector,
         ProviderFactory providers
     ) {
+        this.execOperations = execOperations;
         this.javaInstallationRegistry = javaInstallationRegistry;
         this.metadataDetector = metadataDetector;
         this.providers = providers;
@@ -249,7 +253,7 @@ private File findRuntimeJavaHome() {
     }
 
     private String findJavaHome(String version) {
-        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv").forUseAtConfigurationTime();
+        Provider<String> javaHomeNames = providers.gradleProperty("org.gradle.java.installations.fromEnv");
         String javaHomeEnvVar = getJavaHomeEnvVarName(version);
 
         // Provide a useful error if we're looking for a Java home version that we haven't told Gradle about yet
@@ -318,15 +322,18 @@ private static int findDefaultParallel(Project project) {
                 }
                 _defaultParallel = socketToCore.values().stream().mapToInt(i -> i).sum();
             } else if (OS.current() == OS.MAC) {
-                // Ask macOS to count physical CPUs for us
-                ByteArrayOutputStream stdout = new ByteArrayOutputStream();
-                project.exec(spec -> {
-                    spec.setExecutable("sysctl");
-                    spec.args("-n", "hw.physicalcpu");
-                    spec.setStandardOutput(stdout);
-                });
-
-                _defaultParallel = Integer.parseInt(stdout.toString().trim());
+                // On Apple silicon, we only want to use the performance cores
+                boolean isAppleSilicon = project.getProviders().systemProperty("os.arch").getOrElse("").equals("aarch64");
+                String query = isAppleSilicon && isMontereyOrNewer(project.getProviders())
+                        ? "hw.perflevel0.physicalcpu"
+                        : "hw.physicalcpu";
+
+                String stdout = project.getProviders().exec(execSpec ->
+                        execSpec.commandLine("sysctl", "-n", query)
+                ).getStandardOutput().getAsText().get();
+
+
+                _defaultParallel = Integer.parseInt(stdout.trim());
             }
 
             _defaultParallel = Runtime.getRuntime().availableProcessors() / 2;
@@ -335,6 +342,20 @@ private static int findDefaultParallel(Project project) {
         return _defaultParallel;
     }
 
+    private final static int MACOS_MONTEREY_MAJOR_VERSION = 12;
+
+    private static boolean isMontereyOrNewer(ProviderFactory providers) {
+        String rawVersion = providers.systemProperty("os.version").getOrElse("").trim();
+        if (rawVersion.isEmpty()) {
+            LOGGER.warn("Failed to validate MacOs version.");
+            return false;
+        }
+
+        String majorVersion = rawVersion.split("\\.")[0];
+
+        return Integer.parseInt(majorVersion) >= MACOS_MONTEREY_MAJOR_VERSION;
+    }
+
     public static GitInfo gitInfo(File rootDir) {
         try {
             /*

buildSrc/src/main/java/org/elasticsearch/hadoop/gradle/scala/SparkVariantPlugin.java

@@ -39,12 +39,9 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.SourceDirectorySet;
 import org.gradle.api.internal.artifacts.repositories.resolver.ComponentMetadataDetailsAdapter;
-import org.gradle.api.internal.tasks.DefaultScalaSourceSet;
 import org.gradle.api.plugins.BasePlugin;
-import org.gradle.api.plugins.Convention;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPlugin;
-import org.gradle.api.plugins.JavaPluginConvention;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.tasks.SourceSet;
 import org.gradle.api.tasks.SourceSetContainer;

gradle/wrapper/gradle-wrapper.properties

@@ -1,7 +1,7 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionSha256Sum=443c9c8ee2ac1ee0e11881a40f2376d79c66386264a44b24a9f8ca67e633375f
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.2-all.zip
+distributionSha256Sum=f759b8dd5204e2e3fa4ca3e73f452f087153cf81bac9561eeb854229cc2c5365
+distributionUrl=https\://services.gradle.org/distributions/gradle-9.0.0-all.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

gradlew

@@ -114,7 +114,7 @@ case "$( uname )" in                #(
   NONSTOP* )        nonstop=true ;;
 esac
 
-CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+CLASSPATH="\\\"\\\""
 
 
 # Determine the Java command to use to start the JVM.
@@ -213,7 +213,7 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
 set -- \
         "-Dorg.gradle.appname=$APP_BASE_NAME" \
         -classpath "$CLASSPATH" \
-        org.gradle.wrapper.GradleWrapperMain \
+        -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \
         "$@"
 
 # Stop when "xargs" is not available.

gradlew.bat

@@ -70,11 +70,11 @@ goto fail
 :execute
 @rem Setup the command line
 
-set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+set CLASSPATH=
 
 
 @rem Execute Gradle
-"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %*
 
 :end
 @rem End local scope for the variables with windows NT shell

thirdparty/build.gradle

@@ -1,26 +1,10 @@
 import org.elasticsearch.hadoop.gradle.BuildPlugin
 
-
-
-buildscript {
-  repositories {
-    maven {
-      url = 'https://jitpack.io'
-    }
-    mavenCentral()
-  }
-  dependencies {
-    classpath 'com.github.breskeby:shadow:3b035f2'
-  }
-}
-
 plugins {
     id 'es.hadoop.build'
+    id 'com.gradleup.shadow' version "9.0.1"
 }
 
-apply plugin: 'com.github.johnrengelman.shadow'
-
-
 description = "Elasticsearch Hadoop Shaded Third-Party Libraries"
 
 configurations {