Generate hardware-specific defaults for gradle parallelism on the fir…

…st build run (any task). Add some explanations on how to tweak local settings even further (gradlew :helpLocalSettings

.gitignore

@@ -33,3 +33,6 @@ __pycache__
 build/
 .gradle/
 .idea/
+
+# Ignore the generated local settings file.
+gradle.properties

build.gradle

@@ -16,6 +16,8 @@ allprojects {
 // if the build file is incorrectly written and evaluates something
 // eagerly).
 
+apply from: file('gradle/generate-defaults.gradle')
+
 // CI systems.
 apply from: file('gradle/ci/buildscan.gradle')
 apply from: file('gradle/ci/travis.gradle')

gradle/defaults.gradle

@@ -15,4 +15,19 @@ allprojects {
   // Artifacts will have names after full gradle project path
   // so :solr:core will have solr-core.jar, etc.
   project.archivesBaseName = project.path.replaceAll("^:", "").replace(':', '-')
+
+  ext {
+    // Utility method to support passing overrides via -P or -D.
+    propertyOrDefault = { propName, defValue ->
+      def result
+      if (project.hasProperty(propName)) {
+        result = project.getProperty(propName)
+      } else if (System.properties.containsKey(propName)) {
+        result = System.properties.get(propName)
+      } else {
+        result = defValue
+      }
+      return result
+    }
+  }
 }

gradle/generate-defaults.gradle

@@ -0,0 +1,73 @@
+
+// This script tries to guess sensible defaults for gradle parallelism
+// and local machine's resources and save them under 'gradle.properties'.
+
+def hasDefaults = rootProject.file("gradle.properties").exists()
+
+// If we don't have the defaults yet, create them and re-run the build
+// recursively with the same parameters as originally passed.
+//
+// Sadly, the recursive build doesn't seem to pick up the parallelism
+// tweaks from gradle.properties file.
+
+if (!hasDefaults) {
+  configure(rootProject) {
+    task setupLocalDefaultsOnce(type: GradleBuild) {
+      // Approximate a common-sense default for running gradle with parallel
+      // workers: half the count of available cpus but not more than 12.
+      def cpus = Runtime.runtime.availableProcessors()
+      def maxWorkers = (int) Math.max(1d, Math.min(cpus * 0.5d, 12))
+      def testsJvms = (int) Math.max(1d, Math.min(cpus * 0.5d, 4))
+
+      // Reuse the same set of parameters for the recursive invocation and apply
+      // some of these eagerly.
+      def startParams = gradle.startParameter.newInstance()
+      startParams.setParallelProjectExecutionEnabled(true)
+      startParams.setMaxWorkerCount(maxWorkers)
+      startParameter(startParams)
+
+      // Write the defaults for this machine.
+      rootProject.file("gradle.properties").write(
+          [
+              "# These settings have been generated automatically on the first run.",
+              "# See gradlew :helpLocalSettings for more information.",
+              "systemProp.file.encoding=UTF-8",
+              "org.gradle.daemon=true",
+              "org.gradle.jvmargs=-Xmx1g",
+              "org.gradle.parallel=true",
+              "org.gradle.priority=normal",
+              "",
+              "# Maximum number of parallel gradle workers.",
+              "org.gradle.workers.max=${maxWorkers}",
+              "",
+              "# Maximum number of test JVMs forked per test task.",
+              "tests.jvms=${testsJvms}"
+          ].join("\n"), "UTF-8")
+
+      doFirst {
+        logger.log(LogLevel.WARN, "\nIMPORTANT. This is the first time you ran the build. " +
+            "I wrote some sane defaults (for this machine) to 'gradle.properties', " +
+            "they will be picked up on consecutive gradle invocations (not this one).\n\n" +
+            "Run gradlew :helpLocalSettings for more information.")
+      }
+    }
+  }
+
+  // Disable any tasks in this build, they were forked recursively.
+  gradle.taskGraph.whenReady { graph ->
+    graph.allTasks.each { task ->
+      if (task != rootProject.setupLocalDefaultsOnce) {
+        task.enabled = false
+      }
+    }
+  }
+
+  // Make all tasks depend on local setup to make sure it'll run though.
+  allprojects {
+    tasks.all { task ->
+      if (task != rootProject.setupLocalDefaultsOnce) {
+        task.dependsOn rootProject.setupLocalDefaultsOnce
+      }
+    }
+  }
+}

gradle/help.gradle

@@ -6,6 +6,7 @@ configure(rootProject) {
       ["Ant", "help/ant.txt", "Ant-gradle migration help."],
       ["Tests", "help/tests.txt", "Tests, filtering, beasting, etc."],
       ["ForbiddenApis", "help/forbiddenApis.txt", "How to add/apply rules for forbidden APIs."],
+      ["LocalSettings", "help/localSettings.txt", "Local settings, overrides and build performance tweaks."]
   ]
 
   helpFiles.each { section, path, sectionInfo ->
@@ -24,7 +25,8 @@ configure(rootProject) {
       println "This is an experimental Lucene/Solr gradle build. See some"
       println "guidelines, ant-equivalent commands etc. under help/*; or type:"
       helpFiles.each { section, path, sectionInfo ->
-        println "  gradlew :help${section}  # ${sectionInfo}"
+        println String.format(Locale.ROOT,
+            "  gradlew :help%-14s # %s", section, sectionInfo)
       }
     }
   }

gradle/testing/defaults-tests.gradle

@@ -13,8 +13,7 @@ allprojects {
 
       useJUnit()
 
-      // Set up default parallel execution limits.
-      maxParallelForks = (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 3.0))
+      maxParallelForks = propertyOrDefault("tests.jvms", (int) Math.max(1, Math.min(Runtime.runtime.availableProcessors() / 2.0, 4.0)))
 
       minHeapSize = "256m"
       maxHeapSize = "512m"

gradle/testing/randomization.gradle

@@ -1,23 +1,6 @@
 
 // Configure test randomization seeds and derived test properties.
 
-allprojects {
-  ext {
-    // Support passing overrides via -P or -D.
-    propertyOrDefault = { propName, defValue ->
-      def result
-      if (project.hasProperty(propName)) {
-        result = project.getProperty(propName)
-      } else if (System.properties.containsKey(propName)) {
-        result = System.properties.get(propName)
-      } else {
-        result = defValue
-      }
-      return result
-    }
-  }
-}
-
 // Pick the "root" seed from which everything else is derived.
 configure(rootProject) {
   ext {

gradle/testing/slowest-tests-at-end.gradle

@@ -8,7 +8,7 @@ allprojects {
       def duration = (result.getEndTime() - result.getStartTime())
 
       allTests << [
-        name    : "${desc.className.replaceAll('.+\\.', "")}.${desc.name} (${project.name})",
+        name    : "${desc.className.replaceAll('.+\\.', "")}.${desc.name} (${project.path})",
         duration: duration
       ]
     }

help/localSettings.txt

@@ -0,0 +1,46 @@
+Local developer settings
+========================
+
+The first invocation of any task in Lucene/Solr gradle build will generate
+and save a project-local 'gradle.properties' file. This file contains
+the defaults you may (but don't have to) tweak for your particular hardware
+(or taste).
+
+This is an overview of some of these settings.
+
+Parallelism
+-----------
+
+Gradle build can run tasks in parallel but by default it consumes all CPU cores which
+is too optimistic a default for Lucene/Solr tests. You can disable the parallelism
+entirely or assign it a 'low' priority with these properties:
+
+org.gradle.parallel=[true, false]
+org.gradle.priority=[normal, low]
+
+The default level of parallelism is computed based on the number of cores on
+your machine (on the first run of gradle build). By default these are fairly conservative
+settings (half the number of cores for workers, for example):
+
+org.gradle.workers.max=[X]
+tests.jvms=[N <= X]
+
+The number of test JVMs can be lower than the number of workers: this just means
+that two projects can run tests in parallel to saturate all the workers. The I/O and memory
+bandwidth limits will kick in quickly so even if you have a very beefy machine bumping
+it too high may not help.
+
+You can always override these settings locally using command line as well:
+gradlew -Ptests.jvms=N --max-workers=X
+
+Gradle Daemon
+-------------
+
+The gradle daemon is a background process that keeps an evaluated copy of the project
+structure, some caches, etc. It speeds up repeated builds quite a bit but if you don't
+like the idea of having a (sizeable) background process running in the background,
+disable it.
+
+org.gradle.daemon=[true, false]
+org.gradle.jvmargs=...
+