Merge branch '6.x' into ccr-6.x

* 6.x:
  Docs: Remove duplicate test setup
  Docs: Fix description of percentile ranks example example (#31652)
  Remove deprecation warnings to prepare for Gradle 5 (sourceSets.main.output.classesDirs) (#30389)
  Do not check for Azure container existence (#31617)
  Merge AwsS3Service and InternalAwsS3Service in a S3Service class (#31580)
  Remove item listed in 6.3 notes that's not in 6.3 (#31623)
  remove unused import
  Upgrade gradle wrapper to 4.8 (#31525)
  Only set vm.max_map_count if greater than default (#31512)
  QA: Merge query-builder-bwc to restart test (#30979)
  Docs: Skip xpack snippet tests if no xpack (#31619)
  [TEST] Fix RemoteClusterConnectionTests
  Remove legacy MetaDataStateFormat (#31603)
  [TEST] call yaml client close method from test suite (#31591)
  [TEST] Close additional clients created while running yaml tests (#31575)
  Node selector per client rather than per request (#31471)
  Preserve thread context when connecting to remote cluster (#31574)
  Remove redundant 'minimum_should_match'
  Unify headers for full text queries
  JDBC driver prepared statement set* methods  (#31494)
  add logging breaking changes from 5.3 to 6.0 (#31583)
  Fix a formatting issue in the docvalue_fields documentation. (#31563)
  Improve robustness of geo shape parser for malformed shapes
  QA: Create xpack yaml features (#31403)

Author: dnhatn

benchmarks/build.gradle

@@ -29,10 +29,11 @@ buildscript {
 }
 
 apply plugin: 'elasticsearch.build'
-// build an uberjar with all benchmarks
-apply plugin: 'com.github.johnrengelman.shadow'
-// have the shadow plugin provide the runShadow task
-apply plugin: 'application'
+
+// order of this section matters, see: https://github.com/johnrengelman/shadow/issues/336
+apply plugin: 'application' // have the shadow plugin provide the runShadow task
+mainClassName = 'org.openjdk.jmh.Main'
+apply plugin: 'com.github.johnrengelman.shadow' // build an uberjar with all benchmarks
 
 // Not published so no need to assemble
 tasks.remove(assemble)

build.gradle

@@ -320,6 +320,9 @@ gradle.projectsEvaluated {
       // :test:framework:test cannot run before and after :server:test
       return
     }
+    if (tasks.findByPath('test') != null && tasks.findByPath('integTest') != null) {
+      integTest.mustRunAfter test
+    }
     configurations.all { Configuration configuration ->
       /*
        * The featureAwarePlugin configuration has a dependency on x-pack:plugin:core and x-pack:plugin:core has a dependency on the
@@ -569,3 +572,28 @@ gradle.projectsEvaluated {
     }
   }
 }
+
+if (System.properties.get("build.compare") != null) {
+  apply plugin: 'compare-gradle-builds'
+  compareGradleBuilds {
+    ext.referenceProject = System.properties.get("build.compare")
+    doFirst {
+      if (file(referenceProject).exists() == false) {
+        throw new GradleException(
+                "Use git worktree to check out a version to compare against to ../elasticsearch_build_reference"
+        )
+      }
+    }
+    sourceBuild {
+      gradleVersion = "4.7" // does not default to gradle weapper of project dir, but current version
+      projectDir = referenceProject
+      tasks = ["clean", "assemble"]
+      arguments = ["-Dbuild.compare_friendly=true"]
+    }
+    targetBuild {
+      tasks = ["clean", "assemble"]
+      // use -Dorg.gradle.java.home= to alter jdk versions
+      arguments = ["-Dbuild.compare_friendly=true"]
+    }
+  }
+}

buildSrc/build.gradle

@@ -17,11 +17,13 @@
  * under the License.
  */
 
-import java.nio.file.Files
 
-import org.gradle.util.GradleVersion
+import java.nio.file.Files
 
-apply plugin: 'groovy'
+plugins {
+  id 'java-gradle-plugin'
+  id 'groovy'
+}
 
 group = 'org.elasticsearch.gradle'
 
@@ -83,9 +85,10 @@ repositories {
 }
 
 dependencies {
-  compile gradleApi()
   compile localGroovy()
   compile "com.carrotsearch.randomizedtesting:junit4-ant:${props.getProperty('randomizedrunner')}"
+  compile "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
+
   compile("junit:junit:${props.getProperty('junit')}") {
     transitive = false
   }
@@ -97,31 +100,30 @@ dependencies {
   compile 'de.thetaphi:forbiddenapis:2.5'
   compile 'org.apache.rat:apache-rat:0.11'
   compile "org.elasticsearch:jna:4.5.1"
+  testCompile "junit:junit:${props.getProperty('junit')}"
 }
 
+
 // Gradle 2.14+ removed ProgressLogger(-Factory) classes from the public APIs
 // Use logging dependency instead
 // Gradle 4.3.1 stopped releasing the logging jars to jcenter, just use the last available one
 GradleVersion logVersion = GradleVersion.current() > GradleVersion.version('4.3') ? GradleVersion.version('4.3') : GradleVersion.current()
 
 dependencies {
   compileOnly "org.gradle:gradle-logging:${logVersion.getVersion()}"
-  compile 'ru.vyarus:gradle-animalsniffer-plugin:1.2.0' // Gradle 2.14 requires a version > 1.0.1
 }
 
 /*****************************************************************************
  *                         Bootstrap repositories                            *
  *****************************************************************************/
 // this will only happen when buildSrc is built on its own during build init
 if (project == rootProject) {
-
   repositories {
     if (System.getProperty("repos.mavenLocal") != null) {
       mavenLocal()
     }
     mavenCentral()
   }
-  test.exclude 'org/elasticsearch/test/NamingConventionsCheckBadClasses*'
 }
 
 /*****************************************************************************
@@ -146,9 +148,6 @@ if (project != rootProject) {
   jarHell.enabled = false
   thirdPartyAudit.enabled = false
 
-  // test for elasticsearch.build tries to run with ES...
-  test.enabled = false
-
   // TODO: re-enable once randomizedtesting gradle code is published and removed from here
   licenseHeaders.enabled = false
 
@@ -159,14 +158,7 @@ if (project != rootProject) {
   }
 
   namingConventions {
-    testClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$UnitTestCase'
-    integTestClass = 'org.elasticsearch.test.NamingConventionsCheckBadClasses$IntegTestCase'
-  }
-
-  task namingConventionsMain(type: org.elasticsearch.gradle.precommit.NamingConventionsTask) {
-    checkForTestsInMain = true
-    testClass = namingConventions.testClass
-    integTestClass = namingConventions.integTestClass
+    testClass = 'org.elasticsearch.gradle.test.GradleUnitTestCase'
+    integTestClass = 'org.elasticsearch.gradle.test.GradleIntegrationTestCase'
   }
-  precommit.dependsOn namingConventionsMain
 }

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingPlugin.groovy

@@ -1,20 +1,44 @@
 package com.carrotsearch.gradle.junit4
 
 import com.carrotsearch.ant.tasks.junit4.JUnit4
-import org.gradle.api.AntBuilder
+import org.gradle.api.GradleException
 import org.gradle.api.Plugin
 import org.gradle.api.Project
 import org.gradle.api.Task
+import org.gradle.api.UnknownTaskException
 import org.gradle.api.plugins.JavaBasePlugin
 import org.gradle.api.tasks.TaskContainer
+import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.tasks.testing.Test
 
+import java.util.concurrent.atomic.AtomicBoolean
+
 class RandomizedTestingPlugin implements Plugin<Project> {
 
+    static private AtomicBoolean sanityCheckConfigured = new AtomicBoolean(false)
+
     void apply(Project project) {
         setupSeed(project)
         replaceTestTask(project.tasks)
         configureAnt(project.ant)
+        configureSanityCheck(project)
+    }
+
+    private static void configureSanityCheck(Project project) {
+        // Check the task graph to confirm tasks were indeed replaced
+        // https://github.com/elastic/elasticsearch/issues/31324
+        if (sanityCheckConfigured.getAndSet(true) == false) {
+            project.rootProject.getGradle().getTaskGraph().whenReady {
+                List<Task> nonConforming = project.getGradle().getTaskGraph().allTasks
+                        .findAll { it.name == "test" }
+                        .findAll { (it instanceof RandomizedTestingTask) == false}
+                        .collect { "${it.path} -> ${it.class}" }
+                if (nonConforming.isEmpty() == false) {
+                    throw new GradleException("Found the ${nonConforming.size()} `test` tasks:" +
+                            "\n  ${nonConforming.join("\n  ")}")
+                }
+            }
+        }
     }
 
     /**
@@ -45,29 +69,32 @@ class RandomizedTestingPlugin implements Plugin<Project> {
     }
 
     static void replaceTestTask(TaskContainer tasks) {
-        Test oldTestTask = tasks.findByPath('test')
-        if (oldTestTask == null) {
+        // Gradle 4.8 introduced lazy tasks, thus we deal both with the `test` task as well as it's provider
+        // https://github.com/gradle/gradle/issues/5730#issuecomment-398822153
+        // since we can't be sure if the task was ever realized, we remove both the provider and the task
+        TaskProvider<Test> oldTestProvider
+        try {
+            oldTestProvider = tasks.getByNameLater(Test, 'test')
+        } catch (UnknownTaskException unused) {
             // no test task, ok, user will use testing task on their own
             return
         }
-        tasks.remove(oldTestTask)
+        Test oldTestTask = oldTestProvider.get()
 
-        Map properties = [
-            name: 'test',
-            type: RandomizedTestingTask,
-            dependsOn: oldTestTask.dependsOn,
-            group: JavaBasePlugin.VERIFICATION_GROUP,
-            description: 'Runs unit tests with the randomized testing framework'
-        ]
-        RandomizedTestingTask newTestTask = tasks.create(properties)
-        newTestTask.classpath = oldTestTask.classpath
-        newTestTask.testClassesDir = oldTestTask.project.sourceSets.test.output.classesDir
-        // since gradle 4.5, tasks immutable dependencies are "hidden" (do not show up in dependsOn)
-        // so we must explicitly add a dependency on generating the test classpath
-        newTestTask.dependsOn('testClasses')
+        // we still have to use replace here despite the remove above because the task container knows about the provider
+        // by the same name
+        RandomizedTestingTask newTestTask = tasks.replace('test', RandomizedTestingTask)
+        newTestTask.configure{
+            group =  JavaBasePlugin.VERIFICATION_GROUP
+            description = 'Runs unit tests with the randomized testing framework'
+            dependsOn oldTestTask.dependsOn, 'testClasses'
+            classpath = oldTestTask.classpath
+            testClassesDirs = oldTestTask.project.sourceSets.test.output.classesDirs
+        }
 
         // hack so check task depends on custom test
-        Task checkTask = tasks.findByPath('check')
+        Task checkTask = tasks.getByName('check')
+        checkTask.dependsOn.remove(oldTestProvider)
         checkTask.dependsOn.remove(oldTestTask)
         checkTask.dependsOn.add(newTestTask)
     }

buildSrc/src/main/groovy/com/carrotsearch/gradle/junit4/RandomizedTestingTask.groovy

@@ -6,18 +6,20 @@ import groovy.xml.NamespaceBuilder
 import groovy.xml.NamespaceBuilderSupport
 import org.apache.tools.ant.BuildException
 import org.apache.tools.ant.DefaultLogger
+import org.apache.tools.ant.Project
 import org.apache.tools.ant.RuntimeConfigurable
 import org.apache.tools.ant.UnknownElement
+import org.elasticsearch.gradle.BuildPlugin
 import org.gradle.api.DefaultTask
 import org.gradle.api.InvalidUserDataException
 import org.gradle.api.file.FileCollection
 import org.gradle.api.file.FileTreeElement
-import org.gradle.api.internal.tasks.options.Option
 import org.gradle.api.specs.Spec
 import org.gradle.api.tasks.Input
 import org.gradle.api.tasks.InputDirectory
 import org.gradle.api.tasks.Optional
 import org.gradle.api.tasks.TaskAction
+import org.gradle.api.tasks.options.Option
 import org.gradle.api.tasks.util.PatternFilterable
 import org.gradle.api.tasks.util.PatternSet
 import org.gradle.internal.logging.progress.ProgressLoggerFactory
@@ -43,8 +45,8 @@ class RandomizedTestingTask extends DefaultTask {
     @Input
     String parallelism = '1'
 
-    @InputDirectory
-    File testClassesDir
+    @Input
+    FileCollection testClassesDirs
 
     @Optional
     @Input
@@ -220,15 +222,15 @@ class RandomizedTestingTask extends DefaultTask {
                 listener = new DefaultLogger(
                         errorPrintStream: System.err,
                         outputPrintStream: System.out,
-                        messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
+                        messageOutputLevel: Project.MSG_INFO)
             } else {
                 // we want to buffer the info, and emit it if the test fails
                 antLoggingBuffer = new ByteArrayOutputStream()
                 PrintStream stream = new PrintStream(antLoggingBuffer, true, "UTF-8")
                 listener = new DefaultLogger(
                         errorPrintStream: stream,
                         outputPrintStream: stream,
-                        messageOutputLevel: org.apache.tools.ant.Project.MSG_INFO)
+                        messageOutputLevel: Project.MSG_INFO)
             }
             project.ant.project.addBuildListener(listener)
         }
@@ -251,12 +253,10 @@ class RandomizedTestingTask extends DefaultTask {
                 if (argLine != null) {
                     jvmarg(line: argLine)
                 }
-                fileset(dir: testClassesDir) {
-                    for (String includePattern : patternSet.getIncludes()) {
-                        include(name: includePattern)
-                    }
-                    for (String excludePattern : patternSet.getExcludes()) {
-                        exclude(name: excludePattern)
+                testClassesDirs.each { testClassDir ->
+                    fileset(dir: testClassDir) {
+                        patternSet.getIncludes().each { include(name: it) }
+                        patternSet.getExcludes().each { exclude(name: it) }
                     }
                 }
                 for (Map.Entry<String, Object> prop : systemProperties) {

buildSrc/src/main/groovy/org/elasticsearch/gradle/BuildPlugin.groovy

@@ -348,7 +348,9 @@ class BuildPlugin implements Plugin<Project> {
                 // just a self contained test-fixture configuration, likely transitive and hellacious
                 return
             }
-            configuration.resolutionStrategy.failOnVersionConflict()
+            configuration.resolutionStrategy { 
+                failOnVersionConflict()
+            }
         })
 
         // force all dependencies added directly to compile/testCompile to be non-transitive, except for ES itself
@@ -475,13 +477,17 @@ class BuildPlugin implements Plugin<Project> {
                 }
             }
 
-            project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
-                // place the pom next to the jar it is for
-                t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
-                // build poms with assemble (if the assemble task exists)
-                Task assemble = project.tasks.findByName('assemble')
-                if (assemble) {
-                    assemble.dependsOn(t)
+            // Work around Gradle 4.8 issue until we `enableFeaturePreview('STABLE_PUBLISHING')`
+            // https://github.com/gradle/gradle/issues/5696#issuecomment-396965185
+            project.getGradle().getTaskGraph().whenReady {
+                project.tasks.withType(GenerateMavenPom.class) { GenerateMavenPom t ->
+                    // place the pom next to the jar it is for
+                    t.destination = new File(project.buildDir, "distributions/${project.archivesBaseName}-${project.version}.pom")
+                    // build poms with assemble (if the assemble task exists)
+                    Task assemble = project.tasks.findByName('assemble')
+                    if (assemble) {
+                        assemble.dependsOn(t)
+                    }
                 }
             }
         }
@@ -625,6 +631,10 @@ class BuildPlugin implements Plugin<Project> {
                         jarTask.manifest.attributes('Change': shortHash)
                     }
                 }
+                // Force manifest entries that change by nature to a constant to be able to compare builds more effectively
+                if (System.properties.getProperty("build.compare_friendly", "false") == "true") {
+                    jarTask.manifest.getAttributes().clear()
+                }
             }
             // add license/notice files
             project.afterEvaluate {
@@ -741,7 +751,7 @@ class BuildPlugin implements Plugin<Project> {
         project.extensions.add('additionalTest', { String name, Closure config ->
             RandomizedTestingTask additionalTest = project.tasks.create(name, RandomizedTestingTask.class)
             additionalTest.classpath = test.classpath
-            additionalTest.testClassesDir = test.testClassesDir
+            additionalTest.testClassesDirs = test.testClassesDirs
             additionalTest.configure(commonTestConfig(project))
             additionalTest.configure(config)
             test.dependsOn(additionalTest)