Merge remote-tracking branch 'origin/ccr' into pr/30123

* origin/ccr: (166 commits)
  Introduce soft-deletes retention policy based on global checkpoint (elastic#30335)
  Enable MockHttpTransport in ShardChangsIT
  Remove old sha files from dated Lucene snapshot
  Update InternalEngine tests on ccr side for elastic#30121
  Set the new lucene version for 6.4.0
  [ML][TEST] Clean up jobs in ModelPlotIT
  Upgrade to 7.4.0-snapshot-1ed95c097b (elastic#30357)
  Watcher: Ensure trigger service pauses execution (elastic#30363)
  [DOCS] Added coming qualifiers in changelog
  [DOCS] Commented out empty sections in the changelog to fix the doc build. (elastic#30372)
  Security: reduce garbage during index resolution (elastic#30180)
  Make RepositoriesMetaData contents unmodifiable (elastic#30361)
  Change quad tree max levels to 29. Closes elastic#21191 (elastic#29663)
  Test: use trial license in qa tests with security
  [ML] Add integration test for model plots (elastic#30359)
  SQL: Fix bug caused by empty composites (elastic#30343)
  [ML] Account for gaps in data counts after job is reopened (elastic#30294)
  InternalEngineTests.testConcurrentOutOfOrderDocsOnReplica should use two documents (elastic#30121)
  Change signature of Get Repositories Response (elastic#30333)
  Tests: Use different watch ids per test in smoke test (elastic#30331)
  ...

Author: jasontedor

.gitignore

@@ -20,10 +20,8 @@ nbactions.xml
 .gradle/
 build/
 
-# maven stuff (to be removed when trunk becomes 4.x)
-*-execution-hints.log
-target/
-dependency-reduced-pom.xml
+# vscode stuff
+.vscode/
 
 # testing stuff
 **/.local*
@@ -43,4 +41,3 @@ html_docs
 # random old stuff that we should look at the necessity of...
 /tmp/
 eclipse-build
-

CONTRIBUTING.md

@@ -107,6 +107,8 @@ We support development in the Eclipse and IntelliJ IDEs. For Eclipse, the
 minimum version that we support is [Eclipse Oxygen][eclipse] (version 4.7). For
 IntelliJ, the minimum version that we support is [IntelliJ 2017.2][intellij].
 
+### Configuring IDEs And Running Tests
+
 Eclipse users can automatically configure their IDE: `./gradlew eclipse`
 then `File: Import: Existing Projects into Workspace`. Select the
 option `Search for nested projects`. Additionally you will want to
@@ -144,6 +146,9 @@ For IntelliJ, go to
 For Eclipse, go to `Preferences->Java->Installed JREs` and add `-ea` to
 `VM Arguments`.
 
+
+### Java Language Formatting Guidelines
+
 Please follow these formatting guidelines:
 
 * Java indent is 4 spaces
@@ -155,6 +160,33 @@ Please follow these formatting guidelines:
  * IntelliJ: `Preferences/Settings->Editor->Code Style->Java->Imports`. There are two configuration options: `Class count to use import with '*'` and `Names count to use static import with '*'`. Set their values to 99999 or some other absurdly high value.
 * Don't worry too much about import order. Try not to change it but don't worry about fighting your IDE to stop it from doing so.
 
+### License Headers
+
+We require license headers on all Java files. You will notice that all the Java files in
+the top-level `x-pack` directory contain a separate license from the rest of the repository. This
+directory contains commercial code that is associated with a separate license. It can be helpful
+to have the IDE automatically insert the appropriate license header depending which part of the project
+contributions are made to.
+
+#### IntelliJ: Copyright & Scope Profiles
+
+To have IntelliJ insert the correct license, it is necessary to create to copyright profiles.
+These may potentially be called `apache2` and `commercial`. These can be created in
+`Preferences/Settings->Editor->Copyright->Copyright Profiles`. To associate these profiles to
+their respective directories, two "Scopes" will need to be created. These can be created in
+`Preferences/Settings->Appearances & Behavior->Scopes`. When creating scopes, be sure to choose
+the `shared` scope type. Create a scope, `apache2`, with
+the associated pattern of `!file[group:x-pack]:*/`. This pattern will exclude all the files contained in
+the `x-pack` directory. The other scope, `commercial`, will have the inverse pattern of `file[group:x-pack]:*/`.
+The two scopes, together, should account for all the files in the project. To associate the scopes
+with their copyright-profiles, go into `Preferences/Settings->Editor>Copyright` and use the `+` to add
+the associations `apache2/apache2` and `commercial/commercial`.
+
+Configuring these options in IntelliJ can be quite buggy, so do not be alarmed if you have to open/close
+the settings window and/or restart IntelliJ to see your changes take effect.
+
+### Creating A Distribution
+
 To create a distribution from the source, simply run:
 
 ```sh
@@ -169,6 +201,8 @@ The archive distributions (tar and zip) can be found under:
 `./distribution/archives/(tar|zip)/build/distributions/`
 
 
+### Running The Full Test Suite
+
 Before submitting your changes, run the test suite to make sure that nothing is broken, with:
 
 ```sh

Vagrantfile

@@ -337,6 +337,7 @@ export BATS=/project/build/bats
 export BATS_UTILS=/project/build/packaging/bats/utils
 export BATS_TESTS=/project/build/packaging/bats/tests
 export PACKAGING_ARCHIVES=/project/build/packaging/archives
+export PACKAGING_TESTS=/project/build/packaging/tests
 VARS
     cat \<\<SUDOERS_VARS > /etc/sudoers.d/elasticsearch_vars
 Defaults   env_keep += "ZIP"
@@ -347,6 +348,7 @@ Defaults   env_keep += "BATS"
 Defaults   env_keep += "BATS_UTILS"
 Defaults   env_keep += "BATS_TESTS"
 Defaults   env_keep += "PACKAGING_ARCHIVES"
+Defaults   env_keep += "PACKAGING_TESTS"
 SUDOERS_VARS
     chmod 0440 /etc/sudoers.d/elasticsearch_vars
   SHELL

build.gradle

@@ -19,7 +19,9 @@
 
 
 import org.apache.tools.ant.taskdefs.condition.Os
+import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.BuildPlugin
+import org.elasticsearch.gradle.LoggedExec
 import org.elasticsearch.gradle.Version
 import org.elasticsearch.gradle.VersionCollection
 import org.elasticsearch.gradle.VersionProperties
@@ -30,6 +32,7 @@ import org.gradle.api.tasks.wrapper.Wrapper.DistributionType
 import org.gradle.util.GradleVersion
 import org.gradle.util.DistributionLocator
 
+import java.nio.file.Files
 import java.nio.file.Path
 import java.security.MessageDigest
 
@@ -404,10 +407,19 @@ allprojects {
       }
     }
   }
+
+  File licenseHeaderFile;
+  if (eclipse.project.name.startsWith(':x-pack')) {
+      licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/elastic-license-header.txt')
+  } else {
+      licenseHeaderFile = new File(project.rootDir, 'buildSrc/src/main/resources/license-headers/oss-license-header.txt')
+  }
+  String licenseHeader = licenseHeaderFile.getText('UTF-8').replace('\n', '\\\\n')
   task copyEclipseSettings(type: Copy) {
     // TODO: "package this up" for external builds
     from new File(project.rootDir, 'buildSrc/src/main/resources/eclipse.settings')
     into '.settings'
+    filter{ it.replaceAll('@@LICENSE_HEADER_TEXT@@', licenseHeader)}
   }
   // otherwise .settings is not nuked entirely
   task wipeEclipseSettings(type: Delete) {
@@ -459,6 +471,59 @@ gradle.projectsEvaluated {
 
 }
 
+static void assertLinesInFile(final Path path, final List<String> expectedLines) {
+  final List<String> actualLines = Files.readAllLines(path)
+  int line = 0
+  for (final String expectedLine : expectedLines) {
+    final String actualLine = actualLines.get(line)
+    if (expectedLine != actualLine) {
+      throw new GradleException("expected line [${line + 1}] in [${path}] to be [${expectedLine}] but was [${actualLine}]")
+    }
+    line++
+  }
+}
+
+/*
+ * Check that all generated JARs have our NOTICE.txt and an appropriate
+ * LICENSE.txt in them. We configurate this in gradle but we'd like to
+ * be extra paranoid.
+ */
+subprojects { project ->
+  project.tasks.withType(Jar).whenTaskAdded { jarTask ->
+    final Task extract = project.task("extract${jarTask.name.capitalize()}", type: LoggedExec) {
+      dependsOn jarTask
+      ext.destination = project.buildDir.toPath().resolve("jar-extracted/${jarTask.name}")
+      commandLine "${->new File(rootProject.compilerJavaHome, 'bin/jar')}",
+          'xf', "${-> jarTask.outputs.files.singleFile}", 'META-INF/LICENSE.txt', 'META-INF/NOTICE.txt'
+      workingDir destination
+      doFirst {
+        project.delete(destination)
+        Files.createDirectories(destination)
+      }
+    }
+
+    final Task checkNotice = project.task("verify${jarTask.name.capitalize()}Notice") {
+      dependsOn extract
+      doLast {
+        final List<String> noticeLines = Files.readAllLines(project.noticeFile.toPath())
+        final Path noticePath = extract.destination.resolve('META-INF/NOTICE.txt')
+        assertLinesInFile(noticePath, noticeLines)
+      }
+    }
+    project.check.dependsOn checkNotice
+
+    final Task checkLicense = project.task("verify${jarTask.name.capitalize()}License") {
+      dependsOn extract
+      doLast {
+        final List<String> licenseLines = Files.readAllLines(project.licenseFile.toPath())
+        final Path licensePath = extract.destination.resolve('META-INF/LICENSE.txt')
+        assertLinesInFile(licensePath, licenseLines)
+      }
+    }
+    project.check.dependsOn checkLicense
+  }
+}
+
 /* Remove assemble on all qa projects because we don't need to publish
  * artifacts for them. */
 gradle.projectsEvaluated {

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -64,7 +64,7 @@ class ClusterConfiguration {
     boolean debug = false
 
     /**
-     * Configuration of the setting <tt>discovery.zen.minimum_master_nodes</tt> on the nodes.
+     * Configuration of the setting {@code discovery.zen.minimum_master_nodes} on the nodes.
      * In case of more than one node, this defaults to the number of nodes
      */
     @Input

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterFormationTasks.groovy

@@ -563,16 +563,17 @@ class ClusterFormationTasks {
 
     /** Adds a task to execute a command to help setup the cluster */
     static Task configureExecTask(String name, Project project, Task setup, NodeInfo node, Object[] execArgs) {
-        return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) {
-            workingDir node.cwd
+        return project.tasks.create(name: name, type: LoggedExec, dependsOn: setup) { Exec exec ->
+            exec.workingDir node.cwd
+            exec.environment 'JAVA_HOME', node.getJavaHome()
             if (Os.isFamily(Os.FAMILY_WINDOWS)) {
-                executable 'cmd'
-                args '/C', 'call'
+                exec.executable 'cmd'
+                exec.args '/C', 'call'
                 // On Windows the comma character is considered a parameter separator:
                 // argument are wrapped in an ExecArgWrapper that escapes commas
-                args execArgs.collect { a -> new EscapeCommaWrapper(arg: a) }
+                exec.args execArgs.collect { a -> new EscapeCommaWrapper(arg: a) }
             } else {
-                commandLine execArgs
+                exec.commandLine execArgs
             }
         }
     }

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantPropertiesExtension.groovy

@@ -41,6 +41,9 @@ class VagrantPropertiesExtension {
     @Input
     Boolean inheritTestUtils
 
+    @Input
+    String testClass
+
     VagrantPropertiesExtension(List<String> availableBoxes) {
         this.boxes = availableBoxes
         this.batsDir = 'src/test/resources/packaging'

buildSrc/src/main/groovy/org/elasticsearch/gradle/vagrant/VagrantTestPlugin.groovy

@@ -37,13 +37,21 @@ class VagrantTestPlugin implements Plugin<Project> {
             'ubuntu-1404',
     ]
 
-    /** All onboarded archives by default, available for Bats tests even if not used **/
-    static List<String> DISTRIBUTION_ARCHIVES = ['tar', 'rpm', 'deb', 'oss-rpm', 'oss-deb']
+    /** All distributions to bring into test VM, whether or not they are used **/
+    static List<String> DISTRIBUTIONS = [
+            'archives:tar',
+            'archives:oss-tar',
+            'packages:rpm',
+            'packages:oss-rpm',
+            'packages:deb',
+            'packages:oss-deb'
+    ]
 
     /** Packages onboarded for upgrade tests **/
     static List<String> UPGRADE_FROM_ARCHIVES = ['rpm', 'deb']
 
     private static final PACKAGING_CONFIGURATION = 'packaging'
+    private static final PACKAGING_TEST_CONFIGURATION = 'packagingTest'
     private static final BATS = 'bats'
     private static final String BATS_TEST_COMMAND ="cd \$PACKAGING_ARCHIVES && sudo bats --tap \$BATS_TESTS/*.$BATS"
     private static final String PLATFORM_TEST_COMMAND ="rm -rf ~/elasticsearch && rsync -r /elasticsearch/ ~/elasticsearch && cd ~/elasticsearch && ./gradlew test integTest"
@@ -59,6 +67,7 @@ class VagrantTestPlugin implements Plugin<Project> {
 
         // Creates custom configurations for Bats testing files (and associated scripts and archives)
         createPackagingConfiguration(project)
+        project.configurations.create(PACKAGING_TEST_CONFIGURATION)
 
         // Creates all the main Vagrant tasks
         createVagrantTasks(project)
@@ -117,13 +126,8 @@ class VagrantTestPlugin implements Plugin<Project> {
             upgradeFromVersion = Version.fromString(upgradeFromVersionRaw)
         }
 
-        DISTRIBUTION_ARCHIVES.each {
+        DISTRIBUTIONS.each {
             // Adds a dependency for the current version
-            if (it == 'tar') {
-                it = 'archives:tar'
-            } else {
-                it = "packages:${it}"
-            }
             project.dependencies.add(PACKAGING_CONFIGURATION,
                     project.dependencies.project(path: ":distribution:${it}", configuration: 'default'))
         }
@@ -142,10 +146,12 @@ class VagrantTestPlugin implements Plugin<Project> {
     }
 
     private static void createCleanTask(Project project) {
-        project.tasks.create('clean', Delete.class) {
-            description 'Clean the project build directory'
-            group 'Build'
-            delete project.buildDir
+        if (project.tasks.findByName('clean') == null) {
+            project.tasks.create('clean', Delete.class) {
+                description 'Clean the project build directory'
+                group 'Build'
+                delete project.buildDir
+            }
         }
     }
 
@@ -172,6 +178,18 @@ class VagrantTestPlugin implements Plugin<Project> {
             from project.configurations[PACKAGING_CONFIGURATION]
         }
 
+        File testsDir = new File(packagingDir, 'tests')
+        Copy copyPackagingTests = project.tasks.create('copyPackagingTests', Copy) {
+            into testsDir
+            from project.configurations[PACKAGING_TEST_CONFIGURATION]
+        }
+
+        Task createTestRunnerScript = project.tasks.create('createTestRunnerScript', FileContentsTask) {
+            dependsOn copyPackagingTests
+            file "${testsDir}/run-tests.sh"
+            contents "java -cp \"\$PACKAGING_TESTS/*\" org.junit.runner.JUnitCore ${-> project.extensions.esvagrant.testClass}"
+        }
+
         Task createVersionFile = project.tasks.create('createVersionFile', FileContentsTask) {
             dependsOn copyPackagingArchives
             file "${archivesDir}/version"
@@ -232,7 +250,8 @@ class VagrantTestPlugin implements Plugin<Project> {
 
         Task vagrantSetUpTask = project.tasks.create('setupPackagingTest')
         vagrantSetUpTask.dependsOn 'vagrantCheckVersion'
-        vagrantSetUpTask.dependsOn copyPackagingArchives, createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile
+        vagrantSetUpTask.dependsOn copyPackagingArchives, copyPackagingTests, createTestRunnerScript
+        vagrantSetUpTask.dependsOn createVersionFile, createUpgradeFromFile, createUpgradeIsOssFile
         vagrantSetUpTask.dependsOn copyBatsTests, copyBatsUtils
     }
 
@@ -391,20 +410,29 @@ class VagrantTestPlugin implements Plugin<Project> {
                 packagingTest.dependsOn(batsPackagingTest)
             }
 
-            // This task doesn't do anything yet. In the future it will execute a jar containing tests on the vm
-            Task groovyPackagingTest = project.tasks.create("vagrant${boxTask}#groovyPackagingTest")
-            groovyPackagingTest.dependsOn(up)
-            groovyPackagingTest.finalizedBy(halt)
+            Task javaPackagingTest = project.tasks.create("vagrant${boxTask}#javaPackagingTest", VagrantCommandTask) {
+                command 'ssh'
+                boxName box
+                environmentVars vagrantEnvVars
+                dependsOn up, setupPackagingTest
+                finalizedBy halt
+                args '--command', "bash \"\$PACKAGING_TESTS/run-tests.sh\""
+            }
+
+            // todo remove this onlyIf after all packaging tests are consolidated
+            javaPackagingTest.onlyIf {
+                project.extensions.esvagrant.testClass != null
+            }
 
-            TaskExecutionAdapter groovyPackagingReproListener = createReproListener(project, groovyPackagingTest.path)
-            groovyPackagingTest.doFirst {
-                project.gradle.addListener(groovyPackagingReproListener)
+            TaskExecutionAdapter javaPackagingReproListener = createReproListener(project, javaPackagingTest.path)
+            javaPackagingTest.doFirst {
+                project.gradle.addListener(javaPackagingReproListener)
             }
-            groovyPackagingTest.doLast {
-                project.gradle.removeListener(groovyPackagingReproListener)
+            javaPackagingTest.doLast {
+                project.gradle.removeListener(javaPackagingReproListener)
             }
             if (project.extensions.esvagrant.boxes.contains(box)) {
-                packagingTest.dependsOn(groovyPackagingTest)
+                packagingTest.dependsOn(javaPackagingTest)
             }
 
             Task platform = project.tasks.create("vagrant${boxTask}#platformTest", VagrantCommandTask) {

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -535,7 +535,6 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReplicaShardAllocatorTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]gateway[/\\]ReusePeerRecoverySharedTest.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]get[/\\]GetActionIT.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexServiceTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]IndexingSlowLogTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]MergePolicySettingsTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]SearchSlowLogTests.java" checks="LineLength" />