SQL: Update branch to include the fields API master changes

.ci/bwcVersions

@@ -28,7 +28,7 @@ BWC_VERSION:
   - "7.10.0"
   - "7.10.1"
   - "7.10.2"
-  - "7.10.3"
   - "7.11.0"
+  - "7.11.1"
   - "7.12.0"
   - "8.0.0"

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/DistributionDownloadPluginFuncTest.groovy

@@ -51,19 +51,19 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
         when:
         def guh = new File(testProjectDir.getRoot(), "gradle-user-home").absolutePath;
         def runner = gradleRunner('clean', 'setupDistro', '-i', '-g', guh)
+        def unpackingMessage = "Unpacking elasticsearch-${version}-linux-${Architecture.current().classifier}.tar.gz " +
+                "using SymbolicLinkPreservingUntarTransform"
         def result = withMockedDistributionDownload(version, platform, runner) {
             // initial run
             def firstRun = build()
-            assertOutputContains(firstRun.output, "Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " +
-                    "using SymbolicLinkPreservingUntarTransform")
+            assertOutputContains(firstRun.output, unpackingMessage)
             // 2nd invocation
             build()
         }
 
         then:
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
-        assertOutputMissing(result.output, "Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " +
-                "using SymbolicLinkPreservingUntarTransform")
+        assertOutputMissing(result.output, unpackingMessage)
     }
 
     def "transforms are reused across projects"() {
@@ -100,7 +100,7 @@ class DistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
         then:
         result.tasks.size() == 3
-        result.output.count("Unpacking elasticsearch-${version}-linux-x86_64.tar.gz " +
+        result.output.count("Unpacking elasticsearch-${version}-linux-${Architecture.current().classifier}.tar.gz " +
                 "using SymbolicLinkPreservingUntarTransform") == 1
     }
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/JdkDownloadPluginFuncTest.groovy

@@ -29,14 +29,16 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
     private static final String OPENJDK_VERSION_OLD = "1+99"
     private static final String ADOPT_JDK_VERSION = "12.0.2+10"
+    private static final String ADOPT_JDK_VERSION_11 = "11.0.10+9"
+    private static final String ADOPT_JDK_VERSION_15 = "15.0.2+7"
     private static final String OPEN_JDK_VERSION = "12.0.1+99@123456789123456789123456789abcde"
     private static final String AZUL_AARCH_VERSION = "15.0.1+99@123456789123456789123456789abcde"
     private static final Pattern JDK_HOME_LOGLINE = Pattern.compile("JDK HOME: (.*)");
 
     @Unroll
     def "jdk #jdkVendor for #platform#suffix are downloaded and extracted"() {
         given:
-        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform);
+        def mockRepoUrl = urlPath(jdkVendor, jdkVersion, platform, arch);
         def mockedContent = filebytes(jdkVendor, platform)
         buildFile.text = """
             plugins {
@@ -70,20 +72,22 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         assertExtraction(result.output, expectedJavaBin);
 
         where:
-        platform  | arch      | jdkVendor           | jdkVersion          | expectedJavaBin          | suffix
-        "linux"   | "x64"     | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION   | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION    | "bin/java"               | ""
-        "linux"   | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD | "bin/java"               | "(old version)"
-        "windows" | "x64"     | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION   | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION    | "bin/java"               | ""
-        "windows" | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD | "bin/java"               | "(old version)"
-        "darwin"  | "x64"     | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION   | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION    | "Contents/Home/bin/java" | ""
-        "darwin"  | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
-        "mac"     | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION    | "Contents/Home/bin/java" | ""
-        "mac"     | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD | "Contents/Home/bin/java" | "(old version)"
-        "darwin"  | "aarch64" | VENDOR_AZUL         | AZUL_AARCH_VERSION  | "Contents/Home/bin/java" | ""
-        "linux"   | "aarch64" | VENDOR_AZUL         | AZUL_AARCH_VERSION  | "bin/java"               | ""
+        platform  | arch      | jdkVendor           | jdkVersion           | expectedJavaBin          | suffix
+        "linux"   | "x64"     | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION    | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "bin/java"               | ""
+        "linux"   | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "bin/java"               | "(old version)"
+        "windows" | "x64"     | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION    | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "bin/java"               | ""
+        "windows" | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "bin/java"               | "(old version)"
+        "darwin"  | "x64"     | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION    | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "Contents/Home/bin/java" | ""
+        "darwin"  | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "Contents/Home/bin/java" | "(old version)"
+        "mac"     | "x64"     | VENDOR_OPENJDK      | OPEN_JDK_VERSION     | "Contents/Home/bin/java" | ""
+        "mac"     | "x64"     | VENDOR_OPENJDK      | OPENJDK_VERSION_OLD  | "Contents/Home/bin/java" | "(old version)"
+        "darwin"  | "aarch64" | VENDOR_AZUL         | AZUL_AARCH_VERSION   | "Contents/Home/bin/java" | ""
+        "linux"   | "aarch64" | VENDOR_AZUL         | AZUL_AARCH_VERSION   | "bin/java"               | ""
+        "linux"   | "aarch64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION_11 | "bin/java"               | "(jdk 11)"
+        "linux"   | "aarch64" | VENDOR_ADOPTOPENJDK | ADOPT_JDK_VERSION_15 | "bin/java"               | "(jdk 15)"
     }
 
     def "transforms are reused across projects"() {
@@ -195,10 +199,13 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         true
     }
 
-    private static String urlPath(final String vendor, final String version, final String platform) {
+    private static String urlPath(final String vendor,
+                                  final String version,
+                                  final String platform,
+                                  final String arch = 'x64') {
         if (vendor.equals(VENDOR_ADOPTOPENJDK)) {
             final String module = isMac(platform) ? "mac" : platform;
-            return "/jdk-12.0.2+10/" + module + "/x64/jdk/hotspot/normal/adoptopenjdk";
+            return "/jdk-" + version + "/" + module + "/${arch}/jdk/hotspot/normal/adoptopenjdk";
         } else if (vendor.equals(VENDOR_OPENJDK)) {
             final String effectivePlatform = isMac(platform) ? "osx" : platform;
             final boolean isOld = version.equals(OPENJDK_VERSION_OLD);
@@ -208,7 +215,7 @@ class JdkDownloadPluginFuncTest extends AbstractGradleFuncTest {
         } else if (vendor.equals(VENDOR_AZUL)) {
             final String module = isMac(platform) ? "macosx" : platform;
             // we only test zulu 15 darwin aarch64 for now
-            return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_aarch64.tar.gz";
+            return "/zulu${module.equals('linux') ? '-embedded' : ''}/bin/zulu15.29.15-ca-jdk15.0.2-${module}_${arch}.tar.gz";
         }
     }
 

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/fixtures/DistributionDownloadFixture.groovy

@@ -44,8 +44,7 @@ class DistributionDownloadFixture {
     private static String urlPath(String version,ElasticsearchDistribution.Platform platform) {
         String fileType = ((platform == ElasticsearchDistribution.Platform.LINUX ||
                 platform == ElasticsearchDistribution.Platform.DARWIN)) ? "tar.gz" : "zip"
-        String arch = Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64"
-        "/downloads/elasticsearch/elasticsearch-${version}-${platform}-${arch}.$fileType"
+        "/downloads/elasticsearch/elasticsearch-${version}-${platform}-${Architecture.current().classifier}.$fileType"
     }
 
     private static byte[] filebytes(String urlPath) throws IOException {

buildSrc/src/integTest/groovy/org/elasticsearch/gradle/internal/InternalDistributionDownloadPluginFuncTest.groovy

@@ -8,14 +8,11 @@
 
 package org.elasticsearch.gradle.internal
 
+import org.elasticsearch.gradle.Architecture
 import org.elasticsearch.gradle.VersionProperties
 import org.elasticsearch.gradle.fixtures.AbstractGradleFuncTest
-import org.gradle.testkit.runner.GradleRunner
 import org.gradle.testkit.runner.TaskOutcome
-import org.junit.Rule
-import org.junit.rules.TemporaryFolder
 
-import java.lang.management.ManagementFactory
 
 class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest {
 
@@ -61,7 +58,7 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
         def result = gradleRunner("setupDistro", '-g', testProjectDir.newFolder('GUH').path).build()
 
         then:
-        result.task(":distribution:archives:linux-tar:buildExpanded").outcome == TaskOutcome.SUCCESS
+        result.task(":distribution:archives:${testArchiveProjectName}:buildExpanded").outcome == TaskOutcome.SUCCESS
         result.task(":setupDistro").outcome == TaskOutcome.SUCCESS
         assertExtractedDistroIsCreated("build/distro", 'current-marker.txt')
     }
@@ -133,24 +130,24 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
             apply plugin:'base'
 
             // packed distro
-            configurations.create("linux-tar")
+            configurations.create("${testArchiveProjectName}")
             tasks.register("buildBwcTask", Tar) {
                 from('bwc-marker.txt')
                 archiveExtension = "tar.gz"
                 compression = Compression.GZIP
             }
             artifacts {
-                it.add("linux-tar", buildBwcTask)
+                it.add("${testArchiveProjectName}", buildBwcTask)
             }
 
             // expanded distro
-            configurations.create("expanded-linux-tar")
+            configurations.create("expanded-${testArchiveProjectName}")
             def expandedTask = tasks.register("buildBwcExpandedTask", Copy) {
                 from('bwc-marker.txt')
                 into('build/install/elastic-distro')
             }
             artifacts {
-                it.add("expanded-linux-tar", file('build/install')) {
+                it.add("expanded-${testArchiveProjectName}", file('build/install')) {
                     builtBy expandedTask
                     type = 'directory'
                 }
@@ -160,9 +157,9 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
 
     private void localDistroSetup() {
         settingsFile << """
-        include ":distribution:archives:linux-tar"
+        include ":distribution:archives:${testArchiveProjectName}"
         """
-        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", "linux-tar")
+        def bwcSubProjectFolder = testProjectDir.newFolder("distribution", "archives", testArchiveProjectName)
         new File(bwcSubProjectFolder, 'current-marker.txt') << "current"
         new File(bwcSubProjectFolder, 'build.gradle') << """
             import org.gradle.api.internal.artifacts.ArtifactAttributes;
@@ -190,10 +187,12 @@ class InternalDistributionDownloadPluginFuncTest extends AbstractGradleFuncTest
                 it.add("extracted", buildExpanded)
             }
         """
-        buildFile << """
-        """
     }
 
+    String getTestArchiveProjectName() {
+        def archSuffix = Architecture.current() == Architecture.AARCH64 ? '-aarch64' : ''
+        return "linux${archSuffix}-tar"
+    }
     boolean assertExtractedDistroIsCreated(String relativeDistroPath, String markerFileName) {
         File extractedFolder = new File(testProjectDir.root, relativeDistroPath)
         assert extractedFolder.exists()

buildSrc/src/integTest/resources/org/elasticsearch/gradle/internal/fake_git/remote/.ci/java-versions.properties

@@ -5,6 +5,6 @@
 # in compliance with, at your election, the Elastic License 2.0 or the Server
 # Side Public License, v 1.
 #
-ES_BUILD_JAVA=openjdk12
-ES_RUNTIME_JAVA=openjdk12
+ES_BUILD_JAVA=openjdk11
+ES_RUNTIME_JAVA=openjdk11
 GRADLE_TASK=build

buildSrc/src/main/java/org/elasticsearch/gradle/Architecture.java

@@ -10,8 +10,14 @@
 
 public enum Architecture {
 
-    X64,
-    AARCH64;
+    X64("x86_64"),
+    AARCH64("aarch64");
+
+    public final String classifier;
+
+    Architecture(String classifier) {
+        this.classifier = classifier;
+    }
 
     public static Architecture current() {
         final String architecture = System.getProperty("os.arch", "");

buildSrc/src/main/java/org/elasticsearch/gradle/DistributionDownloadPlugin.java

@@ -166,14 +166,11 @@ private String dependencyNotation(ElasticsearchDistribution distribution) {
 
         Version distroVersion = Version.fromString(distribution.getVersion());
         String extension = distribution.getType().toString();
-        String classifier = ":" + (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64");
+        String classifier = ":" + Architecture.current().classifier;
         if (distribution.getType() == Type.ARCHIVE) {
             extension = distribution.getPlatform() == Platform.WINDOWS ? "zip" : "tar.gz";
             if (distroVersion.onOrAfter("7.0.0")) {
-                classifier = ":"
-                    + distribution.getPlatform()
-                    + "-"
-                    + (Architecture.current() == Architecture.AARCH64 ? "aarch64" : "x86_64");
+                classifier = ":" + distribution.getPlatform() + "-" + Architecture.current().classifier;
             } else {
                 classifier = "";
             }

buildSrc/src/test/java/org/elasticsearch/gradle/BwcVersionsTests.java

@@ -1,6 +1,8 @@
 package org.elasticsearch.gradle;
 
 import org.elasticsearch.gradle.test.GradleUnitTestCase;
+import org.junit.Assume;
+import org.junit.BeforeClass;
 import org.junit.Rule;
 import org.junit.Test;
 import org.junit.rules.ExpectedException;
@@ -279,6 +281,11 @@ public class BwcVersionsTests extends GradleUnitTestCase {
         sampleVersions.put("7.1.0", asList("7_1_0", "7_0_0", "6_7_0", "6_6_1", "6_6_0"));
     }
 
+    @BeforeClass
+    public static void setupAll() {
+        Assume.assumeFalse(Architecture.current() == Architecture.AARCH64);
+    }
+
     @Test(expected = IllegalArgumentException.class)
     public void testExceptionOnEmpty() {
         new BwcVersions(asList("foo", "bar"), Version.fromString("7.0.0"));

docs/reference/analysis/token-graphs.asciidoc

@@ -39,6 +39,10 @@ record the `positionLength` for multi-position tokens. This filters include:
 * <<analysis-synonym-graph-tokenfilter,`synonym_graph`>>
 * <<analysis-word-delimiter-graph-tokenfilter,`word_delimiter_graph`>>
 
+Some tokenizers, such as the
+{plugin}/analysis-nori-tokenizer.html[`nori_tokenizer`], also accurately
+decompose compound tokens into multi-position tokens.
+
 In the following graph, `domain name system` and its synonym, `dns`, both have a
 position of `0`. However, `dns` has a `positionLength` of `3`. Other tokens in
 the graph have a default `positionLength` of `1`.

docs/reference/query-dsl/simple-query-string-query.asciidoc

@@ -86,9 +86,10 @@ query string into tokens. Defaults to the
 `default_field`. If no analyzer is mapped, the index's default analyzer is used.
 
 `auto_generate_synonyms_phrase_query`::
-(Optional, Boolean) If `true`, <<query-dsl-match-query-phrase,match phrase>>
-queries are automatically created for multi-term synonyms. Defaults to `true`.
-See <<simple-query-string-synonyms>> for an example.
+(Optional, Boolean) If `true`, the parser creates a
+<<query-dsl-match-query-phrase,`match_phrase`>> query for each
+<<token-graphs-multi-position-tokens,multi-position token>>. Defaults to `true`.
+For examples, see <<simple-query-string-synonyms>>.
 
 `flags`::
 (Optional, string) List of enabled operators for the
@@ -273,33 +274,36 @@ GET /_search
 <1> The `subject` field is three times as important as the `message` field.
 
 [[simple-query-string-synonyms]]
-===== Synonyms
+===== Multi-position tokens
 
-The `simple_query_string` query supports multi-terms synonym expansion with the <<analysis-synonym-graph-tokenfilter,
-synonym_graph>> token filter. When this filter is used, the parser creates a phrase query for each multi-terms synonyms.
-For example, the following synonym: `"ny, new york"` would produce:
+By default, the `simple_query_string` query parser creates a
+<<query-dsl-match-query-phrase,`match_phrase`>> query for each
+<<token-graphs-multi-position-tokens,multi-position token>> in the query string.
+For example, the parser creates a `match_phrase` query for the multi-word
+synonym `ny, new york`:
 
 `(ny OR ("new york"))`
 
-It is also possible to match multi terms synonyms with conjunctions instead:
+To match multi-position tokens with an `AND` conjunction instead, set
+`auto_generate_synonyms_phrase_query` to `false`:
 
 [source,console]
---------------------------------------------------
+----
 GET /_search
 {
-   "query": {
-       "simple_query_string" : {
-           "query" : "ny city",
-           "auto_generate_synonyms_phrase_query" : false
-       }
-   }
+  "query": {
+    "simple_query_string": {
+      "query": "ny city",
+      "auto_generate_synonyms_phrase_query": false
+    }
+  }
 }
---------------------------------------------------
+----
 
-The example above creates a boolean query:
+For the above example, the parser creates the following
+<<query-dsl-bool-query,`bool`>> query:
 
 `(ny OR (new AND york)) city)`
 
-that matches documents with the term `ny` or the conjunction `new AND york`.
-By default the parameter `auto_generate_synonyms_phrase_query` is set to `true`.
-
+This `bool` query matches documents with the term `ny` or the conjunction
+`new AND york`.

docs/reference/sql/endpoints/translate.asciidoc

@@ -22,20 +22,22 @@ Which returns:
 --------------------------------------------------
 {
   "size": 10,
-  "docvalue_fields": [
+  "_source": false,
+  "fields": [
+    {
+      "field": "author"
+    },
+    {
+      "field": "name"
+    },
+    {
+      "field": "page_count"
+    },
     {
       "field": "release_date",
       "format": "epoch_millis"
     }
   ],
-  "_source": {
-    "includes": [
-      "author",
-      "name",
-      "page_count"
-    ],
-    "excludes": []
-  },
   "sort": [
     {
       "page_count": {