Merge branch '6.x' into ccr-6.x

* 6.x:
  SQL: Fix build on Java 10
  [Tests] Mutualize fixtures code in BaseHttpFixture (#31210)
  [TEST] Fix RemoteClusterClientTests#testEnsureWeReconnect
  [ML] Update test thresholds to account for changes to memory control (#31289)
  Reenable Checkstyle's unused import rule (#31270)
  [ML] Check licence when datafeeds use cross cluster search  (#31247)
  Fix non-REST doc snippet
  [DOC] Extend SQL docs
  [DOCS] Shortens ML API intros
  Use quotes in the call invocation (#31249)
  move security ingest processors to a sub ingest directory (#31306)
  SQL: Whitelist SQL utility class for better scripting (#30681)
  Add 5.6.11 version constant.
  Fix version detection.
  [Docs] All Rollup docs experimental, agg limitations, clarify DeleteJob (#31299)
  Add missing release notes.
  Security: fix token bwc with pre 6.0.0-beta2 (#31254)
  Fix compilation error in UpdateSettingsIT (#31304)
  Test: Remove broken yml test feature (#31255)
  Add unreleased version 6.3.1
  [Rollup] Metric config parser must use builder so validation runs (#31159)
  Removes experimental tag from scripted_metric aggregation (#31298)
  [DOCS] Removes coming tag from 6.3.0 release notes
  6.3 release notes.
  Add notion of internal index settings (#31286)
  REST high-level client: add Cluster Health API (#29331)
  Remove leftover usage of deprecated client API
  SyncedFlushResponse to implement ToXContentObject (#31155)
  Add Get Aliases API to the high-level REST client (#28799)
  HLRest: Add get index templates API (#31161)
  Log warnings when cluster state publication failed to some nodes (#31233)
  Fix AntFixture waiting condition (#31272)
  [TEST] Mute RecoveryIT.testHistoryUUIDIsGenerated
  Ignore numeric shard count if waiting for ALL (#31265)
  Update checkstyle to 8.10.1 (#31269)
  Set analyzer version in PreBuiltAnalyzerProviderFactory (#31202)
  Revert upgrade to Netty 4.1.25.Final (#31282)
  Use armored input stream for reading public key (#31229)
  [DOCS] Added 'fail_on_unsupported_field' param to MLT. Closes #28008 (#31160)
  Fix Netty 4 Server Transport tests. Again.
  [DOCS] Fixed typo.
  [DOCS] Added release highlights for 6.3 (#31256)
  [DOCS] Mark SQL feature as experimental
  [DOCS] Updates machine learning custom URL screenshots (#31222)
  Fix naming conventions check for XPackTestCase
  Fix security Netty 4 transport tests
  Fix race in clear scroll (#31259)
  [DOCS] Clarify audit index settings when remote indexing (#30923)
  [ML][TEST] Mute tests using rules (#31204)
  Support RequestedAuthnContext (#31238)
  Validate xContentType in PutWatchRequest. (#31088)
  [INGEST] Interrupt the current thread if evaluation grok expressions take too long (#31024)
  Upgrade to Netty 4.1.25.Final (#31232)
  Suppress extras FS on caching directory tests
  Revert "[DOCS] Added 6.3 info & updated the upgrade table. (#30940)"
  Revert "Fix snippets in upgrade docs"
  Fix snippets in upgrade docs
  [DOCS] Added 6.3 info & updated the upgrade table. (#30940)
  Enable custom credentials for core REST tests (#31235)
  Move ESIndexLevelReplicationTestCase to test framework (#31243)
  Encapsulate Translog in Engine (#31220)
  [DOCS] Adds machine learning 6.3.0 release notes (#31217)
  Remove all unused imports and fix CRLF (#31207)
  [TEST] Fix testRecoveryAfterPrimaryPromotion
  [Docs] Remove mention pattern files in Grok processor (#31170)
  Use stronger write-once semantics for Azure repository (#30437)
  Don't swallow exceptions on replication (#31179)
  Compliant SAML Response destination check (#31175)
  Move java version checker back to its own jar (#30708)
  TEST:  Retry synced-flush if ongoing ops on primary (#30978)
  [test] add fix for rare virtualbox error (#31212)

Author: dnhatn

Vagrantfile

@@ -31,6 +31,9 @@ Vagrant.configure(2) do |config|
     # Give the box more memory and cpu because our tests are beasts!
     vbox.memory = Integer(ENV['VAGRANT_MEMORY'] || 8192)
     vbox.cpus = Integer(ENV['VAGRANT_CPUS'] || 4)
+
+    # see https://github.com/hashicorp/vagrant/issues/9524
+    vbox.customize ["modifyvm", :id, "--audio", "none"]
   end
 
   # Switch the default share for the project root from /vagrant to

build.gradle

@@ -121,7 +121,7 @@ task verifyVersions {
     new URL('https://repo1.maven.org/maven2/org/elasticsearch/elasticsearch/maven-metadata.xml').openStream().withStream { s ->
         xml = new XmlParser().parse(s)
     }
-    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d\.\d\.\d/ }.collect { Version.fromString(it) })
+    Set<Version> knownVersions = new TreeSet<>(xml.versioning.versions.version.collect { it.text() }.findAll { it ==~ /\d+\.\d+\.\d+/ }.collect { Version.fromString(it) })
 
     // Limit the known versions to those that should be index compatible, and are not future versions
     knownVersions = knownVersions.findAll { it.major >= bwcVersions.currentVersion.major - 1 && it.before(VersionProperties.elasticsearch) }

buildSrc/src/main/groovy/org/elasticsearch/gradle/precommit/PrecommitTasks.groovy

@@ -140,7 +140,7 @@ class PrecommitTasks {
             configProperties = [
                 suppressions: checkstyleSuppressions
             ]
-            toolVersion = 7.5
+            toolVersion = '8.10.1'
         }
 
         project.tasks.withType(Checkstyle) { task ->

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/AntFixture.groovy

@@ -149,11 +149,11 @@ public class AntFixture extends AntTask implements Fixture {
         }
 
         // the process is started (has a pid) and is bound to a network interface
-        // so now wait undil the waitCondition has been met
+        // so now evaluates if the waitCondition is successful
         // TODO: change this to a loop?
         boolean success
         try {
-            success = waitCondition(this, ant) == false
+            success = waitCondition(this, ant)
         } catch (Exception e) {
             String msg = "Wait condition caught exception for ${name}"
             logger.error(msg, e)

buildSrc/src/main/resources/checkstyle.xml

@@ -26,13 +26,9 @@
     </module>
 
     <module name="AvoidStarImport" />
-    <!-- Doesn't pass but we could make it pass pretty quick.
-    <module name="UnusedImports">
-      The next property is optional. If we remove it then imports that are
-      only referenced by Javadoc cause the check to fail.
-      <property name="processJavadoc" value="true" />
-    </module>
-    -->
+
+    <!-- Unused imports are forbidden -->
+    <module name="UnusedImports" />
 
     <!-- Non-inner classes must be in files that match their names. -->
     <module name="OuterTypeFilename" />

buildSrc/src/main/resources/checkstyle_suppressions.xml

@@ -600,7 +600,6 @@
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanMultiTermQueryBuilderTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]SpanNotQueryBuilderTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]query[/\\]functionscore[/\\]FunctionScoreTests.java" checks="LineLength" />
-  <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]replication[/\\]ESIndexLevelReplicationTestCase.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]MultiMatchQueryTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]geo[/\\]GeoUtilsTests.java" checks="LineLength" />
   <suppress files="server[/\\]src[/\\]test[/\\]java[/\\]org[/\\]elasticsearch[/\\]index[/\\]search[/\\]nested[/\\]AbstractNumberNestedSortingTestCase.java" checks="LineLength" />

client/benchmark/src/main/java/org/elasticsearch/client/benchmark/AbstractBenchmark.java

@@ -28,8 +28,6 @@
 import java.lang.management.GarbageCollectorMXBean;
 import java.lang.management.ManagementFactory;
 import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.stream.Collectors;
 

client/rest-high-level/src/main/java/org/elasticsearch/client/ClusterClient.java

@@ -21,12 +21,16 @@
 
 import org.apache.http.Header;
 import org.elasticsearch.action.ActionListener;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthRequest;
+import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsRequest;
 import org.elasticsearch.action.admin.cluster.settings.ClusterUpdateSettingsResponse;
+import org.elasticsearch.rest.RestStatus;
 
 import java.io.IOException;
 
 import static java.util.Collections.emptySet;
+import static java.util.Collections.singleton;
 
 /**
  * A wrapper for the {@link RestHighLevelClient} that provides methods for accessing the Cluster API.
@@ -95,4 +99,34 @@ public void putSettingsAsync(ClusterUpdateSettingsRequest clusterUpdateSettingsR
         restHighLevelClient.performRequestAsyncAndParseEntity(clusterUpdateSettingsRequest, RequestConverters::clusterPutSettings,
                 ClusterUpdateSettingsResponse::fromXContent, listener, emptySet(), headers);
     }
+
+    /**
+     * Get cluster health using the Cluster Health API.
+     * See
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html"> Cluster Health API on elastic.co</a>
+     * <p>
+     * If timeout occurred, {@link ClusterHealthResponse} will have isTimedOut() == true and status() == RestStatus.REQUEST_TIMEOUT
+     * @param healthRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @return the response
+     * @throws IOException in case there is a problem sending the request or parsing back the response
+     */
+    public ClusterHealthResponse health(ClusterHealthRequest healthRequest, RequestOptions options) throws IOException {
+        return restHighLevelClient.performRequestAndParseEntity(healthRequest, RequestConverters::clusterHealth, options,
+                ClusterHealthResponse::fromXContent, singleton(RestStatus.REQUEST_TIMEOUT.getStatus()));
+    }
+
+    /**
+     * Asynchronously get cluster health using the Cluster Health API.
+     * See
+     * <a href="https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html"> Cluster Health API on elastic.co</a>
+     * If timeout occurred, {@link ClusterHealthResponse} will have isTimedOut() == true and status() == RestStatus.REQUEST_TIMEOUT
+     * @param healthRequest the request
+     * @param options the request options (e.g. headers), use {@link RequestOptions#DEFAULT} if nothing needs to be customized
+     * @param listener the listener to be notified upon request completion
+     */
+    public void healthAsync(ClusterHealthRequest healthRequest, RequestOptions options, ActionListener<ClusterHealthResponse> listener) {
+        restHighLevelClient.performRequestAsyncAndParseEntity(healthRequest, RequestConverters::clusterHealth, options,
+                ClusterHealthResponse::fromXContent, listener, singleton(RestStatus.REQUEST_TIMEOUT.getStatus()));
+    }
 }

client/rest-high-level/src/main/java/org/elasticsearch/client/GetAliasesResponse.java

@@ -0,0 +1,199 @@
+/*
+ * Licensed to Elasticsearch under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.elasticsearch.client;
+
+import org.elasticsearch.ElasticsearchException;
+import org.elasticsearch.action.ActionResponse;
+import org.elasticsearch.cluster.metadata.AliasMetaData;
+import org.elasticsearch.common.xcontent.StatusToXContentObject;
+import org.elasticsearch.common.xcontent.ToXContent;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.common.xcontent.XContentParser.Token;
+import org.elasticsearch.rest.RestStatus;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import static org.elasticsearch.common.xcontent.XContentParserUtils.ensureExpectedToken;
+
+/**
+ * Response obtained from the get aliases API.
+ * The format is pretty horrible as it holds aliases, but at the same time errors can come back through the status and error fields.
+ * Such errors are mostly 404 - NOT FOUND for aliases that were specified but not found. In such case the client won't throw exception
+ * so it allows to retrieve the returned aliases, while at the same time checking if errors were returned.
+ * There's also the case where an exception is returned, like for instance an {@link org.elasticsearch.index.IndexNotFoundException}.
+ * We would usually throw such exception, but we configure the client to not throw for 404 to support the case above, hence we also not
+ * throw in case an index is not found, although it is a hard error that doesn't come back with aliases.
+ */
+public class GetAliasesResponse extends ActionResponse implements StatusToXContentObject {
+
+    private final RestStatus status;
+    private final String error;
+    private final ElasticsearchException exception;
+
+    private final Map<String, Set<AliasMetaData>> aliases;
+
+    GetAliasesResponse(RestStatus status, String error, Map<String, Set<AliasMetaData>> aliases) {
+        this.status = status;
+        this.error = error;
+        this.aliases = aliases;
+        this.exception = null;
+    }
+
+    private GetAliasesResponse(RestStatus status, ElasticsearchException exception) {
+        this.status = status;
+        this.error = null;
+        this.aliases = Collections.emptyMap();
+        this.exception = exception;
+    }
+
+    @Override
+    public RestStatus status() {
+        return status;
+    }
+
+    /**
+     * Return the possibly returned error, null otherwise
+     */
+    public String getError() {
+        return error;
+    }
+
+    /**
+     * Return the exception that may have been returned
+     */
+    public ElasticsearchException getException() {
+        return exception;
+    }
+
+    /**
+     * Return the requested aliases
+     */
+    public Map<String, Set<AliasMetaData>> getAliases() {
+        return aliases;
+    }
+
+    @Override
+    public XContentBuilder toXContent(XContentBuilder builder, Params params) throws IOException {
+        builder.startObject();
+        {
+            if (status != RestStatus.OK) {
+                builder.field("error", error);
+                builder.field("status", status.getStatus());
+            }
+
+            for (Map.Entry<String, Set<AliasMetaData>> entry : aliases.entrySet()) {
+                builder.startObject(entry.getKey());
+                {
+                    builder.startObject("aliases");
+                    {
+                        for (final AliasMetaData alias : entry.getValue()) {
+                            AliasMetaData.Builder.toXContent(alias, builder, ToXContent.EMPTY_PARAMS);
+                        }
+                    }
+                    builder.endObject();
+                }
+                builder.endObject();
+            }
+        }
+        builder.endObject();
+        return builder;
+    }
+
+    /**
+     * Parse the get aliases response
+     */
+    public static GetAliasesResponse fromXContent(XContentParser parser) throws IOException {
+        if (parser.currentToken() == null) {
+            parser.nextToken();
+        }
+        ensureExpectedToken(Token.START_OBJECT, parser.currentToken(), parser::getTokenLocation);
+        Map<String, Set<AliasMetaData>> aliases = new HashMap<>();
+
+        String currentFieldName;
+        Token token;
+        String error = null;
+        ElasticsearchException exception = null;
+        RestStatus status = RestStatus.OK;
+
+        while (parser.nextToken() != Token.END_OBJECT) {
+            if (parser.currentToken() == Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+
+                if ("status".equals(currentFieldName)) {
+                    if ((token = parser.nextToken()) != Token.FIELD_NAME) {
+                        ensureExpectedToken(Token.VALUE_NUMBER, token, parser::getTokenLocation);
+                        status = RestStatus.fromCode(parser.intValue());
+                    }
+                } else if ("error".equals(currentFieldName)) {
+                    token = parser.nextToken();
+                    if (token == Token.VALUE_STRING) {
+                        error = parser.text();
+                    } else if (token == Token.START_OBJECT) {
+                        parser.nextToken();
+                        exception = ElasticsearchException.innerFromXContent(parser, true);
+                    } else if (token == Token.START_ARRAY) {
+                        parser.skipChildren();
+                    }
+                } else {
+                    String indexName = parser.currentName();
+                    if (parser.nextToken() == Token.START_OBJECT) {
+                        Set<AliasMetaData> parseInside = parseAliases(parser);
+                        aliases.put(indexName, parseInside);
+                    }
+                }
+            }
+        }
+        if (exception != null) {
+            assert error == null;
+            assert aliases.isEmpty();
+            return new GetAliasesResponse(status, exception);
+        }
+        return new GetAliasesResponse(status, error, aliases);
+    }
+
+    private static Set<AliasMetaData> parseAliases(XContentParser parser) throws IOException {
+        Set<AliasMetaData> aliases = new HashSet<>();
+        Token token;
+        String currentFieldName = null;
+        while ((token = parser.nextToken()) != Token.END_OBJECT) {
+            if (token == Token.FIELD_NAME) {
+                currentFieldName = parser.currentName();
+            } else if (token == Token.START_OBJECT) {
+                if ("aliases".equals(currentFieldName)) {
+                    while (parser.nextToken() != Token.END_OBJECT) {
+                        AliasMetaData fromXContent = AliasMetaData.Builder.fromXContent(parser);
+                        aliases.add(fromXContent);
+                    }
+                } else {
+                    parser.skipChildren();
+                }
+            } else if (token == Token.START_ARRAY) {
+                parser.skipChildren();
+            }
+        }
+        return aliases;
+    }
+}