Merge remote-tracking branch 'elastic/master' into animal-sniffer-vestiges

* elastic/master:
  [DOCS] Fixes broken link in auditing settings
  QA: Better seed nodes for rolling restart
  [DOCS] Moves ML content to stack-docs
  [DOCS] Clarifies recommendation for audit index output type (elastic#31146)
  Add nio-transport as option for http smoke tests (elastic#31162)
  QA: Set better node names on rolling restart tests
  Add support for ignore_unmapped to geo sort (elastic#31153)

Author: jasontedor

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -87,8 +87,9 @@ class ClusterConfiguration {
      * A closure to call which returns the unicast host to connect to for cluster formation.
      *
      * This allows multi node clusters, or a new cluster to connect to an existing cluster.
-     * The closure takes two arguments, the NodeInfo for the first node in the cluster, and
-     * an AntBuilder which may be used to wait on conditions before returning.
+     * The closure takes three arguments, the NodeInfo for the first node in the cluster,
+     * the NodeInfo for the node current being configured, an AntBuilder which may be used
+     * to wait on conditions before returning.
      */
     @Input
     Closure unicastTransportUri = { NodeInfo seedNode, NodeInfo node, AntBuilder ant ->

docs/reference/search/request/sort.asciidoc

@@ -288,8 +288,9 @@ GET /_search
                 "pin.location" : [-70, 40],
                 "order" : "asc",
                 "unit" : "km",
-		"mode" : "min",
-		"distance_type" : "arc"
+                "mode" : "min",
+                "distance_type" : "arc",
+                "ignore_unmapped": true
             }
         }
     ],
@@ -317,6 +318,12 @@ GET /_search
 
     The unit to use when computing sort values. The default is `m` (meters).
 
+
+`ignore_unmapped`::
+
+    Indicates if the unmapped field should be treated as a missing value. Setting it to `true` is equivalent to specifying
+    an `unmapped_type` in the field sort. The default is `false` (unmapped field are causing the search to fail).
+
 NOTE: geo distance sorting does not support configurable missing values: the
 distance will always be considered equal to +Infinity+ when a document does not
 have values for the field that is used for distance computation.

docs/reference/settings/audit-settings.asciidoc

@@ -87,7 +87,7 @@ Controls how often to roll over to a new index: `hourly`, `daily`, `weekly`, or
 `xpack.security.audit.index.events.include`::
 Specifies the audit events to be indexed. The default value is
 `anonymous_access_denied, authentication_failed, realm_authentication_failed, access_granted, access_denied, tampered_request, connection_granted, connection_denied, run_as_granted, run_as_denied`.
-See {xpack-ref}/auditing.html#audit-event-types[Audit Entry Types] for the
+See {xpack-ref}/audit-event-types.html[Audit Entry Types] for the
 complete list.
 
 `xpack.security.audit.index.events.exclude`::

modules/transport-netty4/src/main/java/org/elasticsearch/http/netty4/Netty4HttpServerTransport.java

@@ -202,15 +202,7 @@ public Netty4HttpServerTransport(Settings settings, NetworkService networkServic
         this.maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings);
         this.maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings);
         this.pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings);
-        this.httpHandlingSettings = new HttpHandlingSettings(Math.toIntExact(maxContentLength.getBytes()),
-            Math.toIntExact(maxChunkSize.getBytes()),
-            Math.toIntExact(maxHeaderSize.getBytes()),
-            Math.toIntExact(maxInitialLineLength.getBytes()),
-            SETTING_HTTP_RESET_COOKIES.get(settings),
-            SETTING_HTTP_COMPRESSION.get(settings),
-            SETTING_HTTP_COMPRESSION_LEVEL.get(settings),
-            SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings),
-            pipeliningMaxEvents);
+        this.httpHandlingSettings = HttpHandlingSettings.fromSettings(settings);
 
         this.maxCompositeBufferComponents = SETTING_HTTP_NETTY_MAX_COMPOSITE_BUFFER_COMPONENTS.get(settings);
         this.workerCount = SETTING_HTTP_WORKER_COUNT.get(settings);
@@ -446,7 +438,7 @@ protected void initChannel(Channel ch) throws Exception {
             if (handlingSettings.isCompression()) {
                 ch.pipeline().addLast("encoder_compress", new HttpContentCompressor(handlingSettings.getCompressionLevel()));
             }
-            if (SETTING_CORS_ENABLED.get(transport.settings())) {
+            if (handlingSettings.isCorsEnabled()) {
                 ch.pipeline().addLast("cors", new Netty4CorsHandler(transport.getCorsConfig()));
             }
             ch.pipeline().addLast("pipelining", new Netty4HttpPipeliningHandler(transport.logger, transport.pipeliningMaxEvents));

plugins/transport-nio/src/main/java/org/elasticsearch/http/nio/NioHttpServerTransport.java

@@ -137,15 +137,7 @@ public NioHttpServerTransport(Settings settings, NetworkService networkService,
         ByteSizeValue maxHeaderSize = SETTING_HTTP_MAX_HEADER_SIZE.get(settings);
         ByteSizeValue maxInitialLineLength = SETTING_HTTP_MAX_INITIAL_LINE_LENGTH.get(settings);
         int pipeliningMaxEvents = SETTING_PIPELINING_MAX_EVENTS.get(settings);
-        this.httpHandlingSettings = new HttpHandlingSettings(Math.toIntExact(maxContentLength.getBytes()),
-            Math.toIntExact(maxChunkSize.getBytes()),
-            Math.toIntExact(maxHeaderSize.getBytes()),
-            Math.toIntExact(maxInitialLineLength.getBytes()),
-            SETTING_HTTP_RESET_COOKIES.get(settings),
-            SETTING_HTTP_COMPRESSION.get(settings),
-            SETTING_HTTP_COMPRESSION_LEVEL.get(settings),
-            SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings),
-            pipeliningMaxEvents);
+        this.httpHandlingSettings = HttpHandlingSettings.fromSettings(settings);;
         this.corsConfig = buildCorsConfig(settings);
 
         this.tcpNoDelay = SETTING_HTTP_TCP_NO_DELAY.get(settings);

plugins/transport-nio/src/test/java/org/elasticsearch/http/nio/HttpReadWriteHandlerTests.java

@@ -56,6 +56,7 @@
 import java.util.List;
 import java.util.function.BiConsumer;
 
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED;
@@ -94,7 +95,8 @@ public void setMocks() {
             SETTING_HTTP_COMPRESSION.getDefault(settings),
             SETTING_HTTP_COMPRESSION_LEVEL.getDefault(settings),
             SETTING_HTTP_DETAILED_ERRORS_ENABLED.getDefault(settings),
-            SETTING_PIPELINING_MAX_EVENTS.getDefault(settings));
+            SETTING_PIPELINING_MAX_EVENTS.getDefault(settings),
+            SETTING_CORS_ENABLED.getDefault(settings));
         ThreadContext threadContext = new ThreadContext(settings);
         nioSocketChannel = mock(NioSocketChannel.class);
         handler = new HttpReadWriteHandler(nioSocketChannel, transport, httpHandlingSettings, NamedXContentRegistry.EMPTY,

qa/rolling-upgrade/build.gradle

@@ -83,13 +83,15 @@ for (Version version : bwcVersions.wireCompatible) {
       * just stopped's data directory. */
       dataDir = { nodeNumber -> oldClusterTest.nodes[stopNode].dataDir }
       setting 'repositories.url.allowed_urls', 'http://snapshot.test*'
+      setting 'node.name', "upgraded-node-${stopNode}"
     }
   }
 
   Task oneThirdUpgradedTest = tasks.create(name: "${baseName}#oneThirdUpgradedTest", type: RestIntegTestTask)
 
-  configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner,
-      0, { oldClusterTest.nodes.get(1).transportUri() })
+  configureUpgradeCluster("oneThirdUpgradedTestCluster", oldClusterTestRunner, 0,
+    // Use all running nodes as seed nodes so there is no race between pinging and the tests
+    { oldClusterTest.nodes.get(1).transportUri() + ',' + oldClusterTest.nodes.get(2).transportUri() })
 
   Task oneThirdUpgradedTestRunner = tasks.getByName("${baseName}#oneThirdUpgradedTestRunner")
   oneThirdUpgradedTestRunner.configure {
@@ -100,8 +102,9 @@ for (Version version : bwcVersions.wireCompatible) {
 
   Task twoThirdsUpgradedTest = tasks.create(name: "${baseName}#twoThirdsUpgradedTest", type: RestIntegTestTask)
 
-  configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner,
-      1, { oneThirdUpgradedTest.nodes.get(0).transportUri() })
+  configureUpgradeCluster("twoThirdsUpgradedTestCluster", oneThirdUpgradedTestRunner, 1,
+    // Use all running nodes as seed nodes so there is no race between pinging and the tests
+    { oldClusterTest.nodes.get(2).transportUri() + ',' + oneThirdUpgradedTest.nodes.get(0).transportUri() })
 
   Task twoThirdsUpgradedTestRunner = tasks.getByName("${baseName}#twoThirdsUpgradedTestRunner")
   twoThirdsUpgradedTestRunner.configure {
@@ -112,8 +115,9 @@ for (Version version : bwcVersions.wireCompatible) {
 
   Task upgradedClusterTest = tasks.create(name: "${baseName}#upgradedClusterTest", type: RestIntegTestTask)
 
-  configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner,
-      2, { twoThirdsUpgradedTest.nodes.get(0).transportUri() })
+  configureUpgradeCluster("upgradedClusterTestCluster", twoThirdsUpgradedTestRunner, 2,
+    // Use all running nodes as seed nodes so there is no race between pinging and the tests
+    { oneThirdUpgradedTest.nodes.get(0).transportUri() + ',' + twoThirdsUpgradedTest.nodes.get(0).transportUri() })
 
   Task upgradedClusterTestRunner = tasks.getByName("${baseName}#upgradedClusterTestRunner")
   upgradedClusterTestRunner.configure {

qa/smoke-test-http/build.gradle

@@ -23,6 +23,7 @@ apply plugin: 'elasticsearch.test-with-dependencies'
 
 dependencies {
     testCompile project(path: ':modules:transport-netty4', configuration: 'runtime') // for http
+    testCompile project(path: ':plugins:transport-nio', configuration: 'runtime') // for http
 }
 
 integTestRunner {

qa/smoke-test-http/src/test/java/org/elasticsearch/http/HttpSmokeTestCase.java

@@ -25,6 +25,7 @@
 import org.elasticsearch.transport.MockTcpTransportPlugin;
 import org.elasticsearch.transport.Netty4Plugin;
 import org.elasticsearch.transport.nio.MockNioTransportPlugin;
+import org.elasticsearch.transport.nio.NioTransportPlugin;
 import org.junit.BeforeClass;
 
 import java.util.Arrays;
@@ -39,22 +40,33 @@ public abstract class HttpSmokeTestCase extends ESIntegTestCase {
     @SuppressWarnings("unchecked")
     @BeforeClass
     public static void setUpTransport() {
-        nodeTransportTypeKey = getTypeKey(randomFrom(getTestTransportPlugin(), Netty4Plugin.class));
-        nodeHttpTypeKey = getTypeKey(Netty4Plugin.class);
-        clientTypeKey = getTypeKey(randomFrom(getTestTransportPlugin(), Netty4Plugin.class));
+        nodeTransportTypeKey = getTypeKey(randomFrom(getTestTransportPlugin(), Netty4Plugin.class, NioTransportPlugin.class));
+        nodeHttpTypeKey = getHttpTypeKey(randomFrom(Netty4Plugin.class, NioTransportPlugin.class));
+        clientTypeKey = getTypeKey(randomFrom(getTestTransportPlugin(), Netty4Plugin.class, NioTransportPlugin.class));
     }
 
     private static String getTypeKey(Class<? extends Plugin> clazz) {
         if (clazz.equals(MockTcpTransportPlugin.class)) {
             return MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME;
         } else if (clazz.equals(MockNioTransportPlugin.class)) {
             return MockNioTransportPlugin.MOCK_NIO_TRANSPORT_NAME;
+        } else if (clazz.equals(NioTransportPlugin.class)) {
+            return NioTransportPlugin.NIO_TRANSPORT_NAME;
         } else {
             assert clazz.equals(Netty4Plugin.class);
             return Netty4Plugin.NETTY_TRANSPORT_NAME;
         }
     }
 
+    private static String getHttpTypeKey(Class<? extends Plugin> clazz) {
+        if (clazz.equals(NioTransportPlugin.class)) {
+            return NioTransportPlugin.NIO_HTTP_TRANSPORT_NAME;
+        } else {
+            assert clazz.equals(Netty4Plugin.class);
+            return Netty4Plugin.NETTY_HTTP_TRANSPORT_NAME;
+        }
+    }
+
     @Override
     protected boolean addMockHttpTransport() {
         return false; // enable http
@@ -70,12 +82,12 @@ protected Settings nodeSettings(int nodeOrdinal) {
 
     @Override
     protected Collection<Class<? extends Plugin>> nodePlugins() {
-        return Arrays.asList(getTestTransportPlugin(), Netty4Plugin.class);
+        return Arrays.asList(getTestTransportPlugin(), Netty4Plugin.class, NioTransportPlugin.class);
     }
 
     @Override
     protected Collection<Class<? extends Plugin>> transportClientPlugins() {
-        return Arrays.asList(getTestTransportPlugin(), Netty4Plugin.class);
+        return Arrays.asList(getTestTransportPlugin(), Netty4Plugin.class, NioTransportPlugin.class);
     }
 
     @Override

server/src/main/java/org/elasticsearch/http/HttpHandlingSettings.java

@@ -22,6 +22,7 @@
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.unit.ByteSizeValue;
 
+import static org.elasticsearch.http.HttpTransportSettings.SETTING_CORS_ENABLED;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_COMPRESSION_LEVEL;
 import static org.elasticsearch.http.HttpTransportSettings.SETTING_HTTP_DETAILED_ERRORS_ENABLED;
@@ -47,7 +48,7 @@ public class HttpHandlingSettings {
 
     public HttpHandlingSettings(int maxContentLength, int maxChunkSize, int maxHeaderSize, int maxInitialLineLength,
                                 boolean resetCookies, boolean compression, int compressionLevel, boolean detailedErrorsEnabled,
-                                int pipeliningMaxEvents) {
+                                int pipeliningMaxEvents, boolean corsEnabled) {
         this.maxContentLength = maxContentLength;
         this.maxChunkSize = maxChunkSize;
         this.maxHeaderSize = maxHeaderSize;
@@ -57,6 +58,7 @@ public HttpHandlingSettings(int maxContentLength, int maxChunkSize, int maxHeade
         this.compressionLevel = compressionLevel;
         this.detailedErrorsEnabled = detailedErrorsEnabled;
         this.pipeliningMaxEvents = pipeliningMaxEvents;
+        this.corsEnabled = corsEnabled;
     }
 
     public static HttpHandlingSettings fromSettings(Settings settings) {
@@ -68,7 +70,8 @@ public static HttpHandlingSettings fromSettings(Settings settings) {
             SETTING_HTTP_COMPRESSION.get(settings),
             SETTING_HTTP_COMPRESSION_LEVEL.get(settings),
             SETTING_HTTP_DETAILED_ERRORS_ENABLED.get(settings),
-            SETTING_PIPELINING_MAX_EVENTS.get(settings));
+            SETTING_PIPELINING_MAX_EVENTS.get(settings),
+            SETTING_CORS_ENABLED.get(settings));
     }
 
     public int getMaxContentLength() {