Merge remote-tracking branch 'es/master' into ccr

* es/master: (24 commits)
  Reduce synchronization on field data cache
  add json-processor support for non-map json types (#27335)
  Properly format IndexGraveyard deletion date as date (#27362)
  Upgrade AWS SDK Jackson Databind to 2.6.7.1
  Stop responding to ping requests before master abdication (#27329)
  [Test] Fix POI version in packaging tests
  Allow affix settings to specify dependencies (#27161)
  Tests: Improve size regex in documentation test (#26879)
  reword comment
  Remove unnecessary logger creation for doc values field data
  [Geo] Decouple geojson parse logic from ShapeBuilders
  [DOCS] Fixed link to docker content
  Plugins: Add versionless alias to all security policy codebase properties (#26756)
  [Test] #27342 Fix SearchRequests#testValidate
  [DOCS] Move X-Pack-specific Docker content (#27333)
  Fail queries with scroll that explicitely set request_cache (#27342)
  [Test] Fix S3BlobStoreContainerTests.testNumberOfMultiparts()
  Set minimum_master_nodes to all nodes for REST tests (#27344)
  [Tests] Relax allowed delta in extended_stats aggregation (#27171)
  Remove S3 output stream (#27280)
  ...

Author: martijnvg

buildSrc/src/main/groovy/org/elasticsearch/gradle/test/ClusterConfiguration.groovy

@@ -64,10 +64,10 @@ class ClusterConfiguration {
 
     /**
      * Configuration of the setting <tt>discovery.zen.minimum_master_nodes</tt> on the nodes.
-     * In case of more than one node, this defaults to (number of nodes / 2) + 1
+     * In case of more than one node, this defaults to the number of nodes
      */
     @Input
-    Closure<Integer> minimumMasterNodes = { getNumNodes() > 1 ? getNumNodes().intdiv(2) + 1 : -1 }
+    Closure<Integer> minimumMasterNodes = { getNumNodes() > 1 ? getNumNodes() : -1 }
 
     @Input
     String jvmArgs = "-Xms" + System.getProperty('tests.heap.size', '512m') +

client/rest-high-level/src/test/java/org/elasticsearch/client/documentation/QueryDSLDocumentationTests.java

@@ -23,7 +23,7 @@
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.builders.CoordinatesBuilder;
-import org.elasticsearch.common.geo.builders.ShapeBuilders;
+import org.elasticsearch.common.geo.builders.MultiPointBuilder;
 import org.elasticsearch.common.unit.DistanceUnit;
 import org.elasticsearch.index.query.GeoShapeQueryBuilder;
 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
@@ -189,7 +189,7 @@ public void testGeoShape() throws IOException {
             // tag::geo_shape
             GeoShapeQueryBuilder qb = geoShapeQuery(
                     "pin.location",                                      // <1>
-                    ShapeBuilders.newMultiPoint(                         // <2>
+                    new MultiPointBuilder(                         // <2>
                             new CoordinatesBuilder()
                         .coordinate(0, 0)
                         .coordinate(0, 10)

core/src/main/java/org/elasticsearch/action/admin/cluster/settings/SettingsUpdater.java

@@ -54,15 +54,23 @@ synchronized ClusterState updateSettings(final ClusterState currentState, Settin
         transientSettings.put(currentState.metaData().transientSettings());
         changed |= clusterSettings.updateDynamicSettings(transientToApply, transientSettings, transientUpdates, "transient");
 
+
         Settings.Builder persistentSettings = Settings.builder();
         persistentSettings.put(currentState.metaData().persistentSettings());
         changed |= clusterSettings.updateDynamicSettings(persistentToApply, persistentSettings, persistentUpdates, "persistent");
 
         final ClusterState clusterState;
         if (changed) {
+            Settings transientFinalSettings = transientSettings.build();
+            Settings persistentFinalSettings = persistentSettings.build();
+            // both transient and persistent settings must be consistent by itself we can't allow dependencies to be
+            // in either of them otherwise a full cluster restart will break the settings validation
+            clusterSettings.validate(transientFinalSettings, true);
+            clusterSettings.validate(persistentFinalSettings, true);
+
             MetaData.Builder metaData = MetaData.builder(currentState.metaData())
-                    .persistentSettings(persistentSettings.build())
-                    .transientSettings(transientSettings.build());
+                    .persistentSettings(persistentFinalSettings)
+                    .transientSettings(transientFinalSettings);
 
             ClusterBlocks.Builder blocks = ClusterBlocks.builder().blocks(currentState.blocks());
             boolean updatedReadOnly = MetaData.SETTING_READ_ONLY_SETTING.get(metaData.persistentSettings())

core/src/main/java/org/elasticsearch/action/admin/indices/template/put/TransportPutIndexTemplateAction.java

@@ -77,7 +77,7 @@ protected void masterOperation(final PutIndexTemplateRequest request, final Clus
         }
         final Settings.Builder templateSettingsBuilder = Settings.builder();
         templateSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
-        indexScopedSettings.validate(templateSettingsBuilder);
+        indexScopedSettings.validate(templateSettingsBuilder.build(), true); // templates must be consistent with regards to dependencies
         indexTemplateService.putTemplate(new MetaDataIndexTemplateService.PutRequest(cause, request.name())
                 .patterns(request.patterns())
                 .order(request.order())

core/src/main/java/org/elasticsearch/action/search/SearchRequest.java

@@ -169,6 +169,10 @@ public ActionRequestValidationException validate() {
             validationException =
                 addValidationError("using [from] is not allowed in a scroll context", validationException);
         }
+        if (requestCache != null && requestCache && scroll() != null) {
+            validationException =
+                addValidationError("[request_cache] cannot be used in a a scroll context", validationException);
+        }
         return validationException;
     }
 

core/src/main/java/org/elasticsearch/bootstrap/Security.java

@@ -199,28 +199,28 @@ static Policy readPolicy(URL policyFile, Set<URL> codebases) {
             try {
                 // set codebase properties
                 for (URL url : codebases) {
-                    String shortName = PathUtils.get(url.toURI()).getFileName().toString();
-                    if (shortName.endsWith(".jar") == false) {
+                    String fileName = PathUtils.get(url.toURI()).getFileName().toString();
+                    if (fileName.endsWith(".jar") == false) {
                         continue; // tests :(
                     }
-                    String property = "codebase." + shortName;
-                    if (shortName.startsWith("elasticsearch-rest-client")) {
-                        // The rest client is currently the only example where we have an elasticsearch built artifact
-                        // which needs special permissions in policy files when used. This temporary solution is to
-                        // pass in an extra system property that omits the -version.jar suffix the other properties have.
-                        // That allows the snapshots to reference snapshot builds of the client, and release builds to
-                        // referenced release builds of the client, all with the same grant statements.
-                        final String esVersion = Version.CURRENT + (Build.CURRENT.isSnapshot() ? "-SNAPSHOT" : "");
-                        final int index = property.indexOf("-" + esVersion + ".jar");
-                        assert index >= 0;
-                        String restClientAlias = property.substring(0, index);
-                        propertiesSet.add(restClientAlias);
-                        System.setProperty(restClientAlias, url.toString());
+                    // We attempt to use a versionless identifier for each codebase. This assumes a specific version
+                    // format in the jar filename. While we cannot ensure all jars in all plugins use this format, nonconformity
+                    // only means policy grants would need to include the entire jar filename as they always have before.
+                    String property = "codebase." + fileName;
+                    String aliasProperty = "codebase." + fileName.replaceFirst("-\\d+\\.\\d+.*\\.jar", "");
+                    if (aliasProperty.equals(property) == false) {
+                        propertiesSet.add(aliasProperty);
+                        String previous = System.setProperty(aliasProperty, url.toString());
+                        if (previous != null) {
+                            throw new IllegalStateException("codebase property already set: " + aliasProperty + " -> " + previous +
+                                                            ", cannot set to " + url.toString());
+                        }
                     }
                     propertiesSet.add(property);
                     String previous = System.setProperty(property, url.toString());
                     if (previous != null) {
-                        throw new IllegalStateException("codebase property already set: " + shortName + "->" + previous);
+                        throw new IllegalStateException("codebase property already set: " + property + " -> " + previous +
+                                                        ", cannot set to " + url.toString());
                     }
                 }
                 return Policy.getInstance("JavaPolicy", new URIParameter(policyFile.toURI()));

core/src/main/java/org/elasticsearch/cluster/metadata/IndexGraveyard.java

@@ -34,6 +34,8 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.Index;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
 
 import java.io.IOException;
 import java.util.ArrayList;
@@ -432,7 +434,7 @@ public XContentBuilder toXContent(final XContentBuilder builder, final Params pa
             builder.startObject();
             builder.field(INDEX_KEY);
             index.toXContent(builder, params);
-            builder.timeValueField(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis, TimeUnit.MILLISECONDS);
+            builder.dateField(DELETE_DATE_IN_MILLIS_KEY, DELETE_DATE_KEY, deleteDateInMillis);
             return builder.endObject();
         }
 

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataCreateIndexService.java

@@ -220,10 +220,9 @@ public void createIndex(final CreateIndexClusterStateUpdateRequest request,
     private void onlyCreateIndex(final CreateIndexClusterStateUpdateRequest request,
                                  final ActionListener<ClusterStateUpdateResponse> listener) {
         Settings.Builder updatedSettingsBuilder = Settings.builder();
-        updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX);
-        indexScopedSettings.validate(updatedSettingsBuilder);
-        request.settings(updatedSettingsBuilder.build());
-
+        Settings build = updatedSettingsBuilder.put(request.settings()).normalizePrefix(IndexMetaData.INDEX_SETTING_PREFIX).build();
+        indexScopedSettings.validate(build, true); // we do validate here - index setting must be consistent
+        request.settings(build);
         clusterService.submitStateUpdateTask("create-index [" + request.index() + "], cause [" + request.cause() + "]",
             new IndexCreationTask(logger, allocationService, request, listener, indicesService, aliasValidator, xContentRegistry, settings,
                 this::validate));
@@ -420,7 +419,6 @@ public ClusterState execute(ClusterState currentState) throws Exception {
                         tmpImdBuilder.primaryTerm(shardId, primaryTerm);
                     }
                 }
-
                 // Set up everything, now locally create the index to see that things are ok, and apply
                 final IndexMetaData tmpImd = tmpImdBuilder.build();
                 ActiveShardCount waitForActiveShards = request.waitForActiveShards();

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataIndexTemplateService.java

@@ -276,7 +276,7 @@ private void validate(PutRequest request) {
         }
 
         try {
-            indexScopedSettings.validate(request.settings);
+            indexScopedSettings.validate(request.settings, true); // templates must be consistent with regards to dependencies
         } catch (IllegalArgumentException iae) {
             validationErrors.add(iae.getMessage());
             for (Throwable t : iae.getSuppressed()) {

core/src/main/java/org/elasticsearch/cluster/metadata/MetaDataUpdateSettingsService.java

@@ -54,6 +54,7 @@
 import java.util.Locale;
 import java.util.Map;
 import java.util.Set;
+import java.util.function.Predicate;
 
 import static org.elasticsearch.action.support.ContextPreservingActionListener.wrapPreservingContext;
 
@@ -163,7 +164,7 @@ public void updateSettings(final UpdateSettingsClusterStateUpdateRequest request
         Settings.Builder settingsForOpenIndices = Settings.builder();
         final Set<String> skippedSettings = new HashSet<>();
 
-        indexScopedSettings.validate(normalizedSettings);
+        indexScopedSettings.validate(normalizedSettings, false); // don't validate dependencies here we check it below
         // never allow to change the number of shards
         for (String key : normalizedSettings.keySet()) {
             Setting setting = indexScopedSettings.get(key);
@@ -240,7 +241,9 @@ public ClusterState execute(ClusterState currentState) {
                             if (preserveExisting) {
                                 indexSettings.put(indexMetaData.getSettings());
                             }
-                            metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
+                            Settings finalSettings = indexSettings.build();
+                            indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
+                            metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(finalSettings));
                         }
                     }
                 }
@@ -254,7 +257,9 @@ public ClusterState execute(ClusterState currentState) {
                             if (preserveExisting) {
                                 indexSettings.put(indexMetaData.getSettings());
                             }
-                            metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(indexSettings));
+                            Settings finalSettings = indexSettings.build();
+                            indexScopedSettings.validate(finalSettings.filter(k -> indexScopedSettings.isPrivateSetting(k) == false), true);
+                            metaDataBuilder.put(IndexMetaData.builder(indexMetaData).settings(finalSettings));
                         }
                     }
                 }