elastic · jpountz · Aug 29, 2017
diff --git a/core/src/main/java/org/elasticsearch/action/ActionModule.java b/core/src/main/java/org/elasticsearch/action/ActionModule.java
@@ -203,8 +203,12 @@
 import org.elasticsearch.common.logging.ESLoggerFactory;
 import org.elasticsearch.common.settings.ClusterSettings;
 import org.elasticsearch.common.settings.IndexScopedSettings;
+import org.elasticsearch.common.settings.Setting;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.settings.SettingsFilter;
+import org.elasticsearch.common.settings.Setting.Property;
+import org.elasticsearch.common.unit.ByteSizeUnit;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.indices.breaker.CircuitBreakerService;
 import org.elasticsearch.plugins.ActionPlugin;
 import org.elasticsearch.plugins.ActionPlugin.ActionHandler;
@@ -318,6 +322,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Consumer;
 import java.util.function.Supplier;
 import java.util.function.UnaryOperator;
@@ -333,6 +338,10 @@ public class ActionModule extends AbstractModule {
 
     private static final Logger logger = ESLoggerFactory.getLogger(ActionModule.class);
 
+    public static final Setting<ByteSizeValue> SETTING_SEARCH_MAX_CONTENT_LENGTH =
+            Setting.byteSizeSetting("http.search.max_content_length", new ByteSizeValue(1, ByteSizeUnit.MB),
+                    Property.NodeScope, Property.Dynamic);
+
     private final boolean transportClient;
     private final Settings settings;
     private final IndexNameExpressionResolver indexNameExpressionResolver;
@@ -345,6 +354,7 @@ public class ActionModule extends AbstractModule {
     private final AutoCreateIndex autoCreateIndex;
     private final DestructiveOperations destructiveOperations;
     private final RestController restController;
+    private final AtomicReference<ByteSizeValue> maxSearchContentLength;
 
     public ActionModule(boolean transportClient, Settings settings, IndexNameExpressionResolver indexNameExpressionResolver,
                         IndexScopedSettings indexScopedSettings, ClusterSettings clusterSettings, SettingsFilter settingsFilter,
@@ -378,9 +388,10 @@ public ActionModule(boolean transportClient, Settings settings, IndexNameExpress
         } else {
             restController = new RestController(settings, headers, restWrapper, nodeClient, circuitBreakerService, usageService);
         }
+        maxSearchContentLength = new AtomicReference<>(clusterSettings.get(SETTING_SEARCH_MAX_CONTENT_LENGTH));
+        clusterSettings.addSettingsUpdateConsumer(SETTING_SEARCH_MAX_CONTENT_LENGTH, maxSearchContentLength::set);
     }
 
-
     public Map<String, ActionHandler<?, ?>> getActions() {
         return actions;
     }
@@ -587,10 +598,10 @@ public void initRestHandlers(Supplier<DiscoveryNodes> nodesInCluster) {
         registerHandler.accept(new RestBulkAction(settings, restController));
         registerHandler.accept(new RestUpdateAction(settings, restController));
 
-        registerHandler.accept(new RestSearchAction(settings, restController));
+        registerHandler.accept(new RestSearchAction(settings, restController, maxSearchContentLength));
         registerHandler.accept(new RestSearchScrollAction(settings, restController));
         registerHandler.accept(new RestClearScrollAction(settings, restController));
-        registerHandler.accept(new RestMultiSearchAction(settings, restController));
+        registerHandler.accept(new RestMultiSearchAction(settings, restController, maxSearchContentLength));
 
         registerHandler.accept(new RestValidateQueryAction(settings, restController));
 

diff --git a/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java b/core/src/main/java/org/elasticsearch/common/settings/ClusterSettings.java
@@ -18,6 +18,7 @@
  */
 package org.elasticsearch.common.settings;
 
+import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.admin.indices.close.TransportCloseIndexAction;
 import org.elasticsearch.action.search.TransportSearchAction;
 import org.elasticsearch.action.support.AutoCreateIndex;
@@ -410,6 +411,7 @@ public void apply(Settings value, Settings current, Settings previous) {
                     ThreadPool.ESTIMATED_TIME_INTERVAL_SETTING,
                     FastVectorHighlighter.SETTING_TV_HIGHLIGHT_MULTI_VALUE,
                     Node.BREAKER_TYPE_KEY,
-                    IndexGraveyard.SETTING_MAX_TOMBSTONES
+                    IndexGraveyard.SETTING_MAX_TOMBSTONES,
+                    ActionModule.SETTING_SEARCH_MAX_CONTENT_LENGTH
             )));
 }
diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestMultiSearchAction.java
@@ -20,6 +20,7 @@
 package org.elasticsearch.rest.action.search;
 
 import org.elasticsearch.ElasticsearchParseException;
+import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.search.MultiSearchRequest;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
@@ -28,6 +29,7 @@
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.collect.Tuple;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContent;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.common.xcontent.XContentType;
@@ -42,6 +44,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.BiConsumer;
 
 import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeBooleanValue;
@@ -55,9 +58,11 @@ public class RestMultiSearchAction extends BaseRestHandler {
     private static final Set<String> RESPONSE_PARAMS = Collections.singleton(RestSearchAction.TYPED_KEYS_PARAM);
 
     private final boolean allowExplicitIndex;
+    private final AtomicReference<ByteSizeValue> maxSearchContentLength;
 
-    public RestMultiSearchAction(Settings settings, RestController controller) {
+    public RestMultiSearchAction(Settings settings, RestController controller, AtomicReference<ByteSizeValue> maxSearchContentLength) {
         super(settings);
+        this.maxSearchContentLength = maxSearchContentLength;
 
         controller.registerHandler(GET, "/_msearch", this);
         controller.registerHandler(POST, "/_msearch", this);
@@ -76,14 +81,15 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
-        MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex);
+        MultiSearchRequest multiSearchRequest = parseRequest(request, allowExplicitIndex, maxSearchContentLength.get());
         return channel -> client.multiSearch(multiSearchRequest, new RestToXContentListener<>(channel));
     }
 
     /**
      * Parses a {@link RestRequest} body and returns a {@link MultiSearchRequest}
      */
-    public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex) throws IOException {
+    public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean allowExplicitIndex,
+            ByteSizeValue maxSearchContentLength) throws IOException {
         MultiSearchRequest multiRequest = new MultiSearchRequest();
         if (restRequest.hasParam("max_concurrent_searches")) {
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
@@ -92,14 +98,15 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a
         int preFilterShardSize = restRequest.paramAsInt("pre_filter_shard_size", SearchRequest.DEFAULT_PRE_FILTER_SHARD_SIZE);
 
 
-        parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, (searchRequest, parser) -> {
-            try {
-                searchRequest.source(SearchSourceBuilder.fromXContent(parser));
-                multiRequest.add(searchRequest);
-            } catch (IOException e) {
-                throw new ElasticsearchParseException("Exception when parsing search request", e);
-            }
-        });
+        parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex, maxSearchContentLength,
+                (searchRequest, parser) -> {
+                    try {
+                        searchRequest.source(SearchSourceBuilder.fromXContent(parser));
+                        multiRequest.add(searchRequest);
+                    } catch (IOException e) {
+                        throw new ElasticsearchParseException("Exception when parsing search request", e);
+                    }
+                });
         List<SearchRequest> requests = multiRequest.requests();
         preFilterShardSize = Math.max(1, preFilterShardSize / (requests.size()+1));
         for (SearchRequest request : requests) {
@@ -113,7 +120,7 @@ public static MultiSearchRequest parseRequest(RestRequest restRequest, boolean a
      * Parses a multi-line {@link RestRequest} body, instantiating a {@link SearchRequest} for each line and applying the given consumer.
      */
     public static void parseMultiLineRequest(RestRequest request, IndicesOptions indicesOptions, boolean allowExplicitIndex,
-            BiConsumer<SearchRequest, XContentParser> consumer) throws IOException {
+            ByteSizeValue maxSearchContentLength, BiConsumer<SearchRequest, XContentParser> consumer) throws IOException {
 
         String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         String[] types = Strings.splitStringByCommaToArray(request.param("type"));
@@ -193,7 +200,14 @@ public static void parseMultiLineRequest(RestRequest request, IndicesOptions ind
             if (nextMarker == -1) {
                 break;
             }
-            BytesReference bytes = data.slice(from, nextMarker - from);
+            final int reqLength = nextMarker - from;
+            if (reqLength > maxSearchContentLength.getBytes()) {
+                throw new IllegalArgumentException("Search request body has a size of [" + new ByteSizeValue(reqLength)
+                        + "] which is larger than the configured limit of [" + maxSearchContentLength
+                        + "]. If you really need to send such large requests, you can update the ["
+                        + ActionModule.SETTING_SEARCH_MAX_CONTENT_LENGTH.getKey() +"] cluster setting to a higher value.");
+            }
+            BytesReference bytes = data.slice(from, reqLength);
             try (XContentParser parser = xContent.createParser(request.getXContentRegistry(), bytes)) {
                 consumer.accept(searchRequest, parser);
             }

diff --git a/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java b/core/src/main/java/org/elasticsearch/rest/action/search/RestSearchAction.java
@@ -19,11 +19,13 @@
 
 package org.elasticsearch.rest.action.search;
 
+import org.elasticsearch.action.ActionModule;
 import org.elasticsearch.action.search.SearchRequest;
 import org.elasticsearch.action.support.IndicesOptions;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.XContentParser;
 import org.elasticsearch.index.query.QueryBuilder;
 import org.elasticsearch.rest.BaseRestHandler;
@@ -44,6 +46,7 @@
 import java.util.Arrays;
 import java.util.Collections;
 import java.util.Set;
+import java.util.concurrent.atomic.AtomicReference;
 
 import static org.elasticsearch.common.unit.TimeValue.parseTimeValue;
 import static org.elasticsearch.rest.RestRequest.Method.GET;
@@ -55,8 +58,11 @@ public class RestSearchAction extends BaseRestHandler {
     public static final String TYPED_KEYS_PARAM = "typed_keys";
     private static final Set<String> RESPONSE_PARAMS = Collections.singleton(TYPED_KEYS_PARAM);
 
-    public RestSearchAction(Settings settings, RestController controller) {
+    private final AtomicReference<ByteSizeValue> maxSearchContentLength;
+
+    public RestSearchAction(Settings settings, RestController controller, AtomicReference<ByteSizeValue> maxSearchContentLength) {
         super(settings);
+        this.maxSearchContentLength = maxSearchContentLength;
         controller.registerHandler(GET, "/_search", this);
         controller.registerHandler(POST, "/_search", this);
         controller.registerHandler(GET, "/{index}/_search", this);
@@ -72,6 +78,17 @@ public String getName() {
 
     @Override
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
+        if (request.hasContent()) {
+            final int length = request.requiredContent().length();
+            final ByteSizeValue maxSearchContentLength = this.maxSearchContentLength.get();
+            if (length > maxSearchContentLength.getBytes()) {
+                throw new IllegalArgumentException("Search request body has a size of [" + new ByteSizeValue(length)
+                        + "] which is larger than the configured limit of [" + maxSearchContentLength
+                        + "]. If you really need to send such large requests, you can update the ["
+                        + ActionModule.SETTING_SEARCH_MAX_CONTENT_LENGTH.getKey() +"] cluster setting to a higher value.");
+            }
+        }
+
         SearchRequest searchRequest = new SearchRequest();
         request.withContentOrSourceParamParserOrNull(parser ->
             parseSearchRequest(searchRequest, request, parser));

diff --git a/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java b/core/src/test/java/org/elasticsearch/action/search/MultiSearchRequestTests.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.common.ParseField;
 import org.elasticsearch.common.Strings;
 import org.elasticsearch.common.bytes.BytesArray;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.common.xcontent.NamedXContentRegistry;
 import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -85,7 +86,7 @@ public void testSimpleAddWithCarriageReturn() throws Exception {
             "{\"query\" : {\"match_all\" :{}}}\r\n";
         FakeRestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withContent(new BytesArray(requestContent), XContentType.JSON).build();
-        MultiSearchRequest request = RestMultiSearchAction.parseRequest(restRequest, true);
+        MultiSearchRequest request = RestMultiSearchAction.parseRequest(restRequest, true, new ByteSizeValue(16_384));
         assertThat(request.requests().size(), equalTo(1));
         assertThat(request.requests().get(0).indices()[0], equalTo("test"));
         assertThat(request.requests().get(0).indicesOptions(),
@@ -177,21 +178,21 @@ public void testMsearchTerminatedByNewline() throws Exception {
         RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
                 .withContent(new BytesArray(mserchAction.getBytes(StandardCharsets.UTF_8)), XContentType.JSON).build();
         IllegalArgumentException expectThrows = expectThrows(IllegalArgumentException.class,
-                () -> RestMultiSearchAction.parseRequest(restRequest, true));
+                () -> RestMultiSearchAction.parseRequest(restRequest, true, new ByteSizeValue(16_384)));
         assertEquals("The msearch request must be terminated by a newline [\n]", expectThrows.getMessage());
 
         String mserchActionWithNewLine = mserchAction + "\n";
         RestRequest restRequestWithNewLine = new FakeRestRequest.Builder(xContentRegistry())
                 .withContent(new BytesArray(mserchActionWithNewLine.getBytes(StandardCharsets.UTF_8)), XContentType.JSON).build();
-        MultiSearchRequest msearchRequest = RestMultiSearchAction.parseRequest(restRequestWithNewLine, true);
+        MultiSearchRequest msearchRequest = RestMultiSearchAction.parseRequest(restRequestWithNewLine, true, new ByteSizeValue(16_384));
         assertEquals(3, msearchRequest.requests().size());
     }
 
     private MultiSearchRequest parseMultiSearchRequest(String sample) throws IOException {
         byte[] data = StreamsUtils.copyToBytesFromClasspath(sample);
         RestRequest restRequest = new FakeRestRequest.Builder(xContentRegistry())
             .withContent(new BytesArray(data), XContentType.JSON).build();
-        return RestMultiSearchAction.parseRequest(restRequest, true);
+        return RestMultiSearchAction.parseRequest(restRequest, true, new ByteSizeValue(16_384));
     }
 
     @Override

diff --git a/...stache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java b/...stache/src/main/java/org/elasticsearch/script/mustache/RestMultiSearchTemplateAction.java
@@ -23,6 +23,7 @@
 import org.elasticsearch.ElasticsearchParseException;
 import org.elasticsearch.client.node.NodeClient;
 import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.common.unit.ByteSizeValue;
 import org.elasticsearch.rest.BaseRestHandler;
 import org.elasticsearch.rest.RestController;
 import org.elasticsearch.rest.RestRequest;
@@ -75,8 +76,10 @@ public static MultiSearchTemplateRequest parseRequest(RestRequest restRequest, b
             multiRequest.maxConcurrentSearchRequests(restRequest.paramAsInt("max_concurrent_searches", 0));
         }
 
+        // ignore the limit for now, we will evaluate it after the script is executed
+        ByteSizeValue limit = new ByteSizeValue(Long.MAX_VALUE);
         RestMultiSearchAction.parseMultiLineRequest(restRequest, multiRequest.indicesOptions(), allowExplicitIndex,
-                (searchRequest, bytes) -> {
+                limit, (searchRequest, bytes) -> {
                     try {
                         SearchTemplateRequest searchTemplateRequest = RestSearchTemplateAction.parse(bytes);
                         if (searchTemplateRequest.getScript() != null) {