Deprecate size in cluster state response

client/rest-high-level/src/test/java/org/elasticsearch/client/ClusterClientIT.java

@@ -78,7 +78,7 @@ public void testClusterPutSettings() throws IOException {
         assertThat(setResponse.getPersistentSettings().get(persistentSettingKey), notNullValue());
         assertThat(setResponse.getPersistentSettings().get(persistentSettingKey), equalTo(persistentSettingValue));
 
-        Map<String, Object> setMap = getAsMap("/_cluster/settings");
+        Map<String, Object> setMap = requestAsMap(newGetClusterSettingsRequest());
         String transientSetValue = (String) XContentMapValues.extractValue("transient." + transientSettingKey, setMap);
         assertThat(transientSetValue, equalTo(transientSettingValue + ByteSizeUnit.BYTES.getSuffix()));
         String persistentSetValue = (String) XContentMapValues.extractValue("persistent." + persistentSettingKey, setMap);
@@ -96,7 +96,7 @@ public void testClusterPutSettings() throws IOException {
         assertThat(resetResponse.getTransientSettings(), equalTo(Settings.EMPTY));
         assertThat(resetResponse.getPersistentSettings(), equalTo(Settings.EMPTY));
 
-        Map<String, Object> resetMap = getAsMap("/_cluster/settings");
+        Map<String, Object> resetMap = requestAsMap(newGetClusterSettingsRequest());
         String transientResetValue = (String) XContentMapValues.extractValue("transient." + transientSettingKey, resetMap);
         assertThat(transientResetValue, equalTo(null));
         String persistentResetValue = (String) XContentMapValues.extractValue("persistent." + persistentSettingKey, resetMap);

docs/reference/cluster/state.asciidoc

@@ -10,10 +10,8 @@ GET /_cluster/state
 --------------------------------------------------
 // CONSOLE
 
-The response provides the cluster name, the total compressed size
-of the cluster state (its size when serialized for transmission over
-the network), and the cluster state itself, which can be filtered to
-only retrieve the parts of interest, as described below.
+The response provides the cluster name, and the cluster state itself, which can
+be filtered to only retrieve the parts of interest, as described below.
 
 The cluster's `cluster_uuid` is also returned as part of the top-level
 response, in addition to the `metadata` section. added[6.4.0]
@@ -26,6 +24,13 @@ ensure that the latest cluster state is returned.
 For debugging purposes, you can retrieve the cluster state local to a
 particular node by adding `local=true` to the  query string.
 
+By default the cluster state response includes the compressed size of the
+cluster state as transported over the network to new nodes. This can be
+expensive to compute. This calculation can be avoided by adding
+`?compressed_cluster_state_size=false` to the query string. deprecated[8.0.0,
+In a future version this information will be removed. The default will be
+`?compressed_cluster_state_size=false` and other values will be ignored.]
+
 [float]
 === Response Filters
 

modules/repository-url/src/test/java/org/elasticsearch/repositories/url/RepositoryURLClientYamlTestSuiteIT.java

@@ -70,7 +70,7 @@ public static Iterable<Object[]> parameters() throws Exception {
      **/
     @Before
     public void registerRepositories() throws IOException {
-        Request clusterSettingsRequest = new Request("GET", "/_cluster/settings");
+        Request clusterSettingsRequest = newGetClusterSettingsRequest();
         clusterSettingsRequest.addParameter("include_defaults", "true");
         clusterSettingsRequest.addParameter("filter_path", "defaults.path.repo,defaults.repositories.url.allowed_urls");
         Response clusterSettingsResponse = client().performRequest(clusterSettingsRequest);

modules/transport-netty4/src/test/java/org/elasticsearch/rest/Netty4BadRequestIT.java

@@ -77,7 +77,7 @@ public void testBadRequest() throws IOException {
     }
 
     public void testInvalidParameterValue() throws IOException {
-        final Request request = new Request("GET", "/_cluster/settings");
+        final Request request = newGetClusterSettingsRequest();
         request.addParameter("pretty", "neither-true-nor-false");
         final ResponseException e = expectThrows(ResponseException.class, () -> client().performRequest(request));
         final Response response = e.getResponse();
@@ -89,7 +89,7 @@ public void testInvalidParameterValue() throws IOException {
     }
 
     public void testInvalidHeaderValue() throws IOException {
-        final Request request = new Request("GET", "/_cluster/settings");
+        final Request request = newGetClusterSettingsRequest();
         final RequestOptions.Builder options = request.getOptions().toBuilder();
         options.addHeader("Content-Type", "\t");
         request.setOptions(options);

plugins/repository-hdfs/src/test/java/org/elasticsearch/repositories/hdfs/HdfsTests.java

@@ -142,7 +142,7 @@ public void testSimpleWorkflow() {
         assertThat(restoreSnapshotResponse.getRestoreInfo().totalShards(), greaterThan(0));
         ensureGreen();
         assertThat(count(client, "test-idx-1"), equalTo(100L));
-        ClusterState clusterState = client.admin().cluster().prepareState().get().getState();
+        ClusterState clusterState = client.admin().cluster().prepareState().setCompressedClusterStateSize(false).get().getState();
         assertThat(clusterState.getMetaData().hasIndex("test-idx-1"), equalTo(true));
         assertThat(clusterState.getMetaData().hasIndex("test-idx-2"), equalTo(false));
     }

qa/full-cluster-restart/src/test/java/org/elasticsearch/upgrades/FullClusterRestartIT.java

@@ -34,7 +34,6 @@
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.json.JsonXContent;
 import org.elasticsearch.common.xcontent.support.XContentMapValues;
-
 import org.elasticsearch.test.NotEqualMessageBuilder;
 import org.elasticsearch.test.rest.ESRestTestCase;
 import org.elasticsearch.test.rest.yaml.ObjectPath;
@@ -232,7 +231,7 @@ public void testClusterState() throws Exception {
         }
 
         // verifying if we can still read some properties from cluster state api:
-        Map<String, Object> clusterState = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
+        Map<String, Object> clusterState = entityAsMap(client().performRequest(newGetClusterStateRequest()));
 
         // Check some global properties:
         String clusterName = (String) clusterState.get("cluster_name");
@@ -1010,7 +1009,7 @@ public void testClosedIndices() throws Exception {
      */
     @SuppressWarnings("unchecked")
     private void assertClosedIndex(final String index, final boolean checkRoutingTable) throws IOException {
-        final Map<String, ?> state = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
+        final Map<String, ?> state = entityAsMap(client().performRequest(newGetClusterStateRequest()));
 
         final Map<String, ?> metadata = (Map<String, Object>) XContentMapValues.extractValue("metadata.indices." + index, state);
         assertThat(metadata, notNullValue());
@@ -1107,7 +1106,7 @@ private void checkSnapshot(final String snapshotName, final int count, final Ver
         client().performRequest(new Request("DELETE", "/restored_*"));
 
         // Check settings added by the restore process
-        Request clusterSettingsRequest = new Request("GET", "/_cluster/settings");
+        Request clusterSettingsRequest = newGetClusterSettingsRequest();
         clusterSettingsRequest.addParameter("flat_settings", "true");
         Map<String, Object> clusterSettingsResponse = entityAsMap(client().performRequest(clusterSettingsRequest));
         @SuppressWarnings("unchecked") final Map<String, Object> persistentSettings =

qa/mixed-cluster/src/test/java/org/elasticsearch/backwards/IndexingIT.java

@@ -354,7 +354,7 @@ private Nodes buildNodeAndVersions() throws IOException {
                 Version.fromString(objectPath.evaluate("nodes." + id + ".version")),
                 HttpHost.create(objectPath.evaluate("nodes." + id + ".http.publish_address"))));
         }
-        response = client().performRequest(new Request("GET", "_cluster/state"));
+        response = client().performRequest(newGetClusterStateRequest());
         nodes.setMasterNodeId(ObjectPath.createFromResponse(response).evaluate("master_node"));
         return nodes;
     }

qa/rolling-upgrade/src/test/java/org/elasticsearch/upgrades/RecoveryIT.java

@@ -419,7 +419,7 @@ private static Version minimumNodeVersion() throws IOException {
      */
     @SuppressWarnings("unchecked")
     private void assertClosedIndex(final String index, final boolean checkRoutingTable) throws IOException {
-        final Map<String, ?> state = entityAsMap(client().performRequest(new Request("GET", "/_cluster/state")));
+        final Map<String, ?> state = entityAsMap(client().performRequest(newGetClusterStateRequest()));
 
         final Map<String, ?> metadata = (Map<String, Object>) XContentMapValues.extractValue("metadata.indices." + index, state);
         assertThat(metadata, notNullValue());

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequest.java

@@ -40,6 +40,7 @@ public class ClusterStateRequest extends MasterNodeReadRequest<ClusterStateReque
     private boolean metaData = true;
     private boolean blocks = true;
     private boolean customs = true;
+    private boolean compressedClusterStateSize = true;
     private Long waitForMetaDataVersion;
     private TimeValue waitForTimeout = DEFAULT_WAIT_FOR_NODE_TIMEOUT;
     private String[] indices = Strings.EMPTY_ARRAY;
@@ -61,6 +62,9 @@ public ClusterStateRequest(StreamInput in) throws IOException {
             waitForTimeout = in.readTimeValue();
             waitForMetaDataVersion = in.readOptionalLong();
         }
+        if (in.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO aiming for V_6_7_0 after backport
+            compressedClusterStateSize = in.readBoolean();
+        }
     }
 
     @Override
@@ -77,6 +81,9 @@ public void writeTo(StreamOutput out) throws IOException {
             out.writeTimeValue(waitForTimeout);
             out.writeOptionalLong(waitForMetaDataVersion);
         }
+        if (out.getVersion().onOrAfter(Version.V_8_0_0)) { // TODO aiming for V_6_7_0 after backport
+            out.writeBoolean(compressedClusterStateSize);
+        }
     }
 
     @Override
@@ -192,6 +199,15 @@ public ClusterStateRequest waitForMetaDataVersion(long waitForMetaDataVersion) {
         return this;
     }
 
+    public boolean compressedClusterStateSize() {
+        return compressedClusterStateSize;
+    }
+
+    public ClusterStateRequest compressedClusterStateSize(boolean compressedClusterStateSize) {
+        this.compressedClusterStateSize = compressedClusterStateSize;
+        return this;
+    }
+
     @Override
     public void readFrom(StreamInput in) throws IOException {
         throw new UnsupportedOperationException("usage of Streamable is to be replaced by Writeable");

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateRequestBuilder.java

@@ -100,4 +100,12 @@ public ClusterStateRequestBuilder setIndicesOptions(IndicesOptions indicesOption
         request.indicesOptions(indicesOptions);
         return this;
     }
+
+    /**
+     * Should the cluster state result include the size of the compressed cluster state in bytes. Defaults to {@code true}.
+     */
+    public ClusterStateRequestBuilder setCompressedClusterStateSize(boolean compressedClusterStateSize) {
+        request.compressedClusterStateSize(compressedClusterStateSize);
+        return this;
+    }
 }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/ClusterStateResponse.java

@@ -75,6 +75,7 @@ public ClusterName getClusterName() {
      * of the cluster state as it would be transmitted over the network during
      * intra-node communication.
      */
+    @Deprecated
     public ByteSizeValue getTotalCompressedSize() {
         return totalCompressedSize;
     }

server/src/main/java/org/elasticsearch/action/admin/cluster/state/TransportClusterStateAction.java

@@ -20,6 +20,8 @@
 package org.elasticsearch.action.admin.cluster.state;
 
 import com.carrotsearch.hppc.cursors.ObjectObjectCursor;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
 import org.elasticsearch.Version;
 import org.elasticsearch.action.ActionListener;
 import org.elasticsearch.action.support.ActionFilters;
@@ -35,7 +37,9 @@
 import org.elasticsearch.cluster.routing.RoutingTable;
 import org.elasticsearch.cluster.service.ClusterService;
 import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.logging.DeprecationLogger;
 import org.elasticsearch.common.unit.TimeValue;
+import org.elasticsearch.common.util.concurrent.AbstractRunnable;
 import org.elasticsearch.node.NodeClosedException;
 import org.elasticsearch.threadpool.ThreadPool;
 import org.elasticsearch.transport.TransportService;
@@ -45,6 +49,10 @@
 
 public class TransportClusterStateAction extends TransportMasterNodeReadAction<ClusterStateRequest, ClusterStateResponse> {
 
+    private static final Logger logger = LogManager.getLogger(TransportClusterStateAction.class);
+    private static final DeprecationLogger deprecationLogger = new DeprecationLogger(logger);
+    public static final String COMPRESSED_CLUSTER_STATE_SIZE_DEPRECATION_MESSAGE = "Reporting the compressed cluster state size " +
+        "alongside the cluster state is deprecated and will be removed in the next major version.";
 
     @Inject
     public TransportClusterStateAction(TransportService transportService, ClusterService clusterService,
@@ -79,9 +87,8 @@ protected void masterOperation(final ClusterStateRequest request, final ClusterS
                                    final ActionListener<ClusterStateResponse> listener) throws IOException {
 
         if (request.waitForMetaDataVersion() != null) {
-            final Predicate<ClusterState> metadataVersionPredicate = clusterState -> {
-              return clusterState.metaData().version() >= request.waitForMetaDataVersion();
-            };
+            final Predicate<ClusterState> metadataVersionPredicate
+                = clusterState -> clusterState.metaData().version() >= request.waitForMetaDataVersion();
             final ClusterStateObserver observer =
                 new ClusterStateObserver(clusterService, request.waitForTimeout(), logger, threadPool.getThreadContext());
             final ClusterState clusterState = observer.setAndGetObservedState();
@@ -121,7 +128,7 @@ public void onTimeout(TimeValue timeout) {
 
     private void buildResponse(final ClusterStateRequest request,
                                final ClusterState currentState,
-                               final ActionListener<ClusterStateResponse> listener) throws IOException {
+                               final ActionListener<ClusterStateResponse> listener) {
         logger.trace("Serving cluster state request using version {}", currentState.version());
         ClusterState.Builder builder = ClusterState.builder(currentState.getClusterName());
         builder.version(currentState.version());
@@ -182,9 +189,23 @@ private void buildResponse(final ClusterStateRequest request,
                 }
             }
         }
-        listener.onResponse(new ClusterStateResponse(currentState.getClusterName(), builder.build(),
-            PublicationTransportHandler.serializeFullClusterState(currentState, Version.CURRENT).length(), false));
-    }
 
+        if (request.compressedClusterStateSize()) {
+            deprecationLogger.deprecated(COMPRESSED_CLUSTER_STATE_SIZE_DEPRECATION_MESSAGE);
+            threadPool.generic().execute(new AbstractRunnable() {
+                @Override
+                protected void doRun() throws Exception {
+                    listener.onResponse(new ClusterStateResponse(currentState.getClusterName(), builder.build(),
+                        PublicationTransportHandler.serializeFullClusterState(currentState, Version.CURRENT).length(), false));
+                }
 
+                @Override
+                public void onFailure(Exception e) {
+                    listener.onFailure(e);
+                }
+            });
+        } else {
+            listener.onResponse(new ClusterStateResponse(currentState.getClusterName(), builder.build(), 0, false));
+        }
+    }
 }

server/src/main/java/org/elasticsearch/client/transport/TransportClientNodesService.java

@@ -508,7 +508,7 @@ protected void doRun() throws Exception {
                                 pingConnection = connectionToClose;
                             }
                             transportService.sendRequest(pingConnection, ClusterStateAction.NAME,
-                                Requests.clusterStateRequest().clear().nodes(true).local(true),
+                                Requests.clusterStateRequest().clear().compressedClusterStateSize(false).nodes(true).local(true),
                                 TransportRequestOptions.builder().withType(TransportRequestOptions.Type.STATE)
                                     .withTimeout(pingTimeout).build(),
                                 new TransportResponseHandler<ClusterStateResponse>() {

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterGetSettingsAction.java

@@ -65,7 +65,8 @@ public String getName() {
     public RestChannelConsumer prepareRequest(final RestRequest request, final NodeClient client) throws IOException {
         ClusterStateRequest clusterStateRequest = Requests.clusterStateRequest()
                 .routingTable(false)
-                .nodes(false);
+                .nodes(false)
+                .compressedClusterStateSize(false);
         final boolean renderDefaults = request.paramAsBoolean("include_defaults", false);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));

server/src/main/java/org/elasticsearch/rest/action/admin/cluster/RestClusterStateAction.java

@@ -71,6 +71,7 @@ public RestChannelConsumer prepareRequest(final RestRequest request, final NodeC
             clusterStateRequest.waitForMetaDataVersion(request.paramAsLong("wait_for_metadata_version", 0));
         }
         clusterStateRequest.waitForTimeout(request.paramAsTime("wait_for_timeout", ClusterStateRequest.DEFAULT_WAIT_FOR_NODE_TIMEOUT));
+        clusterStateRequest.compressedClusterStateSize(request.paramAsBoolean("compressed_cluster_state_size", true));
 
         final String[] indices = Strings.splitStringByCommaToArray(request.param("indices", "_all"));
         boolean isAllIndicesOnly = indices.length == 1 && "_all".equals(indices[0]);
@@ -102,8 +103,10 @@ public RestResponse buildResponse(ClusterStateResponse response, XContentBuilder
                     builder.field(Fields.WAIT_FOR_TIMED_OUT, response.isWaitForTimedOut());
                 }
                 builder.field(Fields.CLUSTER_NAME, response.getClusterName().value());
-                builder.humanReadableField(Fields.CLUSTER_STATE_SIZE_IN_BYTES, Fields.CLUSTER_STATE_SIZE,
+                if (clusterStateRequest.compressedClusterStateSize()) {
+                    builder.humanReadableField(Fields.CLUSTER_STATE_SIZE_IN_BYTES, Fields.CLUSTER_STATE_SIZE,
                         response.getTotalCompressedSize());
+                }
                 response.getState().toXContent(builder, request);
                 builder.endObject();
                 return new BytesRestResponse(RestStatus.OK, builder);

server/src/main/java/org/elasticsearch/rest/action/cat/RestAllocationAction.java

@@ -64,7 +64,7 @@ protected void documentation(StringBuilder sb) {
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] nodes = Strings.splitStringByCommaToArray(request.param("nodes", "data:true"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
-        clusterStateRequest.clear().routingTable(true);
+        clusterStateRequest.compressedClusterStateSize(false).clear().routingTable(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
 

server/src/main/java/org/elasticsearch/rest/action/cat/RestIndicesAction.java

@@ -86,7 +86,7 @@ protected void documentation(StringBuilder sb) {
     public RestChannelConsumer doCatRequest(final RestRequest request, final NodeClient client) {
         final String[] indices = Strings.splitStringByCommaToArray(request.param("index"));
         final ClusterStateRequest clusterStateRequest = new ClusterStateRequest();
-        clusterStateRequest.clear().indices(indices).metaData(true);
+        clusterStateRequest.compressedClusterStateSize(false).clear().indices(indices).metaData(true);
         clusterStateRequest.local(request.paramAsBoolean("local", clusterStateRequest.local()));
         clusterStateRequest.masterNodeTimeout(request.paramAsTime("master_timeout", clusterStateRequest.masterNodeTimeout()));
         final IndicesOptions strictExpandIndicesOptions = IndicesOptions.strictExpand();