[Backport 2.x] Remove LinkedDeque and replace with LinkedHashMap, Add…

… filecache support in clear indices cache API (#7595) * Remove LinkedDeque and replace with LinkedHashMap (#6968) * Remove LinkedDeque and replace with LinkedHashMap After the recent changes the usage of the LinkedDeque fits quite well with the insertion order semantics of LinkedHashMap, which also allows for constant time additions and removals. Signed-off-by: Andrew Ross <andrross@amazon.com> * Use class member reference now that lock is final Signed-off-by: Andrew Ross <andrross@amazon.com> --------- Signed-off-by: Andrew Ross <andrross@amazon.com> (cherry picked from commit 65443ad) * Add filecache support in clear indices cache API (#7498) Signed-off-by: Kunal Kotwani <kkotwani@amazon.com> (cherry picked from commit a1e42b1) Signed-off-by: Kunal Kotwani <kkotwani@amazon.com> --------- Signed-off-by: Kunal Kotwani <kkotwani@amazon.com> Co-authored-by: Andrew Ross <andrross@amazon.com>
opensearch-project · May 17, 2023 · 7c744cb · 7c744cb
1 parent b963ea4
commit 7c744cb
Show file tree

Hide file tree

Showing 18 changed files with 226 additions and 535 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add descending order search optimization through reverse segment read. ([#7244](https://github.com/opensearch-project/OpenSearch/pull/7244))
 - Adds ExtensionsManager.lookupExtensionSettingsById ([#7466](https://github.com/opensearch-project/OpenSearch/pull/7466))
 - Provide mechanism to configure XContent parsing constraints (after update to Jackson 2.15.0 and above) ([#7550](https://github.com/opensearch-project/OpenSearch/pull/7550))
+- Support to clear filecache using clear indices cache API ([#7498](https://github.com/opensearch-project/OpenSearch/pull/7498))
 
 ### Dependencies
 - Bump `com.netflix.nebula:gradle-info-plugin` from 12.0.0 to 12.1.3 (#7564)

diff --git a/client/rest-high-level/src/main/java/org/opensearch/client/IndicesRequestConverters.java b/client/rest-high-level/src/main/java/org/opensearch/client/IndicesRequestConverters.java
@@ -261,6 +261,7 @@ static Request clearCache(ClearIndicesCacheRequest clearIndicesCacheRequest) {
         parameters.withIndicesOptions(clearIndicesCacheRequest.indicesOptions());
         parameters.putParam("query", Boolean.toString(clearIndicesCacheRequest.queryCache()));
         parameters.putParam("fielddata", Boolean.toString(clearIndicesCacheRequest.fieldDataCache()));
+        parameters.putParam("file", Boolean.toString(clearIndicesCacheRequest.fileCache()));
         parameters.putParam("request", Boolean.toString(clearIndicesCacheRequest.requestCache()));
         parameters.putParam("fields", String.join(",", clearIndicesCacheRequest.fields()));
         request.addParameters(parameters.asMap());

diff --git a/...nt/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java b/...nt/rest-high-level/src/test/java/org/opensearch/client/IndicesRequestConvertersTests.java
@@ -596,6 +596,10 @@ public void testClearCache() {
             clearIndicesCacheRequest.fields(RequestConvertersTests.randomIndicesNames(1, 5));
             expectedParams.put("fields", String.join(",", clearIndicesCacheRequest.fields()));
         }
+        if (OpenSearchTestCase.randomBoolean()) {
+            clearIndicesCacheRequest.fileCache(OpenSearchTestCase.randomBoolean());
+        }
+        expectedParams.put("file", Boolean.toString(clearIndicesCacheRequest.fileCache()));
 
         Request request = IndicesRequestConverters.clearCache(clearIndicesCacheRequest);
         StringJoiner endpoint = new StringJoiner("/", "/", "");

diff --git a/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json b/rest-api-spec/src/main/resources/rest-api-spec/api/indices.clear_cache.json
@@ -67,6 +67,10 @@
       "request":{
         "type":"boolean",
         "description":"Clear request cache"
+      },
+      "file":{
+        "type":"boolean",
+        "description":"Clear filecache"
       }
     }
   }

diff --git a/...rTest/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java b/...rTest/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheBlocksIT.java
@@ -36,6 +36,7 @@
 import org.opensearch.test.OpenSearchIntegTestCase.ClusterScope;
 
 import java.util.Arrays;
+import java.util.Collections;
 
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_METADATA;
 import static org.opensearch.cluster.metadata.IndexMetadata.SETTING_BLOCKS_READ;
@@ -89,4 +90,42 @@ public void testClearIndicesCacheWithBlocks() {
             }
         }
     }
+
+    public void testClearIndicesFileCacheWithBlocks() {
+        createIndex("test");
+        ensureGreen("test");
+
+        NumShards numShards = getNumShards("test");
+
+        // Request is not blocked
+        for (String blockSetting : Arrays.asList(
+            SETTING_BLOCKS_READ,
+            SETTING_BLOCKS_WRITE,
+            SETTING_READ_ONLY,
+            SETTING_READ_ONLY_ALLOW_DELETE
+        )) {
+            try {
+                enableIndexBlock("test", blockSetting);
+                ClearIndicesCacheResponse clearIndicesCacheResponse = client().admin()
+                    .indices()
+                    .prepareClearCache("test")
+                    .setFileCache(true)
+                    .execute()
+                    .actionGet();
+                assertNoFailures(clearIndicesCacheResponse);
+                assertThat(clearIndicesCacheResponse.getSuccessfulShards(), equalTo(numShards.totalNumShards));
+            } finally {
+                disableIndexBlock("test", blockSetting);
+            }
+        }
+
+        for (String blockSetting : Collections.singletonList(SETTING_BLOCKS_METADATA)) {
+            try {
+                enableIndexBlock("test", blockSetting);
+                assertBlocked(client().admin().indices().prepareClearCache("test").setQueryCache(true).setFileCache(true));
+            } finally {
+                disableIndexBlock("test", blockSetting);
+            }
+        }
+    }
 }
diff --git a/...c/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java b/...c/main/java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequest.java
@@ -32,6 +32,7 @@
 
 package org.opensearch.action.admin.indices.cache.clear;
 
+import org.opensearch.Version;
 import org.opensearch.action.support.broadcast.BroadcastRequest;
 import org.opensearch.common.Strings;
 import org.opensearch.common.io.stream.StreamInput;
@@ -49,6 +50,7 @@ public class ClearIndicesCacheRequest extends BroadcastRequest<ClearIndicesCache
     private boolean queryCache = false;
     private boolean fieldDataCache = false;
     private boolean requestCache = false;
+    private boolean fileCache = false;
     private String[] fields = Strings.EMPTY_ARRAY;
 
     public ClearIndicesCacheRequest(StreamInput in) throws IOException {
@@ -57,6 +59,9 @@ public ClearIndicesCacheRequest(StreamInput in) throws IOException {
         fieldDataCache = in.readBoolean();
         fields = in.readStringArray();
         requestCache = in.readBoolean();
+        if (in.getVersion().onOrAfter(Version.V_2_8_0)) {
+            fileCache = in.readBoolean();
+        }
     }
 
     public ClearIndicesCacheRequest(String... indices) {
@@ -90,6 +95,15 @@ public ClearIndicesCacheRequest fieldDataCache(boolean fieldDataCache) {
         return this;
     }
 
+    public boolean fileCache() {
+        return this.fileCache;
+    }
+
+    public ClearIndicesCacheRequest fileCache(boolean fileCache) {
+        this.fileCache = fileCache;
+        return this;
+    }
+
     public ClearIndicesCacheRequest fields(String... fields) {
         this.fields = fields == null ? Strings.EMPTY_ARRAY : fields;
         return this;
@@ -106,5 +120,8 @@ public void writeTo(StreamOutput out) throws IOException {
         out.writeBoolean(fieldDataCache);
         out.writeStringArrayNullable(fields);
         out.writeBoolean(requestCache);
+        if (out.getVersion().onOrAfter(Version.V_2_8_0)) {
+            out.writeBoolean(fileCache);
+        }
     }
 }
diff --git a/...java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java b/...java/org/opensearch/action/admin/indices/cache/clear/ClearIndicesCacheRequestBuilder.java
@@ -59,6 +59,11 @@ public ClearIndicesCacheRequestBuilder setRequestCache(boolean requestCache) {
         return this;
     }
 
+    public ClearIndicesCacheRequestBuilder setFileCache(boolean fileCache) {
+        request.fileCache(fileCache);
+        return this;
+    }
+
     public ClearIndicesCacheRequestBuilder setFieldDataCache(boolean fieldDataCache) {
         request.fieldDataCache(fieldDataCache);
         return this;

diff --git a/...ava/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java b/...ava/org/opensearch/action/admin/indices/cache/clear/TransportClearIndicesCacheAction.java
@@ -44,12 +44,16 @@
 import org.opensearch.cluster.service.ClusterService;
 import org.opensearch.common.inject.Inject;
 import org.opensearch.common.io.stream.StreamInput;
+import org.opensearch.index.shard.ShardPath;
 import org.opensearch.indices.IndicesService;
+import org.opensearch.node.Node;
 import org.opensearch.threadpool.ThreadPool;
 import org.opensearch.transport.TransportService;
 
 import java.io.IOException;
+import java.nio.file.Path;
 import java.util.List;
+import java.util.function.Predicate;
 
 /**
  * Indices clear cache action.
@@ -63,11 +67,14 @@ public class TransportClearIndicesCacheAction extends TransportBroadcastByNodeAc
 
     private final IndicesService indicesService;
 
+    private final Node node;
+
     @Inject
     public TransportClearIndicesCacheAction(
         ClusterService clusterService,
         TransportService transportService,
         IndicesService indicesService,
+        Node node,
         ActionFilters actionFilters,
         IndexNameExpressionResolver indexNameExpressionResolver
     ) {
@@ -82,6 +89,7 @@ public TransportClearIndicesCacheAction(
             false
         );
         this.indicesService = indicesService;
+        this.node = node;
     }
 
     @Override
@@ -109,6 +117,14 @@ protected ClearIndicesCacheRequest readRequestFrom(StreamInput in) throws IOExce
 
     @Override
     protected EmptyResult shardOperation(ClearIndicesCacheRequest request, ShardRouting shardRouting) {
+        if (request.fileCache()) {
+            if (node.fileCache() != null) {
+                ShardPath shardPath = ShardPath.loadFileCachePath(node.getNodeEnvironment(), shardRouting.shardId());
+                Predicate<Path> pathStartsWithShardPathPredicate = path -> path.startsWith(shardPath.getDataPath());
+                node.fileCache().prune(pathStartsWithShardPathPredicate);
+            }
+        }
+
         indicesService.clearIndexShardCache(
             shardRouting.shardId(),
             request.queryCache(),

diff --git a/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java b/server/src/main/java/org/opensearch/index/store/remote/filecache/FileCache.java
@@ -22,6 +22,7 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.function.BiFunction;
+import java.util.function.Predicate;
 
 import static org.opensearch.index.store.remote.directory.RemoteSnapshotDirectoryFactory.LOCAL_STORE_LOCATION;
 
@@ -121,6 +122,11 @@ public long prune() {
         return theCache.prune();
     }
 
+    @Override
+    public long prune(Predicate<Path> keyPredicate) {
+        return theCache.prune(keyPredicate);
+    }
+
     @Override
     public CacheUsage usage() {
         return theCache.usage();