[Enhancement] Make the number of thread that async lazy refresh the r…

…emote file cache configurable. (#21676) Signed-off-by: stephen <stephen5217@163.com> (cherry picked from commit cb8ff23)
StarRocks · Apr 17, 2023 · 99a5f79 · 99a5f79
1 parent 9f385a3
commit 99a5f79
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/CachingRemoteFileConf.java b/fe/fe-core/src/main/java/com/starrocks/connector/CachingRemoteFileConf.java
@@ -11,13 +11,15 @@ public class CachingRemoteFileConf {
     private final long cacheRefreshIntervalSec;
     private long cacheMaxSize = 1000000L;
     private final int perQueryCacheMaxSize = 10000;
+    private final int refreshMaxThreadNum;
 
     public CachingRemoteFileConf(Map<String, String> conf) {
         this.cacheTtlSec = Long.parseLong(conf.getOrDefault("remote_file_cache_ttl_sec",
                 String.valueOf(Config.remote_file_cache_ttl_s)));
         this.cacheRefreshIntervalSec = Long.parseLong(conf.getOrDefault("remote_file_cache_refresh_interval_sec",
                 String.valueOf(Config.remote_file_cache_refresh_interval_s)));
         this.cacheMaxSize = Long.parseLong(conf.getOrDefault("remote_file_cache_max_num", String.valueOf(cacheMaxSize)));
+        this.refreshMaxThreadNum = Integer.parseInt(conf.getOrDefault("async_refresh_max_thread_num", "32"));
     }
 
     public long getCacheTtlSec() {
@@ -35,4 +37,8 @@ public long getCacheMaxSize() {
     public int getPerQueryCacheMaxSize() {
         return perQueryCacheMaxSize;
     }
+
+    public int getRefreshMaxThreadNum() {
+        return refreshMaxThreadNum;
+    }
 }
diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveConnectorInternalMgr.java b/fe/fe-core/src/main/java/com/starrocks/connector/hive/HiveConnectorInternalMgr.java
@@ -103,7 +103,7 @@ public RemoteFileIO createRemoteFileIO() {
                     new ThreadFactoryBuilder().setNameFormat("hive-remote-files-refresh-%d").build());
             baseRemoteFileIO = CachingRemoteFileIO.createCatalogLevelInstance(
                     remoteFileIO,
-                    new ReentrantExecutor(refreshRemoteFileExecutor, remoteFileConf.getPerQueryCacheMaxSize()),
+                    new ReentrantExecutor(refreshRemoteFileExecutor, remoteFileConf.getRefreshMaxThreadNum()),
                     remoteFileConf.getCacheTtlSec(),
                     enableHmsEventsIncrementalSync ? NEVER_REFRESH : remoteFileConf.getCacheRefreshIntervalSec(),
                     remoteFileConf.getCacheMaxSize());

diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/hudi/HudiConnectorInternalMgr.java b/fe/fe-core/src/main/java/com/starrocks/connector/hudi/HudiConnectorInternalMgr.java
@@ -102,7 +102,7 @@ public RemoteFileIO createRemoteFileIO() {
                     new ThreadFactoryBuilder().setNameFormat("hudi-remote-files-refresh-%d").build());
             baseRemoteFileIO = CachingRemoteFileIO.createCatalogLevelInstance(
                     remoteFileIO,
-                    new ReentrantExecutor(refreshRemoteFileExecutor, remoteFileConf.getPerQueryCacheMaxSize()),
+                    new ReentrantExecutor(refreshRemoteFileExecutor, remoteFileConf.getRefreshMaxThreadNum()),
                     remoteFileConf.getCacheTtlSec(),
                     remoteFileConf.getCacheRefreshIntervalSec(),
                     remoteFileConf.getCacheMaxSize());