[enhancement](memory) reduce memory usage for failed broker loads (ap…

…ache#15895) * [enhancement](memory) reduce memory usage for failed broker loads
adonis0147 · Jan 30, 2023 · ec4a569 · ec4a569
1 parent 69e748b
commit ec4a569
Show file tree

Hide file tree

Showing 5 changed files with 29 additions and 2 deletions.
diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md
@@ -2583,4 +2583,13 @@ MasterOnly：false
 
 The default dir to put jdbc drivers.
 
+#### `max_error_tablet_of_broker_load`
+
+Default: 3;
+
+IsMutable：true
+
+MasterOnly：true
+
+Maximum number of error tablet showed in broker load.
 
diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md
@@ -2583,4 +2583,12 @@ SmallFileMgr 中存储的最大文件数
 
 用于存放默认的 jdbc drivers
 
+#### `max_error_tablet_of_broker_load`
 
+默认值：3;
+
+是否可以动态配置: true
+
+是否为 Master FE 节点独有的配置项：true
+
+broker load job 保存的失败tablet 信息的最大数量
diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java
@@ -1956,6 +1956,12 @@ public class Config extends ConfigBase {
     @ConfField(masterOnly = true)
     public static int hms_events_polling_interval_ms = 10000;
 
+    /**
+     * Maximum number of error tablets showed in broker load
+     */
+    @ConfField(masterOnly = true, mutable = true)
+    public static int max_error_tablet_of_broker_load = 3;
+
     @ConfField(mutable = false)
     public static int topn_two_phase_limit_threshold = 512;
 }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java
@@ -24,6 +24,7 @@
 import org.apache.doris.catalog.OlapTable;
 import org.apache.doris.catalog.Table;
 import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.Config;
 import org.apache.doris.common.DataQualityException;
 import org.apache.doris.common.DuplicatedRequestException;
 import org.apache.doris.common.LabelAlreadyUsedException;
@@ -59,6 +60,7 @@
 import java.util.Map;
 import java.util.UUID;
 import java.util.concurrent.RejectedExecutionException;
+import java.util.stream.Collectors;
 
 /**
  * There are 3 steps in BrokerLoadJob: BrokerPendingTask, LoadLoadingTask, CommitAndPublishTxn.
@@ -351,7 +353,8 @@ private void updateLoadingStatus(BrokerLoadingTaskAttachment attachment) {
             loadingStatus.setTrackingUrl(attachment.getTrackingUrl());
         }
         commitInfos.addAll(attachment.getCommitInfoList());
-        errorTabletInfos.addAll(attachment.getErrorTabletInfos());
+        errorTabletInfos.addAll(attachment.getErrorTabletInfos().stream().limit(Config.max_error_tablet_of_broker_load)
+                .collect(Collectors.toList()));
 
         progress = (int) ((double) finishedTaskIds.size() / idToTasks.size() * 100);
         if (progress == 100) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java
@@ -797,7 +797,8 @@ public List<Comparable> getShowInfo() throws DdlException {
 
     public String errorTabletsToJson() {
         Map<Long, String> map = Maps.newHashMap();
-        errorTabletInfos.stream().limit(3).forEach(p -> map.put(p.getTabletId(), p.getMsg()));
+        errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load)
+            .forEach(p -> map.put(p.getTabletId(), p.getMsg()));
         Gson gson = new GsonBuilder().disableHtmlEscaping().create();
         return gson.toJson(map);
     }