diff --git a/docs/en/docs/admin-manual/config/fe-config.md b/docs/en/docs/admin-manual/config/fe-config.md index 9b9c9794ec250d..c23db572845507 100644 --- a/docs/en/docs/admin-manual/config/fe-config.md +++ b/docs/en/docs/admin-manual/config/fe-config.md @@ -2583,4 +2583,13 @@ MasterOnly:false The default dir to put jdbc drivers. +#### `max_error_tablet_of_broker_load` + +Default: 3; + +IsMutable:true + +MasterOnly:true + +Maximum number of error tablet showed in broker load. diff --git a/docs/zh-CN/docs/admin-manual/config/fe-config.md b/docs/zh-CN/docs/admin-manual/config/fe-config.md index f5ac3ca8e4950b..59a5dcd031ef2e 100644 --- a/docs/zh-CN/docs/admin-manual/config/fe-config.md +++ b/docs/zh-CN/docs/admin-manual/config/fe-config.md @@ -2583,4 +2583,12 @@ SmallFileMgr 中存储的最大文件数 用于存放默认的 jdbc drivers +#### `max_error_tablet_of_broker_load` +默认值:3; + +是否可以动态配置: true + +是否为 Master FE 节点独有的配置项:true + +broker load job 保存的失败tablet 信息的最大数量 diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 706e3543f6dd63..86215c463a8499 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -1956,6 +1956,12 @@ public class Config extends ConfigBase { @ConfField(masterOnly = true) public static int hms_events_polling_interval_ms = 10000; + /** + * Maximum number of error tablets showed in broker load + */ + @ConfField(masterOnly = true, mutable = true) + public static int max_error_tablet_of_broker_load = 3; + @ConfField(mutable = false) public static int topn_two_phase_limit_threshold = 512; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java index 076abf51b99311..01bbd795f45c62 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/BrokerLoadJob.java @@ -24,6 +24,7 @@ import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Table; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.DataQualityException; import org.apache.doris.common.DuplicatedRequestException; import org.apache.doris.common.LabelAlreadyUsedException; @@ -59,6 +60,7 @@ import java.util.Map; import java.util.UUID; import java.util.concurrent.RejectedExecutionException; +import java.util.stream.Collectors; /** * There are 3 steps in BrokerLoadJob: BrokerPendingTask, LoadLoadingTask, CommitAndPublishTxn. @@ -351,7 +353,8 @@ private void updateLoadingStatus(BrokerLoadingTaskAttachment attachment) { loadingStatus.setTrackingUrl(attachment.getTrackingUrl()); } commitInfos.addAll(attachment.getCommitInfoList()); - errorTabletInfos.addAll(attachment.getErrorTabletInfos()); + errorTabletInfos.addAll(attachment.getErrorTabletInfos().stream().limit(Config.max_error_tablet_of_broker_load) + .collect(Collectors.toList())); progress = (int) ((double) finishedTaskIds.size() / idToTasks.size() * 100); if (progress == 100) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java index 523597a11eaf0d..198d388ef73c26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadJob.java @@ -797,7 +797,8 @@ public List getShowInfo() throws DdlException { public String errorTabletsToJson() { Map map = Maps.newHashMap(); - errorTabletInfos.stream().limit(3).forEach(p -> map.put(p.getTabletId(), p.getMsg())); + errorTabletInfos.stream().limit(Config.max_error_tablet_of_broker_load) + .forEach(p -> map.put(p.getTabletId(), p.getMsg())); Gson gson = new GsonBuilder().disableHtmlEscaping().create(); return gson.toJson(map); }