[fix](stats) Don't save colToPartitions anymore to save mem (#27880)

pick from master #27879
apache · Dec 1, 2023 · 5402da0 · 5402da0
1 parent 1363d7c
commit 5402da0
Show file tree

Hide file tree

Showing 4 changed files with 24 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
@@ -97,10 +97,8 @@ public enum ScheduleType {
     public final long tblId;
 
     // TODO: Map here is wired, List is enough
-    @SerializedName("colToPartitions")
     public final Map<String, Set<String>> colToPartitions;
 
-    @SerializedName("partitionNames")
     public final Set<String> partitionNames;
 
     @SerializedName("colName")

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisJob.java
@@ -179,6 +179,18 @@ public void cancel() {
     public void deregisterJob() {
         analysisManager.removeJob(jobInfo.jobId);
         analysisManager.analysisJobIdToTaskMap.remove(jobInfo.jobId);
+        for (BaseAnalysisTask task : queryingTask) {
+            task.info.colToPartitions.clear();
+            if (task.info.partitionNames != null) {
+                task.info.partitionNames.clear();
+            }
+        }
+        for (BaseAnalysisTask task : queryFinished) {
+            task.info.colToPartitions.clear();
+            if (task.info.partitionNames != null) {
+                task.info.partitionNames.clear();
+            }
+        }
     }
 
     protected void syncLoadStats() {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisManager.java
@@ -621,6 +621,10 @@ public void updateTableStats(AnalysisInfo jobInfo) {
             tableStats.update(jobInfo, tbl);
             logCreateTableStats(tableStats);
         }
+        jobInfo.colToPartitions.clear();
+        if (jobInfo.partitionNames != null) {
+            jobInfo.partitionNames.clear();
+        }
     }
 
     public List<AnalysisInfo> showAnalysisJob(ShowAnalyzeStmt stmt) {
@@ -800,13 +804,19 @@ public void replayCreateAnalysisJob(AnalysisInfo jobInfo) {
         while (analysisJobInfoMap.size() >= Config.analyze_record_limit) {
             analysisJobInfoMap.remove(analysisJobInfoMap.pollFirstEntry().getKey());
         }
+        if (jobInfo.message != null && jobInfo.message.length() >= StatisticConstants.MSG_LEN_UPPER_BOUND) {
+            jobInfo.message = jobInfo.message.substring(0, StatisticConstants.MSG_LEN_UPPER_BOUND);
+        }
         this.analysisJobInfoMap.put(jobInfo.jobId, jobInfo);
     }
 
     public void replayCreateAnalysisTask(AnalysisInfo taskInfo) {
         while (analysisTaskInfoMap.size() >= Config.analyze_record_limit) {
             analysisTaskInfoMap.remove(analysisTaskInfoMap.pollFirstEntry().getKey());
         }
+        if (taskInfo.message != null && taskInfo.message.length() >= StatisticConstants.MSG_LEN_UPPER_BOUND) {
+            taskInfo.message = taskInfo.message.substring(0, StatisticConstants.MSG_LEN_UPPER_BOUND);
+        }
         this.analysisTaskInfoMap.put(taskInfo.taskId, taskInfo);
     }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticConstants.java
@@ -100,6 +100,8 @@ public class StatisticConstants {
 
     public static final int AUTO_ANALYZE_TABLE_WIDTH_THRESHOLD = 70;
 
+    public static final int MSG_LEN_UPPER_BOUND = 1024;
+
     static {
         SYSTEM_DBS.add(SystemInfoService.DEFAULT_CLUSTER
                 + ClusterNamespace.CLUSTER_DELIMITER + FeConstants.INTERNAL_DB_NAME);