From 006c88827f16aa527825456486cfd1ffd4817368 Mon Sep 17 00:00:00 2001
From: AKIRA <33112463+Kikyou1997@users.noreply.github.com>
Date: Fri, 25 Aug 2023 21:30:59 +0800
Subject: [PATCH] [fix](stats) Fix auto analyze (#20426)

We only reanalyze those partition that lastVisibleTime is later than job's updatetime, so we shouldn't set this field when creat e system jobs
---
 .../doris/statistics/StatisticsAutoAnalyzer.java   | 14 ++++++++------
 .../statistics/StatisticsAutoAnalyzerTest.java     |  2 +-
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
index e67fc55e64bc67..e07a4c1ddf5610 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoAnalyzer.java
@@ -134,7 +134,6 @@ public List<AnalysisInfo> constructAnalysisInfo(DatabaseIf<? extends TableIf> db
                     .setScheduleType(AnalysisInfo.ScheduleType.ONCE)
                     .setState(AnalysisState.PENDING)
                     .setTaskIds(new ArrayList<>())
-                    .setLastExecTimeInMs(System.currentTimeMillis())
                     .setJobType(JobType.SYSTEM).build();
             analysisInfos.add(jobInfo);
         }
@@ -155,8 +154,7 @@ private void analyzePeriodically() {
     }
 
     @VisibleForTesting
-    public AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
-        long lastExecTimeInMs = jobInfo.lastExecTimeInMs;
+    protected AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
         TableIf table = StatisticsUtil
                 .findTable(jobInfo.catalogName, jobInfo.dbName, jobInfo.tblName);
         TableStats tblStats = Env.getCurrentEnv().getAnalysisManager().findTableStatsStatus(table.getId());
@@ -165,7 +163,7 @@ public AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
             return null;
         }
 
-        Set<String> needRunPartitions = findReAnalyzeNeededPartitions(table, lastExecTimeInMs);
+        Set<String> needRunPartitions = findReAnalyzeNeededPartitions(table, tblStats);
 
         if (needRunPartitions.isEmpty()) {
             return null;
@@ -175,12 +173,16 @@ public AnalysisInfo getReAnalyzeRequiredPart(AnalysisInfo jobInfo) {
     }
 
     @VisibleForTesting
-    public Set<String> findReAnalyzeNeededPartitions(TableIf table, long lastExecTimeInMs) {
+    protected Set<String> findReAnalyzeNeededPartitions(TableIf table, TableStats tableStats) {
+        if (tableStats == null) {
+            return table.getPartitionNames().stream().map(table::getPartition)
+                    .filter(Partition::hasData).map(Partition::getName).collect(Collectors.toSet());
+        }
         return table.getPartitionNames().stream()
                 .map(table::getPartition)
                 .filter(Partition::hasData)
                 .filter(partition ->
-                        partition.getVisibleVersionTime() >= lastExecTimeInMs).map(Partition::getName)
+                        partition.getVisibleVersionTime() >= tableStats.updatedTime).map(Partition::getName)
                 .collect(Collectors.toSet());
     }
 
diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoAnalyzerTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoAnalyzerTest.java
index 81337a1580ba6a..3368a5a6692b70 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoAnalyzerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoAnalyzerTest.java
@@ -177,7 +177,7 @@ public TableStats findTableStatsStatus(long tblId) {
 
         new MockUp<StatisticsAutoAnalyzer>() {
             @Mock
-            public Set<String> findReAnalyzeNeededPartitions(TableIf table, long lastExecTimeInMs) {
+            protected Set<String> findReAnalyzeNeededPartitions(TableIf table, TableStats tableStats)  {
                 Set<String> partitionNames = new HashSet<>();
                 partitionNames.add("p1");
                 partitionNames.add("p2");