[Enhancement] add explain costs in query detail (#51439)

Signed-off-by: Murphy <mofei@starrocks.com>
(cherry picked from commit 92dfb5d)

# Conflicts:
#	fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java
#	fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzerUtils.java
#	fe/fe-core/src/main/java/com/starrocks/sql/ast/StatementBase.java

fe/fe-core/src/main/java/com/starrocks/common/Config.java

@@ -2313,6 +2313,9 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true)
     public static boolean enable_collect_query_detail_info = false;
 
+    @ConfField(mutable = true, comment = "explain level of query plan in this detail")
+    public static String query_detail_explain_level = "COSTS";
+
     /**
      *  StarRocks-manager pull queries every 1 second
      *  metrics calculate query latency every 15 second

fe/fe-core/src/main/java/com/starrocks/connector/parser/trino/AstBuilder.java

@@ -279,7 +279,7 @@ protected ParseNode visitExplain(Explain node, ParseTreeContext context) {
             } else if (type == ExplainType.Type.DISTRIBUTED) {
                 queryStatement.setIsExplain(true, StatementBase.ExplainLevel.VERBOSE);
             } else if (type == ExplainType.Type.IO) {
-                queryStatement.setIsExplain(true, StatementBase.ExplainLevel.COST);
+                queryStatement.setIsExplain(true, StatementBase.ExplainLevel.COSTS);
             } else {
                 queryStatement.setIsExplain(true, StatementBase.ExplainLevel.NORMAL);
             }

fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java

@@ -1070,11 +1070,28 @@ private void handleQueryStmt(ExecPlan execPlan) throws Exception {
         } else if (isSchedulerExplain) {
             // Do nothing.
         } else if (parsedStmt.isExplain()) {
+<<<<<<< HEAD
             handleExplainStmt(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT));
+=======
+            String explainString = buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT,
+                    parsedStmt.getExplainLevel());
+            if (executeInFe) {
+                explainString = "EXECUTE IN FE\n" + explainString;
+            }
+            handleExplainStmt(explainString);
+>>>>>>> 92dfb5d4dc ([Enhancement] add explain costs in query detail (#51439))
             return;
         }
+
+        // Generate a query plan for query detail
+        // Explaining internal table is very quick, we prefer to use EXPLAIN COSTS
+        // But explaining external table is expensive, may need to access lots of metadata, so have to use EXPLAIN
         if (context.getQueryDetail() != null) {
-            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT));
+            StatementBase.ExplainLevel level = AnalyzerUtils.hasExternalTables(parsedStmt) ?
+                    StatementBase.ExplainLevel.defaultValue() :
+                    StatementBase.ExplainLevel.parse(Config.query_detail_explain_level);
+            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT,
+                    level));
         }
 
         StatementBase queryStmt = parsedStmt;
@@ -1621,7 +1638,8 @@ private void handleExplainStmt(String explainString) throws IOException {
         context.getState().setEof();
     }
 
-    private String buildExplainString(ExecPlan execPlan, ResourceGroupClassifier.QueryType queryType) {
+    private String buildExplainString(ExecPlan execPlan, ResourceGroupClassifier.QueryType queryType,
+                                      StatementBase.ExplainLevel explainLevel) {
         String explainString = "";
         if (parsedStmt.getExplainLevel() == StatementBase.ExplainLevel.VERBOSE) {
             TWorkGroup resourceGroup = CoordinatorPreprocessor.prepareResourceGroup(context, queryType);
@@ -1644,7 +1662,16 @@ private String buildExplainString(ExecPlan execPlan, ResourceGroupClassifier.Que
             } else if (parsedStmt.getTraceMode() == Tracers.Mode.REASON) {
                 explainString += Tracers.printReasons();
             } else {
+<<<<<<< HEAD
                 explainString += execPlan.getExplainString(parsedStmt.getExplainLevel());
+=======
+                OperatorTuningGuides.OptimizedRecord optimizedRecord = PlanTuningAdvisor.getInstance()
+                        .getOptimizedRecord(context.getQueryId());
+                if (optimizedRecord != null) {
+                    explainString += optimizedRecord.getExplainString();
+                }
+                explainString += execPlan.getExplainString(explainLevel);
+>>>>>>> 92dfb5d4dc ([Enhancement] add explain costs in query detail (#51439))
             }
         }
         return explainString;
@@ -1867,11 +1894,17 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception {
         } else if (isSchedulerExplain) {
             // Do nothing.
         } else if (stmt.isExplain()) {
-            handleExplainStmt(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT));
+            handleExplainStmt(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT,
+                    parsedStmt.getExplainLevel()));
             return;
         }
+
         if (context.getQueryDetail() != null) {
-            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT));
+            StatementBase.ExplainLevel level = AnalyzerUtils.hasExternalTables(parsedStmt) ?
+                    StatementBase.ExplainLevel.defaultValue() :
+                    StatementBase.ExplainLevel.parse(Config.query_detail_explain_level);
+            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT,
+                    level));
         }
 
         // special handling for delete of non-primary key table, using old handler

fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzerUtils.java

@@ -17,6 +17,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
 import com.google.common.collect.ArrayListMultimap;
@@ -488,6 +489,24 @@ public static boolean areTablesCopySafe(StatementBase statementBase) {
         return nonOlapTables.isEmpty();
     }
 
+<<<<<<< HEAD
+=======
+    public static boolean hasTemporaryTables(StatementBase statementBase) {
+        Map<TableName, Table> tables = new HashMap<>();
+        new AnalyzerUtils.TableCollector(tables).visit(statementBase);
+        return tables.values().stream().anyMatch(t -> t.isTemporaryTable());
+    }
+
+    /**
+     * Whether this statement access external tables
+     */
+    public static boolean hasExternalTables(StatementBase statementBase) {
+        List<Table> tables = Lists.newArrayList();
+        collectSpecifyExternalTables(statementBase, tables, Predicates.alwaysTrue());
+        return !tables.isEmpty();
+    }
+
+>>>>>>> 92dfb5d4dc ([Enhancement] add explain costs in query detail (#51439))
     public static void copyOlapTable(StatementBase statementBase, Set<OlapTable> olapTables) {
         new AnalyzerUtils.OlapTableCollector(olapTables).visit(statementBase);
     }
@@ -701,7 +720,10 @@ public ExternalTableCollector(List<Table> tables, Predicate<Table> filter) {
         @Override
         public Void visitTable(TableRelation node, Void context) {
             Table table = node.getTable();
-            if (predicate.test(table)) {
+            boolean internal = CatalogMgr.isInternalCatalog(table.getCatalogName())
+                    || table.isNativeTableOrMaterializedView()
+                    || table.isOlapView();
+            if (!internal && predicate.test(table)) {
                 tables.add(table);
             }
             return null;

fe/fe-core/src/main/java/com/starrocks/sql/ast/StatementBase.java

@@ -42,6 +42,8 @@
 import com.starrocks.qe.OriginStatement;
 import com.starrocks.sql.parser.NodePosition;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
 
@@ -60,10 +62,30 @@ public enum ExplainLevel {
         // True if the describe_stmt print verbose information, if `isVerbose` is true, `isExplain` must be set to true.
         VERBOSE,
         // True if the describe_stmt print costs information, if `isCosts` is true, `isExplain` must be set to true.
-        COST,
+        COSTS,
         OPTIMIZER,
         REWRITE,
+<<<<<<< HEAD
         SCHEDULER
+=======
+        SCHEDULER,
+        PLAN_ADVISOR;
+
+        public static ExplainLevel defaultValue() {
+            return NORMAL;
+        }
+
+        public static ExplainLevel parse(String value) {
+            if (StringUtils.isEmpty(value)) {
+                return defaultValue();
+            }
+            ExplainLevel result = EnumUtils.getEnumIgnoreCase(ExplainLevel.class, value);
+            if (result == null) {
+                return defaultValue();
+            }
+            return result;
+        }
+>>>>>>> 92dfb5d4dc ([Enhancement] add explain costs in query detail (#51439))
     }
 
     private ExplainLevel explainLevel;
@@ -109,7 +131,7 @@ public String getTraceModule() {
 
     public ExplainLevel getExplainLevel() {
         if (explainLevel == null) {
-            return ExplainLevel.NORMAL;
+            return ExplainLevel.defaultValue();
         } else {
             return explainLevel;
         }

fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java

@@ -6432,7 +6432,7 @@ private static StatementBase.ExplainLevel getExplainType(StarRocksParser.Explain
         } else if (context.VERBOSE() != null) {
             explainLevel = StatementBase.ExplainLevel.VERBOSE;
         } else if (context.COSTS() != null) {
-            explainLevel = StatementBase.ExplainLevel.COST;
+            explainLevel = StatementBase.ExplainLevel.COSTS;
         } else if (context.SCHEDULER() != null) {
             explainLevel = StatementBase.ExplainLevel.SCHEDULER;
         }

fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java

@@ -224,7 +224,7 @@ public String getExplainString(StatementBase.ExplainLevel level) {
             case VERBOSE:
                 tlevel = TExplainLevel.VERBOSE;
                 break;
-            case COST:
+            case COSTS:
                 tlevel = TExplainLevel.COSTS;
                 break;
         }