[Enhancement] add explain costs in query detail

fe/fe-core/src/main/java/com/starrocks/common/Config.java

@@ -2423,6 +2423,9 @@ public class Config extends ConfigBase {
     @ConfField(mutable = true)
     public static boolean enable_collect_query_detail_info = false;
 
+    @ConfField(mutable = true, comment = "explain level of query plan in this detail")
+    public static String query_detail_explain_level = "COSTS";
+
     /**
      * StarRocks-manager pull queries every 1 second
      * metrics calculate query latency every 15 second

fe/fe-core/src/main/java/com/starrocks/connector/parser/trino/AstBuilder.java

@@ -294,7 +294,7 @@ protected ParseNode visitExplain(Explain node, ParseTreeContext context) {
             } else if (type == ExplainType.Type.DISTRIBUTED) {
                 queryStatement.setIsExplain(true, StatementBase.ExplainLevel.VERBOSE);
             } else if (type == ExplainType.Type.IO) {
-                queryStatement.setIsExplain(true, StatementBase.ExplainLevel.COST);
+                queryStatement.setIsExplain(true, StatementBase.ExplainLevel.COSTS);
             } else {
                 queryStatement.setIsExplain(true, StatementBase.ExplainLevel.NORMAL);
             }

fe/fe-core/src/main/java/com/starrocks/qe/StmtExecutor.java

@@ -1153,15 +1153,24 @@ private void handleQueryStmt(ExecPlan execPlan) throws Exception {
         } else if (isSchedulerExplain) {
             // Do nothing.
         } else if (parsedStmt.isExplain()) {
-            String explainString = buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT);
+            String explainString = buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT,
+                    parsedStmt.getExplainLevel());
             if (executeInFe) {
                 explainString = "EXECUTE IN FE\n" + explainString;
             }
             handleExplainStmt(explainString);
             return;
         }
+
+        // Generate a query plan for query detail
+        // Explaining internal table is very quick, we prefer to use EXPLAIN COSTS
+        // But explaining external table is expensive, may need to access lots of metadata, so have to use EXPLAIN
         if (context.getQueryDetail() != null) {
-            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT));
+            StatementBase.ExplainLevel level = AnalyzerUtils.hasExternalTables(parsedStmt) ?
+                    StatementBase.ExplainLevel.defaultValue() :
+                    StatementBase.ExplainLevel.parse(Config.query_detail_explain_level);
+            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.SELECT,
+                    level));
         }
 
         List<PlanFragment> fragments = execPlan.getFragments();
@@ -1780,7 +1789,8 @@ private void handleExplainStmt(String explainString) throws IOException {
         context.getState().setEof();
     }
 
-    private String buildExplainString(ExecPlan execPlan, ResourceGroupClassifier.QueryType queryType) {
+    private String buildExplainString(ExecPlan execPlan, ResourceGroupClassifier.QueryType queryType,
+                                      StatementBase.ExplainLevel explainLevel) {
         String explainString = "";
         if (parsedStmt.getExplainLevel() == StatementBase.ExplainLevel.VERBOSE) {
             TWorkGroup resourceGroup = CoordinatorPreprocessor.prepareResourceGroup(context, queryType);
@@ -1808,7 +1818,7 @@ private String buildExplainString(ExecPlan execPlan, ResourceGroupClassifier.Que
                 if (optimizedRecord != null) {
                     explainString += optimizedRecord.getExplainString();
                 }
-                explainString += execPlan.getExplainString(parsedStmt.getExplainLevel());
+                explainString += execPlan.getExplainString(explainLevel);
             }
         }
         return explainString;
@@ -2086,11 +2096,17 @@ public void handleDMLStmt(ExecPlan execPlan, DmlStmt stmt) throws Exception {
         } else if (isSchedulerExplain) {
             // Do nothing.
         } else if (stmt.isExplain()) {
-            handleExplainStmt(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT));
+            handleExplainStmt(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT,
+                    parsedStmt.getExplainLevel()));
             return;
         }
+
         if (context.getQueryDetail() != null) {
-            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT));
+            StatementBase.ExplainLevel level = AnalyzerUtils.hasExternalTables(parsedStmt) ?
+                    StatementBase.ExplainLevel.defaultValue() :
+                    StatementBase.ExplainLevel.parse(Config.query_detail_explain_level);
+            context.getQueryDetail().setExplain(buildExplainString(execPlan, ResourceGroupClassifier.QueryType.INSERT,
+                    level));
         }
 
         // special handling for delete of non-primary key table, using old handler

fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AnalyzerUtils.java

@@ -17,6 +17,7 @@
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Joiner;
 import com.google.common.base.Predicate;
+import com.google.common.base.Predicates;
 import com.google.common.base.Splitter;
 import com.google.common.base.Strings;
 import com.google.common.collect.ArrayListMultimap;
@@ -734,6 +735,15 @@ public static boolean hasTemporaryTables(StatementBase statementBase) {
         return tables.values().stream().anyMatch(t -> t.isTemporaryTable());
     }
 
+    /**
+     * Whether this statement access external tables
+     */
+    public static boolean hasExternalTables(StatementBase statementBase) {
+        List<Table> tables = Lists.newArrayList();
+        collectSpecifyExternalTables(statementBase, tables, Predicates.alwaysTrue());
+        return !tables.isEmpty();
+    }
+
     public static void copyOlapTable(StatementBase statementBase, Set<OlapTable> olapTables) {
         new AnalyzerUtils.OlapTableCollector(olapTables).visit(statementBase);
     }
@@ -947,7 +957,10 @@ public ExternalTableCollector(List<Table> tables, Predicate<Table> filter) {
         @Override
         public Void visitTable(TableRelation node, Void context) {
             Table table = node.getTable();
-            if (predicate.test(table)) {
+            boolean internal = CatalogMgr.isInternalCatalog(table.getCatalogName())
+                    || table.isNativeTableOrMaterializedView()
+                    || table.isOlapView();
+            if (!internal && predicate.test(table)) {
                 tables.add(table);
             }
             return null;

fe/fe-core/src/main/java/com/starrocks/sql/ast/StatementBase.java

@@ -42,6 +42,8 @@
 import com.starrocks.qe.OriginStatement;
 import com.starrocks.sql.parser.NodePosition;
 import org.apache.commons.collections4.CollectionUtils;
+import org.apache.commons.lang3.EnumUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import java.util.List;
 
@@ -60,11 +62,26 @@ public enum ExplainLevel {
         // True if the describe_stmt print verbose information, if `isVerbose` is true, `isExplain` must be set to true.
         VERBOSE,
         // True if the describe_stmt print costs information, if `isCosts` is true, `isExplain` must be set to true.
-        COST,
+        COSTS,
         OPTIMIZER,
         REWRITE,
         SCHEDULER,
-        PLAN_ADVISOR
+        PLAN_ADVISOR;
+
+        public static ExplainLevel defaultValue() {
+            return NORMAL;
+        }
+
+        public static ExplainLevel parse(String value) {
+            if (StringUtils.isEmpty(value)) {
+                return defaultValue();
+            }
+            ExplainLevel result = EnumUtils.getEnumIgnoreCase(ExplainLevel.class, value);
+            if (result == null) {
+                return defaultValue();
+            }
+            return result;
+        }
     }
 
     private ExplainLevel explainLevel;
@@ -110,7 +127,7 @@ public String getTraceModule() {
 
     public ExplainLevel getExplainLevel() {
         if (explainLevel == null) {
-            return ExplainLevel.NORMAL;
+            return ExplainLevel.defaultValue();
         } else {
             return explainLevel;
         }

fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java

@@ -7239,7 +7239,7 @@ private static StatementBase.ExplainLevel getExplainType(StarRocksParser.Explain
         } else if (context.VERBOSE() != null) {
             explainLevel = StatementBase.ExplainLevel.VERBOSE;
         } else if (context.COSTS() != null) {
-            explainLevel = StatementBase.ExplainLevel.COST;
+            explainLevel = StatementBase.ExplainLevel.COSTS;
         } else if (context.SCHEDULER() != null) {
             explainLevel = StatementBase.ExplainLevel.SCHEDULER;
         }

fe/fe-core/src/main/java/com/starrocks/sql/plan/ExecPlan.java

@@ -241,7 +241,7 @@ public String getExplainString(StatementBase.ExplainLevel level) {
             case VERBOSE:
                 tlevel = TExplainLevel.VERBOSE;
                 break;
-            case COST:
+            case COSTS:
                 tlevel = TExplainLevel.COSTS;
                 break;
         }