From ebc541be6d4e138bb7d5cf4f1b0e1286a110fdb8 Mon Sep 17 00:00:00 2001 From: wyb Date: Sat, 31 Aug 2024 00:00:42 +0800 Subject: [PATCH] [Enhancement] Support describe files() Signed-off-by: wyb --- .../sql/analyzer/AuthorizerStmtVisitor.java | 4 +++ .../sql/analyzer/ShowStmtAnalyzer.java | 25 ++++++++++++++ .../com/starrocks/sql/ast/DescribeStmt.java | 33 ++++++++++++++++++- .../com/starrocks/sql/parser/AstBuilder.java | 11 +++++-- .../com/starrocks/sql/parser/StarRocks.g4 | 2 +- test/sql/test_files/R/csv_format | 10 +++++- test/sql/test_files/T/csv_format | 3 +- 7 files changed, 81 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java index e9b9bc32668602..d41282e502ebb4 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/AuthorizerStmtVisitor.java @@ -1603,6 +1603,10 @@ public Void visitCancelAlterTableStatement(CancelAlterTableStmt statement, Conne @Override public Void visitDescTableStmt(DescribeStmt statement, ConnectContext context) { + if (statement.isTableFunctionTable()) { + return null; + } + try { Authorizer.checkAnyActionOnTable(context.getCurrentUserIdentity(), context.getCurrentRoleIds(), statement.getDbTableName()); diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java index 7c7159d91b72b4..4fcdf48c23f306 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/analyzer/ShowStmtAnalyzer.java @@ -34,8 +34,10 @@ import com.starrocks.catalog.MysqlTable; import com.starrocks.catalog.OlapTable; import com.starrocks.catalog.Table; +import com.starrocks.catalog.TableFunctionTable; import com.starrocks.catalog.Type; import com.starrocks.common.AnalysisException; +import com.starrocks.common.DdlException; import com.starrocks.common.ErrorCode; import com.starrocks.common.ErrorReport; import com.starrocks.common.proc.ExternalTableProcDir; @@ -308,6 +310,11 @@ public Void visitShowDataStatement(ShowDataStmt node, ConnectContext context) { @Override public Void visitDescTableStmt(DescribeStmt node, ConnectContext context) { + if (node.isTableFunctionTable()) { + descTableFunctionTable(node, context); + return null; + } + node.getDbTableName().normalization(context); TableName tableName = node.getDbTableName(); String catalogName = tableName.getCatalog(); @@ -331,6 +338,24 @@ public Void visitDescTableStmt(DescribeStmt node, ConnectContext context) { return null; } + private void descTableFunctionTable(DescribeStmt node, ConnectContext context) { + Table table = null; + try { + table = new TableFunctionTable(node.getTableFunctionProperties()); + } catch (DdlException e) { + throw new StorageAccessException(e); + } + + List columns = table.getFullSchema(); + for (Column column : columns) { + List row = Arrays.asList( + column.getName(), + column.getType().canonicalName().toLowerCase(), + column.isAllowNull() ? "YES" : "NO"); + node.getTotalRows().add(row); + } + } + private void descInternalCatalogTable(DescribeStmt node, ConnectContext context) { Database db = GlobalStateMgr.getCurrentState().getDb(node.getDb()); if (db == null) { diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DescribeStmt.java b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DescribeStmt.java index a78912ea118315..88705cbb55145c 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/ast/DescribeStmt.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/ast/DescribeStmt.java @@ -28,6 +28,7 @@ import java.util.ArrayList; import java.util.LinkedList; import java.util.List; +import java.util.Map; public class DescribeStmt extends ShowStmt { @@ -63,6 +64,13 @@ public class DescribeStmt extends ShowStmt { .addColumn(new Column("Table", ScalarType.createVarchar(30))) .build(); + private static final ShowResultSetMetaData DESC_TABLE_FUNCTION_TABLE_META_DATA = + ShowResultSetMetaData.builder() + .addColumn(new Column("Field", ScalarType.createVarchar(20))) + .addColumn(new Column("Type", ScalarType.createVarchar(20))) + .addColumn(new Column("Null", ScalarType.createVarchar(10))) + .build(); + // empty col num equals to DESC_OLAP_TABLE_ALL_META_DATA.size() public static final List EMPTY_ROW = initEmptyRow(); @@ -75,6 +83,9 @@ public class DescribeStmt extends ShowStmt { private boolean isOlapTable; private boolean isMaterializedView; + private boolean isTableFunctionTable = false; + private Map tableFunctionProperties = null; + public DescribeStmt(TableName dbTableName, boolean isAllTables) { this(dbTableName, isAllTables, NodePosition.ZERO); } @@ -86,6 +97,14 @@ public DescribeStmt(TableName dbTableName, boolean isAllTables, NodePosition pos this.isAllTables = isAllTables; } + public DescribeStmt(Map tableFunctionProperties, NodePosition pos) { + super(pos); + this.dbTableName = null; + this.totalRows = new LinkedList<>(); + this.isTableFunctionTable = true; + this.tableFunctionProperties = tableFunctionProperties; + } + public boolean isAllTables() { return isAllTables; } @@ -126,8 +145,16 @@ public void setOlapTable(boolean olapTable) { isOlapTable = olapTable; } + public boolean isTableFunctionTable() { + return isTableFunctionTable; + } + + public Map getTableFunctionProperties() { + return tableFunctionProperties; + } + public List> getResultRows() throws AnalysisException { - if (isAllTables || isMaterializedView) { + if (isAllTables || isMaterializedView || isTableFunctionTable) { return totalRows; } else { Preconditions.checkNotNull(node); @@ -137,6 +164,10 @@ public List> getResultRows() throws AnalysisException { @Override public ShowResultSetMetaData getMetaData() { + if (isTableFunctionTable) { + return DESC_TABLE_FUNCTION_TABLE_META_DATA; + } + if (!isAllTables) { if (isMaterializedView) { return DESC_OLAP_TABLE_META_DATA; diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java index 4f75c7477941c0..9ea4fd3951705f 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/AstBuilder.java @@ -1245,9 +1245,14 @@ public ParseNode visitShowTemporaryTablesStatement(StarRocksParser.ShowTemporary @Override public ParseNode visitDescTableStatement(StarRocksParser.DescTableStatementContext context) { - QualifiedName qualifiedName = getQualifiedName(context.qualifiedName()); - TableName targetTableName = qualifiedNameToTableName(qualifiedName); - return new DescribeStmt(targetTableName, context.ALL() != null, createPos(context)); + if (context.qualifiedName() != null) { + QualifiedName qualifiedName = getQualifiedName(context.qualifiedName()); + TableName targetTableName = qualifiedNameToTableName(qualifiedName); + return new DescribeStmt(targetTableName, context.ALL() != null, createPos(context)); + } + + Map tableFunctionProperties = getPropertyList(context.propertyList()); + return new DescribeStmt(tableFunctionProperties, createPos(context)); } @Override diff --git a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 index 9e80a854d62c92..ff4b4cf077af40 100644 --- a/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 +++ b/fe/fe-core/src/main/java/com/starrocks/sql/parser/StarRocks.g4 @@ -524,7 +524,7 @@ showAlterStatement ; descTableStatement - : (DESC | DESCRIBE) table=qualifiedName ALL? + : (DESC | DESCRIBE) ((table=qualifiedName ALL?) | (FILES propertyList)) ; createTableLikeStatement diff --git a/test/sql/test_files/R/csv_format b/test/sql/test_files/R/csv_format index 959a859a405281..e9a29ae6617391 100644 --- a/test/sql/test_files/R/csv_format +++ b/test/sql/test_files/R/csv_format @@ -38,4 +38,12 @@ $3 decimal(38,9) YES false None $4 boolean YES false None -- !result -shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null \ No newline at end of file +desc files('path' = 'oss://${oss_bucket}/test_files/csv_format/${uuid0}/*', 'format'='csv', "csv.column_separator"=",","csv.row_delimiter"="\n"); +-- result: +$1 bigint YES +$2 varchar(1048576) YES +$3 double YES +$4 boolean YES +-- !result + +shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null diff --git a/test/sql/test_files/T/csv_format b/test/sql/test_files/T/csv_format index addcf3ace9458d..6c0083ffb0d46d 100644 --- a/test/sql/test_files/T/csv_format +++ b/test/sql/test_files/T/csv_format @@ -16,6 +16,7 @@ select * from files('path' = 'oss://${oss_bucket}/test_files/csv_format/${uuid0} -- check schema desc t1; +desc files('path' = 'oss://${oss_bucket}/test_files/csv_format/${uuid0}/*', 'format'='csv', "csv.column_separator"=",","csv.row_delimiter"="\n"); -- clean -shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null \ No newline at end of file +shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null