Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Enhancement] Support describe files() #50527

Merged
merged 3 commits into from
Sep 10, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 20 additions & 0 deletions fe/fe-core/src/main/java/com/starrocks/common/SchemaConstants.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
// Copyright 2021-present StarRocks, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package com.starrocks.common;

public class SchemaConstants {
public static final String YES = "YES";
public static final String NO = "NO";
}
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
import com.google.common.collect.Lists;
import com.starrocks.catalog.Column;
import com.starrocks.common.AnalysisException;
import com.starrocks.common.SchemaConstants;
import org.apache.commons.lang.StringUtils;

import java.util.Arrays;
Expand Down Expand Up @@ -94,7 +95,7 @@ public ProcResult fetchResult() throws AnalysisException {

List<String> rowList = Arrays.asList(column.getName(),
column.getType().canonicalName().toLowerCase(),
column.isAllowNull() ? "YES" : "NO",
column.isAllowNull() ? SchemaConstants.YES : SchemaConstants.NO,
((Boolean) column.isKey()).toString(),
defaultStr,
extraStr);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.starrocks.catalog.Table;
import com.starrocks.catalog.Type;
import com.starrocks.common.DdlException;
import com.starrocks.common.SchemaConstants;
import com.starrocks.connector.exception.StarRocksConnectorException;

import java.sql.Connection;
Expand Down Expand Up @@ -99,7 +100,7 @@ public List<Column> convertToSRTable(ResultSet columnSet) throws SQLException {
} catch (SQLException ignored) { }

fullSchema.add(new Column(columnSet.getString("COLUMN_NAME"), type,
columnSet.getString("IS_NULLABLE").equals("YES"), comment));
columnSet.getString("IS_NULLABLE").equals(SchemaConstants.YES), comment));
}
return fullSchema;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.starrocks.catalog.Table;
import com.starrocks.catalog.Type;
import com.starrocks.common.DdlException;
import com.starrocks.common.SchemaConstants;

import java.sql.Connection;
import java.sql.ResultSet;
Expand Down Expand Up @@ -56,7 +57,7 @@ public List<Column> convertToSRTable(ResultSet columnSet) throws SQLException {
columnSet.getInt("DECIMAL_DIGITS"));
String columnName = columnSet.getString("COLUMN_NAME");
fullSchema.add(new Column(columnName, type,
columnSet.getString("IS_NULLABLE").equals("YES")));
columnSet.getString("IS_NULLABLE").equals(SchemaConstants.YES)));
}
return fullSchema;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import com.starrocks.catalog.Table;
import com.starrocks.catalog.Type;
import com.starrocks.common.DdlException;
import com.starrocks.common.SchemaConstants;
import com.starrocks.common.util.TimeUtils;

import java.sql.Connection;
Expand Down Expand Up @@ -61,7 +62,7 @@ public List<Column> convertToSRTable(ResultSet columnSet) throws SQLException {
columnName = "\"" + columnName + "\"";
}
fullSchema.add(new Column(columnName, type,
columnSet.getString("IS_NULLABLE").equals("YES")));
columnSet.getString("IS_NULLABLE").equals(SchemaConstants.YES)));
}
return fullSchema;
}
Expand Down
5 changes: 3 additions & 2 deletions fe/fe-core/src/main/java/com/starrocks/qe/ShowExecutor.java
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
import com.starrocks.common.MetaNotFoundException;
import com.starrocks.common.Pair;
import com.starrocks.common.PatternMatcher;
import com.starrocks.common.SchemaConstants;
import com.starrocks.common.proc.BackendsProcDir;
import com.starrocks.common.proc.ComputeNodeProcDir;
import com.starrocks.common.proc.FrontendsProcNode;
Expand Down Expand Up @@ -1002,8 +1003,8 @@ public ShowResultSet visitShowColumnStatement(ShowColumnStmt statement, ConnectC
}
final String columnName = col.getName();
final String columnType = col.getType().canonicalName().toLowerCase();
final String isAllowNull = col.isAllowNull() ? "YES" : "NO";
final String isKey = col.isKey() ? "YES" : "NO";
final String isAllowNull = col.isAllowNull() ? SchemaConstants.YES : SchemaConstants.NO;
final String isKey = col.isKey() ? SchemaConstants.YES : SchemaConstants.NO;
String defaultValue = null;
if (!col.getType().isOnlyMetricType()) {
defaultValue = col.getMetaDefaultValue(Lists.newArrayList());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1604,6 +1604,10 @@ public Void visitCancelAlterTableStatement(CancelAlterTableStmt statement, Conne

@Override
public Void visitDescTableStmt(DescribeStmt statement, ConnectContext context) {
if (statement.isTableFunctionTable()) {
return null;
}

try {
Authorizer.checkAnyActionOnTable(context.getCurrentUserIdentity(), context.getCurrentRoleIds(),
statement.getDbTableName());
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,10 +34,13 @@
import com.starrocks.catalog.MysqlTable;
import com.starrocks.catalog.OlapTable;
import com.starrocks.catalog.Table;
import com.starrocks.catalog.TableFunctionTable;
import com.starrocks.catalog.Type;
import com.starrocks.common.AnalysisException;
import com.starrocks.common.DdlException;
import com.starrocks.common.ErrorCode;
import com.starrocks.common.ErrorReport;
import com.starrocks.common.SchemaConstants;
import com.starrocks.common.proc.ExternalTableProcDir;
import com.starrocks.common.proc.PartitionsProcDir;
import com.starrocks.common.proc.ProcNodeInterface;
Expand Down Expand Up @@ -308,6 +311,11 @@ public Void visitShowDataStatement(ShowDataStmt node, ConnectContext context) {

@Override
public Void visitDescTableStmt(DescribeStmt node, ConnectContext context) {
if (node.isTableFunctionTable()) {
descTableFunctionTable(node, context);
return null;
}

node.getDbTableName().normalization(context);
TableName tableName = node.getDbTableName();
String catalogName = tableName.getCatalog();
Expand All @@ -331,6 +339,24 @@ public Void visitDescTableStmt(DescribeStmt node, ConnectContext context) {
return null;
}

private void descTableFunctionTable(DescribeStmt node, ConnectContext context) {
Table table = null;
try {
table = new TableFunctionTable(node.getTableFunctionProperties());
} catch (DdlException e) {
throw new StorageAccessException(e);
}

List<Column> columns = table.getFullSchema();
for (Column column : columns) {
List<String> row = Arrays.asList(
column.getName(),
column.getType().canonicalName().toLowerCase(),
wyb marked this conversation as resolved.
Show resolved Hide resolved
column.isAllowNull() ? SchemaConstants.YES : SchemaConstants.NO);
node.getTotalRows().add(row);
}
}

private void descInternalCatalogTable(DescribeStmt node, ConnectContext context) {
Database db = GlobalStateMgr.getCurrentState().getLocalMetastore().getDb(node.getDb());
if (db == null) {
wyb marked this conversation as resolved.
Show resolved Hide resolved
Expand Down Expand Up @@ -369,7 +395,7 @@ private void descInternalCatalogTable(DescribeStmt node, ConnectContext context)
// If you do not follow this specification, it may cause the BI system,
// such as superset, to fail to recognize the column type.
column.getType().canonicalName().toLowerCase(),
column.isAllowNull() ? "YES" : "NO",
column.isAllowNull() ? SchemaConstants.YES : SchemaConstants.NO,
((Boolean) column.isKey()).toString(),
defaultStr,
extraStr);
Expand Down Expand Up @@ -448,7 +474,7 @@ private void descInternalCatalogTable(DescribeStmt node, ConnectContext context)
// If you do not follow this specification, it may cause the BI system,
// such as superset, to fail to recognize the column type.
column.getType().canonicalName().toLowerCase(),
column.isAllowNull() ? "YES" : "NO",
column.isAllowNull() ? SchemaConstants.YES : SchemaConstants.NO,
((Boolean) column.isKey()).toString(),
defaultStr,
extraStr);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;

public class DescribeStmt extends ShowStmt {

Expand Down Expand Up @@ -63,6 +64,13 @@ public class DescribeStmt extends ShowStmt {
.addColumn(new Column("Table", ScalarType.createVarchar(30)))
.build();

private static final ShowResultSetMetaData DESC_TABLE_FUNCTION_TABLE_META_DATA =
wyb marked this conversation as resolved.
Show resolved Hide resolved
ShowResultSetMetaData.builder()
.addColumn(new Column("Field", ScalarType.createVarchar(20)))
.addColumn(new Column("Type", ScalarType.createVarchar(20)))
.addColumn(new Column("Null", ScalarType.createVarchar(10)))
.build();

// empty col num equals to DESC_OLAP_TABLE_ALL_META_DATA.size()
public static final List<String> EMPTY_ROW = initEmptyRow();

Expand All @@ -75,6 +83,9 @@ public class DescribeStmt extends ShowStmt {
private boolean isOlapTable;
private boolean isMaterializedView;

private boolean isTableFunctionTable = false;
private Map<String, String> tableFunctionProperties = null;

public DescribeStmt(TableName dbTableName, boolean isAllTables) {
this(dbTableName, isAllTables, NodePosition.ZERO);
}
Expand All @@ -86,6 +97,14 @@ public DescribeStmt(TableName dbTableName, boolean isAllTables, NodePosition pos
this.isAllTables = isAllTables;
}

public DescribeStmt(Map<String, String> tableFunctionProperties, NodePosition pos) {
super(pos);
this.dbTableName = null;
this.totalRows = new LinkedList<>();
this.isTableFunctionTable = true;
wyb marked this conversation as resolved.
Show resolved Hide resolved
this.tableFunctionProperties = tableFunctionProperties;
}

public boolean isAllTables() {
return isAllTables;
}
Expand Down Expand Up @@ -126,8 +145,16 @@ public void setOlapTable(boolean olapTable) {
isOlapTable = olapTable;
}

public boolean isTableFunctionTable() {
return isTableFunctionTable;
}

public Map<String, String> getTableFunctionProperties() {
return tableFunctionProperties;
}

public List<List<String>> getResultRows() throws AnalysisException {
if (isAllTables || isMaterializedView) {
if (isAllTables || isMaterializedView || isTableFunctionTable) {
return totalRows;
} else {
Preconditions.checkNotNull(node);
Expand All @@ -137,6 +164,10 @@ public List<List<String>> getResultRows() throws AnalysisException {

@Override
public ShowResultSetMetaData getMetaData() {
if (isTableFunctionTable) {
return DESC_TABLE_FUNCTION_TABLE_META_DATA;
}

if (!isAllTables) {
if (isMaterializedView) {
return DESC_OLAP_TABLE_META_DATA;
wyb marked this conversation as resolved.
Show resolved Hide resolved
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1285,9 +1285,14 @@ public ParseNode visitShowTemporaryTablesStatement(StarRocksParser.ShowTemporary

@Override
public ParseNode visitDescTableStatement(StarRocksParser.DescTableStatementContext context) {
QualifiedName qualifiedName = getQualifiedName(context.qualifiedName());
TableName targetTableName = qualifiedNameToTableName(qualifiedName);
return new DescribeStmt(targetTableName, context.ALL() != null, createPos(context));
if (context.qualifiedName() != null) {
QualifiedName qualifiedName = getQualifiedName(context.qualifiedName());
TableName targetTableName = qualifiedNameToTableName(qualifiedName);
return new DescribeStmt(targetTableName, context.ALL() != null, createPos(context));
}

Map<String, String> tableFunctionProperties = getPropertyList(context.propertyList());
return new DescribeStmt(tableFunctionProperties, createPos(context));
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -538,7 +538,7 @@ showAlterStatement
;

descTableStatement
: (DESC | DESCRIBE) table=qualifiedName ALL?
: (DESC | DESCRIBE) ((table=qualifiedName ALL?) | (FILES propertyList))
;

createTableLikeStatement
Expand Down
10 changes: 9 additions & 1 deletion test/sql/test_files/R/csv_format
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,12 @@ $3 decimal(38,9) YES false None
$4 boolean YES false None
-- !result

shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null
desc files('path' = 'oss://${oss_bucket}/test_files/csv_format/${uuid0}/*', 'format'='csv', "csv.column_separator"=",","csv.row_delimiter"="\n");
-- result:
$1 bigint YES
$2 varchar(1048576) YES
$3 double YES
$4 boolean YES
-- !result

shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null
3 changes: 2 additions & 1 deletion test/sql/test_files/T/csv_format
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@ select * from files('path' = 'oss://${oss_bucket}/test_files/csv_format/${uuid0}

-- check schema
desc t1;
desc files('path' = 'oss://${oss_bucket}/test_files/csv_format/${uuid0}/*', 'format'='csv', "csv.column_separator"=",","csv.row_delimiter"="\n");

-- clean
shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null
shell: ossutil64 rm -rf oss://${oss_bucket}/test_files/csv_format/${uuid0}/ > /dev/null
Loading