From 97230a54fb78fbf73d60b483f189cd05fbe6fa29 Mon Sep 17 00:00:00 2001 From: Mingyu Chen Date: Mon, 20 Feb 2023 10:32:48 +0800 Subject: [PATCH] [Refactor](auth)(step-2) Add AccessController to support customized authorization (#16802) Support specifying AccessControllerFactory when creating catalog create catalog hive properties( ... "access_controller.class" = "org.apache.doris.mysql.privilege.RangerAccessControllerFactory", "access_controller.properties.prop1" = "xxx", "access_controller.properties.prop2" = "yyy", ... ) So that user can specified their own access controller, such as RangerAccessController Add interface to check column level privilege A new method of CatalogAccessController: checkColsPriv(), for checking column level privileges. TODO: Support grant column level privileges statements in Doris Add TestExternalCatalog/Database/Table/ScanNode These classes are used for FE unit test. In unit test you can create catalog test1 properties( "type" = "test" "catalog_provider.class" = "org.apache.doris.datasource.ColumnPrivTest$MockedCatalogProvider" "access_controller.class" = "org.apache.doris.mysql.privilege.TestAccessControllerFactory", "access_controller.properties.key1" = "val1", "access_controller.properties.key2" = "val2" ); To create a test catalog, and specify catalog_provider to mock database/table/schema metadata Set roles in current user identity in connection context The roles can be used for authorization in access controller. --- .../doris/analysis/DropCatalogStmt.java | 2 +- .../org/apache/doris/analysis/SelectStmt.java | 2 +- .../apache/doris/analysis/SlotDescriptor.java | 11 +- .../org/apache/doris/analysis/TableName.java | 5 + .../apache/doris/analysis/UserIdentity.java | 15 +- .../java/org/apache/doris/catalog/Table.java | 4 + .../org/apache/doris/catalog/TableIf.java | 2 +- .../catalog/external/JdbcExternalTable.java | 1 + .../external/TestExternalDatabase.java | 150 +++++++++ .../catalog/external/TestExternalTable.java | 63 ++++ .../doris/common/AuthorizationException.java | 45 +++ .../doris/common/util/PropertyAnalyzer.java | 18 ++ .../doris/datasource/CatalogFactory.java | 13 +- .../apache/doris/datasource/CatalogIf.java | 2 +- .../apache/doris/datasource/CatalogMgr.java | 4 + .../doris/datasource/ExternalCatalog.java | 45 +++ .../doris/datasource/InitCatalogLog.java | 1 + .../doris/datasource/InitDatabaseLog.java | 1 + .../doris/datasource/JdbcExternalCatalog.java | 1 + .../datasource/test/TestExternalCatalog.java | 155 +++++++++ .../doris/load/loadv2/LoadingTaskPlanner.java | 2 +- .../privilege/AccessControllerFactory.java | 25 ++ .../privilege/AccessControllerManager.java | 64 +++- .../apache/doris/mysql/privilege/Auth.java | 26 +- .../privilege/CatalogAccessController.java | 18 ++ .../InternalCatalogAccessController.java | 9 + .../privilege/RangerAccessController.java | 16 + .../RangerAccessControllerFactory.java | 27 ++ .../translator/PhysicalPlanTranslator.java | 3 +- .../apache/doris/planner/DataGenScanNode.java | 5 + .../apache/doris/planner/OriginalPlanner.java | 74 +++++ .../org/apache/doris/planner/ScanNode.java | 7 + .../doris/planner/SingleNodePlanner.java | 7 +- .../doris/planner/StreamLoadPlanner.java | 2 +- .../planner/TestExternalTableScanNode.java | 95 ++++++ .../external/ExternalFileScanNode.java | 7 +- .../planner/external/ExternalScanNode.java | 12 +- .../planner/external/MetadataScanNode.java | 5 + .../doris/statistics/StatisticalType.java | 1 + .../ExternalFileTableValuedFunction.java | 2 +- .../doris/datasource/ColumnPrivTest.java | 299 ++++++++++++++++++ .../doris/utframe/TestWithFeService.java | 15 +- gensrc/thrift/PlanNodes.thrift | 8 +- gensrc/thrift/Types.thrift | 3 +- 44 files changed, 1217 insertions(+), 55 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/common/AuthorizationException.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/test/TestExternalCatalog.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerFactory.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessControllerFactory.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/planner/TestExternalTableScanNode.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/ColumnPrivTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java index c72989479674cf..f3606c07f29b82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java @@ -59,7 +59,7 @@ public void analyze(Analyzer analyzer) throws UserException { if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv( ConnectContext.get(), catalogName, PrivPredicate.DROP)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED, - analyzer.getQualifiedUser(), catalogName); + ConnectContext.get().getQualifiedUser(), catalogName); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java index f06d0385c89adf..091db335cd0bd3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java @@ -381,7 +381,7 @@ public void getTables(Analyzer analyzer, boolean expandView, Map .checkTblPriv(ConnectContext.get(), tblRef.getName(), PrivPredicate.SELECT)) { ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT", ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(), - dbName + ": " + tableName); + dbName + "." + tableName); } tableMap.put(table.getId(), table); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java index 0fb28f829b52ed..6335a7c5fed9d3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java @@ -67,8 +67,6 @@ public class SlotDescriptor { private ColumnStats stats; // only set if 'column' isn't set private boolean isAgg; private boolean isMultiRef; - // used for load to get more information of varchar and decimal - private Type originType; // If set to false, then such slots will be ignored during // materialize them.Used to optmize to read less data and less memory usage private boolean needMaterialize = true; @@ -162,7 +160,6 @@ public Column getColumn() { public void setColumn(Column column) { this.column = column; this.type = column.getType(); - this.originType = column.getOriginType(); } public void setSrcColumn(Column column) { @@ -254,10 +251,6 @@ public void setLabel(String label) { this.label = label; } - public void setSourceExprs(List exprs) { - sourceExprs = exprs; - } - public void setSourceExpr(Expr expr) { sourceExprs = Collections.singletonList(expr); } @@ -316,11 +309,9 @@ public boolean layoutEquals(SlotDescriptor other) { return true; } - // TODO public TSlotDescriptor toThrift() { - TSlotDescriptor tSlotDescriptor = new TSlotDescriptor(id.asInt(), parent.getId().asInt(), - (originType != null ? originType.toThrift() : type.toThrift()), -1, byteOffset, nullIndicatorByte, + type.toThrift(), -1, byteOffset, nullIndicatorByte, nullIndicatorBit, ((column != null) ? column.getName() : ""), slotIdx, isMaterialized); tSlotDescriptor.setNeedMaterialize(needMaterialize); if (column != null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java index d8360f981ef89b..9fb383feb93682 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java @@ -162,6 +162,11 @@ public boolean equals(Object other) { return false; } + @Override + public int hashCode() { + return Objects.hash(ctl, tbl, db); + } + public String toSql() { StringBuilder stringBuilder = new StringBuilder(); if (ctl != null && !ctl.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java index b4822c197ed848..4b50479f66f2c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java @@ -40,6 +40,7 @@ import java.io.DataInput; import java.io.DataOutput; import java.io.IOException; +import java.util.Set; // https://dev.mysql.com/doc/refman/8.0/en/account-names.html // user name must be literally matched. @@ -52,12 +53,14 @@ public class UserIdentity implements Writable, GsonPostProcessable { @SerializedName(value = "user") private String user; - @SerializedName(value = "host") private String host; - @SerializedName(value = "isDomain") private boolean isDomain; + // The roles which this user belongs to. + // Used for authorization in Access Controller + // This field is only set when getting current user from auth and not need to persist + private Set roles; private boolean isAnalyzed = false; @@ -125,6 +128,14 @@ public void setIsAnalyzed() { this.isAnalyzed = true; } + public void setRoles(Set roles) { + this.roles = roles; + } + + public Set getRoles() { + return roles; + } + public void analyze(String clusterName) throws AnalysisException { if (isAnalyzed) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java index 3f8597f1e09dc6..60547c659a2334 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java @@ -270,6 +270,10 @@ void setQualifiedDbName(String qualifiedDbName) { this.qualifiedDbName = qualifiedDbName; } + public String getQualifiedDbName() { + return qualifiedDbName; + } + public String getQualifiedName() { if (StringUtils.isEmpty(qualifiedDbName)) { return name; diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java index 7a71d4f4e76485..b5ac899d514e03 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java @@ -135,7 +135,7 @@ default int getBaseColumnIdxByName(String colName) { enum TableType { MYSQL, ODBC, OLAP, SCHEMA, INLINE_VIEW, VIEW, BROKER, ELASTICSEARCH, HIVE, ICEBERG, HUDI, JDBC, TABLE_VALUED_FUNCTION, HMS_EXTERNAL_TABLE, ES_EXTERNAL_TABLE, MATERIALIZED_VIEW, JDBC_EXTERNAL_TABLE, - ICEBERG_EXTERNAL_TABLE; + ICEBERG_EXTERNAL_TABLE, TEST_EXTERNAL_TABLE; public String toEngineName() { switch (this) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java index cb7c51bb646980..3d72c197930009 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java @@ -47,6 +47,7 @@ public JdbcExternalTable(long id, String name, String dbName, JdbcExternalCatalo super(id, name, catalog, dbName, TableType.JDBC_EXTERNAL_TABLE); } + @Override protected synchronized void makeSureInitialized() { if (!objectCreated) { jdbcTable = toJdbcTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java new file mode 100644 index 00000000000000..f0451a117ac305 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java @@ -0,0 +1,150 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog.external; + +import org.apache.doris.catalog.Env; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.InitDatabaseLog; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.persist.gson.GsonPostProcessable; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import com.google.gson.annotations.SerializedName; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.locks.ReentrantReadWriteLock; + +public class TestExternalDatabase extends ExternalDatabase implements GsonPostProcessable { + private static final Logger LOG = LogManager.getLogger(TestExternalDatabase.class); + + // Cache of table name to table id. + private Map tableNameToId = Maps.newConcurrentMap(); + @SerializedName(value = "idToTbl") + private Map idToTbl = Maps.newConcurrentMap(); + + public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name) { + super(extCatalog, id, name); + } + + @Override + protected void init() { + InitDatabaseLog initDatabaseLog = new InitDatabaseLog(); + initDatabaseLog.setType(InitDatabaseLog.Type.TEST); + initDatabaseLog.setCatalogId(extCatalog.getId()); + initDatabaseLog.setDbId(id); + List tableNames = extCatalog.listTableNames(null, name); + if (tableNames != null) { + Map tmpTableNameToId = Maps.newConcurrentMap(); + Map tmpIdToTbl = Maps.newHashMap(); + for (String tableName : tableNames) { + long tblId; + if (tableNameToId != null && tableNameToId.containsKey(tableName)) { + tblId = tableNameToId.get(tableName); + tmpTableNameToId.put(tableName, tblId); + TestExternalTable table = idToTbl.get(tblId); + tmpIdToTbl.put(tblId, table); + initDatabaseLog.addRefreshTable(tblId); + } else { + tblId = Env.getCurrentEnv().getNextId(); + tmpTableNameToId.put(tableName, tblId); + TestExternalTable table = new TestExternalTable(tblId, tableName, name, + (TestExternalCatalog) extCatalog); + tmpIdToTbl.put(tblId, table); + initDatabaseLog.addCreateTable(tblId, tableName); + } + } + tableNameToId = tmpTableNameToId; + idToTbl = tmpIdToTbl; + } + initialized = true; + Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog); + } + + public void setTableExtCatalog(ExternalCatalog extCatalog) { + for (TestExternalTable table : idToTbl.values()) { + table.setCatalog(extCatalog); + } + } + + public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) { + Map tmpTableNameToId = Maps.newConcurrentMap(); + Map tmpIdToTbl = Maps.newConcurrentMap(); + for (int i = 0; i < log.getRefreshCount(); i++) { + TestExternalTable table = getTableForReplay(log.getRefreshTableIds().get(i)); + tmpTableNameToId.put(table.getName(), table.getId()); + tmpIdToTbl.put(table.getId(), table); + } + for (int i = 0; i < log.getCreateCount(); i++) { + TestExternalTable table = new TestExternalTable(log.getCreateTableIds().get(i), + log.getCreateTableNames().get(i), name, (TestExternalCatalog) catalog); + tmpTableNameToId.put(table.getName(), table.getId()); + tmpIdToTbl.put(table.getId(), table); + } + tableNameToId = tmpTableNameToId; + idToTbl = tmpIdToTbl; + initialized = true; + } + + // TODO(ftw): drew + @Override + public Set getTableNamesWithLock() { + makeSureInitialized(); + return Sets.newHashSet(tableNameToId.keySet()); + } + + @Override + public List getTables() { + makeSureInitialized(); + return Lists.newArrayList(idToTbl.values()); + } + + @Override + public TestExternalTable getTableNullable(String tableName) { + makeSureInitialized(); + if (!tableNameToId.containsKey(tableName)) { + return null; + } + return idToTbl.get(tableNameToId.get(tableName)); + } + + @Override + public TestExternalTable getTableNullable(long tableId) { + makeSureInitialized(); + return idToTbl.get(tableId); + } + + public TestExternalTable getTableForReplay(long tableId) { + return idToTbl.get(tableId); + } + + @Override + public void gsonPostProcess() throws IOException { + tableNameToId = Maps.newConcurrentMap(); + for (TestExternalTable tbl : idToTbl.values()) { + tableNameToId.put(tbl.getName(), tbl.getId()); + } + rwLock = new ReentrantReadWriteLock(true); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java new file mode 100644 index 00000000000000..331bd9ea98b7d2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java @@ -0,0 +1,63 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog.external; + +import org.apache.doris.catalog.Column; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.thrift.TTableDescriptor; +import org.apache.doris.thrift.TTableType; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; + +/** + * TestExternalTable is a table for unit test. + */ +public class TestExternalTable extends ExternalTable { + private static final Logger LOG = LogManager.getLogger(TestExternalTable.class); + + public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) { + super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE); + } + + @Override + protected synchronized void makeSureInitialized() { + + } + + @Override + public String getMysqlType() { + return type.name(); + } + + @Override + public TTableDescriptor toThrift() { + makeSureInitialized(); + TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.TEST_EXTERNAL_TABLE, + getFullSchema().size(), + 0, getName(), ""); + return tTableDescriptor; + } + + @Override + public List initSchema() { + return ((TestExternalCatalog) catalog).mockedSchema(dbName, name); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/AuthorizationException.java b/fe/fe-core/src/main/java/org/apache/doris/common/AuthorizationException.java new file mode 100644 index 00000000000000..72b002650d1e14 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/common/AuthorizationException.java @@ -0,0 +1,45 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.common; + +/** + * Thrown for authorization errors encountered when accessing Catalog objects. + */ +public class AuthorizationException extends UserException { + + public ErrorCode errorCode = ErrorCode.ERR_COMMON_ERROR; + public Object[] msgs; + + public AuthorizationException(String msg, Throwable cause) { + super(msg, cause); + } + + public AuthorizationException(String msg) { + super(msg); + } + + public AuthorizationException(ErrorCode code, Object... msgs) { + super(code.formatErrorMsg(msgs)); + this.errorCode = code; + this.msgs = msgs; + } + + public String formatErrMsg() { + return errorCode.formatErrorMsg(msgs); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java index 45d983c70947bc..15bed04e3899f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java @@ -31,6 +31,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.policy.Policy; import org.apache.doris.policy.StoragePolicy; import org.apache.doris.resource.Tag; @@ -814,5 +815,22 @@ public static void checkCatalogProperties(Map properties, boolea throw new AnalysisException(e.getMessage()); } } + // validate access controller properties + // eg: + // ( + // "access_controller.class" = "org.apache.doris.mysql.privilege.RangerAccessControllerFactory", + // "access_controller.properties.prop1" = "xxx", + // "access_controller.properties.prop2" = "yyy", + // ) + // 1. get access controller class + String acClass = properties.getOrDefault(CatalogMgr.ACCESS_CONTROLLER_CLASS_PROP, ""); + if (!Strings.isNullOrEmpty(acClass)) { + // 2. check if class exists + try { + Class.forName(acClass); + } catch (ClassNotFoundException e) { + throw new AnalysisException("failed to find class " + acClass, e); + } + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java index 3f9be036ec56eb..52a493683d5964 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java @@ -26,7 +26,9 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Resource; import org.apache.doris.common.DdlException; +import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.iceberg.IcebergExternalCatalogFactory; +import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.parquet.Strings; @@ -37,7 +39,6 @@ * A factory to create catalog instance of log or covert catalog into log. */ public class CatalogFactory { - /** * Convert the sql statement into catalog log. */ @@ -80,11 +81,11 @@ private static CatalogIf constructorCatalog( Resource catalogResource = Optional.ofNullable(Env.getCurrentEnv().getResourceMgr().getResource(resource)) .orElseThrow(() -> new DdlException("Resource doesn't exist: " + resource)); catalogType = catalogResource.getType().name().toLowerCase(); - if (props.containsKey("type")) { + if (props.containsKey(CatalogMgr.CATALOG_TYPE_PROP)) { throw new DdlException("Can not set 'type' when creating catalog with resource"); } } else { - String type = props.get("type"); + String type = props.get(CatalogMgr.CATALOG_TYPE_PROP); if (Strings.isNullOrEmpty(type)) { throw new DdlException("Missing property 'type' in properties"); } @@ -106,6 +107,12 @@ private static CatalogIf constructorCatalog( case "iceberg": catalog = IcebergExternalCatalogFactory.createCatalog(catalogId, name, resource, props); break; + case "test": + if (!FeConstants.runningUnitTest) { + throw new DdlException("test catalog is only for FE unit test"); + } + catalog = new TestExternalCatalog(catalogId, name, resource, props); + break; default: throw new DdlException("Unknown catalog type: " + catalogType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java index 40e63e4526f717..8647a00a8b7d9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java @@ -136,7 +136,7 @@ default T getDbOrAnalysisException(long dbId) throws AnalysisException { s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR)); } + // Called when catalog is dropped default void onClose() { - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 22d483624324a2..d3513feb7a8249 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -76,6 +76,10 @@ public class CatalogMgr implements Writable, GsonPostProcessable { private static final Logger LOG = LogManager.getLogger(CatalogMgr.class); + public static final String ACCESS_CONTROLLER_CLASS_PROP = "access_controller.class"; + public static final String ACCESS_CONTROLLER_PROPERTY_PREFIX_PROP = "access_controller.properties."; + public static final String CATALOG_TYPE_PROP = "type"; + private static final String YES = "yes"; private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index def5756f15a6d3..607c22144daeed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -38,8 +38,10 @@ import com.google.gson.annotations.SerializedName; import lombok.Data; import org.apache.commons.lang.NotImplementedException; +import org.apache.commons.lang3.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.parquet.Strings; import org.jetbrains.annotations.Nullable; import java.io.DataInput; @@ -139,6 +141,7 @@ public final synchronized void makeSureInitialized() { protected final void initLocalObjects() { if (!objectCreated) { initLocalObjectsImpl(); + initAccessController(); objectCreated = true; } } @@ -147,6 +150,38 @@ protected final void initLocalObjects() { // hms client, read properties from hive-site.xml, es client protected abstract void initLocalObjectsImpl(); + + /** + * eg: + * ( + * ""access_controller.class" = "org.apache.doris.mysql.privilege.RangerAccessControllerFactory", + * "access_controller.properties.prop1" = "xxx", + * "access_controller.properties.prop2" = "yyy", + * ) + */ + private void initAccessController() { + Map properties = getCatalogProperty().getProperties(); + // 1. get access controller class name + String className = properties.getOrDefault(CatalogMgr.ACCESS_CONTROLLER_CLASS_PROP, ""); + if (Strings.isNullOrEmpty(className)) { + // not set access controller, use internal access controller + return; + } + + // 2. get access controller properties + Map acProperties = Maps.newHashMap(); + for (Map.Entry entry : properties.entrySet()) { + if (entry.getKey().startsWith(CatalogMgr.ACCESS_CONTROLLER_PROPERTY_PREFIX_PROP)) { + acProperties.put( + StringUtils.removeStart(entry.getKey(), CatalogMgr.ACCESS_CONTROLLER_PROPERTY_PREFIX_PROP), + entry.getValue()); + } + } + + // 3. create access controller + Env.getCurrentEnv().getAccessManager().createAccessController(name, className, acProperties); + } + // init schema related objects protected abstract void init(); @@ -261,6 +296,16 @@ public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); } + @Override + public void onClose() { + removeAccessController(); + CatalogIf.super.onClose(); + } + + private void removeAccessController() { + Env.getCurrentEnv().getAccessManager().removeAccessController(name); + } + public void replayInitCatalog(InitCatalogLog log) { Map tmpDbNameToId = Maps.newConcurrentMap(); Map tmpIdToDb = Maps.newConcurrentMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java index 524e2d9d3fec2d..b9d0ce185daad5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitCatalogLog.java @@ -37,6 +37,7 @@ public enum Type { ES, JDBC, ICEBERG, + TEST, UNKNOWN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java index 24715936261bc6..9036353d303585 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InitDatabaseLog.java @@ -36,6 +36,7 @@ public enum Type { HMS, ES, JDBC, + TEST, UNKNOWN; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java index f196e071545456..8281b735d1af7a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/JdbcExternalCatalog.java @@ -52,6 +52,7 @@ public JdbcExternalCatalog(long catalogId, String name, String resource, Map props) { + super(catalogId, name); + this.type = "test"; + this.catalogProperty = new CatalogProperty(resource, props); + Class providerClazz = null; + try { + providerClazz = Class.forName(props.get("catalog_provider.class")); + this.catalogProvider = (TestCatalogProvider) providerClazz.newInstance(); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + @Override + protected void initLocalObjectsImpl() { + } + + @Override + protected void init() { + Map tmpDbNameToId = Maps.newConcurrentMap(); + Map tmpIdToDb = Maps.newConcurrentMap(); + InitCatalogLog initCatalogLog = new InitCatalogLog(); + initCatalogLog.setCatalogId(id); + initCatalogLog.setType(InitCatalogLog.Type.TEST); + List allDatabaseNames = mockedDatabaseNames(); + for (String dbName : allDatabaseNames) { + long dbId; + if (dbNameToId != null && dbNameToId.containsKey(dbName)) { + dbId = dbNameToId.get(dbName); + tmpDbNameToId.put(dbName, dbId); + ExternalDatabase db = idToDb.get(dbId); + db.setUnInitialized(invalidCacheInInit); + tmpIdToDb.put(dbId, db); + initCatalogLog.addRefreshDb(dbId); + } else { + dbId = Env.getCurrentEnv().getNextId(); + tmpDbNameToId.put(dbName, dbId); + TestExternalDatabase db = new TestExternalDatabase(this, dbId, dbName); + tmpIdToDb.put(dbId, db); + initCatalogLog.addCreateDb(dbId, dbName); + } + } + dbNameToId = tmpDbNameToId; + idToDb = tmpIdToDb; + Env.getCurrentEnv().getEditLog().logInitCatalog(initCatalogLog); + } + + private List mockedDatabaseNames() { + + return Lists.newArrayList(catalogProvider.getMetadata().keySet()); + } + + private List mockedTableNames(String dbName) { + if (!catalogProvider.getMetadata().containsKey(dbName)) { + throw new RuntimeException("unknown database: " + dbName); + } + return Lists.newArrayList(catalogProvider.getMetadata().get(dbName).keySet()); + } + + public List mockedSchema(String dbName, String tblName) { + if (!catalogProvider.getMetadata().containsKey(dbName)) { + throw new RuntimeException("unknown db: " + dbName); + } + if (!catalogProvider.getMetadata().get(dbName).containsKey(tblName)) { + throw new RuntimeException("unknown tbl: " + tblName); + } + return catalogProvider.getMetadata().get(dbName).get(tblName); + } + + @Override + public List listDatabaseNames(SessionContext ctx) { + makeSureInitialized(); + return Lists.newArrayList(dbNameToId.keySet()); + } + + @Override + public List listTableNames(SessionContext ctx, String dbName) { + makeSureInitialized(); + TestExternalDatabase db = (TestExternalDatabase) idToDb.get(dbNameToId.get(dbName)); + if (db != null && db.isInitialized()) { + List names = Lists.newArrayList(); + db.getTables().stream().forEach(table -> names.add(table.getName())); + return names; + } else { + return mockedTableNames(dbName); + } + } + + @Override + public boolean tableExist(SessionContext ctx, String dbName, String tblName) { + makeSureInitialized(); + if (!catalogProvider.getMetadata().containsKey(dbName)) { + return false; + } + if (!catalogProvider.getMetadata().get(dbName).containsKey(tblName)) { + return false; + } + return true; + } + + public interface TestCatalogProvider { + // db name -> (tbl name -> schema) + Map>> getMetadata(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java index 3f4217902330cf..b811ec114fc7ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/load/loadv2/LoadingTaskPlanner.java @@ -163,7 +163,7 @@ public void plan(TUniqueId loadId, List> fileStatusesLis // Generate plan trees // 1. Broker scan node ScanNode scanNode; - scanNode = new ExternalFileScanNode(new PlanNodeId(nextNodeId++), scanTupleDesc); + scanNode = new ExternalFileScanNode(new PlanNodeId(nextNodeId++), scanTupleDesc, false); ((ExternalFileScanNode) scanNode).setLoadInfo(loadJobId, txnId, table, brokerDesc, fileGroups, fileStatusesList, filesAdded, strictMode, loadParallelism, userInfo); scanNode.init(analyzer); diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerFactory.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerFactory.java new file mode 100644 index 00000000000000..d4e0400c9ebcea --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerFactory.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql.privilege; + +import java.util.Map; + +public interface AccessControllerFactory { + + CatalogAccessController createAccessController(Map prop); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerManager.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerManager.java index 2e2ac43505bebe..271ee18f03cff0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/AccessControllerManager.java @@ -20,12 +20,16 @@ import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.UserIdentity; import org.apache.doris.catalog.AuthorizationInfo; +import org.apache.doris.common.UserException; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.mysql.privilege.Auth.PrivLevel; import org.apache.doris.qe.ConnectContext; import com.google.common.base.Preconditions; +import com.google.common.collect.HashMultimap; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; @@ -37,6 +41,8 @@ * And using InternalCatalogAccessController as default. */ public class AccessControllerManager { + private static final Logger LOG = LogManager.getLogger(AccessControllerManager.class); + private SystemAccessController sysAccessController; private CatalogAccessController internalAccessController; private Map ctlToCtlAccessController = Maps.newConcurrentMap(); @@ -51,8 +57,30 @@ private CatalogAccessController getAccessControllerOrDefault(String ctl) { return ctlToCtlAccessController.getOrDefault(ctl, internalAccessController); } - public void addCatalogAccessControl(String ctl, CatalogAccessController controller) { - ctlToCtlAccessController.put(ctl, controller); + public boolean checkIfAccessControllerExist(String ctl) { + return ctlToCtlAccessController.containsKey(ctl); + } + + public void createAccessController(String ctl, String acFactoryClassName, Map prop) { + Class factoryClazz = null; + try { + factoryClazz = Class.forName(acFactoryClassName); + AccessControllerFactory factory = (AccessControllerFactory) factoryClazz.newInstance(); + CatalogAccessController accessController = factory.createAccessController(prop); + ctlToCtlAccessController.put(ctl, accessController); + LOG.info("create access controller {} for catalog {}", ctl, acFactoryClassName); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } catch (InstantiationException e) { + throw new RuntimeException(e); + } catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + } + + public void removeAccessController(String ctl) { + ctlToCtlAccessController.remove(ctl); + LOG.info("remove access controller for catalog {}", ctl); } public Auth getAuth() { @@ -73,10 +101,9 @@ public boolean checkCtlPriv(ConnectContext ctx, String ctl, PrivPredicate wanted return checkCtlPriv(ctx.getCurrentUserIdentity(), ctl, wanted); } - public boolean checkCtlPriv(UserIdentity currentUser, String ctl, PrivPredicate wanted) { + private boolean checkCtlPriv(UserIdentity currentUser, String ctl, PrivPredicate wanted) { boolean hasGlobal = sysAccessController.checkGlobalPriv(currentUser, wanted); - boolean hasCtl = getAccessControllerOrDefault(ctl).checkCtlPriv(hasGlobal, currentUser, ctl, wanted); - return hasGlobal || hasCtl; + return getAccessControllerOrDefault(ctl).checkCtlPriv(hasGlobal, currentUser, ctl, wanted); } // ==== Database ==== @@ -94,16 +121,10 @@ public boolean checkDbPriv(ConnectContext ctx, String ctl, String db, PrivPredic public boolean checkDbPriv(UserIdentity currentUser, String ctl, String db, PrivPredicate wanted) { boolean hasGlobal = sysAccessController.checkGlobalPriv(currentUser, wanted); - boolean hasDb = getAccessControllerOrDefault(ctl).checkDbPriv(hasGlobal, currentUser, ctl, db, wanted); - return hasGlobal || hasDb; + return getAccessControllerOrDefault(ctl).checkDbPriv(hasGlobal, currentUser, ctl, db, wanted); } // ==== Table ==== - public boolean checkTblPriv(ConnectContext ctx, String qualifiedCtl, - String qualifiedDb, String tbl, PrivPredicate wanted) { - return checkTblPriv(ctx.getCurrentUserIdentity(), qualifiedCtl, qualifiedDb, tbl, wanted); - } - public boolean checkTblPriv(ConnectContext ctx, String qualifiedDb, String tbl, PrivPredicate wanted) { return checkTblPriv(ctx, Auth.DEFAULT_CATALOG, qualifiedDb, tbl, wanted); } @@ -113,14 +134,29 @@ public boolean checkTblPriv(ConnectContext ctx, TableName tableName, PrivPredica return checkTblPriv(ctx, tableName.getCtl(), tableName.getDb(), tableName.getTbl(), wanted); } + public boolean checkTblPriv(ConnectContext ctx, String qualifiedCtl, + String qualifiedDb, String tbl, PrivPredicate wanted) { + return checkTblPriv(ctx.getCurrentUserIdentity(), qualifiedCtl, qualifiedDb, tbl, wanted); + } + public boolean checkTblPriv(UserIdentity currentUser, String db, String tbl, PrivPredicate wanted) { return checkTblPriv(currentUser, Auth.DEFAULT_CATALOG, db, tbl, wanted); } public boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, String tbl, PrivPredicate wanted) { boolean hasGlobal = sysAccessController.checkGlobalPriv(currentUser, wanted); - boolean hasTbl = getAccessControllerOrDefault(ctl).checkTblPriv(hasGlobal, currentUser, ctl, db, tbl, wanted); - return hasGlobal || hasTbl; + return getAccessControllerOrDefault(ctl).checkTblPriv(hasGlobal, currentUser, ctl, db, tbl, wanted); + } + + // ==== Column ==== + public void checkColumnsPriv(UserIdentity currentUser, String ctl, HashMultimap tableToColsMap, + PrivPredicate wanted) throws UserException { + boolean hasGlobal = sysAccessController.checkGlobalPriv(currentUser, wanted); + CatalogAccessController accessController = getAccessControllerOrDefault(ctl); + for (TableName tableName : tableToColsMap.keys()) { + accessController.checkColsPriv(hasGlobal, currentUser, ctl, tableName.getDb(), + tableName.getTbl(), tableToColsMap.get(tableName), wanted); + } } // ==== Resource ==== diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java index dfa88bf42314c5..441573dda17663 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/Auth.java @@ -37,6 +37,7 @@ import org.apache.doris.cluster.ClusterNamespace; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.AuthenticationException; +import org.apache.doris.common.AuthorizationException; import org.apache.doris.common.DdlException; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; @@ -182,6 +183,8 @@ public void checkPassword(String remoteUser, String remoteHost, byte[] remotePas readLock(); try { userManager.checkPassword(remoteUser, remoteHost, remotePasswd, randomString, currentUser); + Set roles = userRoleManager.getRolesByUser(currentUser.get(0)); + currentUser.get(0).setRoles(roles); } finally { readUnlock(); } @@ -206,13 +209,17 @@ public void checkPlainPassword(String remoteUser, String remoteHost, String remo throw new AuthenticationException(ErrorCode.ERR_ACCESS_DENIED_ERROR, remoteUser + "@" + remoteHost, Strings.isNullOrEmpty(remotePasswd) ? "NO" : "YES"); } - return; + } else { + readLock(); + try { + userManager.checkPlainPassword(remoteUser, remoteHost, remotePasswd, currentUser); + } finally { + readUnlock(); + } } - readLock(); - try { - userManager.checkPlainPassword(remoteUser, remoteHost, remotePasswd, currentUser); - } finally { - readUnlock(); + if (currentUser != null) { + Set roles = userRoleManager.getRolesByUser(currentUser.get(0)); + currentUser.get(0).setRoles(roles); } } @@ -306,6 +313,13 @@ public boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, Str } } + // ==== Column ==== + public void checkColsPriv(UserIdentity currentUser, String ctl, String db, String tbl, Set cols, + PrivPredicate wanted) throws AuthorizationException { + // TODO: Support column priv + } + + // ==== Resource ==== public boolean checkResourcePriv(UserIdentity currentUser, String resourceName, PrivPredicate wanted) { if (isLdapAuthEnabled() && LdapPrivsChecker.hasResourcePrivFromLdap(currentUser, resourceName, wanted)) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CatalogAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CatalogAccessController.java index afbc177425a19e..9f7c18b4393283 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CatalogAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/CatalogAccessController.java @@ -18,6 +18,9 @@ package org.apache.doris.mysql.privilege; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.common.AuthorizationException; + +import java.util.Set; public interface CatalogAccessController { // ==== Catalog ==== @@ -45,4 +48,19 @@ default boolean checkTblPriv(boolean hasGlobal, UserIdentity currentUser, String } boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, String tbl, PrivPredicate wanted); + + // ==== Column ==== + default void checkColsPriv(boolean hasGlobal, UserIdentity currentUser, String ctl, String db, String tbl, + Set cols, PrivPredicate wanted) throws AuthorizationException { + try { + checkColsPriv(currentUser, ctl, db, tbl, cols, wanted); + } catch (AuthorizationException e) { + if (!hasGlobal) { + throw e; + } + } + } + + void checkColsPriv(UserIdentity currentUser, String ctl, String db, String tbl, + Set cols, PrivPredicate wanted) throws AuthorizationException; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/InternalCatalogAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/InternalCatalogAccessController.java index 9e233cac31689b..175eb08f8e2ba1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/InternalCatalogAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/InternalCatalogAccessController.java @@ -18,6 +18,9 @@ package org.apache.doris.mysql.privilege; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.common.AuthorizationException; + +import java.util.Set; public class InternalCatalogAccessController implements CatalogAccessController { private Auth auth; @@ -40,4 +43,10 @@ public boolean checkDbPriv(UserIdentity currentUser, String ctl, String db, Priv public boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, String tbl, PrivPredicate wanted) { return auth.checkTblPriv(currentUser, ctl, db, tbl, wanted); } + + @Override + public void checkColsPriv(UserIdentity currentUser, String ctl, String db, String tbl, Set cols, + PrivPredicate wanted) throws AuthorizationException { + auth.checkColsPriv(currentUser, ctl, db, tbl, cols, wanted); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessController.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessController.java index b6e9747fb7706b..02152f4fa8d357 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessController.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessController.java @@ -18,8 +18,17 @@ package org.apache.doris.mysql.privilege; import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.common.AuthorizationException; + +import java.util.Map; +import java.util.Set; public class RangerAccessController implements CatalogAccessController { + + public RangerAccessController(Map properties) { + + } + @Override public boolean checkCtlPriv(UserIdentity currentUser, String ctl, PrivPredicate wanted) { // TODO @@ -38,4 +47,11 @@ public boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, Str // TODO return false; } + + @Override + public void checkColsPriv(UserIdentity currentUser, String ctl, String db, String tbl, Set cols, + PrivPredicate wanted) throws AuthorizationException { + // TODO + throw new AuthorizationException("not implemented"); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessControllerFactory.java b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessControllerFactory.java new file mode 100644 index 00000000000000..e5674491c5a9e8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mysql/privilege/RangerAccessControllerFactory.java @@ -0,0 +1,27 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mysql.privilege; + +import java.util.Map; + +public class RangerAccessControllerFactory implements AccessControllerFactory { + @Override + public CatalogAccessController createAccessController(Map prop) { + return new RangerAccessController(prop); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index a7af0a72a6b209..992eb35f4b9ea5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -547,7 +547,8 @@ public PlanFragment visitPhysicalFileScan(PhysicalFileScan fileScan, PlanTransla ExternalTable table = fileScan.getTable(); TupleDescriptor tupleDescriptor = generateTupleDesc(slotList, table, context); tupleDescriptor.setTable(table); - ExternalFileScanNode fileScanNode = new ExternalFileScanNode(context.nextPlanNodeId(), tupleDescriptor); + // TODO(cmy): determine the needCheckColumnPriv param + ExternalFileScanNode fileScanNode = new ExternalFileScanNode(context.nextPlanNodeId(), tupleDescriptor, false); TableName tableName = new TableName(null, "", ""); TableRef ref = new TableRef(tableName, null, null); BaseTableRef tableRef = new BaseTableRef(ref, table, tableName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java index 55f75acc8ad028..94c5c070281bc3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/DataGenScanNode.java @@ -118,4 +118,9 @@ public void finalizeForNereids() { throw new NereidsException("Can not compute shard locations for DataGenScanNode: " + e.getMessage(), e); } } + + @Override + public boolean needToCheckColumnPriv() { + return false; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java index 85b8b7e3397500..e34bf825f8a6c9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java @@ -31,20 +31,32 @@ import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StatementBase; import org.apache.doris.analysis.StorageBackend; +import org.apache.doris.analysis.TableName; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; +import org.apache.doris.catalog.Table; +import org.apache.doris.catalog.TableIf; import org.apache.doris.catalog.Type; +import org.apache.doris.catalog.external.ExternalTable; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.AnalysisException; import org.apache.doris.common.UserException; import org.apache.doris.common.util.VectorizedUtil; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.qe.ConnectContext; import org.apache.doris.rewrite.mvrewrite.MVSelectFailedException; import org.apache.doris.thrift.TQueryOptions; import org.apache.doris.thrift.TRuntimeFilterMode; import com.google.common.base.Preconditions; +import com.google.common.base.Strings; +import com.google.common.collect.HashMultimap; import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -164,6 +176,8 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue insertStmt.prepareExpressions(); } + checkColumnPrivileges(singleNodePlan); + // TODO chenhao16 , no used materialization work // compute referenced slots before calling computeMemLayout() //analyzer.markRefdSlots(analyzer, singleNodePlan, resultExprs, null); @@ -298,6 +312,66 @@ public void createPlanFragments(StatementBase statement, Analyzer analyzer, TQue } } + private void checkColumnPrivileges(PlanNode singleNodePlan) throws UserException { + if (ConnectContext.get() == null) { + return; + } + // 1. collect all columns from all scan nodes + List scanNodes = Lists.newArrayList(); + singleNodePlan.collect((PlanNode planNode) -> planNode instanceof ScanNode, scanNodes); + // catalog : + Map> ctlToTableColumnMap = Maps.newHashMap(); + for (ScanNode scanNode : scanNodes) { + if (!scanNode.needToCheckColumnPriv()) { + continue; + } + TupleDescriptor tupleDesc = scanNode.getTupleDesc(); + TableIf table = tupleDesc.getTable(); + if (table == null) { + continue; + } + TableName tableName = getFullQualifiedTableNameFromTable(table); + for (SlotDescriptor slotDesc : tupleDesc.getSlots()) { + if (!slotDesc.isMaterialized()) { + continue; + } + Column column = slotDesc.getColumn(); + if (column == null) { + continue; + } + HashMultimap tableColumnMap = ctlToTableColumnMap.get(tableName.getCtl()); + if (tableColumnMap == null) { + tableColumnMap = HashMultimap.create(); + ctlToTableColumnMap.put(tableName.getCtl(), tableColumnMap); + } + tableColumnMap.put(tableName, column.getName()); + LOG.debug("collect column {} in {}", column.getName(), tableName); + } + } + // 2. check privs + // TODO: only support SELECT_PRIV now + PrivPredicate wanted = PrivPredicate.SELECT; + for (Map.Entry> entry : ctlToTableColumnMap.entrySet()) { + Env.getCurrentEnv().getAccessManager().checkColumnsPriv(ConnectContext.get().getCurrentUserIdentity(), + entry.getKey(), entry.getValue(), wanted); + } + } + + private TableName getFullQualifiedTableNameFromTable(TableIf table) throws AnalysisException { + if (table instanceof Table) { + String dbName = ClusterNamespace.getNameFromFullName(((Table) table).getQualifiedDbName()); + if (Strings.isNullOrEmpty(dbName)) { + throw new AnalysisException("failed to get db name from table " + table.getName()); + } + return new TableName(InternalCatalog.INTERNAL_CATALOG_NAME, dbName, table.getName()); + } else if (table instanceof ExternalTable) { + ExternalTable extTable = (ExternalTable) table; + return new TableName(extTable.getCatalog().getName(), extTable.getDbName(), extTable.getName()); + } else { + throw new AnalysisException("table " + table.getName() + " is not internal or external table instance"); + } + } + /** * If there are unassigned conjuncts, returns a SelectNode on top of root that evaluate those conjuncts; otherwise * returns root unchanged. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java index 6e4d21cc81b24d..d57a749ab53edb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/ScanNode.java @@ -485,4 +485,11 @@ public String toString() { desc.getTable().getName()).add("keyRanges", "").addValue( super.debugString()).toString(); } + + // Some of scan node(eg, DataGenScanNode) does not need to check column priv + // (because the it has no corresponding catalog/db/table info) + // Subclass may override this method. + public boolean needToCheckColumnPriv() { + return true; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java index 5ebb14a52e5349..6b7162c54f61e4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/SingleNodePlanner.java @@ -1958,7 +1958,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s case HIVE: throw new RuntimeException("Hive external table is not supported, try to use hive catalog please"); case ICEBERG: - scanNode = new ExternalFileScanNode(ctx.getNextNodeId(), tblRef.getDesc()); + scanNode = new ExternalFileScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true); break; case HUDI: throw new UserException( @@ -1971,7 +1971,7 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s break; case HMS_EXTERNAL_TABLE: case ICEBERG_EXTERNAL_TABLE: - scanNode = new ExternalFileScanNode(ctx.getNextNodeId(), tblRef.getDesc()); + scanNode = new ExternalFileScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true); break; case ES_EXTERNAL_TABLE: scanNode = new EsScanNode(ctx.getNextNodeId(), tblRef.getDesc(), "EsScanNode", true); @@ -1979,6 +1979,9 @@ private PlanNode createScanNode(Analyzer analyzer, TableRef tblRef, SelectStmt s case JDBC_EXTERNAL_TABLE: scanNode = new JdbcScanNode(ctx.getNextNodeId(), tblRef.getDesc(), true); break; + case TEST_EXTERNAL_TABLE: + scanNode = new TestExternalTableScanNode(ctx.getNextNodeId(), tblRef.getDesc()); + break; default: break; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java index 8e1bd99ab35535..692e5c755b8e26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/StreamLoadPlanner.java @@ -189,7 +189,7 @@ public TExecPlanFragmentParams plan(TUniqueId loadId) throws UserException { } // create scan node - ExternalFileScanNode fileScanNode = new ExternalFileScanNode(new PlanNodeId(0), scanTupleDesc); + ExternalFileScanNode fileScanNode = new ExternalFileScanNode(new PlanNodeId(0), scanTupleDesc, false); // 1. create file group DataDescription dataDescription = new DataDescription(destTable.getName(), taskInfo); dataDescription.analyzeWithoutCheckPriv(db.getFullName()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TestExternalTableScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TestExternalTableScanNode.java new file mode 100644 index 00000000000000..f537d091be364a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TestExternalTableScanNode.java @@ -0,0 +1,95 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.planner; + +import org.apache.doris.analysis.Analyzer; +import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.common.UserException; +import org.apache.doris.statistics.StatisticalType; +import org.apache.doris.statistics.StatsRecursiveDerive; +import org.apache.doris.thrift.TExplainLevel; +import org.apache.doris.thrift.TPlanNode; +import org.apache.doris.thrift.TPlanNodeType; +import org.apache.doris.thrift.TScanRangeLocations; +import org.apache.doris.thrift.TTestExternalScanNode; + +import com.google.common.base.MoreObjects; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.List; + +public class TestExternalTableScanNode extends ScanNode { + private static final Logger LOG = LogManager.getLogger(TestExternalTableScanNode.class); + private String tableName; + + public TestExternalTableScanNode(PlanNodeId id, TupleDescriptor desc) { + super(id, desc, "TestExternalTableScanNode", StatisticalType.TEST_EXTERNAL_TABLE); + tableName = desc.getTable().getName(); + } + + @Override + public void init(Analyzer analyzer) throws UserException { + super.init(analyzer); + computeStats(analyzer); + } + + @Override + public List getScanRangeLocations(long maxScanRangeLength) { + return null; + } + + @Override + public String getNodeExplainString(String prefix, TExplainLevel detailLevel) { + StringBuilder output = new StringBuilder(); + output.append(prefix).append("TABLE: ").append(tableName).append("\n"); + return output.toString(); + } + + @Override + public void finalize(Analyzer analyzer) throws UserException { + } + + @Override + public void computeStats(Analyzer analyzer) throws UserException { + super.computeStats(analyzer); + // even if current node scan has no data,at least on backend will be assigned when the fragment actually execute + numNodes = numNodes <= 0 ? 1 : numNodes; + StatsRecursiveDerive.getStatsRecursiveDerive().statsRecursiveDerive(this); + cardinality = (long) statsDeriveResult.getRowCount(); + } + + @Override + protected void toThrift(TPlanNode msg) { + msg.node_type = TPlanNodeType.TEST_EXTERNAL_SCAN_NODE; + msg.test_external_scan_node = new TTestExternalScanNode(); + msg.test_external_scan_node.setTupleId(desc.getId().asInt()); + msg.test_external_scan_node.setTableName(tableName); + } + + @Override + protected String debugString() { + MoreObjects.ToStringHelper helper = MoreObjects.toStringHelper(this); + return helper.addValue(super.debugString()).toString(); + } + + @Override + public int getNumInstances() { + return 1; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java index 0ee81dd6347056..68e75640f9e8d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalFileScanNode.java @@ -135,9 +135,12 @@ public enum Type { * External file scan node for: * 1. Query hms table * 2. Load from file + * needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv + * eg: s3 tvf, load scan node. + * These scan nodes do not have corresponding catalog/database/table info, so no need to do priv check */ - public ExternalFileScanNode(PlanNodeId id, TupleDescriptor desc) { - super(id, desc, "EXTERNAL_FILE_SCAN_NODE", StatisticalType.FILE_SCAN_NODE); + public ExternalFileScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) { + super(id, desc, "EXTERNAL_FILE_SCAN_NODE", StatisticalType.FILE_SCAN_NODE, needCheckColumnPriv); } // Only for broker load job. diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java index b86ffd91302bb2..8eabeb9d5d25ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/ExternalScanNode.java @@ -35,8 +35,13 @@ */ public class ExternalScanNode extends ScanNode { - public ExternalScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType) { + // set to false means this scan node does not need to check column priv. + private boolean needCheckColumnPriv; + + public ExternalScanNode(PlanNodeId id, TupleDescriptor desc, String planNodeName, StatisticalType statisticalType, + boolean needCheckColumnPriv) { super(id, desc, planNodeName, statisticalType); + this.needCheckColumnPriv = needCheckColumnPriv; } @Override @@ -48,4 +53,9 @@ public List getScanRangeLocations(long maxScanRangeLength) protected void toThrift(TPlanNode msg) { } + + @Override + public boolean needToCheckColumnPriv() { + return this.needCheckColumnPriv; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java index 96a6bdc9e34979..de4f0cdb56a565 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/MetadataScanNode.java @@ -81,6 +81,11 @@ public void finalize(Analyzer analyzer) throws UserException { buildScanRanges(); } + @Override + public boolean needToCheckColumnPriv() { + return super.needToCheckColumnPriv(); + } + private void buildScanRanges() { if (tvf.getMetaType() == MetadataTableValuedFunction.MetaType.ICEBERG) { IcebergTableValuedFunction icebergTvf = (IcebergTableValuedFunction) tvf; diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java index 47ad4a61616a3a..586c9139cca0bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticalType.java @@ -48,4 +48,5 @@ public enum StatisticalType { FILE_SCAN_NODE, METADATA_SCAN_NODE, JDBC_SCAN_NODE, + TEST_EXTERNAL_TABLE, } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 16aa5aa666b6ed..adfbeceafb943d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -207,7 +207,7 @@ public TFileAttributes getFileAttributes() { @Override public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc) { - return new ExternalFileScanNode(id, desc); + return new ExternalFileScanNode(id, desc, false); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ColumnPrivTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ColumnPrivTest.java new file mode 100644 index 00000000000000..257b6bbad3b639 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ColumnPrivTest.java @@ -0,0 +1,299 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.analysis.CreateCatalogStmt; +import org.apache.doris.analysis.CreateDbStmt; +import org.apache.doris.analysis.CreateRoleStmt; +import org.apache.doris.analysis.CreateTableStmt; +import org.apache.doris.analysis.CreateUserStmt; +import org.apache.doris.analysis.CreateViewStmt; +import org.apache.doris.analysis.DropCatalogStmt; +import org.apache.doris.analysis.GrantStmt; +import org.apache.doris.analysis.ShowCatalogStmt; +import org.apache.doris.analysis.UserIdentity; +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.cluster.ClusterNamespace; +import org.apache.doris.common.AuthorizationException; +import org.apache.doris.common.FeConstants; +import org.apache.doris.datasource.test.TestExternalCatalog.TestCatalogProvider; +import org.apache.doris.mysql.privilege.AccessControllerFactory; +import org.apache.doris.mysql.privilege.Auth; +import org.apache.doris.mysql.privilege.CatalogAccessController; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.ShowResultSet; +import org.apache.doris.utframe.TestWithFeService; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import com.google.common.collect.Sets; +import org.junit.Assert; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; +import java.util.Set; + +public class ColumnPrivTest extends TestWithFeService { + private static Auth auth; + private static Env env; + private CatalogMgr mgr; + private ConnectContext rootCtx; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + mgr = Env.getCurrentEnv().getCatalogMgr(); + rootCtx = createDefaultCtx(); + env = Env.getCurrentEnv(); + auth = env.getAuth(); + + // 1. create test catalog + CreateCatalogStmt testCatalog = (CreateCatalogStmt) parseAndAnalyzeStmt( + "create catalog test1 properties(\n" + + " \"type\" = \"test\",\n" + + " \"catalog_provider.class\" " + + "= \"org.apache.doris.datasource.ColumnPrivTest$MockedCatalogProvider\",\n" + + " \"access_controller.class\" " + + "= \"org.apache.doris.datasource.ColumnPrivTest$TestAccessControllerFactory\",\n" + + " \"access_controller.properties.key1\" = \"val1\",\n" + + " \"access_controller.properties.key2\" = \"val2\"\n" + + ");", + rootCtx); + env.getCatalogMgr().createCatalog(testCatalog); + + // 2. create internal db and tbl + CreateDbStmt createDbStmt = (CreateDbStmt) parseAndAnalyzeStmt("create database innerdb1"); + env.createDb(createDbStmt); + createDbStmt = (CreateDbStmt) parseAndAnalyzeStmt("create database innerdb2"); + env.createDb(createDbStmt); + + CreateTableStmt createTableStmt = (CreateTableStmt) parseAndAnalyzeStmt( + "create table innerdb1.innertbl11\n" + + "(\n" + + " col1 int, \n" + + " col2 string\n" + + ")\n" + + "distributed by hash(col1) buckets 1\n" + + "properties(\"replication_num\" = \"1\");", rootCtx); + env.createTable(createTableStmt); + + createTableStmt = (CreateTableStmt) parseAndAnalyzeStmt( + "create table innerdb1.innertbl12\n" + + "(\n" + + " col3 int, \n" + + " col4 string\n" + + ")\n" + + "distributed by hash(col3) buckets 1\n" + + "properties(\"replication_num\" = \"1\");", rootCtx); + env.createTable(createTableStmt); + + createTableStmt = (CreateTableStmt) parseAndAnalyzeStmt( + "create table innerdb2.innertbl21\n" + + "(\n" + + " col5 int, \n" + + " col6 string\n" + + ")\n" + + "distributed by hash(col5) buckets 1\n" + + "properties(\"replication_num\" = \"1\");", rootCtx); + env.createTable(createTableStmt); + } + + @Override + protected void runAfterAll() throws Exception { + super.runAfterAll(); + rootCtx.setThreadLocalInfo(); + Assert.assertTrue(env.getAccessManager().checkIfAccessControllerExist("test1")); + DropCatalogStmt stmt = (DropCatalogStmt) parseAndAnalyzeStmt("drop catalog test1"); + env.getCatalogMgr().dropCatalog(stmt); + Assert.assertFalse(env.getAccessManager().checkIfAccessControllerExist("test1")); + } + + @Test + public void testColumnPrivs() throws Exception { + String showCatalogSql = "SHOW CATALOGS"; + ShowCatalogStmt showStmt = (ShowCatalogStmt) parseAndAnalyzeStmt(showCatalogSql); + ShowResultSet showResultSet = mgr.showCatalogs(showStmt); + Assertions.assertEquals(2, showResultSet.getResultRows().size()); + + CreateRoleStmt createRole1 = (CreateRoleStmt) parseAndAnalyzeStmt("create role role1;", rootCtx); + auth.createRole(createRole1); + GrantStmt grantRole = (GrantStmt) parseAndAnalyzeStmt("grant select_priv on test1.*.* to role 'role1';", + rootCtx); + auth.grant(grantRole); + grantRole = (GrantStmt) parseAndAnalyzeStmt( + "grant select_priv on internal.innerdb1.innertbl11 to role 'role1';", rootCtx); + auth.grant(grantRole); + grantRole = (GrantStmt) parseAndAnalyzeStmt( + "grant select_priv on internal.innerdb1.v1 to role 'role1';", rootCtx); + auth.grant(grantRole); + auth.createUser((CreateUserStmt) parseAndAnalyzeStmt( + "create user 'user1'@'%' identified by 'pwd1' default role 'role1';", rootCtx)); + // create a view + CreateViewStmt viewStmt = (CreateViewStmt) parseAndAnalyzeStmt( + "create view innerdb1.v1 as select * from test1.db1.tbl11", rootCtx); + env.createView(viewStmt); + + // Now we have + // internal.innerdb1 + // innertbl11: col1(int), col2(string) + // innertbl12: col2(int), col4(string) + // internal.innerdb2 + // innertbl21: col5(int), col6(string) + // test1.db1 + // tbl11: a11(bigint), a12(string), a13(float) + // tbl12: b21(bigint), b22(string), b23(float) + // test1.db2 + // tbl21: c11(bigint), c12(string), c13(float) + + UserIdentity user1 = UserIdentity.createAnalyzedUserIdentWithIp("default_cluster:user1", "%"); + Set roles = Sets.newHashSet(); + roles.add("role1"); + user1.setRoles(roles); + ConnectContext user1Ctx = createCtx(user1, "127.0.0.1"); + + // 1. query inner table + testSql(user1Ctx, "select * from innerdb1.innertbl11", "0:VOlapScanNode"); + // 2. query external table, without a11 column priv + testSql(user1Ctx, "select * from test1.db1.tbl11", "Access deny to column a11"); + // 3. query external table, not query column a12 + testSql(user1Ctx, "select a12 from test1.db1.tbl11", "TABLE: tbl11"); + // change to test1.db1 + user1Ctx.changeDefaultCatalog("test1"); + user1Ctx.setDatabase("default_cluster:db1"); + testSql(user1Ctx, "select a12 from tbl11 where a11 > 0;", "Access deny to column a11"); + testSql(user1Ctx, "select sum(a13) from db1.tbl11 group by a11;", "Access deny to column a11"); + testSql(user1Ctx, "select sum(a13) x from test1.db1.tbl11 group by a11 having x > 0;", + "Access deny to column a11"); + testSql(user1Ctx, "select a12 from tbl11 where abs(a11) > 0;", "Access deny to column a11"); + // TODO: how to handle count(*) when setting column privilege? + // testSql(user1Ctx, "select count(*) from tbl11;", "Access deny to column a11"); + + // change to internal.innerdb1 + user1Ctx.changeDefaultCatalog("internal"); + user1Ctx.setDatabase("default_cluster:innerdb1"); + testSql(user1Ctx, "select sum(a13) x from test1.db1.tbl11 group by a11 having x > 0;", + "Access deny to column a11"); + testSql(user1Ctx, "with cte1 as (select a11 from test1.db1.tbl11) select * from cte1;", + "Access deny to column a11"); + testSql(user1Ctx, "select a12 from (select * from test1.db1.tbl11) x", "TABLE: tbl11"); + testSql(user1Ctx, "select * from v1", "Access deny to column a11"); + + testSql(user1Ctx, "select * from numbers(\"number\" = \"1\");", "0:VDataGenScanNode"); + } + + private void testSql(ConnectContext ctx, String sql, String expectedMsg) throws Exception { + String res = getSQLPlanOrErrorMsg(ctx, "explain " + sql, false); + System.out.println(res); + Assert.assertTrue(res.contains(expectedMsg)); + } + + public static class TestAccessControllerFactory implements AccessControllerFactory { + @Override + public CatalogAccessController createAccessController(Map prop) { + return new TestAccessController(prop); + } + + public static class TestAccessController implements CatalogAccessController { + private Map prop; + + public TestAccessController(Map prop) { + this.prop = prop; + } + + @Override + public boolean checkCtlPriv(UserIdentity currentUser, String ctl, PrivPredicate wanted) { + return false; + } + + @Override + public boolean checkDbPriv(UserIdentity currentUser, String ctl, String db, PrivPredicate wanted) { + return false; + } + + @Override + public boolean checkTblPriv(UserIdentity currentUser, String ctl, String db, String tbl, + PrivPredicate wanted) { + if (ClusterNamespace.getNameFromFullName(currentUser.getQualifiedUser()).equals("user1")) { + if (ctl.equals("test1")) { + if (ClusterNamespace.getNameFromFullName(db).equals("db1")) { + if (tbl.equals("tbl11")) { + return true; + } + } + } + } + return false; + } + + @Override + public void checkColsPriv(UserIdentity currentUser, String ctl, String db, String tbl, Set cols, + PrivPredicate wanted) throws AuthorizationException { + if (currentUser.getQualifiedUser().contains("user1")) { + if (ctl.equals("test1")) { + if (db.equals("db1")) { + if (tbl.equals("tbl11")) { + if (cols.contains("a11")) { + throw new AuthorizationException("Access deny to column a11"); + } + } + } + } + } + } + } + } + + public static class MockedCatalogProvider implements TestCatalogProvider { + public static final Map>> MOCKED_META; + + static { + MOCKED_META = Maps.newHashMap(); + Map> tblSchemaMap1 = Maps.newHashMap(); + // db1 + tblSchemaMap1.put("tbl11", Lists.newArrayList( + new Column("a11", PrimitiveType.BIGINT), + new Column("a12", PrimitiveType.STRING), + new Column("a13", PrimitiveType.FLOAT) + )); + tblSchemaMap1.put("tbl12", Lists.newArrayList( + new Column("b21", PrimitiveType.BIGINT), + new Column("b22", PrimitiveType.STRING), + new Column("b23", PrimitiveType.FLOAT) + )); + MOCKED_META.put("db1", tblSchemaMap1); + // db2 + Map> tblSchemaMap2 = Maps.newHashMap(); + tblSchemaMap2.put("tbl21", Lists.newArrayList( + new Column("c11", PrimitiveType.BIGINT), + new Column("c12", PrimitiveType.STRING), + new Column("c13", PrimitiveType.FLOAT) + )); + MOCKED_META.put("db2", tblSchemaMap2); + } + + @Override + public Map>> getMetadata() { + return MOCKED_META; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java index d6190b18919355..d9c44e8c777271 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java +++ b/fe/fe-core/src/test/java/org/apache/doris/utframe/TestWithFeService.java @@ -441,16 +441,21 @@ public String getSQLPlanOrErrorMsg(String sql) throws Exception { } public String getSQLPlanOrErrorMsg(String sql, boolean isVerbose) throws Exception { - connectContext.getState().reset(); - StmtExecutor stmtExecutor = new StmtExecutor(connectContext, sql); - connectContext.setExecutor(stmtExecutor); + return getSQLPlanOrErrorMsg(connectContext, sql, isVerbose); + } + + public String getSQLPlanOrErrorMsg(ConnectContext ctx, String sql, boolean isVerbose) throws Exception { + ctx.setThreadLocalInfo(); + ctx.getState().reset(); + StmtExecutor stmtExecutor = new StmtExecutor(ctx, sql); + ctx.setExecutor(stmtExecutor); ConnectContext.get().setExecutor(stmtExecutor); stmtExecutor.execute(); - if (connectContext.getState().getStateType() != QueryState.MysqlStateType.ERR) { + if (ctx.getState().getStateType() != QueryState.MysqlStateType.ERR) { Planner planner = stmtExecutor.planner(); return planner.getExplainString(new ExplainOptions(isVerbose, false)); } else { - return connectContext.getState().getErrorMessage(); + return ctx.getState().getErrorMessage(); } } diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 6eb6a657d3d9b4..978c90e1036056 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -56,6 +56,7 @@ enum TPlanNodeType { DATA_GEN_SCAN_NODE, FILE_SCAN_NODE, JDBC_SCAN_NODE, + TEST_EXTERNAL_SCAN_NODE, } // phases of an execution node @@ -430,7 +431,6 @@ struct TJdbcScanNode { 4: optional Types.TOdbcTableType table_type } - struct TBrokerScanNode { 1: required Types.TTupleId tuple_id @@ -537,6 +537,11 @@ struct TMetaScanNode { 4: optional string table } +struct TTestExternalScanNode { + 1: optional Types.TTupleId tuple_id + 2: optional string table_name +} + struct TSortInfo { 1: required list ordering_exprs 2: required list is_asc_order @@ -1061,6 +1066,7 @@ struct TPlanNode { 44: optional TFileScanNode file_scan_node 45: optional TJdbcScanNode jdbc_scan_node 46: optional TNestedLoopJoinNode nested_loop_join_node + 47: optional TTestExternalScanNode test_external_scan_node 101: optional list projections 102: optional Types.TTupleId output_tuple_id diff --git a/gensrc/thrift/Types.thrift b/gensrc/thrift/Types.thrift index 5407952daac41c..641e54f780206c 100644 --- a/gensrc/thrift/Types.thrift +++ b/gensrc/thrift/Types.thrift @@ -567,7 +567,8 @@ enum TTableType { HIVE_TABLE, ICEBERG_TABLE, HUDI_TABLE, - JDBC_TABLE + JDBC_TABLE, + TEST_EXTERNAL_TABLE, } enum TOdbcTableType {