Skip to content

Commit 97230a5

Browse files
morningmanmorningman
authored
[Refactor](auth)(step-2) Add AccessController to support customized authorization (#16802)
Support specifying AccessControllerFactory when creating catalog create catalog hive properties( ... "access_controller.class" = "org.apache.doris.mysql.privilege.RangerAccessControllerFactory", "access_controller.properties.prop1" = "xxx", "access_controller.properties.prop2" = "yyy", ... ) So that user can specified their own access controller, such as RangerAccessController Add interface to check column level privilege A new method of CatalogAccessController: checkColsPriv(), for checking column level privileges. TODO: Support grant column level privileges statements in Doris Add TestExternalCatalog/Database/Table/ScanNode These classes are used for FE unit test. In unit test you can create catalog test1 properties( "type" = "test" "catalog_provider.class" = "org.apache.doris.datasource.ColumnPrivTest$MockedCatalogProvider" "access_controller.class" = "org.apache.doris.mysql.privilege.TestAccessControllerFactory", "access_controller.properties.key1" = "val1", "access_controller.properties.key2" = "val2" ); To create a test catalog, and specify catalog_provider to mock database/table/schema metadata Set roles in current user identity in connection context The roles can be used for authorization in access controller.
1 parent 5291f14 commit 97230a5

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

44 files changed

+1217
-55
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/analysis/DropCatalogStmt.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ public void analyze(Analyzer analyzer) throws UserException {
5959
if (!Env.getCurrentEnv().getAccessManager().checkCtlPriv(
6060
ConnectContext.get(), catalogName, PrivPredicate.DROP)) {
6161
ErrorReport.reportAnalysisException(ErrorCode.ERR_CATALOG_ACCESS_DENIED,
62-
analyzer.getQualifiedUser(), catalogName);
62+
ConnectContext.get().getQualifiedUser(), catalogName);
6363
}
6464
}
6565

‎fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ public void getTables(Analyzer analyzer, boolean expandView, Map<Long, TableIf>
381381
.checkTblPriv(ConnectContext.get(), tblRef.getName(), PrivPredicate.SELECT)) {
382382
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT",
383383
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
384-
dbName + ": " + tableName);
384+
dbName + "." + tableName);
385385
}
386386
tableMap.put(table.getId(), table);
387387
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/SlotDescriptor.java

Lines changed: 1 addition & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -67,8 +67,6 @@ public class SlotDescriptor {
6767
private ColumnStats stats; // only set if 'column' isn't set
6868
private boolean isAgg;
6969
private boolean isMultiRef;
70-
// used for load to get more information of varchar and decimal
71-
private Type originType;
7270
// If set to false, then such slots will be ignored during
7371
// materialize them.Used to optmize to read less data and less memory usage
7472
private boolean needMaterialize = true;
@@ -162,7 +160,6 @@ public Column getColumn() {
162160
public void setColumn(Column column) {
163161
this.column = column;
164162
this.type = column.getType();
165-
this.originType = column.getOriginType();
166163
}
167164

168165
public void setSrcColumn(Column column) {
@@ -254,10 +251,6 @@ public void setLabel(String label) {
254251
this.label = label;
255252
}
256253

257-
public void setSourceExprs(List<Expr> exprs) {
258-
sourceExprs = exprs;
259-
}
260-
261254
public void setSourceExpr(Expr expr) {
262255
sourceExprs = Collections.singletonList(expr);
263256
}
@@ -316,11 +309,9 @@ public boolean layoutEquals(SlotDescriptor other) {
316309
return true;
317310
}
318311

319-
// TODO
320312
public TSlotDescriptor toThrift() {
321-
322313
TSlotDescriptor tSlotDescriptor = new TSlotDescriptor(id.asInt(), parent.getId().asInt(),
323-
(originType != null ? originType.toThrift() : type.toThrift()), -1, byteOffset, nullIndicatorByte,
314+
type.toThrift(), -1, byteOffset, nullIndicatorByte,
324315
nullIndicatorBit, ((column != null) ? column.getName() : ""), slotIdx, isMaterialized);
325316
tSlotDescriptor.setNeedMaterialize(needMaterialize);
326317
if (column != null) {

‎fe/fe-core/src/main/java/org/apache/doris/analysis/TableName.java

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,11 @@ public boolean equals(Object other) {
162162
return false;
163163
}
164164

165+
@Override
166+
public int hashCode() {
167+
return Objects.hash(ctl, tbl, db);
168+
}
169+
165170
public String toSql() {
166171
StringBuilder stringBuilder = new StringBuilder();
167172
if (ctl != null && !ctl.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {

‎fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.io.DataInput;
4141
import java.io.DataOutput;
4242
import java.io.IOException;
43+
import java.util.Set;
4344

4445
// https://dev.mysql.com/doc/refman/8.0/en/account-names.html
4546
// user name must be literally matched.
@@ -52,12 +53,14 @@ public class UserIdentity implements Writable, GsonPostProcessable {
5253

5354
@SerializedName(value = "user")
5455
private String user;
55-
5656
@SerializedName(value = "host")
5757
private String host;
58-
5958
@SerializedName(value = "isDomain")
6059
private boolean isDomain;
60+
// The roles which this user belongs to.
61+
// Used for authorization in Access Controller
62+
// This field is only set when getting current user from auth and not need to persist
63+
private Set<String> roles;
6164

6265
private boolean isAnalyzed = false;
6366

@@ -125,6 +128,14 @@ public void setIsAnalyzed() {
125128
this.isAnalyzed = true;
126129
}
127130

131+
public void setRoles(Set<String> roles) {
132+
this.roles = roles;
133+
}
134+
135+
public Set<String> getRoles() {
136+
return roles;
137+
}
138+
128139
public void analyze(String clusterName) throws AnalysisException {
129140
if (isAnalyzed) {
130141
return;

‎fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,10 @@ void setQualifiedDbName(String qualifiedDbName) {
270270
this.qualifiedDbName = qualifiedDbName;
271271
}
272272

273+
public String getQualifiedDbName() {
274+
return qualifiedDbName;
275+
}
276+
273277
public String getQualifiedName() {
274278
if (StringUtils.isEmpty(qualifiedDbName)) {
275279
return name;

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ default int getBaseColumnIdxByName(String colName) {
135135
enum TableType {
136136
MYSQL, ODBC, OLAP, SCHEMA, INLINE_VIEW, VIEW, BROKER, ELASTICSEARCH, HIVE, ICEBERG, HUDI, JDBC,
137137
TABLE_VALUED_FUNCTION, HMS_EXTERNAL_TABLE, ES_EXTERNAL_TABLE, MATERIALIZED_VIEW, JDBC_EXTERNAL_TABLE,
138-
ICEBERG_EXTERNAL_TABLE;
138+
ICEBERG_EXTERNAL_TABLE, TEST_EXTERNAL_TABLE;
139139

140140
public String toEngineName() {
141141
switch (this) {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public JdbcExternalTable(long id, String name, String dbName, JdbcExternalCatalo
4747
super(id, name, catalog, dbName, TableType.JDBC_EXTERNAL_TABLE);
4848
}
4949

50+
@Override
5051
protected synchronized void makeSureInitialized() {
5152
if (!objectCreated) {
5253
jdbcTable = toJdbcTable();

‎fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog.external;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.datasource.ExternalCatalog;
22+
import org.apache.doris.datasource.InitDatabaseLog;
23+
import org.apache.doris.datasource.test.TestExternalCatalog;
24+
import org.apache.doris.persist.gson.GsonPostProcessable;
25+
26+
import com.google.common.collect.Lists;
27+
import com.google.common.collect.Maps;
28+
import com.google.common.collect.Sets;
29+
import com.google.gson.annotations.SerializedName;
30+
import org.apache.logging.log4j.LogManager;
31+
import org.apache.logging.log4j.Logger;
32+
33+
import java.io.IOException;
34+
import java.util.List;
35+
import java.util.Map;
36+
import java.util.Set;
37+
import java.util.concurrent.locks.ReentrantReadWriteLock;
38+
39+
public class TestExternalDatabase extends ExternalDatabase<TestExternalTable> implements GsonPostProcessable {
40+
private static final Logger LOG = LogManager.getLogger(TestExternalDatabase.class);
41+
42+
// Cache of table name to table id.
43+
private Map<String, Long> tableNameToId = Maps.newConcurrentMap();
44+
@SerializedName(value = "idToTbl")
45+
private Map<Long, TestExternalTable> idToTbl = Maps.newConcurrentMap();
46+
47+
public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
48+
super(extCatalog, id, name);
49+
}
50+
51+
@Override
52+
protected void init() {
53+
InitDatabaseLog initDatabaseLog = new InitDatabaseLog();
54+
initDatabaseLog.setType(InitDatabaseLog.Type.TEST);
55+
initDatabaseLog.setCatalogId(extCatalog.getId());
56+
initDatabaseLog.setDbId(id);
57+
List<String> tableNames = extCatalog.listTableNames(null, name);
58+
if (tableNames != null) {
59+
Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
60+
Map<Long, TestExternalTable> tmpIdToTbl = Maps.newHashMap();
61+
for (String tableName : tableNames) {
62+
long tblId;
63+
if (tableNameToId != null && tableNameToId.containsKey(tableName)) {
64+
tblId = tableNameToId.get(tableName);
65+
tmpTableNameToId.put(tableName, tblId);
66+
TestExternalTable table = idToTbl.get(tblId);
67+
tmpIdToTbl.put(tblId, table);
68+
initDatabaseLog.addRefreshTable(tblId);
69+
} else {
70+
tblId = Env.getCurrentEnv().getNextId();
71+
tmpTableNameToId.put(tableName, tblId);
72+
TestExternalTable table = new TestExternalTable(tblId, tableName, name,
73+
(TestExternalCatalog) extCatalog);
74+
tmpIdToTbl.put(tblId, table);
75+
initDatabaseLog.addCreateTable(tblId, tableName);
76+
}
77+
}
78+
tableNameToId = tmpTableNameToId;
79+
idToTbl = tmpIdToTbl;
80+
}
81+
initialized = true;
82+
Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog);
83+
}
84+
85+
public void setTableExtCatalog(ExternalCatalog extCatalog) {
86+
for (TestExternalTable table : idToTbl.values()) {
87+
table.setCatalog(extCatalog);
88+
}
89+
}
90+
91+
public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
92+
Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
93+
Map<Long, TestExternalTable> tmpIdToTbl = Maps.newConcurrentMap();
94+
for (int i = 0; i < log.getRefreshCount(); i++) {
95+
TestExternalTable table = getTableForReplay(log.getRefreshTableIds().get(i));
96+
tmpTableNameToId.put(table.getName(), table.getId());
97+
tmpIdToTbl.put(table.getId(), table);
98+
}
99+
for (int i = 0; i < log.getCreateCount(); i++) {
100+
TestExternalTable table = new TestExternalTable(log.getCreateTableIds().get(i),
101+
log.getCreateTableNames().get(i), name, (TestExternalCatalog) catalog);
102+
tmpTableNameToId.put(table.getName(), table.getId());
103+
tmpIdToTbl.put(table.getId(), table);
104+
}
105+
tableNameToId = tmpTableNameToId;
106+
idToTbl = tmpIdToTbl;
107+
initialized = true;
108+
}
109+
110+
// TODO(ftw): drew
111+
@Override
112+
public Set<String> getTableNamesWithLock() {
113+
makeSureInitialized();
114+
return Sets.newHashSet(tableNameToId.keySet());
115+
}
116+
117+
@Override
118+
public List<TestExternalTable> getTables() {
119+
makeSureInitialized();
120+
return Lists.newArrayList(idToTbl.values());
121+
}
122+
123+
@Override
124+
public TestExternalTable getTableNullable(String tableName) {
125+
makeSureInitialized();
126+
if (!tableNameToId.containsKey(tableName)) {
127+
return null;
128+
}
129+
return idToTbl.get(tableNameToId.get(tableName));
130+
}
131+
132+
@Override
133+
public TestExternalTable getTableNullable(long tableId) {
134+
makeSureInitialized();
135+
return idToTbl.get(tableId);
136+
}
137+
138+
public TestExternalTable getTableForReplay(long tableId) {
139+
return idToTbl.get(tableId);
140+
}
141+
142+
@Override
143+
public void gsonPostProcess() throws IOException {
144+
tableNameToId = Maps.newConcurrentMap();
145+
for (TestExternalTable tbl : idToTbl.values()) {
146+
tableNameToId.put(tbl.getName(), tbl.getId());
147+
}
148+
rwLock = new ReentrantReadWriteLock(true);
149+
}
150+
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog.external;
19+
20+
import org.apache.doris.catalog.Column;
21+
import org.apache.doris.datasource.test.TestExternalCatalog;
22+
import org.apache.doris.thrift.TTableDescriptor;
23+
import org.apache.doris.thrift.TTableType;
24+
25+
import org.apache.logging.log4j.LogManager;
26+
import org.apache.logging.log4j.Logger;
27+
28+
import java.util.List;
29+
30+
/**
31+
* TestExternalTable is a table for unit test.
32+
*/
33+
public class TestExternalTable extends ExternalTable {
34+
private static final Logger LOG = LogManager.getLogger(TestExternalTable.class);
35+
36+
public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) {
37+
super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE);
38+
}
39+
40+
@Override
41+
protected synchronized void makeSureInitialized() {
42+
43+
}
44+
45+
@Override
46+
public String getMysqlType() {
47+
return type.name();
48+
}
49+
50+
@Override
51+
public TTableDescriptor toThrift() {
52+
makeSureInitialized();
53+
TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.TEST_EXTERNAL_TABLE,
54+
getFullSchema().size(),
55+
0, getName(), "");
56+
return tTableDescriptor;
57+
}
58+
59+
@Override
60+
public List<Column> initSchema() {
61+
return ((TestExternalCatalog) catalog).mockedSchema(dbName, name);
62+
}
63+
}

0 commit comments

Comments
 (0)