Skip to content

Commit 1aba215

Browse files
morningmanmorningman
committed
3
1 parent 3643bf5 commit 1aba215

26 files changed

+975
-29
lines changed

‎fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -381,7 +381,7 @@ public void getTables(Analyzer analyzer, boolean expandView, Map<Long, TableIf>
381381
.checkTblPriv(ConnectContext.get(), tblRef.getName(), PrivPredicate.SELECT)) {
382382
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "SELECT",
383383
ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
384-
dbName + ": " + tableName);
384+
dbName + "." + tableName);
385385
}
386386
tableMap.put(table.getId(), table);
387387
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/UserIdentity.java

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
import java.io.DataInput;
4141
import java.io.DataOutput;
4242
import java.io.IOException;
43+
import java.util.Set;
4344

4445
// https://dev.mysql.com/doc/refman/8.0/en/account-names.html
4546
// user name must be literally matched.
@@ -52,12 +53,14 @@ public class UserIdentity implements Writable, GsonPostProcessable {
5253

5354
@SerializedName(value = "user")
5455
private String user;
55-
5656
@SerializedName(value = "host")
5757
private String host;
58-
5958
@SerializedName(value = "isDomain")
6059
private boolean isDomain;
60+
// The roles which this user belongs to.
61+
// Used for authorization in Access Controller
62+
// This field is only set when getting current user from auth and not need to persist
63+
private Set<String> roles;
6164

6265
private boolean isAnalyzed = false;
6366

@@ -125,6 +128,14 @@ public void setIsAnalyzed() {
125128
this.isAnalyzed = true;
126129
}
127130

131+
public void setRoles(Set<String> roles) {
132+
this.roles = roles;
133+
}
134+
135+
public Set<String> getRoles() {
136+
return roles;
137+
}
138+
128139
public void analyze(String clusterName) throws AnalysisException {
129140
if (isAnalyzed) {
130141
return;

‎fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,7 @@ default int getBaseColumnIdxByName(String colName) {
135135
enum TableType {
136136
MYSQL, ODBC, OLAP, SCHEMA, INLINE_VIEW, VIEW, BROKER, ELASTICSEARCH, HIVE, ICEBERG, HUDI, JDBC,
137137
TABLE_VALUED_FUNCTION, HMS_EXTERNAL_TABLE, ES_EXTERNAL_TABLE, MATERIALIZED_VIEW, JDBC_EXTERNAL_TABLE,
138-
ICEBERG_EXTERNAL_TABLE;
138+
ICEBERG_EXTERNAL_TABLE, TEST_EXTERNAL_TABLE;
139139

140140
public String toEngineName() {
141141
switch (this) {

‎fe/fe-core/src/main/java/org/apache/doris/catalog/external/JdbcExternalTable.java

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@ public JdbcExternalTable(long id, String name, String dbName, JdbcExternalCatalo
4747
super(id, name, catalog, dbName, TableType.JDBC_EXTERNAL_TABLE);
4848
}
4949

50+
@Override
5051
protected synchronized void makeSureInitialized() {
5152
if (!objectCreated) {
5253
jdbcTable = toJdbcTable();

‎fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalDatabase.java

Lines changed: 150 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,150 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog.external;
19+
20+
import org.apache.doris.catalog.Env;
21+
import org.apache.doris.datasource.ExternalCatalog;
22+
import org.apache.doris.datasource.InitDatabaseLog;
23+
import org.apache.doris.datasource.test.TestExternalCatalog;
24+
import org.apache.doris.persist.gson.GsonPostProcessable;
25+
26+
import com.google.common.collect.Lists;
27+
import com.google.common.collect.Maps;
28+
import com.google.common.collect.Sets;
29+
import com.google.gson.annotations.SerializedName;
30+
import org.apache.logging.log4j.LogManager;
31+
import org.apache.logging.log4j.Logger;
32+
33+
import java.io.IOException;
34+
import java.util.List;
35+
import java.util.Map;
36+
import java.util.Set;
37+
import java.util.concurrent.locks.ReentrantReadWriteLock;
38+
39+
public class TestExternalDatabase extends ExternalDatabase<TestExternalTable> implements GsonPostProcessable {
40+
private static final Logger LOG = LogManager.getLogger(TestExternalDatabase.class);
41+
42+
// Cache of table name to table id.
43+
private Map<String, Long> tableNameToId = Maps.newConcurrentMap();
44+
@SerializedName(value = "idToTbl")
45+
private Map<Long, TestExternalTable> idToTbl = Maps.newConcurrentMap();
46+
47+
public TestExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
48+
super(extCatalog, id, name);
49+
}
50+
51+
@Override
52+
protected void init() {
53+
InitDatabaseLog initDatabaseLog = new InitDatabaseLog();
54+
initDatabaseLog.setType(InitDatabaseLog.Type.TEST);
55+
initDatabaseLog.setCatalogId(extCatalog.getId());
56+
initDatabaseLog.setDbId(id);
57+
List<String> tableNames = extCatalog.listTableNames(null, name);
58+
if (tableNames != null) {
59+
Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
60+
Map<Long, TestExternalTable> tmpIdToTbl = Maps.newHashMap();
61+
for (String tableName : tableNames) {
62+
long tblId;
63+
if (tableNameToId != null && tableNameToId.containsKey(tableName)) {
64+
tblId = tableNameToId.get(tableName);
65+
tmpTableNameToId.put(tableName, tblId);
66+
TestExternalTable table = idToTbl.get(tblId);
67+
tmpIdToTbl.put(tblId, table);
68+
initDatabaseLog.addRefreshTable(tblId);
69+
} else {
70+
tblId = Env.getCurrentEnv().getNextId();
71+
tmpTableNameToId.put(tableName, tblId);
72+
TestExternalTable table = new TestExternalTable(tblId, tableName, name,
73+
(TestExternalCatalog) extCatalog);
74+
tmpIdToTbl.put(tblId, table);
75+
initDatabaseLog.addCreateTable(tblId, tableName);
76+
}
77+
}
78+
tableNameToId = tmpTableNameToId;
79+
idToTbl = tmpIdToTbl;
80+
}
81+
initialized = true;
82+
Env.getCurrentEnv().getEditLog().logInitExternalDb(initDatabaseLog);
83+
}
84+
85+
public void setTableExtCatalog(ExternalCatalog extCatalog) {
86+
for (TestExternalTable table : idToTbl.values()) {
87+
table.setCatalog(extCatalog);
88+
}
89+
}
90+
91+
public void replayInitDb(InitDatabaseLog log, ExternalCatalog catalog) {
92+
Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
93+
Map<Long, TestExternalTable> tmpIdToTbl = Maps.newConcurrentMap();
94+
for (int i = 0; i < log.getRefreshCount(); i++) {
95+
TestExternalTable table = getTableForReplay(log.getRefreshTableIds().get(i));
96+
tmpTableNameToId.put(table.getName(), table.getId());
97+
tmpIdToTbl.put(table.getId(), table);
98+
}
99+
for (int i = 0; i < log.getCreateCount(); i++) {
100+
TestExternalTable table = new TestExternalTable(log.getCreateTableIds().get(i),
101+
log.getCreateTableNames().get(i), name, (TestExternalCatalog) catalog);
102+
tmpTableNameToId.put(table.getName(), table.getId());
103+
tmpIdToTbl.put(table.getId(), table);
104+
}
105+
tableNameToId = tmpTableNameToId;
106+
idToTbl = tmpIdToTbl;
107+
initialized = true;
108+
}
109+
110+
// TODO(ftw): drew
111+
@Override
112+
public Set<String> getTableNamesWithLock() {
113+
makeSureInitialized();
114+
return Sets.newHashSet(tableNameToId.keySet());
115+
}
116+
117+
@Override
118+
public List<TestExternalTable> getTables() {
119+
makeSureInitialized();
120+
return Lists.newArrayList(idToTbl.values());
121+
}
122+
123+
@Override
124+
public TestExternalTable getTableNullable(String tableName) {
125+
makeSureInitialized();
126+
if (!tableNameToId.containsKey(tableName)) {
127+
return null;
128+
}
129+
return idToTbl.get(tableNameToId.get(tableName));
130+
}
131+
132+
@Override
133+
public TestExternalTable getTableNullable(long tableId) {
134+
makeSureInitialized();
135+
return idToTbl.get(tableId);
136+
}
137+
138+
public TestExternalTable getTableForReplay(long tableId) {
139+
return idToTbl.get(tableId);
140+
}
141+
142+
@Override
143+
public void gsonPostProcess() throws IOException {
144+
tableNameToId = Maps.newConcurrentMap();
145+
for (TestExternalTable tbl : idToTbl.values()) {
146+
tableNameToId.put(tbl.getName(), tbl.getId());
147+
}
148+
rwLock = new ReentrantReadWriteLock(true);
149+
}
150+
}

‎fe/fe-core/src/main/java/org/apache/doris/catalog/external/TestExternalTable.java

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,63 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.catalog.external;
19+
20+
import org.apache.doris.catalog.Column;
21+
import org.apache.doris.datasource.test.TestExternalCatalog;
22+
import org.apache.doris.thrift.TTableDescriptor;
23+
import org.apache.doris.thrift.TTableType;
24+
25+
import org.apache.logging.log4j.LogManager;
26+
import org.apache.logging.log4j.Logger;
27+
28+
import java.util.List;
29+
30+
/**
31+
* Elasticsearch external table.
32+
*/
33+
public class TestExternalTable extends ExternalTable {
34+
private static final Logger LOG = LogManager.getLogger(TestExternalTable.class);
35+
36+
public TestExternalTable(long id, String name, String dbName, TestExternalCatalog catalog) {
37+
super(id, name, catalog, dbName, TableType.TEST_EXTERNAL_TABLE);
38+
}
39+
40+
@Override
41+
protected synchronized void makeSureInitialized() {
42+
43+
}
44+
45+
@Override
46+
public String getMysqlType() {
47+
return type.name();
48+
}
49+
50+
@Override
51+
public TTableDescriptor toThrift() {
52+
makeSureInitialized();
53+
TTableDescriptor tTableDescriptor = new TTableDescriptor(getId(), TTableType.TEST_EXTERNAL_TABLE,
54+
getFullSchema().size(),
55+
0, getName(), "");
56+
return tTableDescriptor;
57+
}
58+
59+
@Override
60+
public List<Column> initSchema() {
61+
return ((TestExternalCatalog) catalog).mockedSchema(dbName, name);
62+
}
63+
}

‎fe/fe-core/src/main/java/org/apache/doris/common/util/PropertyAnalyzer.java

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@
3131
import org.apache.doris.catalog.Type;
3232
import org.apache.doris.common.AnalysisException;
3333
import org.apache.doris.common.Config;
34+
import org.apache.doris.datasource.CatalogMgr;
3435
import org.apache.doris.policy.Policy;
3536
import org.apache.doris.policy.StoragePolicy;
3637
import org.apache.doris.resource.Tag;
@@ -41,6 +42,7 @@
4142
import org.apache.doris.thrift.TStorageType;
4243
import org.apache.doris.thrift.TTabletType;
4344

45+
import com.google.common.base.Joiner;
4446
import com.google.common.base.Preconditions;
4547
import com.google.common.base.Strings;
4648
import com.google.common.collect.Maps;
@@ -809,5 +811,29 @@ public static void checkCatalogProperties(Map<String, String> properties, boolea
809811
throw new AnalysisException(e.getMessage());
810812
}
811813
}
814+
// validate access controller properties
815+
// eg:
816+
// (
817+
// "access_controller.name" = "my_access",
818+
// "access_controller.my_access.class" = "org.apache.doris.mysql.privilege.RangerAccessControllerFactory",
819+
// "access_controller.my_access.properties.prop1" = "xxx",
820+
// "access_controller.my_access.properties.prop2" = "yyy",
821+
// )
822+
// 1. get access controller name
823+
String acName = properties.getOrDefault(CatalogMgr.ACCESS_CONTROLLER_NAME_PROP, "");
824+
if (!Strings.isNullOrEmpty(acName)) {
825+
// 2. get access controller class name
826+
String className = properties.getOrDefault(
827+
Joiner.on(".").join(CatalogMgr.ACCESS_CONTROLLER_PREFIX_PROP, acName, "class"), "");
828+
if (Strings.isNullOrEmpty(className)) {
829+
throw new AnalysisException("missing factory class name for access controller: " + acName);
830+
}
831+
// 3. check if class exists
832+
try {
833+
Class.forName(className);
834+
} catch (ClassNotFoundException e) {
835+
throw new AnalysisException("failed to find class " + className, e);
836+
}
837+
}
812838
}
813839
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogFactory.java

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,9 @@
2626
import org.apache.doris.catalog.Env;
2727
import org.apache.doris.catalog.Resource;
2828
import org.apache.doris.common.DdlException;
29+
import org.apache.doris.common.FeConstants;
2930
import org.apache.doris.datasource.iceberg.IcebergExternalCatalogFactory;
31+
import org.apache.doris.datasource.test.TestExternalCatalog;
3032

3133
import org.apache.parquet.Strings;
3234

@@ -37,7 +39,6 @@
3739
* A factory to create catalog instance of log or covert catalog into log.
3840
*/
3941
public class CatalogFactory {
40-
4142
/**
4243
* Convert the sql statement into catalog log.
4344
*/
@@ -80,11 +81,11 @@ private static CatalogIf constructorCatalog(
8081
Resource catalogResource = Optional.ofNullable(Env.getCurrentEnv().getResourceMgr().getResource(resource))
8182
.orElseThrow(() -> new DdlException("Resource doesn't exist: " + resource));
8283
catalogType = catalogResource.getType().name().toLowerCase();
83-
if (props.containsKey("type")) {
84+
if (props.containsKey(CatalogMgr.CATALOG_TYPE_PROP)) {
8485
throw new DdlException("Can not set 'type' when creating catalog with resource");
8586
}
8687
} else {
87-
String type = props.get("type");
88+
String type = props.get(CatalogMgr.CATALOG_TYPE_PROP);
8889
if (Strings.isNullOrEmpty(type)) {
8990
throw new DdlException("Missing property 'type' in properties");
9091
}
@@ -106,6 +107,12 @@ private static CatalogIf constructorCatalog(
106107
case "iceberg":
107108
catalog = IcebergExternalCatalogFactory.createCatalog(catalogId, name, resource, props);
108109
break;
110+
case "test":
111+
if (!FeConstants.runningUnitTest) {
112+
throw new DdlException("test catalog is only for FE unit test");
113+
}
114+
catalog = new TestExternalCatalog(catalogId, name, resource, props);
115+
break;
109116
default:
110117
throw new DdlException("Unknown catalog type: " + catalogType);
111118
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@
7676
public class CatalogMgr implements Writable, GsonPostProcessable {
7777
private static final Logger LOG = LogManager.getLogger(CatalogMgr.class);
7878

79+
public static final String ACCESS_CONTROLLER_PREFIX_PROP = "access_controller";
80+
public static final String ACCESS_CONTROLLER_NAME_PROP = "access_controller.name";
81+
public static final String CATALOG_TYPE_PROP = "type";
82+
7983
private static final String YES = "yes";
8084

8185
private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock(true);

0 commit comments

Comments
 (0)