diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java index 379ff148a2fa1..88cd56c050955 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeConnector.java @@ -15,8 +15,6 @@ package com.starrocks.connector.delta; -import com.google.common.base.Preconditions; -import com.starrocks.common.util.Util; import com.starrocks.connector.Connector; import com.starrocks.connector.ConnectorContext; import com.starrocks.connector.ConnectorMetadata; @@ -47,14 +45,6 @@ public DeltaLakeConnector(ConnectorContext context) { this.internalMgr = new DeltaLakeInternalMgr(catalogName, properties, hdfsEnvironment); this.metadataFactory = createMetadataFactory(); // TODO extract to ConnectorConfigFactory - validate(); - onCreate(); - } - - public void validate() { - String hiveMetastoreUris = Preconditions.checkNotNull(properties.get(HIVE_METASTORE_URIS), - "%s must be set in properties when creating hive catalog", HIVE_METASTORE_URIS); - Util.validateMetastoreUris(hiveMetastoreUris); } @Override @@ -68,15 +58,12 @@ private DeltaLakeMetadataFactory createMetadataFactory() { catalogName, metastore, internalMgr.getHiveMetastoreConf(), - properties.get(HIVE_METASTORE_URIS), + properties, internalMgr.getHdfsEnvironment(), internalMgr.getMetastoreType() ); } - public void onCreate() { - } - public CloudConfiguration getCloudConfiguration() { return this.cloudConfiguration; } diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeInternalMgr.java b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeInternalMgr.java index f24363343aec9..228727cf6d350 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeInternalMgr.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeInternalMgr.java @@ -15,6 +15,7 @@ package com.starrocks.connector.delta; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.util.concurrent.ThreadFactoryBuilder; import com.starrocks.common.util.Util; import com.starrocks.connector.HdfsEnvironment; @@ -25,21 +26,25 @@ import com.starrocks.connector.hive.HiveMetaClient; import com.starrocks.connector.hive.HiveMetastore; import com.starrocks.connector.hive.IHiveMetastore; +import com.starrocks.sql.analyzer.SemanticException; +import java.util.List; import java.util.Map; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; -import static com.starrocks.connector.delta.DeltaLakeConnector.HIVE_METASTORE_URIS; +import static com.starrocks.connector.hive.HiveConnector.HIVE_METASTORE_TYPE; +import static com.starrocks.connector.hive.HiveConnector.HIVE_METASTORE_URIS; public class DeltaLakeInternalMgr { + public static final List SUPPORTED_METASTORE_TYPE = ImmutableList.of("hive", "glue"); private final String catalogName; private final Map properties; private final HdfsEnvironment hdfsEnvironment; private final boolean enableMetastoreCache; private final CachingHiveMetastoreConf hmsConf; private ExecutorService refreshHiveMetastoreExecutor; - private MetastoreType metastoreType = MetastoreType.HMS; + private final MetastoreType metastoreType; public DeltaLakeInternalMgr(String catalogName, Map properties, HdfsEnvironment hdfsEnvironment) { this.catalogName = catalogName; @@ -47,9 +52,18 @@ public DeltaLakeInternalMgr(String catalogName, Map properties, this.enableMetastoreCache = Boolean.parseBoolean(properties.getOrDefault("enable_metastore_cache", "false")); this.hmsConf = new CachingHiveMetastoreConf(properties, "delta lake"); this.hdfsEnvironment = hdfsEnvironment; - String hiveMetastoreUris = Preconditions.checkNotNull(properties.get(HIVE_METASTORE_URIS), - "%s must be set in properties when creating hive catalog", HIVE_METASTORE_URIS); - Util.validateMetastoreUris(hiveMetastoreUris); + + String hiveMetastoreType = properties.getOrDefault(HIVE_METASTORE_TYPE, "hive").toLowerCase(); + if (!SUPPORTED_METASTORE_TYPE.contains(hiveMetastoreType)) { + throw new SemanticException("hive metastore type [%s] is not supported", hiveMetastoreType); + } + + if (hiveMetastoreType.equals("hive")) { + String hiveMetastoreUris = Preconditions.checkNotNull(properties.get(HIVE_METASTORE_URIS), + "%s must be set in properties when creating catalog of hive-metastore", HIVE_METASTORE_URIS); + Util.validateMetastoreUris(hiveMetastoreUris); + } + this.metastoreType = MetastoreType.get(hiveMetastoreType); } public IHiveMetastore createHiveMetastore() { diff --git a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeMetadataFactory.java b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeMetadataFactory.java index 88629f8dc7ac0..93b70e02b7d50 100644 --- a/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeMetadataFactory.java +++ b/fe/fe-core/src/main/java/com/starrocks/connector/delta/DeltaLakeMetadataFactory.java @@ -22,6 +22,9 @@ import com.starrocks.connector.hive.IHiveMetastore; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; +import java.util.Map; + +import static com.starrocks.connector.delta.DeltaLakeConnector.HIVE_METASTORE_URIS; import static com.starrocks.connector.hive.CachingHiveMetastore.createQueryLevelInstance; public class DeltaLakeMetadataFactory { @@ -32,12 +35,16 @@ public class DeltaLakeMetadataFactory { private final MetastoreType metastoreType; public DeltaLakeMetadataFactory(String catalogName, IHiveMetastore metastore, CachingHiveMetastoreConf hmsConf, - String uri, HdfsEnvironment hdfsEnvironment, MetastoreType metastoreType) { + Map properties, HdfsEnvironment hdfsEnvironment, + MetastoreType metastoreType) { this.catalogName = catalogName; this.metastore = metastore; this.perQueryMetastoreMaxNum = hmsConf.getPerQueryCacheMaxNum(); this.hdfsEnvironment = hdfsEnvironment; - this.hdfsEnvironment.getConfiguration().set(MetastoreConf.ConfVars.THRIFT_URIS.getHiveName(), uri); + if (properties.containsKey(HIVE_METASTORE_URIS)) { + this.hdfsEnvironment.getConfiguration().set(MetastoreConf.ConfVars.THRIFT_URIS.getHiveName(), + properties.get(HIVE_METASTORE_URIS)); + } this.metastoreType = metastoreType; }