From 4edd3074eb14f966c98e31bfabb075a58240761b Mon Sep 17 00:00:00 2001 From: "Mingyu Chen (Rayner)" Date: Sat, 8 Feb 2025 18:10:28 +0800 Subject: [PATCH] [refactor](plugin) unify the plugin dir (#45936) ### What problem does this PR solve? This pull request primarily updates the directory structure for plugins and JDBC drivers, ensuring compatibility with previous versions. Additionally, it includes changes to various configuration files and scripts to reflect these updates. #### Directory Structure Updates: * [`be/src/common/config.cpp`](diffhunk://#diff-b626e6ab16bc72abf40db76bf5094fcc8ca3c37534c2eb83b63b7805e1b601ffL54-R54): Changed the default directory for `jdbc_drivers_dir` and `trino_connector_plugin_dir` to `${DORIS_HOME}/plugins/*` for better organization. [[1]](diffhunk://#diff-b626e6ab16bc72abf40db76bf5094fcc8ca3c37534c2eb83b63b7805e1b601ffL54-R54) [[2]](diffhunk://#diff-b626e6ab16bc72abf40db76bf5094fcc8ca3c37534c2eb83b63b7805e1b601ffL1288-R1288) * [`build.sh`](diffhunk://#diff-4d2a8eefdf2a9783512a35da4dc7676a66404b6f3826a8af9aad038722da6823L755-R759): Updated the script to create new plugin directories under `${DORIS_OUTPUT}/fe/plugins/` and `${DORIS_OUTPUT}/be/plugins/`. [[1]](diffhunk://#diff-4d2a8eefdf2a9783512a35da4dc7676a66404b6f3826a8af9aad038722da6823L755-R759) [[2]](diffhunk://#diff-4d2a8eefdf2a9783512a35da4dc7676a66404b6f3826a8af9aad038722da6823L882-R890) #### Code Refactoring for Compatibility: * `be/src/runtime/user_function_cache.cpp` and `be/src/runtime/user_function_cache.h`: Introduced `_check_and_return_default_driver_url` method to maintain compatibility with the old default directory for JDBC drivers. [[1]](diffhunk://#diff-08d48161cad0bb4c6ec31c154c3017407caaff2402f0feb599f014f2d87ae9edL328-R354) [[2]](diffhunk://#diff-0cfb7bc4e6df1282c7dc7f9ec6677ffc9a03ed6cd08f4fea11ba43bcc297f0c3R79-R80) * [`fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java`](diffhunk://#diff-19838c8c525ec05716e434f18d1ef270dfa03029385258c53b8fbe7da5ba25b9L43-R43): Added `checkAndReturnPluginDir` method to ensure compatibility with the old plugin directory. [[1]](diffhunk://#diff-19838c8c525ec05716e434f18d1ef270dfa03029385258c53b8fbe7da5ba25b9L43-R43) [[2]](diffhunk://#diff-19838c8c525ec05716e434f18d1ef270dfa03029385258c53b8fbe7da5ba25b9L75-R84) [[3]](diffhunk://#diff-19838c8c525ec05716e434f18d1ef270dfa03029385258c53b8fbe7da5ba25b9R94-R116) #### Configuration Updates: * [`fe/fe-common/src/main/java/org/apache/doris/common/Config.java`](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L144-R144): Updated various configuration paths to use `EnvUtils.getDorisHome()` instead of `System.getenv("DORIS_HOME")` for consistency and maintainability. [[1]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L144-R144) [[2]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L153-R153) [[3]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L204-R207) [[4]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L358-R358) [[5]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L630-R630) [[6]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L639-R642) [[7]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L1281-R1281) [[8]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L2364-R2371) [[9]](diffhunk://#diff-ed7dee981e5322b45ed92fbc3c9e049191574baa990894ae36a97567ce198c74L2903-R2903) #### Minor Cleanup: * [`fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java`](diffhunk://#diff-1481395f49d6c346e93c01ca884ae2b7e65c85247d02cc91a20cd48b98bdf349L271-L273): Removed commented-out code for better readability. * [`fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java`](diffhunk://#diff-24260bdb6ba0b3ac0f5c719e5c48c3ea70d13405af93670adc8692de523730cfR38): Refactored `getFullDriverUrl` method to use `checkAndReturnDefaultDriverUrl` for compatibility with the old JDBC drivers directory. [[1]](diffhunk://#diff-24260bdb6ba0b3ac0f5c719e5c48c3ea70d13405af93670adc8692de523730cfR38) [[2]](diffhunk://#diff-24260bdb6ba0b3ac0f5c719e5c48c3ea70d13405af93670adc8692de523730cfL296-R297) [[3]](diffhunk://#diff-24260bdb6ba0b3ac0f5c719e5c48c3ea70d13405af93670adc8692de523730cfR316-R336) There are following kinds of plugins in Doris: 1. jdbc drivers 2. java udf 3. trino connectors 4. hadoop conf 5. other custom jars After this PR, all these plugins will be located in `${DORIS_HOME}/plugins/`. ``` ${DORIS_HOME}/plugins/jdbc_drivers/ ${DORIS_HOME}/plugins/java_udf/ ${DORIS_HOME}/plugins/connectors/ ${DORIS_HOME}/plugins/hadoop_conf/ ``` I tested: 1. Put jars in old dir 2. Put jars in new dir 3. Put jars in custom dir and set dir in fe/be conf file All works well. --- be/src/common/config.cpp | 4 +-- be/src/runtime/user_function_cache.cpp | 24 ++++++++++++++- be/src/runtime/user_function_cache.h | 2 ++ bin/start_be.sh | 8 +++++ bin/start_fe.sh | 8 +++++ build.sh | 12 ++++++-- .../TrinoConnectorPluginLoader.java | 30 +++++++++++++++++-- .../java/org/apache/doris/common/Config.java | 24 +++++++-------- .../doris/analysis/CreateFunctionStmt.java | 3 -- .../apache/doris/catalog/JdbcResource.java | 25 +++++++++++++++- .../TrinoConnectorPluginLoader.java | 29 ++++++++++++++++-- .../doris/catalog/JdbcResourceTest.java | 6 ++-- 12 files changed, 146 insertions(+), 29 deletions(-) diff --git a/be/src/common/config.cpp b/be/src/common/config.cpp index 216ae0d76b7022..76e9495dd45541 100644 --- a/be/src/common/config.cpp +++ b/be/src/common/config.cpp @@ -51,7 +51,7 @@ namespace doris::config { DEFINE_String(custom_config_dir, "${DORIS_HOME}/conf"); // Dir of jdbc drivers -DEFINE_String(jdbc_drivers_dir, "${DORIS_HOME}/jdbc_drivers"); +DEFINE_String(jdbc_drivers_dir, "${DORIS_HOME}/plugins/jdbc_drivers"); // cluster id DEFINE_Int32(cluster_id, "-1"); @@ -1300,7 +1300,7 @@ DEFINE_mInt64(s3_put_bucket_tokens, "1000000000000000000"); DEFINE_mInt64(s3_put_token_per_second, "1000000000000000000"); DEFINE_mInt64(s3_put_token_limit, "0"); -DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/connectors"); +DEFINE_String(trino_connector_plugin_dir, "${DORIS_HOME}/plugins/connectors"); // ca_cert_file is in this path by default, Normally no modification is required // ca cert default path is different from different OS diff --git a/be/src/runtime/user_function_cache.cpp b/be/src/runtime/user_function_cache.cpp index f04b508e4723f7..62087e6878fa9b 100644 --- a/be/src/runtime/user_function_cache.cpp +++ b/be/src/runtime/user_function_cache.cpp @@ -324,11 +324,33 @@ Status UserFunctionCache::_download_lib(const std::string& url, std::string UserFunctionCache::_get_real_url(const std::string& url) { if (url.find(":/") == std::string::npos) { - return "file://" + config::jdbc_drivers_dir + "/" + url; + return _check_and_return_default_driver_url(url); } return url; } +std::string UserFunctionCache::_check_and_return_default_driver_url(const std::string& url) { + const char* doris_home = std::getenv("DORIS_HOME"); + + std::string default_url = std::string(doris_home) + "/plugins/jdbc_drivers"; + std::string default_old_url = std::string(doris_home) + "/jdbc_drivers"; + + if (config::jdbc_drivers_dir == default_url) { + // If true, which means user does not set `jdbc_drivers_dir` and use the default one. + // Because in 2.1.8, we change the default value of `jdbc_drivers_dir` + // from `DORIS_HOME/jdbc_drivers` to `DORIS_HOME/plugins/jdbc_drivers`, + // so we need to check the old default dir for compatibility. + std::filesystem::path file = default_url + "/" + url; + if (std::filesystem::exists(file)) { + return "file://" + default_url + "/" + url; + } else { + return "file://" + default_old_url + "/" + url; + } + } else { + return "file://" + config::jdbc_drivers_dir + "/" + url; + } +} + std::string UserFunctionCache::_get_file_name_from_url(const std::string& url) const { std::string file_name; size_t last_slash_pos = url.find_last_of('/'); diff --git a/be/src/runtime/user_function_cache.h b/be/src/runtime/user_function_cache.h index 5d1bff8b8664b7..4ead76a3f758e1 100644 --- a/be/src/runtime/user_function_cache.h +++ b/be/src/runtime/user_function_cache.h @@ -76,6 +76,8 @@ class UserFunctionCache { std::string _get_file_name_from_url(const std::string& url) const; std::vector _split_string_by_checksum(const std::string& file); + std::string _check_and_return_default_driver_url(const std::string& url); + private: std::string _lib_dir; void* _current_process_handle = nullptr; diff --git a/bin/start_be.sh b/bin/start_be.sh index 35b453a407c688..de1fea82d385ad 100755 --- a/bin/start_be.sh +++ b/bin/start_be.sh @@ -244,12 +244,20 @@ if [[ -d "${DORIS_HOME}/lib/hadoop_hdfs/" ]]; then fi # add custom_libs to CLASSPATH +# ATTN, custom_libs is deprecated, use plugins/java_extensions if [[ -d "${DORIS_HOME}/custom_lib" ]]; then for f in "${DORIS_HOME}/custom_lib"/*.jar; do DORIS_CLASSPATH="${DORIS_CLASSPATH}:${f}" done fi +# add plugins/java_extensions to CLASSPATH +if [[ -d "${DORIS_HOME}/plugins/java_extensions" ]]; then + for f in "${DORIS_HOME}/plugins/java_extensions"/*.jar; do + CLASSPATH="${CLASSPATH}:${f}" + done +fi + # make sure the preload-extensions-project.jar is at first order, so that some classed # with same qualified name can be loaded priority from preload-extensions-project.jar. DORIS_CLASSPATH="${DORIS_PRELOAD_JAR}:${DORIS_CLASSPATH}" diff --git a/bin/start_fe.sh b/bin/start_fe.sh index b089596a9cdb73..5ae69867329c11 100755 --- a/bin/start_fe.sh +++ b/bin/start_fe.sh @@ -216,12 +216,20 @@ for f in "${DORIS_HOME}/lib"/*.jar; do done # add custom_libs to CLASSPATH +# ATTN, custom_libs is deprecated, use plugins/java_extensions if [[ -d "${DORIS_HOME}/custom_lib" ]]; then for f in "${DORIS_HOME}/custom_lib"/*.jar; do CLASSPATH="${CLASSPATH}:${f}" done fi +# add plugins/java_extensions to CLASSPATH +if [[ -d "${DORIS_HOME}/plugins/java_extensions" ]]; then + for f in "${DORIS_HOME}/plugins/java_extensions"/*.jar; do + CLASSPATH="${CLASSPATH}:${f}" + done +fi + # make sure the doris-fe.jar is at first order, so that some classed # with same qualified name can be loaded priority from doris-fe.jar CLASSPATH="${DORIS_FE_JAR}:${CLASSPATH}" diff --git a/build.sh b/build.sh index ae7ae84dc036e3..5f2b669d85df6e 100755 --- a/build.sh +++ b/build.sh @@ -757,7 +757,11 @@ if [[ "${BUILD_FE}" -eq 1 ]]; then mkdir -p "${DORIS_OUTPUT}/fe/log" mkdir -p "${DORIS_OUTPUT}/fe/doris-meta" mkdir -p "${DORIS_OUTPUT}/fe/conf/ssl" - mkdir -p "${DORIS_OUTPUT}/fe/connectors" + mkdir -p "${DORIS_OUTPUT}/fe/plugins/jdbc_drivers/" + mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_udf/" + mkdir -p "${DORIS_OUTPUT}/fe/plugins/connectors/" + mkdir -p "${DORIS_OUTPUT}/fe/plugins/hadoop_conf/" + mkdir -p "${DORIS_OUTPUT}/fe/plugins/java_extensions/" fi if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then @@ -884,7 +888,11 @@ EOF mkdir -p "${DORIS_OUTPUT}/be/log" mkdir -p "${DORIS_OUTPUT}/be/log/pipe_tracing" mkdir -p "${DORIS_OUTPUT}/be/storage" - mkdir -p "${DORIS_OUTPUT}/be/connectors" + mkdir -p "${DORIS_OUTPUT}/be/plugins/jdbc_drivers/" + mkdir -p "${DORIS_OUTPUT}/be/plugins/java_udf/" + mkdir -p "${DORIS_OUTPUT}/be/plugins/connectors/" + mkdir -p "${DORIS_OUTPUT}/be/plugins/hadoop_conf/" + mkdir -p "${DORIS_OUTPUT}/be/plugins/java_extensions/" fi if [[ "${BUILD_BROKER}" -eq 1 ]]; then diff --git a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java index 472260aa35a9f0..a53afff09c1db7 100644 --- a/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java +++ b/fe/be-java-extensions/trino-connector-scanner/src/main/java/org/apache/doris/trinoconnector/TrinoConnectorPluginLoader.java @@ -40,7 +40,7 @@ public class TrinoConnectorPluginLoader { private static final Logger LOG = LogManager.getLogger(TrinoConnectorPluginLoader.class); - private static String pluginsDir = EnvUtils.getDorisHome() + "/connectors"; + private static String pluginsDir = EnvUtils.getDorisHome() + "/plugins/connectors"; // Suppress default constructor for noninstantiability private TrinoConnectorPluginLoader() { @@ -72,7 +72,7 @@ private static class TrinoConnectorPluginLoad { TypeRegistry typeRegistry = new TypeRegistry(typeOperators, featuresConfig); ServerPluginsProviderConfig serverPluginsProviderConfig = new ServerPluginsProviderConfig() - .setInstalledPluginsDir(new File(pluginsDir)); + .setInstalledPluginsDir(new File(checkAndReturnPluginDir())); ServerPluginsProvider serverPluginsProvider = new ServerPluginsProvider(serverPluginsProviderConfig, MoreExecutors.directExecutor()); HandleResolver handleResolver = new HandleResolver(); @@ -80,7 +80,8 @@ private static class TrinoConnectorPluginLoad { typeRegistry, handleResolver); trinoConnectorPluginManager.loadPlugins(); } catch (Exception e) { - LOG.warn("Failed load trino-connector plugins from " + pluginsDir + ", Exception:" + e.getMessage()); + LOG.warn("Failed load trino-connector plugins from " + checkAndReturnPluginDir() + + ", Exception:" + e.getMessage(), e); } } } @@ -90,6 +91,29 @@ public static void setPluginsDir(String pluginsDir) { TrinoConnectorPluginLoader.pluginsDir = pluginsDir; } + private static String checkAndReturnPluginDir() { + final String defaultDir = System.getenv("DORIS_HOME") + "/plugins/connectors"; + final String defaultOldDir = System.getenv("DORIS_HOME") + "/connectors"; + if (TrinoConnectorPluginLoader.pluginsDir.equals(defaultDir)) { + // If true, which means user does not set `trino_connector_plugin_dir` and use the default one. + // Because in 2.1.8, we change the default value of `trino_connector_plugin_dir` + // from `DORIS_HOME/connectors` to `DORIS_HOME/plugins/connectors`, + // so we need to check the old default dir for compatibility. + File oldDir = new File(defaultOldDir); + if (oldDir.exists() && oldDir.isDirectory()) { + String[] contents = oldDir.list(); + if (contents != null && contents.length > 0) { + // there are contents in old dir, use old one + return defaultOldDir; + } + } + return defaultDir; + } else { + // Return user specified dir directly. + return TrinoConnectorPluginLoader.pluginsDir; + } + } + public static TrinoConnectorPluginManager getTrinoConnectorPluginManager() { return TrinoConnectorPluginLoad.trinoConnectorPluginManager; } diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index b4b385eb970f33..766fedd636ef4f 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -141,7 +141,7 @@ public class Config extends ConfigBase { public static boolean check_table_lock_leaky = false; @ConfField(description = {"插件的安装目录", "The installation directory of the plugin"}) - public static String plugin_dir = System.getenv("DORIS_HOME") + "/plugins"; + public static String plugin_dir = EnvUtils.getDorisHome() + "/plugins"; @ConfField(mutable = true, masterOnly = true, description = {"是否启用插件", "Whether to enable the plugin"}) public static boolean plugin_enable = true; @@ -150,7 +150,7 @@ public class Config extends ConfigBase { "JDBC 驱动的存放路径。在创建 JDBC Catalog 时，如果指定的驱动文件路径不是绝对路径，则会在这个目录下寻找", "The path to save jdbc drivers. When creating JDBC Catalog," + "if the specified driver file path is not an absolute path, Doris will find jars from this path"}) - public static String jdbc_drivers_dir = System.getenv("DORIS_HOME") + "/jdbc_drivers"; + public static String jdbc_drivers_dir = EnvUtils.getDorisHome() + "/plugins/jdbc_drivers"; @ConfField(description = {"JDBC 驱动的安全路径。在创建 JDBC Catalog 时，允许使用的文件或者网络路径，可配置多个，使用分号分隔" + "默认为 * 表示全部允许，如果设置为空也表示全部允许", @@ -201,10 +201,10 @@ public class Config extends ConfigBase { public static int label_clean_interval_second = 1 * 3600; // 1 hours @ConfField(description = {"元数据的存储目录", "The directory to save Doris meta data"}) - public static String meta_dir = System.getenv("DORIS_HOME") + "/doris-meta"; + public static String meta_dir = EnvUtils.getDorisHome() + "/doris-meta"; @ConfField(description = {"临时文件的存储目录", "The directory to save Doris temp data"}) - public static String tmp_dir = System.getenv("DORIS_HOME") + "/temp_dir"; + public static String tmp_dir = EnvUtils.getDorisHome() + "/temp_dir"; @ConfField(description = {"元数据日志的存储类型。BDB: 日志存储在 BDBJE 中。LOCAL：日志存储在本地文件中（仅用于测试）", "The storage type of the metadata log. BDB: Logs are stored in BDBJE. " @@ -359,7 +359,7 @@ public class Config extends ConfigBase { @ConfField(description = {"FE https 服务的 key store 路径", "The key store path of FE https service"}) - public static String key_store_path = System.getenv("DORIS_HOME") + public static String key_store_path = EnvUtils.getDorisHome() + "/conf/ssl/doris_ssl_certificate.keystore"; @ConfField(description = {"FE https 服务的 key store 密码", @@ -631,7 +631,7 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = true, description = {"Spark Load 所使用的 Spark 程序目录", "Spark dir for Spark Load"}) - public static String spark_home_default_dir = System.getenv("DORIS_HOME") + "/lib/spark2x"; + public static String spark_home_default_dir = EnvUtils.getDorisHome() + "/lib/spark2x"; @ConfField(description = {"Spark load 所使用的依赖项目录", "Spark dependencies dir for Spark Load"}) public static String spark_resource_path = ""; @@ -640,10 +640,10 @@ public class Config extends ConfigBase { public static String spark_launcher_log_dir = System.getenv("LOG_DIR") + "/spark_launcher_log"; @ConfField(description = {"Yarn client 的路径", "Yarn client path"}) - public static String yarn_client_path = System.getenv("DORIS_HOME") + "/lib/yarn-client/hadoop/bin/yarn"; + public static String yarn_client_path = EnvUtils.getDorisHome() + "/lib/yarn-client/hadoop/bin/yarn"; @ConfField(description = {"Yarn 配置文件的路径", "Yarn config path"}) - public static String yarn_config_dir = System.getenv("DORIS_HOME") + "/lib/yarn-config"; + public static String yarn_config_dir = EnvUtils.getDorisHome() + "/lib/yarn-config"; @ConfField(mutable = true, masterOnly = true, description = {"Ingestion load 的默认超时时间，单位是秒。", "Default timeout for ingestion load job, in seconds."}) @@ -1293,7 +1293,7 @@ public class Config extends ConfigBase { * Save small files */ @ConfField - public static String small_file_dir = System.getenv("DORIS_HOME") + "/small_files"; + public static String small_file_dir = EnvUtils.getDorisHome() + "/small_files"; /** * This will limit the max recursion depth of hash distribution pruner. @@ -2376,14 +2376,14 @@ public class Config extends ConfigBase { * Default CA certificate file location for mysql ssl connection. */ @ConfField(mutable = false, masterOnly = false) - public static String mysql_ssl_default_ca_certificate = System.getenv("DORIS_HOME") + public static String mysql_ssl_default_ca_certificate = EnvUtils.getDorisHome() + "/mysql_ssl_default_certificate/ca_certificate.p12"; /** * Default server certificate file location for mysql ssl connection. */ @ConfField(mutable = false, masterOnly = false) - public static String mysql_ssl_default_server_certificate = System.getenv("DORIS_HOME") + public static String mysql_ssl_default_server_certificate = EnvUtils.getDorisHome() + "/mysql_ssl_default_certificate/server_certificate.p12"; /** @@ -2915,7 +2915,7 @@ public class Config extends ConfigBase { @ConfField(mutable = true, masterOnly = false, description = {"指定 trino-connector catalog 的插件默认加载路径", "Specify the default plugins loading path for the trino-connector catalog"}) - public static String trino_connector_plugin_dir = EnvUtils.getDorisHome() + "/connectors"; + public static String trino_connector_plugin_dir = EnvUtils.getDorisHome() + "/plugins/connectors"; @ConfField(mutable = true) public static boolean fix_tablet_partition_id_eq_0 = false; diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java index 4a38d06ffe28ff..06cab70d7d9107 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/CreateFunctionStmt.java @@ -268,9 +268,6 @@ private void analyzeCommon(Analyzer analyzer) throws AnalysisException { } userFile = properties.getOrDefault(FILE_KEY, properties.get(OBJECT_FILE_KEY)); - // if (Strings.isNullOrEmpty(userFile)) { - // throw new AnalysisException("No 'file' or 'object_file' in properties"); - // } if (!Strings.isNullOrEmpty(userFile) && binaryType != TFunctionBinaryType.RPC) { try { computeObjectChecksum(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java index 28d58b35297ac3..5fc99b026eb3be 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java @@ -21,6 +21,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.EnvUtils; import org.apache.doris.common.FeConstants; import org.apache.doris.common.proc.BaseProcResult; import org.apache.doris.common.util.Util; @@ -35,6 +36,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.io.File; import java.io.IOException; import java.io.InputStream; import java.net.URI; @@ -293,7 +295,7 @@ public static String getFullDriverUrl(String driverUrl) throws IllegalArgumentEx String schema = uri.getScheme(); checkCloudWhiteList(driverUrl); if (schema == null && !driverUrl.startsWith("/")) { - return "file://" + Config.jdbc_drivers_dir + "/" + driverUrl; + return checkAndReturnDefaultDriverUrl(driverUrl); } if ("*".equals(Config.jdbc_driver_secure_path)) { @@ -312,6 +314,27 @@ public static String getFullDriverUrl(String driverUrl) throws IllegalArgumentEx } } + private static String checkAndReturnDefaultDriverUrl(String driverUrl) { + final String defaultDriverUrl = EnvUtils.getDorisHome() + "/plugins/jdbc_drivers"; + final String defaultOldDriverUrl = EnvUtils.getDorisHome() + "/jdbc_drivers"; + if (Config.jdbc_drivers_dir.equals(defaultDriverUrl)) { + // If true, which means user does not set `jdbc_drivers_dir` and use the default one. + // Because in new version, we change the default value of `jdbc_drivers_dir` + // from `DORIS_HOME/jdbc_drivers` to `DORIS_HOME/plugins/jdbc_drivers`, + // so we need to check the old default dir for compatibility. + File file = new File(defaultDriverUrl + "/" + driverUrl); + if (file.exists()) { + return "file://" + defaultDriverUrl + "/" + driverUrl; + } else { + // use old one + return "file://" + defaultOldDriverUrl + "/" + driverUrl; + } + } else { + // Return user specified driver url directly. + return "file://" + Config.jdbc_drivers_dir + "/" + driverUrl; + } + } + public static String parseDbType(String url) throws DdlException { if (url.startsWith(JDBC_MYSQL) || url.startsWith(JDBC_MARIADB)) { return MYSQL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java index c846e2edf4297c..08761446b2113d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/TrinoConnectorPluginLoader.java @@ -68,15 +68,38 @@ private static class TrinoConnectorPluginLoad { typeRegistry = new TypeRegistry(typeOperators, featuresConfig); ServerPluginsProviderConfig serverPluginsProviderConfig = new ServerPluginsProviderConfig() - .setInstalledPluginsDir(new File(Config.trino_connector_plugin_dir)); + .setInstalledPluginsDir(new File(checkAndReturnPluginDir())); ServerPluginsProvider serverPluginsProvider = new ServerPluginsProvider(serverPluginsProviderConfig, MoreExecutors.directExecutor()); trinoConnectorPluginManager = new TrinoConnectorPluginManager(serverPluginsProvider, typeRegistry, handleResolver); trinoConnectorPluginManager.loadPlugins(); } catch (Exception e) { - LOG.warn("Failed load trino-connector plugins from " + Config.trino_connector_plugin_dir - + ", Exception:" + e.getMessage()); + LOG.warn("Failed load trino-connector plugins from " + checkAndReturnPluginDir() + + ", Exception:" + e.getMessage(), e); + } + } + + private static String checkAndReturnPluginDir() { + final String defaultDir = System.getenv("DORIS_HOME") + "/plugins/connectors"; + final String defaultOldDir = System.getenv("DORIS_HOME") + "/connectors"; + if (Config.trino_connector_plugin_dir.equals(defaultDir)) { + // If true, which means user does not set `trino_connector_plugin_dir` and use the default one. + // Because in 2.1.8, we change the default value of `trino_connector_plugin_dir` + // from `DORIS_HOME/connectors` to `DORIS_HOME/plugins/connectors`, + // so we need to check the old default dir for compatibility. + File oldDir = new File(defaultOldDir); + if (oldDir.exists() && oldDir.isDirectory()) { + String[] contents = oldDir.list(); + if (contents != null && contents.length > 0) { + // there are contents in old dir, use old one + return defaultOldDir; + } + } + return defaultDir; + } else { + // Return user specified dir directly. + return Config.trino_connector_plugin_dir; } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java index 81c2157686ab19..3b2293e6b8cdb6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/JdbcResourceTest.java @@ -22,6 +22,7 @@ import org.apache.doris.analysis.CreateResourceStmt; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.common.EnvUtils; import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.mysql.privilege.AccessControllerManager; @@ -204,11 +205,12 @@ public void testHandleJdbcUrlForSqlServerWithParams() throws DdlException { } @Test - public void testJdbcDriverPtah() { + public void testJdbcDriverPath() { String driverPath = "postgresql-42.5.0.jar"; Config.jdbc_driver_secure_path = ""; + Config.jdbc_drivers_dir = EnvUtils.getDorisHome() + "/plugins/jdbc_drivers"; String fullPath = JdbcResource.getFullDriverUrl(driverPath); - Assert.assertEquals(fullPath, "file://" + Config.jdbc_drivers_dir + "/" + driverPath); + Assert.assertEquals("file://" + EnvUtils.getDorisHome() + "/jdbc_drivers/" + driverPath, fullPath); Config.jdbc_driver_secure_path = "file:///jdbc/;http://jdbc"; String driverPath2 = "file:///postgresql-42.5.0.jar"; Exception exception = Assert.assertThrows(IllegalArgumentException.class, () -> {