From 25ce674242ebc776e1bd84397968746cf2293ae9 Mon Sep 17 00:00:00 2001 From: lexluo Date: Sun, 11 Jun 2023 23:30:56 +0800 Subject: [PATCH] [Enhancement](java-udf) java-udf module split to sub modules --- be/src/util/jni-util.cpp | 8 +- be/src/util/jni_native_method.h | 2 +- be/src/vec/exec/jni_connector.cpp | 10 +- be/src/vec/exec/jni_connector.h | 8 +- be/src/vec/exec/scan/jni_reader.cpp | 4 +- .../vec/exec/scan/max_compute_jni_reader.cpp | 4 +- be/src/vec/exec/scan/paimon_reader.cpp | 4 +- be/src/vec/exec/vjdbc_connector.cpp | 2 +- build.sh | 96 +++++++++------- .../hudi-scanner}/pom.xml | 103 ++---------------- .../apache/doris/hudi/HudiColumnValue.java | 3 +- .../org/apache/doris/hudi/HudiJniScanner.java | 9 +- .../org/apache/doris/hudi/HudiScanParam.java | 3 +- .../org/apache/doris/hudi/HudiScanUtils.java | 0 .../java/org/apache/doris/hudi}/Utils.java | 2 +- .../src/main/resources/package.xml | 0 .../apache/doris/hudi/HudiJniScannerTest.java | 0 fe/be-java-extensions/java-common/pom.xml | 42 +++++++ .../common/exception}/InternalException.java | 2 +- .../exception}/UdfRuntimeException.java | 2 +- .../apache/doris/common}/jni/JniScanner.java | 11 +- .../doris/common}/jni/MockJniScanner.java | 9 +- .../doris/common/jni/utils}/JMXJsonUtil.java | 2 +- .../common/jni/utils}/JNINativeMethod.java | 2 +- .../doris/common/jni/utils}/JniUtil.java | 3 +- .../common/jni/utils}/JvmPauseMonitor.java | 2 +- .../doris/common}/jni/utils/OffHeap.java | 3 +- .../common}/jni/utils/TypeNativeBytes.java | 2 +- .../doris/common/jni/utils}/UdfUtils.java | 7 +- .../doris/common}/jni/vec/ColumnType.java | 2 +- .../doris/common}/jni/vec/ColumnValue.java | 2 +- .../doris/common}/jni/vec/ScanPredicate.java | 9 +- .../doris/common}/jni/vec/VectorColumn.java | 9 +- .../doris/common}/jni/vec/VectorTable.java | 7 +- .../src/main/resources/log4j.properties | 0 .../src/main/resources/package.xml | 41 +++++++ .../doris/common}/jni/JniScannerTest.java | 7 +- fe/be-java-extensions/java-udf/pom.xml | 56 ++++++++++ .../org/apache/doris/udf/BaseExecutor.java | 5 +- .../org/apache/doris/udf/UdafExecutor.java | 4 +- .../org/apache/doris/udf/UdfExecutor.java | 4 +- .../java-udf/src/main/resources/package.xml | 41 +++++++ .../org/apache/doris/udf/ConstantOneUdf.java | 0 .../org/apache/doris/udf/DateTimeUdf.java | 0 .../java/org/apache/doris/udf/DecimalUdf.java | 0 .../org/apache/doris/udf/LargeIntUdf.java | 0 .../org/apache/doris/udf/SimpleAddUdf.java | 0 .../org/apache/doris/udf/StringConcatUdf.java | 0 .../org/apache/doris/udf/UdfExecutorTest.java | 1 + fe/be-java-extensions/jdbc-scanner/pom.xml | 72 ++++++++++++ .../apache/doris/jdbc}/JdbcDataSource.java | 2 +- .../org/apache/doris/jdbc}/JdbcExecutor.java | 14 ++- .../src/main/resources/package.xml | 41 +++++++ .../max-compute-scanner/pom.xml | 86 +++++++++++++++ .../maxcompute}/MaxComputeColumnValue.java | 4 +- .../maxcompute}/MaxComputeJniScanner.java | 8 +- .../src/main/resources/package.xml | 41 +++++++ fe/be-java-extensions/paimon-scanner/pom.xml | 82 ++++++++++++++ .../doris/paimon}/PaimonColumnValue.java | 4 +- .../doris/paimon}/PaimonJniScanner.java | 10 +- .../src/main/resources/package.xml | 41 +++++++ fe/be-java-extensions/pom.xml | 46 ++++++++ fe/fe-core/pom.xml | 1 - fe/pom.xml | 31 +++++- 64 files changed, 797 insertions(+), 219 deletions(-) rename fe/{java-udf => be-java-extensions/hudi-scanner}/pom.xml (57%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiColumnValue.java (98%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiJniScanner.java (97%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiScanParam.java (99%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/java/org/apache/doris/hudi/HudiScanUtils.java (100%) rename fe/{java-udf/src/main/java/org/apache/doris/jni/utils => be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi}/Utils.java (99%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/main/resources/package.xml (100%) rename fe/{java-udf => be-java-extensions/hudi-scanner}/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java (100%) create mode 100644 fe/be-java-extensions/java-common/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception}/InternalException.java (95%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception}/UdfRuntimeException.java (95%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/JniScanner.java (92%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/MockJniScanner.java (96%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JMXJsonUtil.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JNINativeMethod.java (96%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JniUtil.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/JvmPauseMonitor.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/utils/OffHeap.java (98%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/utils/TypeNativeBytes.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils}/UdfUtils.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/ColumnType.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/ColumnValue.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/ScanPredicate.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/VectorColumn.java (99%) rename fe/{java-udf/src/main/java/org/apache/doris => be-java-extensions/java-common/src/main/java/org/apache/doris/common}/jni/vec/VectorTable.java (96%) rename fe/{java-udf => be-java-extensions/java-common}/src/main/resources/log4j.properties (100%) create mode 100644 fe/be-java-extensions/java-common/src/main/resources/package.xml rename fe/{java-udf/src/test/java/org/apache/doris => be-java-extensions/java-common/src/test/java/org/apache/doris/common}/jni/JniScannerTest.java (94%) create mode 100644 fe/be-java-extensions/java-udf/pom.xml rename fe/{ => be-java-extensions}/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java (99%) rename fe/{ => be-java-extensions}/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java (98%) rename fe/{ => be-java-extensions}/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java (98%) create mode 100644 fe/be-java-extensions/java-udf/src/main/resources/package.xml rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java (100%) rename fe/{ => be-java-extensions}/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java (99%) create mode 100644 fe/be-java-extensions/jdbc-scanner/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc}/JdbcDataSource.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris/udf => be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc}/JdbcExecutor.java (99%) create mode 100644 fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml create mode 100644 fe/be-java-extensions/max-compute-scanner/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/jni/vec => be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute}/MaxComputeColumnValue.java (98%) rename fe/{java-udf/src/main/java/org/apache/doris/jni => be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute}/MaxComputeJniScanner.java (98%) create mode 100644 fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml create mode 100644 fe/be-java-extensions/paimon-scanner/pom.xml rename fe/{java-udf/src/main/java/org/apache/doris/jni/vec => be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon}/PaimonColumnValue.java (97%) rename fe/{java-udf/src/main/java/org/apache/doris/jni => be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon}/PaimonJniScanner.java (97%) create mode 100644 fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml create mode 100644 fe/be-java-extensions/pom.xml diff --git a/be/src/util/jni-util.cpp b/be/src/util/jni-util.cpp index 5ee95e7ea592acb..7b1843490a8b135 100644 --- a/be/src/util/jni-util.cpp +++ b/be/src/util/jni-util.cpp @@ -267,7 +267,7 @@ Status JniUtil::Init() { RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); if (env == NULL) return Status::InternalError("Failed to get/create JVM"); // Find JniUtil class and create a global ref. - jclass local_jni_util_cl = env->FindClass("org/apache/doris/udf/JniUtil"); + jclass local_jni_util_cl = env->FindClass("org/apache/doris/common/jni/utils/JniUtil"); if (local_jni_util_cl == NULL) { if (env->ExceptionOccurred()) env->ExceptionDescribe(); return Status::InternalError("Failed to find JniUtil class."); @@ -283,7 +283,8 @@ Status JniUtil::Init() { } // Find InternalException class and create a global ref. - jclass local_internal_exc_cl = env->FindClass("org/apache/doris/udf/InternalException"); + jclass local_internal_exc_cl = + env->FindClass("org/apache/doris/common/exception/InternalException"); if (local_internal_exc_cl == NULL) { if (env->ExceptionOccurred()) env->ExceptionDescribe(); return Status::InternalError("Failed to find JniUtil class."); @@ -299,7 +300,8 @@ Status JniUtil::Init() { } // Find JNINativeMethod class and create a global ref. - jclass local_jni_native_exc_cl = env->FindClass("org/apache/doris/udf/JNINativeMethod"); + jclass local_jni_native_exc_cl = + env->FindClass("org/apache/doris/common/jni/utils/JNINativeMethod"); if (local_jni_native_exc_cl == nullptr) { if (env->ExceptionOccurred()) { env->ExceptionDescribe(); diff --git a/be/src/util/jni_native_method.h b/be/src/util/jni_native_method.h index d208cb0d957fb40..e3dc3fb3dd927df 100644 --- a/be/src/util/jni_native_method.h +++ b/be/src/util/jni_native_method.h @@ -24,7 +24,7 @@ namespace doris { /** - * Java native methods for org.apache.doris.udf.JNINativeMethod. + * Java native methods for org.apache.doris.common.jni.utils.JNINativeMethod. */ struct JavaNativeMethods { /** diff --git a/be/src/vec/exec/jni_connector.cpp b/be/src/vec/exec/jni_connector.cpp index fd5cf52e2b7e392..178bcd8bb7e3093 100644 --- a/be/src/vec/exec/jni_connector.cpp +++ b/be/src/vec/exec/jni_connector.cpp @@ -71,7 +71,7 @@ Status JniConnector::open(RuntimeState* state, RuntimeProfile* profile) { return Status::InternalError("Failed to get/create JVM"); } RETURN_IF_ERROR(_init_jni_scanner(env, state->batch_size())); - // Call org.apache.doris.jni.JniScanner#open + // Call org.apache.doris.common.jni.JniScanner#open env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_open); RETURN_ERROR_IF_EXC(env); return Status::OK(); @@ -82,7 +82,7 @@ Status JniConnector::init( _generate_predicates(colname_to_value_range); if (_predicates_length != 0 && _predicates != nullptr) { int64_t predicates_address = (int64_t)_predicates.get(); - // We can call org.apache.doris.jni.vec.ScanPredicate#parseScanPredicates to parse the + // We can call org.apache.doris.common.jni.vec.ScanPredicate#parseScanPredicates to parse the // serialized predicates in java side. _scanner_params.emplace("push_down_predicates", std::to_string(predicates_address)); } @@ -90,7 +90,7 @@ Status JniConnector::init( } Status JniConnector::get_nex_block(Block* block, size_t* read_rows, bool* eof) { - // Call org.apache.doris.jni.JniScanner#getNextBatchMeta + // Call org.apache.doris.common.jni.JniScanner#getNextBatchMeta // return the address of meta information JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); @@ -123,7 +123,7 @@ Status JniConnector::close() { JNIEnv* env = nullptr; RETURN_IF_ERROR(JniUtil::GetJNIEnv(&env)); // _fill_block may be failed and returned, we should release table in close. - // org.apache.doris.jni.JniScanner#releaseTable is idempotent + // org.apache.doris.common.jni.JniScanner#releaseTable is idempotent env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_release_table); env->CallVoidMethod(_jni_scanner_obj, _jni_scanner_close); env->DeleteGlobalRef(_jni_scanner_obj); @@ -199,7 +199,7 @@ Status JniConnector::_fill_column(ColumnPtr& doris_column, DataTypePtr& data_typ TypeIndex logical_type = remove_nullable(data_type)->get_type_id(); void* null_map_ptr = _next_meta_as_ptr(); if (null_map_ptr == nullptr) { - // org.apache.doris.jni.vec.ColumnType.Type#UNSUPPORTED will set column address as 0 + // org.apache.doris.common.jni.vec.ColumnType.Type#UNSUPPORTED will set column address as 0 return Status::InternalError("Unsupported type {} in java side", getTypeName(logical_type)); } MutableColumnPtr data_column; diff --git a/be/src/vec/exec/jni_connector.h b/be/src/vec/exec/jni_connector.h index 90d5fbcd0a886f5..77be2329e0eff76 100644 --- a/be/src/vec/exec/jni_connector.h +++ b/be/src/vec/exec/jni_connector.h @@ -55,13 +55,13 @@ struct Decimal; namespace doris::vectorized { /** - * Connector to java jni scanner, which should extend org.apache.doris.jni.JniScanner + * Connector to java jni scanner, which should extend org.apache.doris.common.jni.JniScanner */ class JniConnector { public: /** * The predicates that can be pushed down to java side. - * Reference to java class org.apache.doris.jni.vec.ScanPredicate + * Reference to java class org.apache.doris.common.jni.vec.ScanPredicate */ template struct ScanPredicate { @@ -102,7 +102,7 @@ class JniConnector { /** * The value ranges can be stored as byte array as following format: * number_filters(4) | length(4) | column_name | op(4) | scale(4) | num_values(4) | value_length(4) | value | ... - * The read method is implemented in org.apache.doris.jni.vec.ScanPredicate#parseScanPredicates + * The read method is implemented in org.apache.doris.common.jni.vec.ScanPredicate#parseScanPredicates */ int write(std::unique_ptr& predicates, int origin_length) { int num_filters = 0; @@ -232,7 +232,7 @@ class JniConnector { std::unique_ptr _predicates = nullptr; /** - * Set the address of meta information, which is returned by org.apache.doris.jni.JniScanner#getNextBatchMeta + * Set the address of meta information, which is returned by org.apache.doris.common.jni.JniScanner#getNextBatchMeta */ void _set_meta(long meta_addr) { _meta_ptr = static_cast(reinterpret_cast(meta_addr)); diff --git a/be/src/vec/exec/scan/jni_reader.cpp b/be/src/vec/exec/scan/jni_reader.cpp index bbf9a5f8d05d28e..99858c1c42146c5 100644 --- a/be/src/vec/exec/scan/jni_reader.cpp +++ b/be/src/vec/exec/scan/jni_reader.cpp @@ -58,8 +58,8 @@ MockJniReader::MockJniReader(const std::vector& file_slot_descs std::map params = {{"mock_rows", "10240"}, {"required_fields", required_fields.str()}, {"columns_types", columns_types.str()}}; - _jni_connector = std::make_unique("org/apache/doris/jni/MockJniScanner", params, - column_names); + _jni_connector = std::make_unique("org/apache/doris/common/jni/MockJniScanner", + params, column_names); } Status MockJniReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { diff --git a/be/src/vec/exec/scan/max_compute_jni_reader.cpp b/be/src/vec/exec/scan/max_compute_jni_reader.cpp index f5182931d1a5afb..9edd8bfc514b4d2 100644 --- a/be/src/vec/exec/scan/max_compute_jni_reader.cpp +++ b/be/src/vec/exec/scan/max_compute_jni_reader.cpp @@ -70,8 +70,8 @@ MaxComputeJniReader::MaxComputeJniReader(const MaxComputeTableDescriptor* mc_des {"split_size", std::to_string(_range.size)}, {"required_fields", required_fields.str()}, {"columns_types", columns_types.str()}}; - _jni_connector = std::make_unique("org/apache/doris/jni/MaxComputeJniScanner", - params, column_names); + _jni_connector = std::make_unique( + "org/apache/doris/maxcompute/MaxComputeJniScanner", params, column_names); } Status MaxComputeJniReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { diff --git a/be/src/vec/exec/scan/paimon_reader.cpp b/be/src/vec/exec/scan/paimon_reader.cpp index 906973d838c826f..9348ad6b01e8f94 100644 --- a/be/src/vec/exec/scan/paimon_reader.cpp +++ b/be/src/vec/exec/scan/paimon_reader.cpp @@ -55,8 +55,8 @@ PaimonJniReader::PaimonJniReader(const std::vector& file_slot_d params["length_byte"] = range.table_format_params.paimon_params.length_byte; params["split_byte"] = std::to_string((int64_t)range.table_format_params.paimon_params.paimon_split.data()); - _jni_connector = std::make_unique("org/apache/doris/jni/PaimonJniScanner", params, - column_names); + _jni_connector = std::make_unique("org/apache/doris/paimon/PaimonJniScanner", + params, column_names); } Status PaimonJniReader::get_next_block(Block* block, size_t* read_rows, bool* eof) { diff --git a/be/src/vec/exec/vjdbc_connector.cpp b/be/src/vec/exec/vjdbc_connector.cpp index 055ea1e90d0a404..f9743584ab79a71 100644 --- a/be/src/vec/exec/vjdbc_connector.cpp +++ b/be/src/vec/exec/vjdbc_connector.cpp @@ -61,7 +61,7 @@ namespace doris { namespace vectorized { -const char* JDBC_EXECUTOR_CLASS = "org/apache/doris/udf/JdbcExecutor"; +const char* JDBC_EXECUTOR_CLASS = "org/apache/doris/jdbc/JdbcExecutor"; const char* JDBC_EXECUTOR_CTOR_SIGNATURE = "([B)V"; const char* JDBC_EXECUTOR_WRITE_SIGNATURE = "(Ljava/lang/String;)I"; const char* JDBC_EXECUTOR_STMT_WRITE_SIGNATURE = "(Ljava/util/Map;)I"; diff --git a/build.sh b/build.sh index 6095a595a9764a7..381e22e3e69463e 100755 --- a/build.sh +++ b/build.sh @@ -38,23 +38,23 @@ usage() { echo " Usage: $0 Optional options: - [no option] build all components - --fe build Frontend and Spark DPP application. Default ON. - --be build Backend. Default ON. - --meta-tool build Backend meta tool. Default OFF. - --broker build Broker. Default ON. - --audit build audit loader. Default ON. - --spark-dpp build Spark DPP application. Default ON. - --hive-udf build Hive UDF library for Spark Load. Default ON. - --java-udf build Java UDF. Default ON. - --clean clean and build target - --output specify the output directory - -j build Backend parallel + [no option] build all components + --fe build Frontend and Spark DPP application. Default ON. + --be build Backend. Default ON. + --meta-tool build Backend meta tool. Default OFF. + --broker build Broker. Default ON. + --audit build audit loader. Default ON. + --spark-dpp build Spark DPP application. Default ON. + --hive-udf build Hive UDF library for Spark Load. Default ON. + --be-java-extensions build Backend java extensions. Default ON. + --clean clean and build target + --output specify the output directory + -j build Backend parallel Environment variables: USE_AVX2 If the CPU does not support AVX2 instruction set, please set USE_AVX2=0. Default is ON. STRIP_DEBUG_INFO If set STRIP_DEBUG_INFO=ON, the debug information in the compiled binaries will be stored separately in the 'be/lib/debug_info' directory. Default is OFF. - DISABLE_JAVA_UDF If set DISABLE_JAVA_UDF=ON, we will do not build binary with java-udf. Default is OFF. + DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default is OFF. DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE. Eg. $0 build all @@ -119,7 +119,7 @@ if ! OPTS="$(getopt \ -l 'meta-tool' \ -l 'spark-dpp' \ -l 'hive-udf' \ - -l 'java-udf' \ + -l 'be-java-extensions' \ -l 'clean' \ -l 'coverage' \ -l 'help' \ @@ -138,7 +138,7 @@ BUILD_BROKER=0 BUILD_AUDIT=0 BUILD_META_TOOL='OFF' BUILD_SPARK_DPP=0 -BUILD_JAVA_UDF=0 +BUILD_BE_JAVA_EXTENSIONS=0 BUILD_HIVE_UDF=0 CLEAN=0 HELP=0 @@ -154,7 +154,7 @@ if [[ "$#" == 1 ]]; then BUILD_META_TOOL='OFF' BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 CLEAN=0 else while true; do @@ -163,12 +163,12 @@ else BUILD_FE=1 BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 shift ;; --be) BUILD_BE=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 shift ;; --broker) @@ -191,8 +191,8 @@ else BUILD_HIVE_UDF=1 shift ;; - --java-udf) - BUILD_JAVA_UDF=1 + --be-java-extensions) + BUILD_BE_JAVA_EXTENSIONS=1 shift ;; --clean) @@ -239,7 +239,7 @@ else BUILD_META_TOOL='ON' BUILD_SPARK_DPP=1 BUILD_HIVE_UDF=1 - BUILD_JAVA_UDF=1 + BUILD_BE_JAVA_EXTENSIONS=1 CLEAN=0 fi fi @@ -343,8 +343,8 @@ if [[ -z "${OUTPUT_BE_BINARY}" ]]; then OUTPUT_BE_BINARY=${BUILD_BE} fi -if [[ -z "${DISABLE_JAVA_UDF}" ]]; then - DISABLE_JAVA_UDF='OFF' +if [[ -z "${BUILD_BE_JAVA_EXTENSIONS}" ]]; then + BUILD_BE_JAVA_EXTENSIONS='OFF' fi if [[ -z "${DISABLE_JAVA_CHECK_STYLE}" ]]; then @@ -355,7 +355,7 @@ if [[ -z "${RECORD_COMPILER_SWITCHES}" ]]; then RECORD_COMPILER_SWITCHES='OFF' fi -if [[ "${BUILD_JAVA_UDF}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then +if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then if [[ -z "${JAVA_HOME}" ]]; then CAUSE='the environment variable JAVA_HOME is not set' else @@ -369,13 +369,13 @@ if [[ "${BUILD_JAVA_UDF}" -eq 1 && "$(uname -s)" == 'Darwin' ]]; then if [[ -n "${CAUSE}" ]]; then echo -e "\033[33;1mWARNNING: \033[37;1mSkip building with Java UDF due to ${CAUSE}.\033[0m" - BUILD_JAVA_UDF=0 - DISABLE_JAVA_UDF_IN_CONF=1 + BUILD_BE_JAVA_EXTENSIONS=0 + BUILD_BE_JAVA_EXTENSIONS_IN_CONF=1 fi fi -if [[ "${DISABLE_JAVA_UDF}" == "ON" ]]; then - BUILD_JAVA_UDF=0 +if [[ "${BUILD_BE_JAVA_EXTENSIONS}" == "ON" ]]; then + BUILD_BE_JAVA_EXTENSIONS=0 fi echo "Get params: @@ -385,7 +385,7 @@ echo "Get params: BUILD_AUDIT -- ${BUILD_AUDIT} BUILD_META_TOOL -- ${BUILD_META_TOOL} BUILD_SPARK_DPP -- ${BUILD_SPARK_DPP} - BUILD_JAVA_UDF -- ${BUILD_JAVA_UDF} + BUILD_BE_JAVA_EXTENSIONS -- ${BUILD_BE_JAVA_EXTENSIONS} BUILD_HIVE_UDF -- ${BUILD_HIVE_UDF} PARALLEL -- ${PARALLEL} CLEAN -- ${CLEAN} @@ -424,14 +424,19 @@ if [[ "${BUILD_SPARK_DPP}" -eq 1 ]]; then modules+=("fe-common") modules+=("spark-dpp") fi -if [[ "${BUILD_JAVA_UDF}" -eq 1 ]]; then - modules+=("fe-common") - modules+=("java-udf") -fi if [[ "${BUILD_HIVE_UDF}" -eq 1 ]]; then modules+=("fe-common") modules+=("hive-udf") fi +if [[ "${BUILD_BE_JAVA_EXTENSIONS}" -eq 1 ]]; then + modules+=("fe-common") + modules+=("be-java-extensions/hudi-scanner") + modules+=("be-java-extensions/java-common") + modules+=("be-java-extensions/java-udf") + modules+=("be-java-extensions/jdbc-scanner") + modules+=("be-java-extensions/paimon-scanner") + modules+=("be-java-extensions/max-compute-scanner") +fi FE_MODULES="$( IFS=',' echo "${modules[*]}" @@ -596,11 +601,11 @@ if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then rm -rf "${DORIS_OUTPUT}/be/lib/hadoop_hdfs/native/" fi - if [[ "${DISABLE_JAVA_UDF_IN_CONF}" -eq 1 ]]; then + if [[ "${BUILD_BE_JAVA_EXTENSIONS_IN_CONF}" -eq 1 ]]; then echo -e "\033[33;1mWARNNING: \033[37;1mDisable Java UDF support in be.conf due to the BE was built without Java UDF.\033[0m" cat >>"${DORIS_OUTPUT}/be/conf/be.conf" < - 4.0.0 - + be-java-extensions org.apache.doris ${revision} - fe - ../pom.xml - - java-udf - jar + 4.0.0 + hudi-scanner ${basedir}/../../ 1 - 2.7.4-11 - 3.0.0-8 - 0.4-SNAPSHOT - 0.13.0 - com.vesoft - client - 3.0.0 - - - org.apache.paimon - paimon-bundle - ${paimon.version} - - - org.apache.paimon - paimon-hive-connector-2.3 - ${paimon.version} - - - hive-common - org.apache.hive - 2.3.9 - - - com.fasterxml.jackson.core - jackson-core - - - org.apache.velocity - velocity-engine-core - - - org.apache.httpcomponents - httpclient - - - - com.oracle.database.jdbc - ojdbc6 - - - - com.alibaba - druid - - - com.clickhouse - clickhouse-jdbc - all - - - com.aliyun.odps - odps-sdk-core - - - org.codehaus.jackson - jackson-core-asl - - - org.codehaus.jackson - jackson-mapper-asl - - - - - org.apache.arrow - arrow-vector - 9.0.0 - - - com.fasterxml.jackson.core - jackson-databind - - - - - org.apache.arrow - arrow-memory-unsafe - 9.0.0 + org.apache.doris + java-common + ${project.version} + org.apache.hudi hudi-hadoop-mr-bundle @@ -167,16 +88,14 @@ under the License. - - org.apache.doris - fe-common - ${project.version} + commons-io + commons-io - + - java-udf + hudi-scanner org.apache.maven.plugins diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiColumnValue.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiColumnValue.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java index bb7272eedebfe36..4b4a7bbcf798970 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiColumnValue.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiColumnValue.java @@ -17,7 +17,8 @@ package org.apache.doris.hudi; -import org.apache.doris.jni.vec.ColumnValue; + +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector; diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiJniScanner.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiJniScanner.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java index a913329214ddb7d..d7078dc9ca02a56 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiJniScanner.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiJniScanner.java @@ -17,9 +17,9 @@ package org.apache.doris.hudi; -import org.apache.doris.jni.JniScanner; -import org.apache.doris.jni.utils.Utils; -import org.apache.doris.jni.vec.ColumnValue; + +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.serde2.Deserializer; @@ -163,7 +163,8 @@ private void init(JobConf jobConf, Properties properties) throws Exception { executorService.scheduleAtFixedRate(() -> { if (!isKilled.get()) { synchronized (HudiJniScanner.class) { - List pids = Utils.getChildProcessIds(Utils.getCurrentProcId()); + List pids = Utils.getChildProcessIds( + Utils.getCurrentProcId()); for (long pid : pids) { String cmd = Utils.getCommandLine(pid); if (cmd != null && cmd.contains("org.openjdk.jol.vm.sa.AttachMain")) { diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanParam.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanParam.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanParam.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanParam.java index 68a0b88e0af7a0c..4343dd3f492c800 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanParam.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanParam.java @@ -17,7 +17,8 @@ package org.apache.doris.hudi; -import org.apache.doris.jni.vec.ColumnType; + +import org.apache.doris.common.jni.vec.ColumnType; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.hive.serde.serdeConstants; diff --git a/fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanUtils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanUtils.java similarity index 100% rename from fe/java-udf/src/main/java/org/apache/doris/hudi/HudiScanUtils.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/HudiScanUtils.java diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/Utils.java b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/utils/Utils.java rename to fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java index 53805e96f14bd66..c7f3ecf9dab289b 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/Utils.java +++ b/fe/be-java-extensions/hudi-scanner/src/main/java/org/apache/doris/hudi/Utils.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.utils; +package org.apache.doris.hudi; import org.apache.commons.io.FileUtils; import org.apache.hadoop.conf.Configuration; diff --git a/fe/java-udf/src/main/resources/package.xml b/fe/be-java-extensions/hudi-scanner/src/main/resources/package.xml similarity index 100% rename from fe/java-udf/src/main/resources/package.xml rename to fe/be-java-extensions/hudi-scanner/src/main/resources/package.xml diff --git a/fe/java-udf/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java b/fe/be-java-extensions/hudi-scanner/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java rename to fe/be-java-extensions/hudi-scanner/src/test/java/org/apache/doris/hudi/HudiJniScannerTest.java diff --git a/fe/be-java-extensions/java-common/pom.xml b/fe/be-java-extensions/java-common/pom.xml new file mode 100644 index 000000000000000..ba763b21dab07f2 --- /dev/null +++ b/fe/be-java-extensions/java-common/pom.xml @@ -0,0 +1,42 @@ + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + java-common + + + 8 + 8 + + + + + com.vesoft + client + + + org.apache.doris + fe-common + ${project.version} + + + com.fasterxml.jackson.core + jackson-core + + + org.apache.velocity + velocity-engine-core + + + org.apache.httpcomponents + httpclient + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/InternalException.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/InternalException.java similarity index 95% rename from fe/java-udf/src/main/java/org/apache/doris/udf/InternalException.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/InternalException.java index 1b1e335ae219305..ddd94e50486a41f 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/InternalException.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/InternalException.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.exception; public class InternalException extends Exception { public InternalException(String msg, Throwable cause) { diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfRuntimeException.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/UdfRuntimeException.java similarity index 95% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdfRuntimeException.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/UdfRuntimeException.java index ef02ae5c368841e..861b4f46fe2f00d 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfRuntimeException.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/exception/UdfRuntimeException.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.exception; public class UdfRuntimeException extends Exception { public UdfRuntimeException(String msg, Throwable cause) { diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/JniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java similarity index 92% rename from fe/java-udf/src/main/java/org/apache/doris/jni/JniScanner.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java index 3d0223ba7594c9d..7e7391520e1d955 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/JniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/JniScanner.java @@ -15,12 +15,13 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.common.jni; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.ColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; -import org.apache.doris.jni.vec.VectorTable; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.doris.common.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.vec.VectorTable; import java.io.IOException; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/MockJniScanner.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java similarity index 96% rename from fe/java-udf/src/main/java/org/apache/doris/jni/MockJniScanner.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java index c4c4c5f80b52785..b559f2a0af0be8a 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/MockJniScanner.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/MockJniScanner.java @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.common.jni; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.ColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; + +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ColumnValue; +import org.apache.doris.common.jni.vec.ScanPredicate; import org.apache.log4j.Logger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JMXJsonUtil.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JMXJsonUtil.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JMXJsonUtil.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JMXJsonUtil.java index 5089b45a079776f..02cb53232de11ab 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JMXJsonUtil.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JMXJsonUtil.java @@ -18,7 +18,7 @@ // https://github.com/apache/impala/blob/branch-4.0.0/fe/src/main/java/org/apache/impala/util/JMXJsonUtil.java // and modified by Doris -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; import com.fasterxml.jackson.core.JsonFactory; import com.fasterxml.jackson.core.JsonGenerator; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JNINativeMethod.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JNINativeMethod.java similarity index 96% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JNINativeMethod.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JNINativeMethod.java index c496bd5f0f37a4a..50820c2ecb93521 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JNINativeMethod.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JNINativeMethod.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; /** * Native method in doris::JavaNativeMethods. diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java index af1cf6e3b0d7797..1a91df84be6904d 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JniUtil.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JniUtil.java @@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; +import org.apache.doris.common.exception.InternalException; import org.apache.doris.thrift.TGetJMXJsonResponse; import org.apache.doris.thrift.TGetJvmMemoryMetricsResponse; import org.apache.doris.thrift.TGetJvmThreadsInfoRequest; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JvmPauseMonitor.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JvmPauseMonitor.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JvmPauseMonitor.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JvmPauseMonitor.java index 1129c7ac49266f8..85af333156a3d3d 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JvmPauseMonitor.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/JvmPauseMonitor.java @@ -18,7 +18,7 @@ // https://github.com/apache/impala/blob/branch-4.0.0/fe/src/main/java/org/apache/impala/util/JvmPauseMonitor.java // and modified by Doris -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; import com.google.common.base.Joiner; import com.google.common.base.Stopwatch; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/OffHeap.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/OffHeap.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/jni/utils/OffHeap.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/OffHeap.java index 2ad77b7a7e1e917..268f5f6787666b8 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/OffHeap.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/OffHeap.java @@ -15,9 +15,8 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.utils; +package org.apache.doris.common.jni.utils; -import org.apache.doris.udf.JNINativeMethod; import sun.misc.Unsafe; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/TypeNativeBytes.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/utils/TypeNativeBytes.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java index 666f9683299b746..bd2eb79cd295e09 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/utils/TypeNativeBytes.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/TypeNativeBytes.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.utils; +package org.apache.doris.common.jni.utils; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfUtils.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdfUtils.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java index c1248f87ca4a369..09e0dc380462f82 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfUtils.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/utils/UdfUtils.java @@ -15,13 +15,14 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.common.jni.utils; import org.apache.doris.catalog.ArrayType; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.ScalarType; import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; +import org.apache.doris.common.exception.InternalException; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.TScalarType; import org.apache.doris.thrift.TTypeDesc; @@ -193,7 +194,7 @@ public void setItemType(Type type) { } } - protected static Pair fromThrift(TTypeDesc typeDesc, int nodeIdx) throws InternalException { + public static Pair fromThrift(TTypeDesc typeDesc, int nodeIdx) throws InternalException { TTypeNode node = typeDesc.getTypes().get(nodeIdx); Type type = null; switch (node.getType()) { @@ -238,7 +239,7 @@ protected static Pair fromThrift(TTypeDesc typeDesc, int nodeIdx) return Pair.of(type, nodeIdx); } - protected static long getAddressAtOffset(long base, int offset) { + public static long getAddressAtOffset(long base, int offset) { return base + 8L * offset; } diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnType.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnType.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java index a5eee7e43e1f84b..eba69e51c86ed7c 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnType.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnType.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; import java.util.ArrayList; import java.util.Arrays; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnValue.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnValue.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java index 8d190aa21223571..04951792601f3b7 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ColumnValue.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ColumnValue.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ScanPredicate.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/ScanPredicate.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java index e02107f1432b94c..f9b35b2352e25e4 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/ScanPredicate.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/ScanPredicate.java @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.utils.TypeNativeBytes; -import org.apache.doris.jni.vec.ColumnType.Type; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.utils.TypeNativeBytes; +import org.apache.doris.common.jni.vec.ColumnType.Type; import org.apache.commons.lang3.StringUtils; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorColumn.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorColumn.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java index 499619177604ea4..70b358a87f2a6af 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorColumn.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorColumn.java @@ -15,11 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.utils.TypeNativeBytes; -import org.apache.doris.jni.vec.ColumnType.Type; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.utils.TypeNativeBytes; +import org.apache.doris.common.jni.vec.ColumnType.Type; import java.math.BigDecimal; import java.math.BigInteger; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorTable.java b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java similarity index 96% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorTable.java rename to fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java index 3f86ebc7a24e8b8..e70d8f683f6b08f 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/VectorTable.java +++ b/fe/be-java-extensions/java-common/src/main/java/org/apache/doris/common/jni/vec/VectorTable.java @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.common.jni.vec; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.vec.ColumnType.Type; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.ColumnType.Type; /** * Store a batch of data as vector table. diff --git a/fe/java-udf/src/main/resources/log4j.properties b/fe/be-java-extensions/java-common/src/main/resources/log4j.properties similarity index 100% rename from fe/java-udf/src/main/resources/log4j.properties rename to fe/be-java-extensions/java-common/src/main/resources/log4j.properties diff --git a/fe/be-java-extensions/java-common/src/main/resources/package.xml b/fe/be-java-extensions/java-common/src/main/resources/package.xml new file mode 100644 index 000000000000000..4bbb26106033637 --- /dev/null +++ b/fe/be-java-extensions/java-common/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/java-udf/src/test/java/org/apache/doris/jni/JniScannerTest.java b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java similarity index 94% rename from fe/java-udf/src/test/java/org/apache/doris/jni/JniScannerTest.java rename to fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java index 6be67ecba3a6587..e8a6d49df796ee3 100644 --- a/fe/java-udf/src/test/java/org/apache/doris/jni/JniScannerTest.java +++ b/fe/be-java-extensions/java-common/src/test/java/org/apache/doris/common/jni/JniScannerTest.java @@ -15,10 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.common.jni; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.vec.VectorTable; + +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.VectorTable; import org.junit.Assert; import org.junit.Test; diff --git a/fe/be-java-extensions/java-udf/pom.xml b/fe/be-java-extensions/java-udf/pom.xml new file mode 100644 index 000000000000000..bc037fd7151f209 --- /dev/null +++ b/fe/be-java-extensions/java-udf/pom.xml @@ -0,0 +1,56 @@ + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + java-udf + + + 8 + 8 + + + + + + org.apache.doris + java-common + ${project.version} + + + + + java-udf + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + \ No newline at end of file diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java rename to fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java index f1d2e3a9a7c3b02..3dbe10ca27eff90 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/BaseExecutor.java @@ -19,8 +19,11 @@ import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.Type; +import org.apache.doris.common.exception.InternalException; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; -import org.apache.doris.udf.UdfUtils.JavaUdfDataType; import org.apache.log4j.Logger; import org.apache.thrift.TDeserializer; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java rename to fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java index 76e9a09c3d638af..e2d8ab1b75ab4d1 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdafExecutor.java @@ -19,8 +19,10 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; -import org.apache.doris.udf.UdfUtils.JavaUdfDataType; import com.google.common.base.Joiner; import com.google.common.collect.Lists; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java rename to fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java index b888279db3866bd..18eaaf8c452c336 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java +++ b/fe/be-java-extensions/java-udf/src/main/java/org/apache/doris/udf/UdfExecutor.java @@ -19,8 +19,10 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.Pair; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.utils.UdfUtils.JavaUdfDataType; import org.apache.doris.thrift.TJavaUdfExecutorCtorParams; -import org.apache.doris.udf.UdfUtils.JavaUdfDataType; import com.google.common.base.Joiner; import com.google.common.collect.Lists; diff --git a/fe/be-java-extensions/java-udf/src/main/resources/package.xml b/fe/be-java-extensions/java-udf/src/main/resources/package.xml new file mode 100644 index 000000000000000..4bbb26106033637 --- /dev/null +++ b/fe/be-java-extensions/java-udf/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/ConstantOneUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DateTimeUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/DecimalUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/LargeIntUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/SimpleAddUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java similarity index 100% rename from fe/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/StringConcatUdf.java diff --git a/fe/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java similarity index 99% rename from fe/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java rename to fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java index 7330c9b0832c13b..7c725da50e00eeb 100644 --- a/fe/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java +++ b/fe/be-java-extensions/java-udf/src/test/java/org/apache/doris/udf/UdfExecutorTest.java @@ -17,6 +17,7 @@ package org.apache.doris.udf; +import org.apache.doris.common.jni.utils.UdfUtils; import org.apache.doris.thrift.TFunction; import org.apache.doris.thrift.TFunctionBinaryType; import org.apache.doris.thrift.TFunctionName; diff --git a/fe/be-java-extensions/jdbc-scanner/pom.xml b/fe/be-java-extensions/jdbc-scanner/pom.xml new file mode 100644 index 000000000000000..952939ceac9e1a4 --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/pom.xml @@ -0,0 +1,72 @@ + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + jdbc-scanner + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + + + + + com.oracle.database.jdbc + ojdbc6 + + + + com.alibaba + druid + + + com.clickhouse + clickhouse-jdbc + all + + + + + jdbc-scanner + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + + \ No newline at end of file diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcDataSource.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JdbcDataSource.java rename to fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java index 4d6f4f59e667f35..95b3dac585a0f20 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcDataSource.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcDataSource.java @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; +package org.apache.doris.jdbc; import com.alibaba.druid.pool.DruidDataSource; diff --git a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java similarity index 99% rename from fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java rename to fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java index 4f8eb2199d1c86c..11bf3d6d791cbb9 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/udf/JdbcExecutor.java +++ b/fe/be-java-extensions/jdbc-scanner/src/main/java/org/apache/doris/jdbc/JdbcExecutor.java @@ -15,11 +15,15 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.udf; - -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.VectorColumn; -import org.apache.doris.jni.vec.VectorTable; +package org.apache.doris.jdbc; + +import org.apache.doris.common.exception.InternalException; +import org.apache.doris.common.exception.UdfRuntimeException; +import org.apache.doris.common.jni.utils.JNINativeMethod; +import org.apache.doris.common.jni.utils.UdfUtils; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.VectorColumn; +import org.apache.doris.common.jni.vec.VectorTable; import org.apache.doris.thrift.TJdbcExecutorCtorParams; import org.apache.doris.thrift.TJdbcOperation; import org.apache.doris.thrift.TOdbcTableType; diff --git a/fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml b/fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml new file mode 100644 index 000000000000000..4bbb26106033637 --- /dev/null +++ b/fe/be-java-extensions/jdbc-scanner/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/be-java-extensions/max-compute-scanner/pom.xml b/fe/be-java-extensions/max-compute-scanner/pom.xml new file mode 100644 index 000000000000000..14dd75519935b69 --- /dev/null +++ b/fe/be-java-extensions/max-compute-scanner/pom.xml @@ -0,0 +1,86 @@ + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + max-compute-scanner + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + + + com.aliyun.odps + odps-sdk-core + + + org.codehaus.jackson + jackson-core-asl + + + org.codehaus.jackson + jackson-mapper-asl + + + + + org.apache.arrow + arrow-vector + ${arrow.version} + + + com.fasterxml.jackson.core + jackson-databind + + + + + org.apache.arrow + arrow-memory-unsafe + ${arrow.version} + + + + + max-compute-scanner + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/MaxComputeColumnValue.java b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/MaxComputeColumnValue.java rename to fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java index 0945f1f3260c8b2..4c3aff97b470f7b 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/MaxComputeColumnValue.java +++ b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeColumnValue.java @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.maxcompute; + +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.arrow.vector.BigIntVector; import org.apache.arrow.vector.DateDayVector; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/MaxComputeJniScanner.java b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java similarity index 98% rename from fe/java-udf/src/main/java/org/apache/doris/jni/MaxComputeJniScanner.java rename to fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java index 65aa2ccded8af43..95664c93c3d0b11 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/MaxComputeJniScanner.java +++ b/fe/be-java-extensions/max-compute-scanner/src/main/java/org/apache/doris/maxcompute/MaxComputeJniScanner.java @@ -15,11 +15,11 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.maxcompute; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.MaxComputeColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ScanPredicate; import com.aliyun.odps.Column; import com.aliyun.odps.Odps; diff --git a/fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml b/fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml new file mode 100644 index 000000000000000..4bbb26106033637 --- /dev/null +++ b/fe/be-java-extensions/max-compute-scanner/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/be-java-extensions/paimon-scanner/pom.xml b/fe/be-java-extensions/paimon-scanner/pom.xml new file mode 100644 index 000000000000000..219974cf9163057 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/pom.xml @@ -0,0 +1,82 @@ + + + + be-java-extensions + org.apache.doris + ${revision} + + 4.0.0 + + paimon-scanner + + + 8 + 8 + + + + + org.apache.doris + java-common + ${project.version} + + + + hive-common + org.apache.hive + + + + org.apache.paimon + paimon-bundle + ${paimon.version} + + + org.apache.paimon + paimon-hive-connector-2.3 + ${paimon.version} + + + + org.apache.hadoop + hadoop-mapreduce-client-core + + + + org.apache.hadoop + hadoop-mapreduce-client-common + + + + + + paimon-scanner + + + org.apache.maven.plugins + maven-assembly-plugin + + + src/main/resources/package.xml + + + + + + + + + + make-assembly + package + + single + + + + + + + diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/PaimonColumnValue.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/vec/PaimonColumnValue.java rename to fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java index 8e3744b1f123731..e0a89c9db26661d 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/vec/PaimonColumnValue.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonColumnValue.java @@ -15,7 +15,9 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni.vec; +package org.apache.doris.paimon; + +import org.apache.doris.common.jni.vec.ColumnValue; import org.apache.paimon.data.columnar.ColumnarRow; diff --git a/fe/java-udf/src/main/java/org/apache/doris/jni/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java similarity index 97% rename from fe/java-udf/src/main/java/org/apache/doris/jni/PaimonJniScanner.java rename to fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 18d8d1dc4b4de6b..190bdb829a4de90 100644 --- a/fe/java-udf/src/main/java/org/apache/doris/jni/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -15,12 +15,12 @@ // specific language governing permissions and limitations // under the License. -package org.apache.doris.jni; +package org.apache.doris.paimon; -import org.apache.doris.jni.utils.OffHeap; -import org.apache.doris.jni.vec.ColumnType; -import org.apache.doris.jni.vec.PaimonColumnValue; -import org.apache.doris.jni.vec.ScanPredicate; +import org.apache.doris.common.jni.JniScanner; +import org.apache.doris.common.jni.utils.OffHeap; +import org.apache.doris.common.jni.vec.ColumnType; +import org.apache.doris.common.jni.vec.ScanPredicate; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.log4j.Logger; diff --git a/fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml b/fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml new file mode 100644 index 000000000000000..4bbb26106033637 --- /dev/null +++ b/fe/be-java-extensions/paimon-scanner/src/main/resources/package.xml @@ -0,0 +1,41 @@ + + + + jar-with-dependencies + + jar + + false + + + / + true + true + runtime + + + **/Log4j2Plugins.dat + + + + + diff --git a/fe/be-java-extensions/pom.xml b/fe/be-java-extensions/pom.xml new file mode 100644 index 000000000000000..aee63210938cb24 --- /dev/null +++ b/fe/be-java-extensions/pom.xml @@ -0,0 +1,46 @@ + + + + 4.0.0 + + hudi-scanner + java-common + java-udf + jdbc-scanner + paimon-scanner + max-compute-scanner + + + + org.apache.doris + ${revision} + fe + ../pom.xml + + + be-java-extensions + pom + + ${basedir}/../../ + 1 + + + diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 2c264e146b655dc..00959e1fcbecd46 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -34,7 +34,6 @@ under the License. 1 4.9.3 2.17.257 - 0.4-SNAPSHOT diff --git a/fe/pom.xml b/fe/pom.xml index 5cb5f36482da2dc..b54daa21fe55052 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -186,7 +186,7 @@ under the License. spark-dpp fe-core hive-udf - java-udf + be-java-extensions @@ -263,6 +263,8 @@ under the License. 0.8.13 2.4.6 3.1.3 + 2.3.9 + 2.10.1 1.33.0 1.22.0 @@ -270,9 +272,13 @@ under the License. you can find avro version info in iceberg mvn repository --> 1.1.0 0.43.3-public + 9.0.0 1.11.1 0.13.0 + 2.7.4-11 + 3.0.0-8 + 1.13.0 3.2.2 1.22 @@ -299,6 +305,9 @@ under the License. 6.5.1 2.0.3 1.5.4 + 3.0.0 + + 0.4.0-incubating @@ -1269,6 +1278,21 @@ under the License. metastore-client-hive3 ${dlf-metastore-client-hive.version} + + hive-common + org.apache.hive + ${hive.common.version} + + + org.apache.hadoop + hadoop-mapreduce-client-core + ${mapreduce.client.version} + + + org.apache.hadoop + hadoop-mapreduce-client-common + ${mapreduce.client.version} + com.alibaba druid @@ -1297,6 +1321,11 @@ under the License. tomcat-embed-core ${tomcat-embed-core.version} + + com.vesoft + client + ${vesoft.client.version} +