[dependency][enhancement] support build libhdfs in arm cpus (apache#1…

…0018)

Supports native hdfs functionality on arm cpu
This pr mainly upgrades libdfs3 and supports running on arm，and make libhdfs3 with kerberos as default

be/CMakeLists.txt

@@ -85,7 +85,6 @@ set(BASE_DIR "${CMAKE_CURRENT_SOURCE_DIR}")
 set(ENV{DORIS_HOME} "${BASE_DIR}/../")
 set(BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}")
 set(THIRDPARTY_DIR "$ENV{DORIS_THIRDPARTY}/installed/")
-set(HDFS3_KRB5_INSTALL_DIR "$ENV{DORIS_THIRDPARTY}/installed/libhdfs_with_kerberos/")
 set(GENSRC_DIR "${BASE_DIR}/../gensrc/build/")
 set(SRC_DIR "${BASE_DIR}/src/")
 set(TEST_DIR "${CMAKE_SOURCE_DIR}/test/")
@@ -98,7 +97,6 @@ else()
 endif()
 message(STATUS "make test: ${MAKE_TEST}")
 option(WITH_MYSQL "Support access MySQL" ON)
-option(WITH_KERBEROS "Enable or disable Kereberos support" ${WITH_KERBEROS})
 
 # Check gcc
 if (CMAKE_CXX_COMPILER_ID STREQUAL "GNU")
@@ -330,43 +328,32 @@ set_target_properties(minizip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib
 add_library(idn STATIC IMPORTED)
 set_target_properties(idn PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libidn.a)
 
-if (WITH_KERBEROS)
-    # kerberos lib for libhdfs3 
-    add_library(gsasl STATIC IMPORTED)
-    set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgsasl.a)
+add_library(xml2 STATIC IMPORTED)
+set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)
 
-    add_library(krb5support STATIC IMPORTED)
-    set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5support.a)
-
-    add_library(krb5 STATIC IMPORTED)
-    set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libkrb5.a)
+add_library(lzma STATIC IMPORTED)
+set_target_properties(lzma PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/liblzma.a)
 
-    add_library(com_err STATIC IMPORTED)
-    set_target_properties(com_err PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libcom_err.a)
+add_library(gsasl STATIC IMPORTED)
+set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libgsasl.a)
 
-    add_library(k5crypto STATIC IMPORTED)
-    set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libk5crypto.a)
+add_library(krb5support STATIC IMPORTED)
+set_target_properties(krb5support PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libkrb5support.a)
 
-    add_library(gssapi_krb5 STATIC IMPORTED)
-    set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libgssapi_krb5.a)
+add_library(krb5 STATIC IMPORTED)
+set_target_properties(krb5 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libkrb5.a)
 
-    add_library(hdfs3 STATIC IMPORTED)
-    set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${HDFS3_KRB5_INSTALL_DIR}/lib/libhdfs3.a)
-else()
-    add_library(gsasl STATIC IMPORTED)
-    set_target_properties(gsasl PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libgsasl.a)
+add_library(com_err STATIC IMPORTED)
+set_target_properties(com_err PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libcom_err.a)
 
-    add_library(hdfs3 STATIC IMPORTED)
-    set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libhdfs3.a)
-endif()
+add_library(k5crypto STATIC IMPORTED)
+set_target_properties(k5crypto PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libk5crypto.a)
 
-if (ARCH_AMD64)
-    add_library(xml2 STATIC IMPORTED)
-    set_target_properties(xml2 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libxml2.a)
+add_library(gssapi_krb5 STATIC IMPORTED)
+set_target_properties(gssapi_krb5 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libgssapi_krb5.a)
 
-    add_library(lzma STATIC IMPORTED)
-    set_target_properties(lzma PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/liblzma.a)
-endif()
+add_library(hdfs3 STATIC IMPORTED)
+set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libhdfs3.a)
 
 find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)
 
@@ -570,6 +557,7 @@ set(DORIS_LINK_LIBS
     Exec
     Exprs
     Gutil
+    IO
     Olap
     Rowset
     OlapFs
@@ -632,6 +620,9 @@ set(COMMON_THIRDPARTY
     # put this after lz4 to avoid using lz4 lib in librdkafka
     librdkafka_cpp
     librdkafka
+    hdfs3
+    xml2
+    lzma
 )
 
 if (${MAKE_TEST} STREQUAL "ON")
@@ -641,29 +632,13 @@ if (${MAKE_TEST} STREQUAL "ON")
     )
 endif()
 
-# thirdparties dependescies that can only run on X86 platform
-set(X86_DEPENDENCIES
+set(DORIS_DEPENDENCIES
+    ${DORIS_DEPENDENCIES}
+    ${WL_START_GROUP}
     ${COMMON_THIRDPARTY}
-    hdfs3
-    xml2
-    lzma
+    ${KRB5_LIBS}
 )
 
-if(ARCH_AARCH64)
-    # Set thirdparty libraries
-    set(DORIS_DEPENDENCIES
-        ${DORIS_DEPENDENCIES}
-        ${WL_START_GROUP}
-        ${COMMON_THIRDPARTY}
-    )
-else()
-    set(DORIS_DEPENDENCIES
-        ${DORIS_DEPENDENCIES}
-        ${WL_START_GROUP}
-        ${X86_DEPENDENCIES}
-    )
-endif()
-
 if(WITH_LZO)
     set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
         lzo
@@ -676,12 +651,6 @@ if (WITH_MYSQL)
         )
 endif()
 
-if (WITH_KERBEROS)
-    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-    	${KRB5_LIBS}
-        )
-endif()
-
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
 
 message(STATUS "DORIS_DEPENDENCIES is ${DORIS_DEPENDENCIES}")
@@ -755,6 +724,7 @@ add_subdirectory(${SRC_DIR}/exec)
 add_subdirectory(${SRC_DIR}/exprs)
 add_subdirectory(${SRC_DIR}/gen_cpp)
 add_subdirectory(${SRC_DIR}/geo)
+add_subdirectory(${SRC_DIR}/io)
 add_subdirectory(${SRC_DIR}/gutil)
 add_subdirectory(${SRC_DIR}/http)
 add_subdirectory(${SRC_DIR}/olap)

be/src/exec/CMakeLists.txt

@@ -28,7 +28,6 @@ set(EXEC_FILES
     analytic_eval_node.cpp
     blocking_join_node.cpp
     broker_scan_node.cpp
-        ../io/buffered_reader.cpp
     base_scanner.cpp
     broker_scanner.cpp
     cross_join_node.cpp
@@ -98,24 +97,7 @@ set(EXEC_FILES
     json_scanner.cpp
     assert_num_rows_node.cpp
 
-        ../io/local_file_reader.cpp
-        ../io/local_file_writer.cpp
-        ../io/broker_reader.cpp
-        ../io/broker_writer.cpp
-        ../io/s3_reader.cpp
-        ../io/s3_writer.cpp
-        ../io/hdfs_reader_writer.cpp
-        ../io/file_factory.cpp
 )
-
-if (ARCH_AMD64)
-    set(EXEC_FILES
-        ${EXEC_FILES}
-            ../io/hdfs_file_reader.cpp
-            ../io/hdfs_writer.cpp
-        )
-endif()
-
 if (WITH_MYSQL)
     set(EXEC_FILES
         ${EXEC_FILES}

be/src/io/CMakeLists.txt

@@ -21,26 +21,20 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io")
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io")
 
-set(EXEC_FILES
+set(IO_FILES
+     broker_reader.cpp
+     broker_writer.cpp
      buffered_reader.cpp
+     file_factory.cpp
+     hdfs_file_reader.cpp
+     hdfs_reader_writer.cpp
+     hdfs_writer.cpp
      local_file_reader.cpp
      local_file_writer.cpp
-     broker_reader.cpp
-     broker_writer.cpp
      s3_reader.cpp
      s3_writer.cpp
-     hdfs_reader_writer.cpp
-     file_factory.cpp
 )
 
-if (ARCH_AMD64)
-    set(EXEC_FILES
-        ${EXEC_FILES}
-        hdfs_file_reader.cpp
-        hdfs_writer.cpp
-        )
-endif()
-
 add_library(IO STATIC
-    ${EXEC_FILES}
+    ${IO_FILES}
 )

be/src/io/broker_reader.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "broker_reader.h"
+#include "io/broker_reader.h"
 
 #include <sstream>
 

be/src/io/broker_reader.h

@@ -23,9 +23,9 @@
 #include <string>
 
 #include "common/status.h"
-#include "file_reader.h"
 #include "gen_cpp/PaloBrokerService_types.h"
 #include "gen_cpp/Types_types.h"
+#include "io/file_reader.h"
 
 namespace doris {
 

be/src/io/broker_writer.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "broker_writer.h"
+#include "io/broker_writer.h"
 
 #include <sstream>
 

be/src/io/broker_writer.h

@@ -23,9 +23,9 @@
 #include <string>
 
 #include "common/status.h"
-#include "file_writer.h"
 #include "gen_cpp/PaloBrokerService_types.h"
 #include "gen_cpp/Types_types.h"
+#include "io/file_writer.h"
 
 namespace doris {
 

be/src/io/buffered_reader.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "buffered_reader.h"
+#include "io/buffered_reader.h"
 
 #include <algorithm>
 #include <sstream>

be/src/io/buffered_reader.h

@@ -22,7 +22,7 @@
 #include <memory>
 
 #include "common/status.h"
-#include "file_reader.h"
+#include "io/file_reader.h"
 #include "olap/olap_define.h"
 #include "util/runtime_profile.h"
 

be/src/io/file_factory.cpp

@@ -15,18 +15,18 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "file_factory.h"
+#include "io/file_factory.h"
 
-#include "broker_reader.h"
-#include "broker_writer.h"
-#include "buffered_reader.h"
-#include "hdfs_reader_writer.h"
-#include "local_file_reader.h"
-#include "local_file_writer.h"
+#include "io/broker_reader.h"
+#include "io/broker_writer.h"
+#include "io/buffered_reader.h"
+#include "io/hdfs_reader_writer.h"
+#include "io/local_file_reader.h"
+#include "io/local_file_writer.h"
+#include "io/s3_reader.h"
+#include "io/s3_writer.h"
 #include "runtime/exec_env.h"
 #include "runtime/stream_load/load_stream_mgr.h"
-#include "s3_reader.h"
-#include "s3_writer.h"
 
 doris::Status doris::FileFactory::create_file_writer(
         TFileType::type type, doris::ExecEnv* env,

be/src/io/file_factory.h

@@ -16,10 +16,10 @@
 // under the License.
 #pragma once
 
-#include "file_reader.h"
-#include "file_writer.h"
 #include "gen_cpp/PlanNodes_types.h"
 #include "gen_cpp/Types_types.h"
+#include "io/file_reader.h"
+#include "io/file_writer.h"
 
 namespace doris {
 class ExecEnv;

be/src/io/hdfs_file_reader.cpp

@@ -14,7 +14,7 @@
 // KIND, either express or implied.  See the License for the
 // specific language governing permissions and limitations
 // under the License.
-#include "hdfs_file_reader.h"
+#include "io/hdfs_file_reader.h"
 
 #include <sys/stat.h>
 #include <unistd.h>

be/src/io/hdfs_file_reader.h

@@ -19,8 +19,8 @@
 
 #include <hdfs/hdfs.h>
 
-#include "file_reader.h"
 #include "gen_cpp/PlanNodes_types.h"
+#include "io/file_reader.h"
 
 namespace doris {
 

be/src/io/hdfs_reader_writer.cpp

@@ -15,34 +15,24 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "hdfs_reader_writer.h"
+#include "io/hdfs_reader_writer.h"
 
-#if defined(__x86_64__)
-#include "hdfs_file_reader.h"
-#include "hdfs_writer.h"
-#endif
+#include "io/hdfs_file_reader.h"
+#include "io/hdfs_writer.h"
 
 namespace doris {
 
 Status HdfsReaderWriter::create_reader(const THdfsParams& hdfs_params, const std::string& path,
                                        int64_t start_offset, FileReader** reader) {
-#if defined(__x86_64__)
     *reader = new HdfsFileReader(hdfs_params, path, start_offset);
     return Status::OK();
-#else
-    return Status::InternalError("HdfsFileReader do not support on non x86 platform");
-#endif
 }
 
 Status HdfsReaderWriter::create_writer(std::map<std::string, std::string>& properties,
                                        const std::string& path,
                                        std::unique_ptr<FileWriter>& writer) {
-#if defined(__x86_64__)
     writer.reset(new HDFSWriter(properties, path));
     return Status::OK();
-#else
-    return Status::InternalError("HdfsWriter do not support on non x86 platform");
-#endif
 }
 
 } // namespace doris

be/src/io/hdfs_reader_writer.h

@@ -17,9 +17,9 @@
 
 #pragma once
 
-#include "file_reader.h"
-#include "file_writer.h"
 #include "gen_cpp/PlanNodes_types.h"
+#include "io/file_reader.h"
+#include "io/file_writer.h"
 
 namespace doris {
 

be/src/io/hdfs_writer.cpp

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include "hdfs_writer.h"
+#include "io/hdfs_writer.h"
 
 #include <filesystem>