[enhancement](macOS M1) Support building from source on macOS (M1) (#…

…13195)

# Proposed changes

This PR fixed lots of issues when building from source on macOS with Apple M1 chip.

## ATTENTION

The job for supporting macOS with Apple M1 chip is too big and there are lots of unresolved issues during runtime:
1. Some errors with memory tracker occur when BE (RELEASE) starts.
2. Some UT cases fail.
...

Temporarily, the following changes are made on macOS to start BE successfully.
1. Disable memory tracker.
2. Use tcmalloc instead of jemalloc.

This PR kicks off the job. Guys who are interested in this job can continue to fix these runtime issues.

## Use case

```shell
./build.sh -j 8 --be --clean

cd output/be/bin
ulimit -n 60000
./start_be.sh --daemon
```

## Something else

It takes around _**10+**_ minutes to build BE (with prebuilt third-parties) on macOS with M1 chip. We will improve the  development experience on macOS greatly when we finish the adaptation job.

be/CMakeLists.txt

@@ -26,7 +26,7 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 if (CMAKE_SYSTEM_PROCESSOR MATCHES "amd64|x86_64")
     set (ARCH_AMD64 1)
 endif ()
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*)")
+if (CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64.*|AARCH64.*|arm64.*)")
     set (ARCH_AARCH64 1)
 endif ()
 if (ARCH_AARCH64 OR CMAKE_SYSTEM_PROCESSOR MATCHES "arm")
@@ -58,8 +58,12 @@ elseif (CMAKE_CXX_COMPILER_ID MATCHES "Clang")
 endif ()
 
 # Set boost/stacktrace use backtrace api to unwind
-add_definitions(-DBOOST_STACKTRACE_USE_BACKTRACE)
-#add_definitions(-DPRINT_ALL_ERR_STATUS_STACKTRACE)
+if (NOT OS_MACOSX)
+    add_definitions(-DBOOST_STACKTRACE_USE_BACKTRACE)
+    #add_definitions(-DPRINT_ALL_ERR_STATUS_STACKTRACE)
+else()
+    add_definitions(-DBOOST_STACKTRACE_USE_NOOP)
+endif()
 
 option(GLIBC_COMPATIBILITY "Enable compatibility with older glibc libraries." ON)
 message(STATUS "GLIBC_COMPATIBILITY is ${GLIBC_COMPATIBILITY}")
@@ -90,11 +94,7 @@ set(SRC_DIR "${BASE_DIR}/src/")
 set(TEST_DIR "${CMAKE_SOURCE_DIR}/test/")
 set(OUTPUT_DIR "${BASE_DIR}/output")
 
-if (APPLE)
-    set(MAKE_TEST "ON")
-else()
-    option(MAKE_TEST "ON for make unit test or OFF for not" OFF)
-endif()
+option(MAKE_TEST "ON for make unit test or OFF for not" OFF)
 message(STATUS "make test: ${MAKE_TEST}")
 option(WITH_MYSQL "Support access MySQL" ON)
 
@@ -122,11 +122,13 @@ set(Boost_DEBUG FALSE)
 set(Boost_USE_MULTITHREADED ON)
 set(BOOST_ROOT ${THIRDPARTY_DIR})
 set(Boost_NO_BOOST_CMAKE OFF)
+set(BOOST_VERSION "1.73.0")
 
 if (NOT APPLE)
-    find_package(Boost 1.73.0 REQUIRED COMPONENTS system date_time)
+    find_package(Boost ${BOOST_VERSION} REQUIRED COMPONENTS system date_time)
 else()
-    find_package(Boost 1.73.0 COMPONENTS system date_time)
+    find_package(Boost ${BOOST_VERSION} COMPONENTS system date_time)
+    find_package(Boost ${BOOST_VERSION} COMPONENTS system container)
 endif()
 
 set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
@@ -325,8 +327,10 @@ set_target_properties(aws-checksums PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DI
 add_library(aws-c-s3 STATIC IMPORTED)
 set_target_properties(aws-c-s3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libaws-c-s3.a)
 
-add_library(aws-s2n STATIC IMPORTED)
-set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libs2n.a)
+if (NOT OS_MACOSX)
+    add_library(aws-s2n STATIC IMPORTED)
+    set_target_properties(aws-s2n PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libs2n.a)
+endif()
 
 add_library(minizip STATIC IMPORTED)
 set_target_properties(minizip PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib64/libminizip.a)
@@ -399,6 +403,17 @@ set_target_properties(hdfs3 PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/l
 
 find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)
 
+if (OS_MACOSX)
+    add_library(bfd STATIC IMPORTED)
+    set_target_properties(bfd PROPERTIES IMPORTED_LOCATION "${THIRDPARTY_DIR}/lib/libbfd.a")
+
+    add_library(iberty STATIC IMPORTED)
+    set_target_properties(iberty PROPERTIES IMPORTED_LOCATION "${THIRDPARTY_DIR}/lib/libiberty.a")
+
+    add_library(intl STATIC IMPORTED)
+    set_target_properties(intl PROPERTIES IMPORTED_LOCATION "${THIRDPARTY_DIR}/lib/libintl.a")
+endif()
+
 # Check if functions are supported in this platform. All flags will generated
 # in gensrc/build/common/env_config.h.
 # You can check funcion here which depends on platform. Don't forget add this
@@ -447,7 +462,10 @@ if (NOT CUSTUM_LINKER_COMMAND STREQUAL "ld")
 endif()
 
 if (USE_LIBCPP AND COMPILER_CLANG)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++ -lstdc++")
+    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -stdlib=libc++")
+    if (NOT OS_MACOSX)
+        set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -lstdc++")
+    endif()
     add_definitions(-DUSE_LIBCPP)
 endif()
 
@@ -494,7 +512,7 @@ if (USE_MEM_TRACKER)
 endif()
 
 # Compile with jemalloc.
-# Adding the option `USE_JEMALLOC=ON sh build.sh` when compiling can turn on building with jemalloc 
+# Adding the option `USE_JEMALLOC=ON sh build.sh` when compiling can turn on building with jemalloc
 if (USE_JEMALLOC)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DUSE_JEMALLOC")
 endif()
@@ -522,7 +540,12 @@ if (USE_DWARF)
 endif()
 
 # For CMAKE_BUILD_TYPE=Debug
-set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -O0")
+if (OS_MACOSX AND ARCH_ARM)
+    # Using -O0 may meet ARM64 branch out of range errors when linking with tcmalloc.
+    set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -Og")
+else()
+    set(CXX_FLAGS_DEBUG "${CXX_GCC_FLAGS} -O0")
+endif()
 
 # For CMAKE_BUILD_TYPE=Release
 #   -O3: Enable all compiler optimizations
@@ -597,9 +620,10 @@ if (BUILD_JAVA_UDF)
     endif()
 endif()
 
-set(WL_START_GROUP "-Wl,--start-group")
-set(WL_END_GROUP "-Wl,--end-group")
-
+if (NOT OS_MACOSX)
+    set(WL_START_GROUP "-Wl,--start-group")
+    set(WL_END_GROUP "-Wl,--end-group")
+endif()
 
 set(KRB5_LIBS
     krb5support
@@ -626,6 +650,10 @@ set(AWS_LIBS
     aws-c-mqtt
     aws-sdk-s3-crt)
 
+if (OS_MACOSX)
+    list(REMOVE_ITEM AWS_LIBS aws-s2n)
+endif()
+
 # Set Doris libraries
 set(DORIS_LINK_LIBS
     ${WL_START_GROUP}
@@ -649,7 +677,6 @@ set(DORIS_LINK_LIBS
     ${WL_END_GROUP}
 )
 
-
 # COMMON_THIRDPARTY are thirdparty dependencies that can run on all platform
 # When adding new dependencies, If you don’t know if it can run on all platforms,
 # add it here first.
@@ -716,6 +743,16 @@ set(COMMON_THIRDPARTY
     simdjson
 )
 
+if (OS_MACOSX)
+    set(COMMON_THIRDPARTY
+        ${COMMON_THIRDPARTY}
+        Boost::container
+        bfd
+        iberty
+        intl
+    )
+endif()
+
 if (${MAKE_TEST} STREQUAL "ON")
     set(COMMON_THIRDPARTY
         ${COMMON_THIRDPARTY}
@@ -731,15 +768,11 @@ set(DORIS_DEPENDENCIES
 )
 
 if(WITH_LZO)
-    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-        lzo
-    )
+    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} lzo)
 endif()
 
 if (WITH_MYSQL)
-    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES}
-        mysql
-        )
+    set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} mysql)
 endif()
 
 set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_END_GROUP})
@@ -748,13 +781,23 @@ message(STATUS "DORIS_DEPENDENCIES is ${DORIS_DEPENDENCIES}")
 
 # Add all external dependencies. They should come after the palo libs.
 # static link gcc's lib
-set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
-    ${DORIS_DEPENDENCIES}
-    -static-libstdc++
-    -static-libgcc
-    -lstdc++fs
-    -lresolv
-)
+if (NOT OS_MACOSX)
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
+        ${DORIS_DEPENDENCIES}
+        -static-libstdc++
+        -static-libgcc
+        -lstdc++fs
+        -lresolv
+    )
+else()
+    set(DORIS_LINK_LIBS
+        ${DORIS_LINK_LIBS}
+        ${DORIS_DEPENDENCIES}
+        -lapple_nghttp2
+        -lresolv
+        -liconv
+    )
+endif()
 
 if (USE_JEMALLOC)
     set(MALLOCLIB jemalloc)
@@ -793,9 +836,26 @@ if (GLIBC_COMPATIBILITY)
     set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS} glibc-compatibility-explicit glibc-compatibility)
 endif()
 
-set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
-    -lrt -l:libbfd.a -liberty -lc -lm -ldl -pthread
-)
+if (NOT OS_MACOSX)
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
+        -lrt -l:libbfd.a -liberty -lc -lm -ldl -pthread
+    )
+else()
+    set(DORIS_LINK_LIBS ${DORIS_LINK_LIBS}
+        "-framework CoreFoundation"
+        "-framework CoreGraphics"
+        "-framework CoreText"
+        "-framework Foundation"
+        "-framework SystemConfiguration"
+        "-framework Security"
+    )
+    if (USE_JEMALLOC OR (NOT CMAKE_BUILD_TYPE STREQUAL "DEBUG" AND NOT CMAKE_BUILD_TYPE STREQUAL "RELEASE"))
+        set(DORIS_LINK_LIBS
+            ${DORIS_LINK_LIBS}
+            "-Wl,-U,_MallocExtension_ReleaseFreeMemory"
+        )
+    endif()
+endif()
 
 # Set libraries for test
 set (TEST_LINK_LIBS ${DORIS_LINK_LIBS}
@@ -805,12 +865,21 @@ set (TEST_LINK_LIBS ${DORIS_LINK_LIBS}
     ${WL_END_GROUP}
 )
 
+if (NOT MAKE_TEST)
+    message(STATUS "Link Flags: ${DORIS_LINK_LIBS}")
+else()
+    message(STATUS "Link Flags: ${TEST_LINK_LIBS}")
+endif()
+
 # Only build static libs
 set(BUILD_SHARED_LIBS OFF)
 
 if (${MAKE_TEST} STREQUAL "ON")
     SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -fprofile-arcs -ftest-coverage -DGTEST_USE_OWN_TR1_TUPLE=0")
-    SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage -lgcov")
+    SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -fprofile-arcs -ftest-coverage")
+    if (NOT OS_MACOSX)
+        SET(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} -lgcov")
+    endif()
     add_definitions(-DBE_TEST)
 endif ()
 

be/src/agent/CMakeLists.txt

@@ -20,13 +20,20 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/agent")
 
 # where to put generated binaries
 set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/agent")
-  
-add_library(Agent STATIC
+
+set(AGENT_SOURCES
     agent_server.cpp
     heartbeat_server.cpp
     task_worker_pool.cpp
     utils.cpp
     cgroups_mgr.cpp
     topic_subscriber.cpp
     user_resource_listener.cpp
-)
+)
+
+if (OS_MACOSX)
+    list(REMOVE_ITEM AGENT_SOURCES cgroups_mgr.cpp user_resource_listener.cpp)
+    list(APPEND AGENT_SOURCES cgroups_mgr_mac.cpp)
+endif()
+
+add_library(Agent STATIC ${AGENT_SOURCES})

be/src/agent/agent_server.cpp

@@ -94,7 +94,7 @@ AgentServer::AgentServer(ExecEnv* exec_env, const TMasterInfo& master_info)
 #undef CREATE_AND_START_POOL
 #undef CREATE_AND_START_THREAD
 
-#ifndef BE_TEST
+#if !defined(BE_TEST) && !defined(__APPLE__)
     // Add subscriber here and register listeners
     TopicListener* user_resource_listener = new UserResourceListener(exec_env, master_info);
     LOG(INFO) << "Register user resource listener";

be/src/agent/cgroups_mgr_mac.cpp

@@ -0,0 +1,37 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "agent/cgroups_mgr.h"
+
+namespace doris {
+
+const std::string CgroupsMgr::_s_system_user = "system";
+const std::string CgroupsMgr::_s_system_group = "normal";
+
+std::map<TResourceType::type, std::string> CgroupsMgr::_s_resource_cgroups;
+
+CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const std::string& root_cgroups_path) {}
+
+CgroupsMgr::~CgroupsMgr() = default;
+
+Status CgroupsMgr::init_cgroups() {
+    return Status::OK();
+}
+
+void CgroupsMgr::apply_cgroup(const std::string& user_name, const std::string& level) {}
+
+} // namespace doris

be/src/env/env_posix.cpp

@@ -82,9 +82,15 @@ static Status io_error(const std::string& context, int err_number) {
 }
 
 static Status do_sync(int fd, const string& filename) {
+#ifdef __APPLE__
+    if (fcntl(fd, F_FULLFSYNC) < 0) {
+        return io_error(filename, errno);
+    }
+#else
     if (fdatasync(fd) < 0) {
         return io_error(filename, errno);
     }
+#endif
     return Status::OK();
 }
 
@@ -306,6 +312,9 @@ class PosixWritableFile : public WritableFile {
     }
 
     Status pre_allocate(uint64_t size) override {
+#ifdef __APPLE__
+        return io_error(_filename, ENOSYS);
+#else
         uint64_t offset = std::max(_filesize, _pre_allocated_size);
         int ret;
         RETRY_ON_EINTR(ret, fallocate(_fd, 0, offset, size));
@@ -320,6 +329,7 @@ class PosixWritableFile : public WritableFile {
         }
         _pre_allocated_size = offset + size;
         return Status::OK();
+#endif
     }
 
     Status close() override {

be/src/exec/analytic_eval_node.cpp

@@ -426,15 +426,16 @@ inline void AnalyticEvalNode::try_remove_rows_before_window(int64_t stream_idx)
     // The start of the window may have been before the current partition, in which case
     // there is no tuple to remove in _window_tuples. Check the index of the row at which
     // tuples from _window_tuples should begin to be removed.
-    int64_t remove_idx = stream_idx - _rows_end_offset + std::min(_rows_start_offset, 0L) - 1;
+    int64_t remove_idx =
+            stream_idx - _rows_end_offset + std::min<int64_t>(_rows_start_offset, 0L) - 1;
 
     if (remove_idx < _curr_partition_idx) {
         return;
     }
 
     VLOG_ROW << id() << " Remove idx=" << remove_idx << " stream_idx=" << stream_idx;
     DCHECK(!_window_tuples.empty()) << debug_state_string(true);
-    DCHECK_EQ(remove_idx + std::max(_rows_start_offset, 0L), _window_tuples.front().first)
+    DCHECK_EQ(remove_idx + std::max<int64_t>(_rows_start_offset, 0L), _window_tuples.front().first)
             << debug_state_string(true);
     TupleRow* remove_row = reinterpret_cast<TupleRow*>(&_window_tuples.front().second);
     AggFnEvaluator::remove(_evaluators, _fn_ctxs, remove_row, _curr_tuple);