[Feature] Add native format writer to access StarRocks data bypass BE…

… Server.

Signed-off-by: plotor <zhenchao.wang@hotmail.com>

be/CMakeLists.txt

@@ -859,7 +859,7 @@ set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
 set_target_properties(aws-cpp-sdk-core PROPERTIES INTERFACE_LINK_LIBRARIES AWS::aws-crt-cpp)
 
 if (STARROCKS_JIT_ENABLE)
-    set(STARROCKS_DEPENDENCIES 
+    set(STARROCKS_DEPENDENCIES
         ${STARROCKS_DEPENDENCIES}
         ${WL_START_GROUP}
         ${LLVM_LIBRARIES}
@@ -969,7 +969,6 @@ endif()
 # Add all external dependencies. They should come after the starrocks libs.
 
 set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS}
-    ${STARROCKS_LINK_LIBS}
     ${STARROCKS_DEPENDENCIES}
     hdfs
     jvm
@@ -979,33 +978,28 @@ set(BUILD_FOR_SANITIZE "OFF")
 # Add sanitize static link flags or jemalloc
 if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
     message("use jemalloc")
-    set(STARROCKS_MEMORY_DEPENDENCIES_LIBS jemalloc)
+    set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} jemalloc)
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ASAN")
     if (CMAKE_CXX_COMPILER_ID STREQUAL "Clang")
-        set(STARROCKS_MEMORY_DEPENDENCIES_LIBS -static-libsan)
+        set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} -static-libsan)
     else()
-        set(STARROCKS_MEMORY_DEPENDENCIES_LIBS -static-libasan)
+        set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} -static-libasan)
     endif()
     set(BUILD_FOR_SANITIZE "ON")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "LSAN")
-    set(STARROCKS_MEMORY_DEPENDENCIES_LIBS -static-liblsan)
-     set(BUILD_FOR_SANITIZE "ON")
+    set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} -static-liblsan)
+    set(BUILD_FOR_SANITIZE "ON")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "UBSAN")
     message("use jemalloc")
-    set(STARROCKS_MEMORY_DEPENDENCIES_LIBS -static-libubsan jemalloc)
+    set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} -static-libubsan jemalloc)
     set(BUILD_FOR_SANITIZE "ON")
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "TSAN")
-    set(STARROCKS_MEMORY_DEPENDENCIES_LIBS -static-libtsan)
+    set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} -static-libtsan)
     set(BUILD_FOR_SANITIZE "ON")
 else()
     message(FATAL_ERROR "Unknown build type: ${CMAKE_BUILD_TYPE}")
 endif()
 
-if (NOT BUILD_FORMAT_LIB)
-    # skip the STARROCKS_MEMORY_DEPENDENCIES_LIBS only when BUILD_FORMAT_LIB=ON
-    set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} ${STARROCKS_MEMORY_DEPENDENCIES_LIBS})
-endif()
-
 if (NOT ("${MAKE_TEST}" STREQUAL "ON" AND "${BUILD_FOR_SANITIZE}" STREQUAL "ON"))
     # In other words, turn to dynamic link when MAKE_TEST and BUILD_TYPE == *SAN
     # otherwise do static link gcc's lib

be/src/column/field.h

@@ -84,6 +84,7 @@ class Field {
               _name(rhs._name),
               _type(rhs._type),
               _sub_fields(rhs._sub_fields ? new std::vector<Field>(*rhs._sub_fields) : nullptr),
+              _length(rhs._length),
               _short_key_length(rhs._short_key_length),
               _flags(rhs._flags),
               _uid(rhs._uid) {}
@@ -95,6 +96,7 @@ class Field {
               _name(std::move(rhs._name)),
               _type(std::move(rhs._type)),
               _sub_fields(rhs._sub_fields),
+              _length(rhs._length),
               _short_key_length(rhs._short_key_length),
               _flags(rhs._flags),
               _uid(rhs._uid) {
@@ -108,6 +110,7 @@ class Field {
             _name = rhs._name;
             _type = rhs._type;
             _agg_method = rhs._agg_method;
+            _length = rhs._length;
             _agg_state_desc = rhs._agg_state_desc;
             _short_key_length = rhs._short_key_length;
             _flags = rhs._flags;
@@ -123,6 +126,7 @@ class Field {
             _name = std::move(rhs._name);
             _type = std::move(rhs._type);
             _agg_method = rhs._agg_method;
+            _length = rhs._length;
             _agg_state_desc = rhs._agg_state_desc;
             _short_key_length = rhs._short_key_length;
             _flags = rhs._flags;

be/src/exec/olap_scan_prepare.cpp

@@ -797,7 +797,9 @@ Status ChunkPredicateBuilder<E, Type>::normalize_predicate(const SlotDescriptor&
     RETURN_IF_ERROR((normalize_not_in_or_not_equal_predicate<SlotType, RangeValueType, Negative>(slot, range)));
     RETURN_IF_ERROR(normalize_is_null_predicate(slot));
     // Must handle join runtime filter last
-    RETURN_IF_ERROR((normalize_join_runtime_filter<SlotType, RangeValueType, Negative>(slot, range)));
+    if (nullptr != _opts.runtime_filters) {
+        RETURN_IF_ERROR((normalize_join_runtime_filter<SlotType, RangeValueType, Negative>(slot, range)));
+    }
 
     return Status::OK();
 }

be/src/exprs/function_context.cpp

@@ -38,7 +38,9 @@ FunctionContext* FunctionContext::create_context(RuntimeState* state, MemPool* p
     ctx->_mem_pool = pool;
     ctx->_return_type = return_type;
     ctx->_arg_types = arg_types;
+#if !defined(BUILD_FORMAT_LIB)
     ctx->_jvm_udaf_ctxs = std::make_unique<JavaUDAFContext>();
+#endif
     return ctx;
 }
 
@@ -52,7 +54,9 @@ FunctionContext* FunctionContext::create_context(RuntimeState* state, MemPool* p
     ctx->_mem_pool = pool;
     ctx->_return_type = return_type;
     ctx->_arg_types = arg_types;
+#if !defined(BUILD_FORMAT_LIB)
     ctx->_jvm_udaf_ctxs = std::make_unique<JavaUDAFContext>();
+#endif
     ctx->_is_distinct = is_distinct;
     ctx->_is_asc_order = is_asc_order;
     ctx->_nulls_first = nulls_first;

be/src/starrocks_format/CMakeLists.txt

@@ -49,12 +49,14 @@ set(STARROCKS_LIBS
 add_library(hdfs_so SHARED IMPORTED GLOBAL)
 set_target_properties(hdfs_so PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/hadoop/lib/native/libhdfs.so)
 
-set(STARROCKS_THIRDPARTY_DEPENDENCIES
+SET(STARROCKS_FORMAT_LIBS
+    ${STARROCKS_LIBS}
     ${STARROCKS_DEPENDENCIES}
-    ${STARROCKS_STD_DEPENDENCIES}
-    ${STARROCKS_MEMORY_DEPENDENCIES_LIBS}
-    ${STARROCKS_DYNAMIC_DEPENDENCIES_LIBS}
+    -static-libstdc++ -static-libgcc -lbfd
+    -lresolv -liberty -lc -lm -ldl -rdynamic -pthread -Wl,-wrap=__cxa_throw
+    hdfs_so
 )
+message(STATUS "STARROCKS_FORMAT_LIBS is ${STARROCKS_FORMAT_LIBS}")
 
 # only build starrocks_be when TEST is off
 if (NOT ${MAKE_TEST} STREQUAL "ON")
@@ -64,15 +66,7 @@ if (NOT ${MAKE_TEST} STREQUAL "ON")
         )
 
     # This permits libraries loaded by dlopen to link to the symbols in the program.
-    target_link_libraries(starrocks_format
-        # -Wl,--whole-archive
-         ${STARROCKS_LIBS}
-        #   -Wl,--no-whole-archive
-        ${STARROCKS_DEPENDENCIES}
-        ${STARROCKS_STD_DEPENDENCIES}
-        ${STARROCKS_DYNAMIC_DEPENDENCIES_LIBS}
-        hdfs_so
-    )
+    target_link_libraries(starrocks_format ${STARROCKS_FORMAT_LIBS})
 
     install(DIRECTORY DESTINATION ${OUTPUT_DIR}/format-lib/)
 

be/src/types/date_value.cpp

@@ -14,6 +14,7 @@
 
 #include "types/date_value.hpp"
 
+#include "date_value.h"
 #include "gutil/strings/substitute.h"
 #include "types/timestamp_value.h"
 
@@ -41,6 +42,14 @@ int32_t DateValue::to_date_literal() const {
     return year * 10000 + month * 100 + day;
 }
 
+// return milliseconds since UNIX epoch.
+int64_t DateValue::to_unixtime() const {
+    int64_t result = (int64_t)_julian * SECS_PER_DAY;
+    result -= timestamp::UNIX_EPOCH_SECONDS;
+    result *= 1000L;
+    return result;
+}
+
 void DateValue::from_date_literal(int64_t date_literal) {
     _julian = date::from_date_literal(date_literal);
 }

be/src/types/date_value.h

@@ -44,6 +44,8 @@ class DateValue {
 
     int32_t to_date_literal() const;
 
+    int64_t to_unixtime() const;
+
     void from_date_literal(int64_t date_literal);
 
     bool from_date_literal_with_check(int64_t date_literal);

be/src/types/timestamp_value.cpp

@@ -846,6 +846,13 @@ int64_t TimestampValue::to_unixtime() const {
     return result;
 }
 
+int64_t TimestampValue::to_unixtime(const cctz::time_zone& ctz) const {
+    int64_t offset = TimezoneUtils::to_utc_offset(ctz);
+    int64_t result = to_unixtime();
+    result -= offset * MILLIS_PER_SEC;
+    return result;
+}
+
 bool TimestampValue::from_unixtime(int64_t second, const std::string& timezone) {
     cctz::time_zone ctz;
     if (!TimezoneUtils::find_cctz_time_zone(timezone, ctz)) {

be/src/types/timestamp_value.h

@@ -121,6 +121,7 @@ class TimestampValue {
 
     int64_t to_unix_second() const;
     int64_t to_unixtime() const;
+    int64_t to_unixtime(const cctz::time_zone& ctz) const;
 
     bool from_unixtime(int64_t second, const std::string& timezone);
     void from_unixtime(int64_t second, const cctz::time_zone& ctz);

be/test/column/timestamp_value_test.cpp

@@ -111,13 +111,15 @@ TEST(TimestampValueTest, calculate) {
 
 TEST(TimestampValueTest, cast) {
     auto v = TimestampValue::create(2004, 2, 29, 23, 30, 30);
-    ASSERT_EQ("2004-02-29", ((DateValue)v).to_string());
+    DateValue date = (DateValue)v;
+    ASSERT_EQ("2004-02-29", date.to_string());
+    ASSERT_EQ(1078012800000, date.to_unixtime());
 }
 
 TEST(TimestampValueTest, unixTime) {
     auto v = TimestampValue::create(2004, 2, 29, 23, 30, 30);
-    std::cout << "unix time " << v.to_unixtime() << std::endl;
     ASSERT_EQ(1078097430000, v.to_unixtime());
+    ASSERT_EQ(1078097430000, v.to_unixtime(cctz::utc_time_zone()));
 }
 
 } // namespace starrocks

build.sh

@@ -294,7 +294,7 @@ if [ ${BUILD_BE} -eq 1 ] && [ ${BUILD_FORMAT_LIB} -eq 1 ]; then
     exit 1
 fi
 if [ ${BUILD_FORMAT_LIB} -eq 1 ]; then
-    echo "do not build java extendsions when build format-lib."
+    echo "do not build java extensions when build format-lib."
     BUILD_JAVA_EXT=OFF
 fi
 
@@ -373,9 +373,11 @@ if [ ${BUILD_BE} -eq 1 ] || [ ${BUILD_FORMAT_LIB} -eq 1 ] ; then
         echo "Error: cmake is not found"
         exit 1
     fi
+
     # When build starrocks format lib, USE_STAROS must be ON
     if [ ${BUILD_FORMAT_LIB} -eq 1 ] ; then
         USE_STAROS=ON
+        WITH_TENANN=OFF
     fi
 
     CMAKE_BUILD_TYPE=$BUILD_TYPE

format-sdk/.gitignore

@@ -0,0 +1,44 @@
+target/
+!.mvn/wrapper/maven-wrapper.jar
+!**/src/main/**/target/
+!**/src/test/**/target/
+checkstyle.xml
+checkstyle-header.txt
+
+### IntelliJ IDEA & CLion ###
+.idea/modules.xml
+.idea/jarRepositories.xml
+.idea/compiler.xml
+.idea/libraries/
+*.iws
+*.iml
+*.ipr
+**/.clion.source.upload.marker
+
+### Eclipse ###
+.apt_generated
+.classpath
+.factorypath
+.project
+.settings
+.springBeans
+.sts4-cache
+
+### NetBeans ###
+/nbproject/private/
+/nbbuild/
+/dist/
+/nbdist/
+/.nb-gradle/
+build/
+!**/src/main/**/build/
+!**/src/test/**/build/
+
+### VS Code ###
+.vscode/
+
+### Mac OS ###
+.DS_Store
+
+### C++ ###
+!CMakeLists.txt

format-sdk/README.md

@@ -0,0 +1 @@
+# StarRocks Format Library

format-sdk/build.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -ex
+
+BASE_DIR=`dirname "$0"`
+BASE_DIR=`cd "$BASE_DIR"; pwd`
+
+STARROCKS_HOME=${STARROCKS_HOME:-$BASE_DIR/..}
+
+if [ -z "${STARROCKS_THIRDPARTY}" ]; then
+  echo "ERR: Unknown STARROCKS_THIRDPARTY env."
+  exit 1
+fi
+
+BUILD_TYPE=RELEASE
+
+# build format-sdk
+cd $STARROCKS_HOME/format-sdk
+
+cmake -S src/main/cpp \
+      -B target/build-jni/${BUILD_TYPE} \
+      -DTARGET_NAME=starrocks_format_wrapper \
+      -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+      -DWITH_STARCACHE=ON
+
+cmake --build target/build-jni/${BUILD_TYPE} --config ${BUILD_TYPE}
+
+mkdir -p target/classes/native && cp target/build-jni/${BUILD_TYPE}/*.so target/classes/native