[Feature] Use mariadb-connector api access mysql-external table inste…

…ad of mysql-client (#215)
StarRocks · Sep 14, 2021 · 6cfb84c · 6cfb84c
1 parent 7360603
commit 6cfb84c
Show file tree

Hide file tree

Showing 16 changed files with 50 additions and 176 deletions.
diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt
@@ -68,7 +68,6 @@ else()
 endif()
 message(STATUS "make test: ${MAKE_TEST}")
 
-option(WITH_MYSQL "Support access MySQL" ON)
 option(WITH_GCOV "Build binary with gcov to get code coverage" OFF)
 
 # Check gcc
@@ -178,10 +177,8 @@ set_target_properties(thrift PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/
 add_library(thriftnb STATIC IMPORTED)
 set_target_properties(thriftnb PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libthriftnb.a)
 
-if (WITH_MYSQL)
-    add_library(mysql STATIC IMPORTED)
-    set_target_properties(mysql PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/libmysqlclient.a)
-endif()
+add_library(mysql STATIC IMPORTED)
+set_target_properties(mysql PROPERTIES IMPORTED_LOCATION ${THIRDPARTY_DIR}/lib/mariadb/libmariadbclient.a)
 
 if (WITH_HDFS)
     add_library(hdfs STATIC IMPORTED)
@@ -317,10 +314,6 @@ if (WITH_HDFS)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DSTARROCKS_WITH_HDFS")
 endif()
 
-if (WITH_MYSQL)
-    set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -DSTARROCKS_WITH_MYSQL")
-endif()
-
 if (CMAKE_CXX_COMPILER_VERSION VERSION_GREATER 7.0)
     set(CXX_COMMON_FLAGS "${CXX_COMMON_FLAGS} -faligned-new")
 endif()
@@ -438,6 +431,17 @@ set(STARROCKS_LINK_LIBS
     ${WL_END_GROUP}
 )
 
+set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
+    mysql
+)
+
+if (WITH_HDFS)
+    set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
+        hdfs
+        jvm
+        )
+endif()
+
 # Set thirdparty libraries
 set(STARROCKS_DEPENDENCIES
     ${STARROCKS_DEPENDENCIES}
@@ -483,19 +487,6 @@ set(STARROCKS_DEPENDENCIES
     ${WL_END_GROUP}
 )
 
-if (WITH_MYSQL)
-    set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
-        mysql
-        )
-endif()
-
-if (WITH_HDFS)
-    set(STARROCKS_DEPENDENCIES ${STARROCKS_DEPENDENCIES}
-        hdfs
-        jvm
-        )
-endif()
-
 # Add all external dependencies. They should come after the starrocks libs.
 # static link gcc's lib
 set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS}
@@ -504,14 +495,8 @@ set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS}
     -static-libgcc
 )
 
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "BCC")
-    set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS}
-        -Wl,--dynamic-linker=/lib64/ld-linux-x86-64.so.2
-    )
-endif()
-
 # Add sanitize static link flags or tcmalloc
-if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE" OR "${CMAKE_BUILD_TYPE}" STREQUAL "BCC")
+if ("${CMAKE_BUILD_TYPE}" STREQUAL "DEBUG" OR "${CMAKE_BUILD_TYPE}" STREQUAL "RELEASE")
     set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} tcmalloc)
 elseif ("${CMAKE_BUILD_TYPE}" STREQUAL "ASAN")
     set(STARROCKS_LINK_LIBS ${STARROCKS_LINK_LIBS} -static-libasan)

diff --git a/be/src/exec/CMakeLists.txt b/be/src/exec/CMakeLists.txt
@@ -187,13 +187,11 @@ set(EXEC_FILES
     pipeline/aggregate/aggregate_distinct_streaming_source_operator.cpp
 )
 
-if (WITH_MYSQL)
-    set(EXEC_FILES
-        ${EXEC_FILES}
-        mysql_scan_node.cpp
-        mysql_scanner.cpp
-        )
-endif()
+set(EXEC_FILES
+    ${EXEC_FILES}
+    mysql_scan_node.cpp
+    mysql_scanner.cpp
+)
 
 IF (WITH_HDFS)
     set(EXEC_FILES ${EXEC_FILES}

diff --git a/be/src/exec/data_sink.cpp b/be/src/exec/data_sink.cpp
@@ -77,7 +77,6 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink
         sink->reset(tmp_sink);
         break;
     case TDataSinkType::MYSQL_TABLE_SINK: {
-#ifdef STARROCKS_WITH_MYSQL
         if (!thrift_sink.__isset.mysql_table_sink) {
             return Status::InternalError("Missing data buffer sink.");
         }
@@ -86,10 +85,6 @@ Status DataSink::create_data_sink(ObjectPool* pool, const TDataSink& thrift_sink
         MysqlTableSink* mysql_tbl_sink = new MysqlTableSink(pool, row_desc, output_exprs);
         sink->reset(mysql_tbl_sink);
         break;
-#else
-        return Status::InternalError(
-                "Don't support MySQL table, you should rebuild StarRocks with WITH_MYSQL option ON");
-#endif
     }
 
     case TDataSinkType::EXPORT_SINK: {

diff --git a/be/src/exec/exec_node.cpp b/be/src/exec/exec_node.cpp
@@ -546,13 +546,8 @@ Status ExecNode::create_vectorized_node(starrocks::RuntimeState* state, starrock
         return Status::InternalError("Don't support HDFS table, you should rebuild StarRocks with WITH_HDFS option ON");
 #endif
     case TPlanNodeType::MYSQL_SCAN_NODE:
-#ifdef STARROCKS_WITH_MYSQL
         *node = pool->add(new vectorized::MysqlScanNode(pool, tnode, descs));
         return Status::OK();
-#else
-        return Status::InternalError(
-                "Don't support MySQL table, you should rebuild StarRocks with WITH_MYSQL option ON");
-#endif
     case TPlanNodeType::ES_HTTP_SCAN_NODE:
         *node = pool->add(new vectorized::EsHttpScanNode(pool, tnode, descs));
         return Status::OK();
@@ -576,14 +571,8 @@ Status ExecNode::create_node(RuntimeState* state, ObjectPool* pool, const TPlanN
 
     switch (tnode.node_type) {
     case TPlanNodeType::MYSQL_SCAN_NODE:
-#ifdef STARROCKS_WITH_MYSQL
         *node = pool->add(new MysqlScanNode(pool, tnode, descs));
         return Status::OK();
-#else
-        return Status::InternalError(
-                "Don't support MySQL table, you should rebuild StarRocks with WITH_MYSQL option ON");
-#endif
-
     case TPlanNodeType::ES_HTTP_SCAN_NODE:
         *node = pool->add(new EsHttpScanNode(pool, tnode, descs));
         return Status::OK();

diff --git a/be/src/exec/mysql_scanner.cpp b/be/src/exec/mysql_scanner.cpp
@@ -19,7 +19,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <mysql/mysql.h>
+#include <mariadb/mysql.h>
 
 #define __StarRocksMysql MYSQL
 #define __StarRocksMysqlRes MYSQL_RES

diff --git a/be/src/runtime/CMakeLists.txt b/be/src/runtime/CMakeLists.txt
@@ -113,12 +113,10 @@ set(RUNTIME_FILES
     runtime_filter_worker.cpp
 )
 
-if (WITH_MYSQL)
-    set(RUNTIME_FILES ${RUNTIME_FILES}
-        mysql_table_writer.cpp
-        mysql_table_sink.cpp
-        )
-endif()
+set(RUNTIME_FILES ${RUNTIME_FILES}
+    mysql_table_writer.cpp
+    mysql_table_sink.cpp
+)
 
 add_library(Runtime STATIC
     ${RUNTIME_FILES}

diff --git a/be/src/runtime/mysql_table_writer.cpp b/be/src/runtime/mysql_table_writer.cpp
@@ -19,7 +19,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <mysql/mysql.h>
+#include <mariadb/mysql.h>
 
 #define __StarRocksMysql MYSQL
 #include <sstream>

diff --git a/be/src/util/CMakeLists.txt b/be/src/util/CMakeLists.txt
@@ -105,11 +105,9 @@ set(UTIL_FILES
   hdfs_util.cpp
 )
 
-if (WITH_MYSQL)
-    set(UTIL_FILES ${UTIL_FILES}
-        mysql_load_error_hub.cpp
-        )
-endif()
+set(UTIL_FILES ${UTIL_FILES}
+    mysql_load_error_hub.cpp
+)
 
 add_library(Util STATIC
     ${UTIL_FILES}

diff --git a/be/src/util/load_error_hub.cpp b/be/src/util/load_error_hub.cpp
@@ -45,15 +45,9 @@ Status LoadErrorHub::create_hub(ExecEnv* env, const TLoadErrorHubInfo* t_hub_inf
 
     switch (t_hub_info->type) {
     case TErrorHubType::MYSQL:
-#ifdef STARROCKS_WITH_MYSQL
         tmp_hub = new MysqlLoadErrorHub(t_hub_info->mysql_info);
         tmp_hub->prepare();
         hub->reset(tmp_hub);
-        break;
-#else
-        return Status::InternalError(
-                "Don't support MySQL table, you should rebuild StarRocks with WITH_MYSQL option ON");
-#endif
     case TErrorHubType::BROKER: {
         // the origin file name may contains __shard_0/xxx
         // replace the '/' with '_'

diff --git a/be/src/util/mysql_load_error_hub.cpp b/be/src/util/mysql_load_error_hub.cpp
@@ -19,7 +19,7 @@
 // specific language governing permissions and limitations
 // under the License.
 
-#include <mysql/mysql.h>
+#include <mariadb/mysql.h>
 
 #define __StarRocksMysql MYSQL
 #include "mysql_load_error_hub.h"

diff --git a/build.sh b/build.sh
@@ -36,7 +36,7 @@ export STARROCKS_HOME=${ROOT}
 . ${STARROCKS_HOME}/env.sh
 
 #build thirdparty libraries if necessary
-if [[ ! -f ${STARROCKS_THIRDPARTY}/installed/lib/libhs.a ]]; then
+if [[ ! -f ${STARROCKS_THIRDPARTY}/installed/lib/mariadb/libmariadbclient.a ]]; then
     echo "Thirdparty libraries need to be build ..."
     ${STARROCKS_THIRDPARTY}/build-thirdparty.sh
 fi
@@ -54,8 +54,6 @@ Usage: $0 <options>
      --clean            clean and build target
      --with-gcov        build Backend with gcov, has an impact on performance
      --without-gcov     build Backend without gcov(default)
-     --with-mysql       enable MySQL support(default)
-     --without-mysql    disable MySQL support
      --with-lzo         enable LZO compress support(default)
      --without-lzo      disable LZO compress  support
      --with-hdfs        enable hdfs support
@@ -64,8 +62,7 @@ Usage: $0 <options>
   Eg.
     $0                                      build all
     $0 --be                                 build Backend without clean
-    $0 --be --without-mysql                 build Backend with MySQL disable
-    $0 --be --without-mysql --without-lzo   build Backend with both MySQL and LZO disable
+    $0 --be --without-lzo                   build Backend with LZO disable
     $0 --fe --clean                         clean and build Frontend and Spark Dpp application
     $0 --fe --be --clean                    clean and build Frontend, Spark Dpp application and Backend
     $0 --spark-dpp                          build Spark DPP application alone
@@ -83,8 +80,6 @@ OPTS=$(getopt \
   -l 'clean' \
   -l 'with-gcov' \
   -l 'without-gcov' \
-  -l 'with-mysql' \
-  -l 'without-mysql' \
   -l 'with-lzo' \
   -l 'without-lzo' \
   -l 'with-hdfs' \
@@ -103,7 +98,6 @@ BUILD_FE=
 BUILD_SPARK_DPP=
 CLEAN=
 RUN_UT=
-WITH_MYSQL=ON
 WITH_LZO=ON
 WITH_GCOV=OFF
 WITH_HDFS=ON
@@ -130,8 +124,6 @@ else
             --spark-dpp) BUILD_SPARK_DPP=1 ; shift ;;
             --clean) CLEAN=1 ; shift ;;
             --ut) RUN_UT=1   ; shift ;;
-            --with-mysql) WITH_MYSQL=ON; shift ;;
-            --without-mysql) WITH_MYSQL=OFF; shift ;;
             --with-gcov) WITH_GCOV=ON; shift ;;
             --without-gcov) WITH_GCOV=OFF; shift ;;
             --with-lzo) WITH_LZO=ON; shift ;;
@@ -162,7 +154,6 @@ echo "Get params:
     BUILD_SPARK_DPP     -- $BUILD_SPARK_DPP
     CLEAN               -- $CLEAN
     RUN_UT              -- $RUN_UT
-    WITH_MYSQL          -- $WITH_MYSQL
     WITH_LZO            -- $WITH_LZO
     WITH_GCOV           -- $WITH_GCOV
     WITH_HDFS           -- $WITH_HDFS
@@ -198,7 +189,7 @@ if [ ${BUILD_BE} -eq 1 ] ; then
     fi
     mkdir -p ${CMAKE_BUILD_DIR}
     cd ${CMAKE_BUILD_DIR}
-    ${CMAKE_CMD} .. -DSTARROCKS_THIRDPARTY=${STARROCKS_THIRDPARTY} -DSTARROCKS_HOME=${STARROCKS_HOME} -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DMAKE_TEST=OFF -DWITH_MYSQL=${WITH_MYSQL} -DWITH_HDFS=${WITH_HDFS} -DWITH_LZO=${WITH_LZO} -DWITH_GCOV=${WITH_GCOV} \
+    ${CMAKE_CMD} .. -DSTARROCKS_THIRDPARTY=${STARROCKS_THIRDPARTY} -DSTARROCKS_HOME=${STARROCKS_HOME} -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DMAKE_TEST=OFF -DWITH_HDFS=${WITH_HDFS} -DWITH_LZO=${WITH_LZO} -DWITH_GCOV=${WITH_GCOV} \
         -DWITH_BENCHMARK=${WITH_BENCHMARK}
     time make -j${PARALLEL}
     make install

diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh
@@ -436,38 +436,6 @@ build_boost() {
     ./b2 link=static runtime-link=static -j $PARALLEL --without-mpi --without-graph --without-graph_parallel --without-python cxxflags="-std=c++11 -g -fPIC -I$TP_INCLUDE_DIR -L$TP_LIB_DIR" install
 }
 
-# mysql
-build_mysql() {
-    check_if_source_exist $MYSQL_SOURCE
-    check_if_source_exist $BOOST_SOURCE
-
-    cd $TP_SOURCE_DIR/$MYSQL_SOURCE
-
-    mkdir -p $BUILD_DIR && cd $BUILD_DIR
-    rm -rf CMakeCache.txt CMakeFiles/
-    if [ ! -d $BOOST_SOURCE ]; then
-        cp -rf $TP_SOURCE_DIR/$BOOST_SOURCE ./
-    fi
-
-    $CMAKE_CMD ../ -DWITH_BOOST=`pwd`/$BOOST_SOURCE -DCMAKE_INSTALL_PREFIX=$TP_INSTALL_DIR/mysql/ \
-    -DCMAKE_INCLUDE_PATH=$TP_INCLUDE_DIR -DCMAKE_LIBRARY_PATH=$TP_LIB_DIR -DWITHOUT_SERVER=1 \
-    -DCMAKE_CXX_FLAGS_RELWITHDEBINFO="-O3 -g -fabi-version=2 -fno-omit-frame-pointer -fno-strict-aliasing -std=gnu++11" \
-    -DDISABLE_SHARED=1 -DBUILD_SHARED_LIBS=0
-    make -j$PARALLEL mysqlclient
-
-    # copy headers manually
-    rm -rf ../../../installed/include/mysql/
-    mkdir ../../../installed/include/mysql/ -p
-    cp -R ./include/* ../../../installed/include/mysql/
-    cp -R ../include/* ../../../installed/include/mysql/
-    cp ../libbinlogevents/export/binary_log_types.h ../../../installed/include/mysql/
-    echo "mysql headers are installed."
-
-    # copy libmysqlclient.a
-    cp libmysql/libmysqlclient.a ../../../installed/lib/
-    echo "mysql client lib is installed."
-}
-
 #leveldb
 build_leveldb() {
     check_if_source_exist $LEVELDB_SOURCE
@@ -767,6 +735,15 @@ build_hyperscan() {
     make -j$PARALLEL && make install
 }
 
+#mariadb-connector-c
+build_mariadb() {
+    check_if_source_exist $MARIADB_SOURCE
+    cd $TP_SOURCE_DIR/$MARIADB_SOURCE
+    mkdir -p build && cd build
+    cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR}
+    make -j$PARALLEL && make install
+}
+
 build_libevent
 build_zlib
 build_lz4
@@ -783,7 +760,6 @@ build_snappy
 build_gperftools
 build_curl
 build_re2
-build_mysql
 build_thrift
 build_leveldb
 build_brpc
@@ -803,5 +779,6 @@ build_hadoop
 build_jdk
 build_ragel
 build_hyperscan
+build_mariadb
 echo "Finihsed to build all thirdparties"
 
diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh
@@ -241,16 +241,6 @@ fi
 cd -
 echo "Finished patching $RE2_SOURCE"
 
-# mysql patch
-cd $TP_SOURCE_DIR/$MYSQL_SOURCE
-if [ ! -f $PATCHED_MARK ]; then
-    patch -p0 < $TP_PATCH_DIR/mysql-5.7.18.patch
-	patch -p0 < $TP_PATCH_DIR/mysql-5.7.18-boost.patch
-    touch $PATCHED_MARK
-fi
-cd -
-echo "Finished patching $MYSQL_SOURCE"
-
 # libevent patch
 cd $TP_SOURCE_DIR/$LIBEVENT_SOURCE
 if [ ! -f $PATCHED_MARK ]; then