build: refactor cmake (#6875)

close #6201

.gitignore

@@ -60,21 +60,6 @@ __pycache__
 *-metrika-yandex
 
 test.cpp
-utils/compressor/compressor
-utils/corrector_utf8/corrector_utf8
-utils/iotest/iotest
-utils/iotest/iotest_aio
-utils/iotest/iotest_nonblock
-utils/config-processor/config-processor
-CPackConfig.cmake
-CPackSourceConfig.cmake
-contrib/libpoco/Poco/
-contrib/libpoco/bin/
-contrib/libpoco/cmake_uninstall.cmake
-contrib/libre2/re2_st/
-dbms/src/Client/clickhouse-benchmark
-dbms/src/Client/clickhouse-client
-dbms/src/Client/tests/test-connect
 dbms/src/Common/tests/arena_with_free_lists
 dbms/src/Common/tests/auto_array
 dbms/src/Common/tests/compact_array
@@ -94,26 +79,8 @@ dbms/src/Core/tests/exception
 dbms/src/Core/tests/field
 dbms/src/Core/tests/rvo_test
 dbms/src/Core/tests/string_pool
-dbms/src/DataStreams/tests/aggregating_stream
-dbms/src/DataStreams/tests/block_row_transforms
-dbms/src/DataStreams/tests/block_tab_separated_streams
-dbms/src/DataStreams/tests/collapsing_sorted_stream
-dbms/src/DataStreams/tests/expression_stream
-dbms/src/DataStreams/tests/filter_stream
-dbms/src/DataStreams/tests/filter_stream_hitlog
-dbms/src/DataStreams/tests/fork_streams
-dbms/src/DataStreams/tests/glue_streams
-dbms/src/DataStreams/tests/json_streams
-dbms/src/DataStreams/tests/native_streams
-dbms/src/DataStreams/tests/sorting_stream
-dbms/src/DataStreams/tests/tab_separated_streams
-dbms/src/DataStreams/tests/union_stream
-dbms/src/DataStreams/tests/union_stream2
 dbms/src/DataTypes/tests/data_type_string
 dbms/src/DataTypes/tests/data_types_number_fixed
-dbms/src/Functions/tests/functions_arithmetic
-dbms/src/Functions/tests/logical_functions_performance
-dbms/src/Functions/tests/number_traits
 dbms/src/IO/tests/async_write
 dbms/src/IO/tests/cached_compressed_read_buffer
 dbms/src/IO/tests/compressed_buffer
@@ -137,67 +104,10 @@ dbms/src/IO/tests/var_uint
 dbms/src/IO/tests/write_buffer
 dbms/src/IO/tests/write_buffer_aio
 dbms/src/IO/tests/write_buffer_perf
-dbms/src/Interpreters/tests/address_patterns
-dbms/src/Interpreters/tests/aggregate
-dbms/src/Interpreters/tests/compiler_test
-dbms/src/Interpreters/tests/create_query
-dbms/src/Interpreters/tests/expression
-dbms/src/Interpreters/tests/expression_analyzer
-dbms/src/Interpreters/tests/hash_map
-dbms/src/Interpreters/tests/hash_map2
-dbms/src/Interpreters/tests/hash_map3
-dbms/src/Interpreters/tests/hash_map_string
-dbms/src/Interpreters/tests/hash_map_string_2
-dbms/src/Interpreters/tests/hash_map_string_3
-dbms/src/Interpreters/tests/hash_map_string_small
-dbms/src/Interpreters/tests/in_join_subqueries_preprocessor
-dbms/src/Interpreters/tests/logical_expressions_optimizer
-dbms/src/Interpreters/tests/select_query
-dbms/src/Interpreters/tests/two_level_hash_map
-dbms/src/Interpreters/tests/users
 dbms/src/Parsers/tests/create_parser
 dbms/src/Parsers/tests/select_parser
-dbms/src/Server/clickhouse-server
-dbms/src/Server/clickhouse-server.init
 dbms/src/Storages/tests/hit_log
-dbms/src/Storages/tests/merge_tree
-dbms/src/Storages/tests/part_checker
-dbms/src/Storages/tests/part_name
-dbms/src/Storages/tests/pk_condition
 dbms/src/Storages/tests/seek_speed_test
-dbms/src/Storages/tests/storage_log
-dbms/src/Storages/tests/system_numbers
-libs/libcommon/src/revision.h
-libs/libcommon/src/tests/date_lut2
-libs/libcommon/src/tests/date_lut3
-libs/libcommon/src/tests/date_lut4
-libs/libcommon/src/tests/date_lut_init
-libs/libcommon/src/tests/multi_version
-libs/libmysqlxx/src/tests/failover
-libs/libmysqlxx/src/tests/mysqlxx_test
-libs/libzkutil/src/tests/zkutil_expiration_test
-libs/libzkutil/src/tests/zkutil_test
-libs/libzkutil/src/tests/zkutil_test_async
-libs/libzkutil/src/tests/zkutil_test_commands
-libs/libzkutil/src/tests/zkutil_test_lock
-libs/libzkutil/src/tests/zkutil_zookeeper_holder
-utils/zookeeper-create-entry-to-download-part/zookeeper-create-entry-to-download-part
-utils/zookeeper-dump-tree/zookeeper-dump-tree
-utils/zookeeper-remove-by-list/zookeeper-remove-by-list
-dbms/src/Storages/tests/remove_symlink_directory
-dbms/tests/queries/1_stateful
-debian/control
-debian/copyright
-debian/tmp/
-libs/libcommon/src/tests/json_test
-utils/compressor/zstd_test
-utils/wikistat-loader/wikistat-loader
-dbms/src/Common/tests/pod_array
-debian/clickhouse-benchmark/
-debian/clickhouse-client/
-debian/clickhouse-server-base/
-debian/clickhouse-server-common/
-debian/files
 
 dbms/src/Server/data/*
 dbms/src/Server/metadata/*

cmake/print_include_directories.cmake

@@ -18,7 +18,7 @@
 get_property (dirs1 TARGET dbms PROPERTY INCLUDE_DIRECTORIES)
 list(APPEND dirs ${dirs1})
 
-get_property (dirs1 TARGET clickhouse_common_io PROPERTY INCLUDE_DIRECTORIES)
+get_property (dirs1 TARGET tiflash_common_io PROPERTY INCLUDE_DIRECTORIES)
 list(APPEND dirs ${dirs1})
 
 get_property (dirs1 TARGET common PROPERTY INCLUDE_DIRECTORIES)

dbms/CMakeLists.txt

@@ -54,10 +54,10 @@ set(dbms_sources)
 
 include(${TiFlash_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
-add_headers_and_sources(clickhouse_common_io src/Common)
-add_headers_and_sources(clickhouse_common_io src/Common/HashTable)
-add_headers_and_sources(clickhouse_common_io src/Common/SyncPoint)
-add_headers_and_sources(clickhouse_common_io src/IO)
+add_headers_and_sources(tiflash_common_io src/Common)
+add_headers_and_sources(tiflash_common_io src/Common/HashTable)
+add_headers_and_sources(tiflash_common_io src/Common/SyncPoint)
+add_headers_and_sources(tiflash_common_io src/IO)
 
 add_headers_and_sources(dbms src/Analyzers)
 add_headers_and_sources(dbms src/Core)
@@ -118,9 +118,9 @@ check_then_add_sources_compile_flag (
     src/DataTypes/DataTypeString.cpp
 )
 
-list (APPEND clickhouse_common_io_sources ${CONFIG_BUILD})
-list (APPEND clickhouse_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
-list (APPEND clickhouse_common_io_headers ${fiu_include_dirs})
+list (APPEND tiflash_common_io_sources ${CONFIG_BUILD})
+list (APPEND tiflash_common_io_headers ${CONFIG_VERSION} ${CONFIG_COMMON})
+list (APPEND tiflash_common_io_headers ${fiu_include_dirs})
 
 list (APPEND dbms_sources src/Functions/IFunction.cpp src/Functions/FunctionFactory.cpp src/Functions/FunctionHelpers.cpp)
 list (APPEND dbms_headers src/Functions/IFunction.h src/Functions/FunctionFactory.h src/Functions/FunctionHelpers.h)
@@ -144,11 +144,11 @@ list (APPEND dbms_headers
 list (APPEND dbms_sources src/TableFunctions/ITableFunction.cpp src/TableFunctions/TableFunctionFactory.cpp)
 list (APPEND dbms_headers src/TableFunctions/ITableFunction.h src/TableFunctions/TableFunctionFactory.h)
 
-add_library(clickhouse_common_io ${SPLIT_SHARED} ${clickhouse_common_io_headers} ${clickhouse_common_io_sources})
-target_include_directories(clickhouse_common_io BEFORE PUBLIC ${XXHASH_INCLUDE_DIR})
+add_library(tiflash_common_io ${SPLIT_SHARED} ${tiflash_common_io_headers} ${tiflash_common_io_sources})
+target_include_directories(tiflash_common_io BEFORE PUBLIC ${XXHASH_INCLUDE_DIR})
 
 if (OS_FREEBSD)
-    target_compile_definitions (clickhouse_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
+    target_compile_definitions (tiflash_common_io PUBLIC CLOCK_MONOTONIC_COARSE=CLOCK_MONOTONIC_FAST)
 endif ()
 
 add_subdirectory(src/Common/Config)
@@ -187,7 +187,7 @@ if (NOT ARCH_ARM)
     set (LINK_LIBRARIES_ONLY_ON_X86_64 cpuid)
 endif()
 
-target_link_libraries (clickhouse_common_io
+target_link_libraries (tiflash_common_io
     PUBLIC
     common
     string_utils
@@ -210,12 +210,12 @@ target_link_libraries (clickhouse_common_io
     libsymbolization
 )
 
-target_include_directories (clickhouse_common_io BEFORE PRIVATE ${kvClient_SOURCE_DIR}/include)
-target_compile_definitions(clickhouse_common_io PUBLIC -DTIFLASH_SOURCE_PREFIX=\"${TiFlash_SOURCE_DIR}\")
+target_include_directories (tiflash_common_io BEFORE PRIVATE ${kvClient_SOURCE_DIR}/include)
+target_compile_definitions(tiflash_common_io PUBLIC -DTIFLASH_SOURCE_PREFIX=\"${TiFlash_SOURCE_DIR}\")
 target_link_libraries (dbms
-    clickhouse_parsers
-    clickhouse_common_config
-    clickhouse_common_io
+    tiflash_parsers
+    tiflash_common_config
+    tiflash_common_io
     flash_service
     kvproto
     kv_client
@@ -239,17 +239,17 @@ if (NOT USE_INTERNAL_RE2_LIBRARY)
     target_include_directories (dbms BEFORE PRIVATE ${RE2_INCLUDE_DIR})
 endif ()
 
-target_include_directories (clickhouse_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
+target_include_directories (tiflash_common_io BEFORE PUBLIC ${Boost_INCLUDE_DIRS})
 
 if (Poco_Data_FOUND AND NOT USE_INTERNAL_POCO_LIBRARY)
-    target_include_directories (clickhouse_common_io PRIVATE ${Poco_Data_INCLUDE_DIRS})
+    target_include_directories (tiflash_common_io PRIVATE ${Poco_Data_INCLUDE_DIRS})
     target_include_directories (dbms PRIVATE ${Poco_Data_INCLUDE_DIRS})
 endif()
 
-# NetSSL headers are exposed in clickhouse_common_io's headers.
+# NetSSL headers are exposed in tiflash_common_io's headers.
 # We mark it public to make the headers available.
 if (Poco_NetSSL_FOUND)
-    target_link_libraries (clickhouse_common_io PUBLIC ${Poco_NetSSL_LIBRARY})
+    target_link_libraries (tiflash_common_io PUBLIC ${Poco_NetSSL_LIBRARY})
 endif()
 
 target_link_libraries (dbms ${Poco_Foundation_LIBRARY})
@@ -274,14 +274,14 @@ if (NOT USE_INTERNAL_ZSTD_LIBRARY)
 endif ()
 
 target_include_directories (dbms PUBLIC ${DBMS_INCLUDE_DIR})
-target_include_directories (clickhouse_common_io PUBLIC ${DBMS_INCLUDE_DIR})
-target_include_directories (clickhouse_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
-target_include_directories (clickhouse_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
+target_include_directories (tiflash_common_io PUBLIC ${DBMS_INCLUDE_DIR})
+target_include_directories (tiflash_common_io PUBLIC ${PCG_RANDOM_INCLUDE_DIR})
+target_include_directories (tiflash_common_io BEFORE PUBLIC ${DOUBLE_CONVERSION_INCLUDE_DIR})
 
 # also for copy_headers.sh:
-target_include_directories (clickhouse_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
+target_include_directories (tiflash_common_io BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
 # https://cmake.org/pipermail/cmake/2016-May/063400.html
-target_link_libraries (clickhouse_common_io PUBLIC ${TIFLASH_XXHASH_LIBRARY})
+target_link_libraries (tiflash_common_io PUBLIC ${TIFLASH_XXHASH_LIBRARY})
 
 function(add_target_pch context target)
     if (ENABLE_PCH)
@@ -328,7 +328,7 @@ if (ENABLE_TESTS)
     target_compile_definitions(gtests_dbms PUBLIC DBMS_PUBLIC_GTEST)
     target_compile_definitions(dbms PUBLIC MULTIPLE_CONTEXT_GTEST)
 
-    target_link_libraries(gtests_dbms test_util_gtest_main clickhouse_functions tiflash-dttool-lib)
+    target_link_libraries(gtests_dbms test_util_gtest_main tiflash_functions tiflash-dttool-lib)
 
     install (TARGETS gtests_dbms
             COMPONENT tiflash-gtest
@@ -372,26 +372,19 @@ if (ENABLE_TESTS)
         )
     target_include_directories(bench_dbms BEFORE PRIVATE ${SPARCEHASH_INCLUDE_DIR} ${benchmark_SOURCE_DIR}/include)
     target_compile_definitions(bench_dbms PUBLIC DBMS_PUBLIC_GTEST)
-
-    target_link_libraries(bench_dbms gtest dbms test_util_bench_main benchmark clickhouse_functions)
-    if (ENABLE_TIFLASH_DTWORKLOAD)
-        target_link_libraries(bench_dbms dt-workload-lib)
-    endif ()
-    if (ENABLE_TIFLASH_PAGEWORKLOAD)
-        target_link_libraries(bench_dbms page-workload-lib)
-    endif ()
+    target_link_libraries(bench_dbms gtest dbms test_util_bench_main benchmark tiflash_functions server_for_test)
 
     add_check(bench_dbms)
 endif ()
 
 # dbms
 add_target_pch("pch-dbms.h" flash_service)
-add_target_pch("pch-common.h" clickhouse_common_io clickhouse_aggregate_functions)
+add_target_pch("pch-common.h" tiflash_common_io tiflash_aggregate_functions)
 
-# disable PCH for `clickhouse_functions`, `dbms` temporarily because some source files need to use specific cxx flags
-# add_target_pch("pch-common.h" clickhouse_functions dbms)
+# disable PCH for `tiflash_functions`, `dbms` temporarily because some source files need to use specific cxx flags
+# add_target_pch("pch-common.h" tiflash_functions dbms)
 
-add_target_pch("pch-common.h" clickhouse_parsers clickhouse_storages_system dt-workload-lib clickhouse-server-lib)
+add_target_pch("pch-common.h" tiflash_parsers tiflash_storages_system dt-workload-lib tiflash-server-lib)
 
 # common
 add_target_pch("pch-kvpb.h" kv_client)

dbms/src/AggregateFunctions/CMakeLists.txt

@@ -13,17 +13,17 @@
 # limitations under the License.
 
 include(${TiFlash_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
-add_headers_and_sources(clickhouse_aggregate_functions .)
+add_headers_and_sources(tiflash_aggregate_functions .)
 
-list(REMOVE_ITEM clickhouse_aggregate_functions_sources
+list(REMOVE_ITEM tiflash_aggregate_functions_sources
     AggregateFunctionFactory.cpp
     AggregateFunctionCombinatorFactory.cpp
     AggregateFunctionState.cpp
     parseAggregateFunctionParameters.cpp
     FactoryHelpers.cpp
 )
 
-list(REMOVE_ITEM clickhouse_aggregate_functions_headers
+list(REMOVE_ITEM tiflash_aggregate_functions_headers
     IAggregateFunction.h
     IAggregateFunctionCombinator.h
     AggregateFunctionFactory.h
@@ -33,6 +33,6 @@ list(REMOVE_ITEM clickhouse_aggregate_functions_headers
     FactoryHelpers.h
 )
 
-add_library(clickhouse_aggregate_functions ${clickhouse_aggregate_functions_sources} AggregateFunctionGroupConcat.h)
-target_link_libraries(clickhouse_aggregate_functions dbms)
-target_include_directories (clickhouse_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})
+add_library(tiflash_aggregate_functions ${tiflash_aggregate_functions_sources} AggregateFunctionGroupConcat.h)
+target_link_libraries(tiflash_aggregate_functions dbms)
+target_include_directories (tiflash_aggregate_functions BEFORE PRIVATE ${COMMON_INCLUDE_DIR})

dbms/src/Client/CMakeLists.txt

@@ -12,9 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# TODO: make separate lib datastream, block, ...
-#include(${TiFlash_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
-#add_headers_and_sources(clickhouse_client .)
-#add_library(clickhouse_client ${SPLIT_SHARED} ${clickhouse_client_headers} ${clickhouse_client_sources})
-#target_link_libraries (clickhouse_client clickhouse_common_io ${Poco_Net_LIBRARY})
-#target_include_directories (clickhouse_client PRIVATE ${DBMS_INCLUDE_DIR})

dbms/src/Common/CMakeLists.txt

@@ -13,8 +13,6 @@
 # limitations under the License.
 
 add_subdirectory(StringUtils)
-# after common_io
-#add_subdirectory(ConfigProcessor)
 
 if (ENABLE_TESTS)
     add_subdirectory (tests EXCLUDE_FROM_ALL)

dbms/src/Common/Config/CMakeLists.txt

@@ -15,9 +15,9 @@
 
 include(${TiFlash_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
-add_headers_and_sources(clickhouse_common_config .)
+add_headers_and_sources(tiflash_common_config .)
 
-add_library(clickhouse_common_config ${SPLIT_SHARED} ${clickhouse_common_config_headers} ${clickhouse_common_config_sources})
+add_library(tiflash_common_config ${SPLIT_SHARED} ${tiflash_common_config_headers} ${tiflash_common_config_sources})
 
-target_link_libraries(clickhouse_common_config clickhouse_common_io string_utils cpptoml)
-target_include_directories(clickhouse_common_config PRIVATE ${DBMS_INCLUDE_DIR})
+target_link_libraries(tiflash_common_config tiflash_common_io string_utils cpptoml)
+target_include_directories(tiflash_common_config PRIVATE ${DBMS_INCLUDE_DIR})

dbms/src/Common/StringUtils/CMakeLists.txt

@@ -14,7 +14,7 @@
 
 include(${TiFlash_SOURCE_DIR}/cmake/dbms_glob_sources.cmake)
 
-add_headers_and_sources(clickhouse_common_stringutils .)
+add_headers_and_sources(tiflash_common_stringutils .)
 
-add_library(string_utils ${SPLIT_SHARED} ${clickhouse_common_stringutils_headers} ${clickhouse_common_stringutils_sources})
+add_library(string_utils ${SPLIT_SHARED} ${tiflash_common_stringutils_headers} ${tiflash_common_stringutils_sources})
 target_include_directories (string_utils PRIVATE ${DBMS_INCLUDE_DIR})