Merge branch 'master' into fix_ps_test

pingcap · Mar 7, 2022 · 995843a · 995843a
2 parents 7de5e3a + 444d875
commit 995843a
Show file tree

Hide file tree

Showing 68 changed files with 3,347 additions and 997 deletions.
diff --git a/NOTICE b/NOTICE
@@ -0,0 +1,195 @@
+// ------------------------------------------------------------------
+// NOTICE file corresponding to the section 4d of The Apache License,
+// Version 2.0, in this case for
+// ------------------------------------------------------------------
+
+abseil-cpp
+Revision: 215105818dfde3174fe799600bb0f3cae233d0bf
+Address: https://github.com/abseil/abseil-cpp
+License: https://github.com/abseil/abseil-cpp/blob/master/LICENSE
+--------------------------------------
+benchmark
+Revision: f91b6b42b1b9854772a90ae9501464a161707d1e
+Address: https://github.com/google/benchmark
+License: https://github.com/google/benchmark/blob/main/LICENSE
+--------------------------------------
+boost
+Revision: f7aa8cd1fea2dcd85b54542d19200c8f8957a0aa
+Address: https://github.com/boostorg/boost
+License: https://github.com/boostorg/boost/blob/master/LICENSE_1_0.txt
+--------------------------------------
+boringssl
+Revision: f6ef1c560ae5af51e2df5d8d2175bed207b28b8f
+Address: https://github.com/google/boringssl
+License: https://github.com/google/boringssl/blob/master/LICENSE
+--------------------------------------
+cctz
+Revision: 79e037678a5fce43eb7a3584508a67366dc2111d
+Address: https://github.com/google/cctz
+License: https://github.com/google/cctz/blob/master/LICENSE.txt
+--------------------------------------
+client-c
+Revision: 0c834b9fe189841e078046b7b66d4960acc339fc
+Address: https://github.com/tikv/client-c
+License: https://github.com/tikv/client-c/blob/master/LICENSE
+--------------------------------------
+cpptoml
+Revision: fededad7169e538ca47e11a9ee9251bc361a9a65
+Address: https://github.com/skystrife/cpptoml
+License: https://github.com/skystrife/cpptoml/blob/master/LICENSE
+--------------------------------------
+cpu_features
+Revision: e38dc6d2a8a855dc4bcea421c23eaf68ddd671e6
+Address: https://github.com/google/cpu_features
+License: https://github.com/google/cpu_features/blob/main/LICENSE
+--------------------------------------
+curl
+Revision: 801bd5138ce31aa0d906fa4e2eabfc599d74e793
+Address: https://github.com/curl/curl
+License: https://github.com/curl/curl/blob/master/COPYING
+--------------------------------------
+double-conversion
+Revision: cf2f0f3d547dc73b4612028a155b80536902ba02
+Address: https://github.com/google/double-conversion
+License: https://github.com/google/double-conversion/blob/master/LICENSE
+--------------------------------------
+fmtlib
+Revision: d141cdbeb0fb422a3fb7173b285fd38e0d1772dc
+Address: https://github.com/fmtlib/fmt
+License: https://github.com/fmtlib/fmt/blob/master/LICENSE.rst
+--------------------------------------
+google-test
+Revision: d175c8bf823e709d570772b038757fadf63bc632
+Address: https://github.com/google/googletest
+License: https://github.com/google/googletest/blob/main/LICENSE
+--------------------------------------
+grpc
+Revision: de893acb6aef88484a427e64b96727e4926fdcfd
+Address: https://github.com/grpc/grpc
+License: https://github.com/grpc/grpc/blob/master/LICENSE
+--------------------------------------
+jemalloc
+Revision: ea6b3e973b477b8061e0076bb257dbd7f3faa756
+Address: https://github.com/jemalloc/jemalloc
+License: https://github.com/jemalloc/jemalloc/blob/dev/COPYING
+--------------------------------------
+junction
+Revision: 5ad3be7ce1d3f16b9f7ed6065bbfeacd2d629a08
+Address: https://github.com/preshing/junction
+License: https://github.com/preshing/junction/blob/master/LICENSE
+--------------------------------------
+kvproto
+Revision: 714e05ea3b18f9f89e88ae88db693bd5e0b383a2
+Address: https://github.com/pingcap/kvproto
+License: https://github.com/pingcap/kvproto/blob/master/LICENSE
+--------------------------------------
+btrie
+Revision: ddb0dcd2fd4f093d9622f68ee48b61655348a512
+Address: https://github.com/ClickHouse/ClickHouse/tree/v1.1.54381-stable/contrib/libbtrie
+License: https://github.com/ClickHouse/ClickHouse/blob/v1.1.54381-stable/contrib/libbtrie/LICENSE
+--------------------------------------
+cityhash
+Revision: caf83a650e88f5c1ac3da33dfc9409c5314f4554
+Address: https://github.com/ClickHouse/ClickHouse/tree/v1.1.54381-stable/contrib/libcityhash
+License: https://github.com/ClickHouse/ClickHouse/blob/v1.1.54381-stable/contrib/libcityhash/COPYING
+--------------------------------------
+cpuid
+Revision: ac178bde1fa3c27adb97d8dfcae2992074c88caa
+Address: https://github.com/anrieff/libcpuid
+License: https://github.com/anrieff/libcpuid/blob/master/COPYING
+--------------------------------------
+divide（lbdivide）
+Revision: fdbafd427f329b3c1ccafcdc13a354e88c596787
+Address: https://github.com/ridiculousfish/libdivide
+License: https://github.com/ridiculousfish/libdivide/blob/master/LICENSE.txt
+--------------------------------------
+farmhash
+Revision: afdb2e459d030afd1f3b6116545397509326385e
+Address: https://github.com/google/farmhash
+License: https://github.com/google/farmhash/blob/master/COPYING
+--------------------------------------
+pcg-random
+Revision: 93f1e274fb29b6a7cb20676252085a19d8d5a62d
+Address: https://github.com/ClickHouse/ClickHouse/tree/v1.1.54381-stable/contrib/libpcg-random
+License: https://github.com/ClickHouse/ClickHouse/blob/v1.1.54381-stable/contrib/libpcg-random/LICENSE-APACHE.txt
+--------------------------------------
+metrohash
+Revision: d6833a0d5569a9dbd4d1cdf67d95015712f13cb6
+Address: https://github.com/ClickHouse/ClickHouse/tree/v1.1.54381-stable/contrib/libmetrohash
+License: https://github.com/ClickHouse/ClickHouse/blob/v1.1.54381-stable/contrib/libmetrohash/LICENSE
+--------------------------------------
+sparsehash
+Revision: 21d37dbaa45742c0bd23cc1e5a70b52cbc27f809
+Address: https://github.com/ClickHouse/ClickHouse/tree/b13313eecca9455f4fdc923f597fe863df409742/contrib/libsparsehash
+License: https://github.com/ClickHouse/ClickHouse/blob/b13313eecca9455f4fdc923f597fe863df409742/contrib/libsparsehash/COPYING
+--------------------------------------
+tcmalloc
+Revision: dde32f8bbc95312379f9f5a651799815bb6327c5
+Address: https://github.com/ClickHouse/ClickHouse/tree/b13313eecca9455f4fdc923f597fe863df409742/contrib/libtcmalloc
+License: https://github.com/ClickHouse/ClickHouse/blob/b13313eecca9455f4fdc923f597fe863df409742/contrib/libtcmalloc/COPYING
+--------------------------------------
+libunwind
+Revision: 19a2c01b1e8ac20871ea09d20f596d425ba53aed
+Address: https://github.com/libunwind/libunwind
+License: https://github.com/libunwind/libunwind/blob/master/LICENSE
+--------------------------------------
+lz4
+Revision: d44371841a2f1728a3f36839fd4b7e872d0927d3
+Address: https://github.com/lz4/lz4
+License: https://github.com/lz4/lz4/blob/dev/LICENSE
+--------------------------------------
+mimalloc
+Revision: dc6bce256d4f3ce87761f9337977dff3d8b1776c
+Address: https://github.com/microsoft/mimalloc
+License: https://github.com/microsoft/mimalloc/blob/master/LICENSE
+--------------------------------------
+poco
+Revision: e411ea34492bf26e76b4e4a03ff5813a0c779240
+Address: https://github.com/ClickHouse-Extras/poco
+License: https://github.com/ClickHouse-Extras/poco/blob/clickhouse/LICENSE
+--------------------------------------
+prometheus-cpp
+Revision: ca1f3463e74d957d1cccddd4a1a29e3e5d34bd83
+Address: https://github.com/jupp0r/prometheus-cpp
+License: https://github.com/jupp0r/prometheus-cpp/blob/master/LICENSE
+--------------------------------------
+protobuf
+Revision: 09745575a923640154bcf307fba8aedff47f240a
+Address: https://github.com/protocolbuffers/protobuf
+License: https://github.com/protocolbuffers/protobuf/blob/master/LICENSE
+--------------------------------------
+re2
+Revision: 7cf8b88e8f70f97fd4926b56aa87e7f53b2717e0
+Address: https://github.com/google/re2
+License: https://github.com/google/re2/blob/main/LICENSE
+--------------------------------------
+tiflash-proxy
+Revision: 1e3f15fdb93a7ae41958d81c168d9e25ef3d4570
+Address: https://github.com/pingcap/tidb-engine-ext
+License: https://github.com/pingcap/tidb-engine-ext/blob/raftstore-proxy/LICENSE
+--------------------------------------
+tipb
+Revision: 0e3817b1f556337705053dac55606d04030bf1a0
+Address: https://github.com/pingcap/tipb
+License: https://github.com/pingcap/tipb/blob/master/LICENSE
+--------------------------------------
+xxhash
+Revision: 94e5f23e736f2bb67ebdf90727353e65344f9fc0
+Address: https://github.com/Cyan4973/xxHash
+License: https://github.com/Cyan4973/xxHash/blob/dev/LICENSE
+--------------------------------------
+zlib-ng
+Revision: b56a2fd0b126cfe5f13e68ab9090cd4f6a773286
+Address: https://github.com/zlib-ng/zlib-ng
+License: https://github.com/zlib-ng/zlib-ng/blob/develop/LICENSE.md
+--------------------------------------
+zstd
+Revision: 791626dfb92acf4a3d3ba0342636b0dd82848e01
+Address: https://github.com/facebook/zstd
+License: https://github.com/facebook/zstd/blob/dev/LICENSE
+--------------------------------------
+ClickHouse
+Revision: 30fcaeb2a3fff1bf894aae9c776bed7fd83f783f
+Address: https://github.com/ClickHouse/ClickHouse
+License: https://github.com/ClickHouse/ClickHouse/blob/master/LICENSE
+
diff --git a/contrib/zstd b/contrib/zstd
diff --git a/contrib/zstd-cmake/CMakeLists.txt b/contrib/zstd-cmake/CMakeLists.txt
@@ -50,53 +50,80 @@ GetLibraryVersion("${HEADER_CONTENT}" LIBVER_MAJOR LIBVER_MINOR LIBVER_RELEASE)
 MESSAGE(STATUS "ZSTD VERSION ${LIBVER_MAJOR}.${LIBVER_MINOR}.${LIBVER_RELEASE}")
 
 SET(Sources
+        ${LIBRARY_DIR}/dictBuilder/cover.c
+        ${LIBRARY_DIR}/dictBuilder/zdict.c
+        ${LIBRARY_DIR}/dictBuilder/divsufsort.c
+        ${LIBRARY_DIR}/dictBuilder/fastcover.c
+        ${LIBRARY_DIR}/compress/zstd_fast.c
+        ${LIBRARY_DIR}/compress/fse_compress.c
+        ${LIBRARY_DIR}/compress/hist.c
+        ${LIBRARY_DIR}/compress/zstd_lazy.c
+        ${LIBRARY_DIR}/compress/zstd_compress_superblock.c
+        ${LIBRARY_DIR}/compress/zstdmt_compress.c
+        ${LIBRARY_DIR}/compress/zstd_ldm.c
+        ${LIBRARY_DIR}/compress/zstd_compress_literals.c
+        ${LIBRARY_DIR}/compress/zstd_compress.c
+        ${LIBRARY_DIR}/compress/zstd_opt.c
+        ${LIBRARY_DIR}/compress/zstd_compress_sequences.c
+        ${LIBRARY_DIR}/compress/huf_compress.c
+        ${LIBRARY_DIR}/compress/zstd_double_fast.c
         ${LIBRARY_DIR}/common/entropy_common.c
-        ${LIBRARY_DIR}/common/error_private.c
+        ${LIBRARY_DIR}/common/debug.c
         ${LIBRARY_DIR}/common/fse_decompress.c
         ${LIBRARY_DIR}/common/pool.c
+        ${LIBRARY_DIR}/common/error_private.c
         ${LIBRARY_DIR}/common/threading.c
-        ${LIBRARY_DIR}/common/xxhash.c
         ${LIBRARY_DIR}/common/zstd_common.c
-        ${LIBRARY_DIR}/compress/fse_compress.c
-        ${LIBRARY_DIR}/compress/huf_compress.c
-        ${LIBRARY_DIR}/compress/zstd_compress.c
-        ${LIBRARY_DIR}/compress/zstd_double_fast.c
-        ${LIBRARY_DIR}/compress/zstd_fast.c
-        ${LIBRARY_DIR}/compress/zstd_lazy.c
-        ${LIBRARY_DIR}/compress/zstd_ldm.c
-        ${LIBRARY_DIR}/compress/zstdmt_compress.c
-        ${LIBRARY_DIR}/compress/zstd_opt.c
+        ${LIBRARY_DIR}/common/xxhash.c
+        ${LIBRARY_DIR}/deprecated/zbuff_decompress.c
+        ${LIBRARY_DIR}/deprecated/zbuff_compress.c
+        ${LIBRARY_DIR}/deprecated/zbuff_common.c
+        ${LIBRARY_DIR}/decompress/zstd_decompress_block.c
         ${LIBRARY_DIR}/decompress/huf_decompress.c
         ${LIBRARY_DIR}/decompress/zstd_decompress.c
-        ${LIBRARY_DIR}/deprecated/zbuff_common.c
-        ${LIBRARY_DIR}/deprecated/zbuff_compress.c
-        ${LIBRARY_DIR}/deprecated/zbuff_decompress.c
-        ${LIBRARY_DIR}/dictBuilder/cover.c
-        ${LIBRARY_DIR}/dictBuilder/divsufsort.c
-        ${LIBRARY_DIR}/dictBuilder/zdict.c)
+        ${LIBRARY_DIR}/decompress/zstd_ddict.c
+        ${LIBRARY_DIR}/decompress/huf_decompress_amd64.S)
 
 SET(Headers
-        ${LIBRARY_DIR}/common/bitstream.h
+        ${LIBRARY_DIR}/dictBuilder/cover.h
+        ${LIBRARY_DIR}/dictBuilder/divsufsort.h
+        ${LIBRARY_DIR}/compress/zstd_ldm_geartab.h
+        ${LIBRARY_DIR}/compress/zstd_compress_internal.h
+        ${LIBRARY_DIR}/compress/zstd_ldm.h
+        ${LIBRARY_DIR}/compress/zstd_double_fast.h
+        ${LIBRARY_DIR}/compress/clevels.h
+        ${LIBRARY_DIR}/compress/zstd_lazy.h
+        ${LIBRARY_DIR}/compress/zstdmt_compress.h
+        ${LIBRARY_DIR}/compress/zstd_fast.h
+        ${LIBRARY_DIR}/compress/zstd_opt.h
+        ${LIBRARY_DIR}/compress/hist.h
+        ${LIBRARY_DIR}/compress/zstd_cwksp.h
+        ${LIBRARY_DIR}/compress/zstd_compress_superblock.h
+        ${LIBRARY_DIR}/compress/zstd_compress_sequences.h
+        ${LIBRARY_DIR}/compress/zstd_compress_literals.h
+        ${LIBRARY_DIR}/common/portability_macros.h
         ${LIBRARY_DIR}/common/error_private.h
-        ${LIBRARY_DIR}/common/fse.h
         ${LIBRARY_DIR}/common/huf.h
         ${LIBRARY_DIR}/common/mem.h
+        ${LIBRARY_DIR}/common/compiler.h
+        ${LIBRARY_DIR}/common/bitstream.h
+        ${LIBRARY_DIR}/common/debug.h
         ${LIBRARY_DIR}/common/pool.h
-        ${LIBRARY_DIR}/common/threading.h
+        ${LIBRARY_DIR}/common/fse.h
         ${LIBRARY_DIR}/common/xxhash.h
-        ${LIBRARY_DIR}/common/zstd_errors.h
+        ${LIBRARY_DIR}/common/threading.h
+        ${LIBRARY_DIR}/common/zstd_trace.h
+        ${LIBRARY_DIR}/common/cpu.h
         ${LIBRARY_DIR}/common/zstd_internal.h
-        ${LIBRARY_DIR}/compress/zstd_double_fast.h
-        ${LIBRARY_DIR}/compress/zstd_fast.h
-        ${LIBRARY_DIR}/compress/zstd_lazy.h
-        ${LIBRARY_DIR}/compress/zstd_ldm.h
-        ${LIBRARY_DIR}/compress/zstdmt_compress.h
-        ${LIBRARY_DIR}/compress/zstd_opt.h
-        ${LIBRARY_DIR}/compress/zstd_ldm.h
+        ${LIBRARY_DIR}/common/zstd_deps.h
         ${LIBRARY_DIR}/deprecated/zbuff.h
-        ${LIBRARY_DIR}/dictBuilder/divsufsort.h
-        ${LIBRARY_DIR}/dictBuilder/zdict.h
-        ${LIBRARY_DIR}/zstd.h)
+        ${LIBRARY_DIR}/zstd.h
+        ${LIBRARY_DIR}/legacy/zstd_legacy.h
+        ${LIBRARY_DIR}/zdict.h
+        ${LIBRARY_DIR}/zstd_errors.h
+        ${LIBRARY_DIR}/decompress/zstd_decompress_block.h
+        ${LIBRARY_DIR}/decompress/zstd_ddict.h
+        ${LIBRARY_DIR}/decompress/zstd_decompress_internal.h)
 
 SET(ZSTD_LEGACY_SUPPORT true)
 
@@ -124,7 +151,7 @@ IF (ZSTD_LEGACY_SUPPORT)
             ${LIBRARY_LEGACY_DIR}/zstd_v06.h
             ${LIBRARY_LEGACY_DIR}/zstd_v07.h)
 ENDIF (ZSTD_LEGACY_SUPPORT)
-
+ENABLE_LANGUAGE(ASM)
 ADD_LIBRARY(zstd ${Sources} ${Headers})
 
 target_include_directories (zstd PUBLIC ${LIBRARY_DIR})
diff --git a/dbms/src/Common/DynamicThreadPool.cpp b/dbms/src/Common/DynamicThreadPool.cpp
@@ -1,4 +1,5 @@
 #include <Common/DynamicThreadPool.h>
+#include <Common/TiFlashMetrics.h>
 
 namespace DB
 {
@@ -80,24 +81,32 @@ void DynamicThreadPool::scheduledToNewDynamicThread(TaskPtr & task)
     t.detach();
 }
 
+void executeTask(const std::unique_ptr<IExecutableTask> & task)
+{
+    UPDATE_CUR_AND_MAX_METRIC(tiflash_thread_count, type_active_threads_of_thdpool, type_max_active_threads_of_thdpool);
+    task->execute();
+}
+
 void DynamicThreadPool::fixedWork(size_t index)
 {
+    UPDATE_CUR_AND_MAX_METRIC(tiflash_thread_count, type_total_threads_of_thdpool, type_max_threads_of_thdpool);
     Queue * queue = fixed_queues[index].get();
     while (true)
     {
         TaskPtr task;
         queue->pop(task);
         if (!task)
             break;
-        task->execute();
+        executeTask(task);
 
         idle_fixed_queues.push(queue);
     }
 }
 
 void DynamicThreadPool::dynamicWork(TaskPtr initial_task)
 {
-    initial_task->execute();
+    UPDATE_CUR_AND_MAX_METRIC(tiflash_thread_count, type_total_threads_of_thdpool, type_max_threads_of_thdpool);
+    executeTask(initial_task);
 
     DynamicNode node;
     while (true)
@@ -114,7 +123,7 @@ void DynamicThreadPool::dynamicWork(TaskPtr initial_task)
 
         if (!node.task) // may be timeout or cancelled
             break;
-        node.task->execute();
+        executeTask(node.task);
         node.task.reset();
     }
     alive_dynamic_threads.fetch_sub(1);

diff --git a/dbms/src/Common/Exception.cpp b/dbms/src/Common/Exception.cpp
@@ -61,7 +61,7 @@ void tryLogCurrentException(Poco::Logger * logger, const std::string & start_of_
 {
     try
     {
-        LOG_ERROR(logger, start_of_message << (start_of_message.empty() ? "" : ": ") << getCurrentExceptionMessage(true));
+        LOG_FMT_ERROR(logger, "{}{}{}", start_of_message, (start_of_message.empty() ? "" : ": "), getCurrentExceptionMessage(true));
     }
     catch (...)
     {

diff --git a/dbms/src/Common/Stopwatch.h b/dbms/src/Common/Stopwatch.h
@@ -18,6 +18,10 @@ inline UInt64 nanoseconds(clockid_t clock_type)
     clock_gettime(clock_type, &ts);
     return ts.tv_sec * 1000000000ULL + ts.tv_nsec;
 }
+inline UInt64 seconds(clockid_t clock_type)
+{
+    return nanoseconds(clock_type) / 1000000000ULL;
+}
 } // namespace StopWatchDetail