Reduce UT binary size (#314)

* Reduce UT binary size Almost every module depend on ExecEnv, and ExecEnv contains all singleton, which make UT binary contains all object files. This patch seperate ExecEnv's initial and destory to anthor file to avoid other file's dependence. And status.cc include debug_util.h which depend tuple.h tuple_row.h, and I move get_stack_trace() to stack_util.cpp to reduce status.cc's dependence. I add USE_RTTI=1 to build rocksdb to avoid linking librocksdb.a Issue: #292 * Update
apache · Nov 15, 2018 · a2b299e · a2b299e
1 parent d7ee57e
commit a2b299e
Show file tree

Hide file tree

Showing 72 changed files with 843 additions and 742 deletions.
diff --git a/.gitignore b/.gitignore
@@ -6,3 +6,4 @@ output
 docs/build
 gensrc/build
 fe/target
+thirdparty/src
diff --git a/be/src/agent/heartbeat_server.cpp b/be/src/agent/heartbeat_server.cpp
@@ -18,15 +18,17 @@
 #include "agent/heartbeat_server.h"
 #include <ctime>
 #include <fstream>
-#include "boost/filesystem.hpp"
-#include "thrift/TProcessor.h"
-#include "gen_cpp/HeartbeatService.h"
-#include "gen_cpp/Status_types.h"
+
+#include <boost/filesystem.hpp>
+#include <thrift/TProcessor.h>
 
 #include "common/status.h"
+#include "gen_cpp/HeartbeatService.h"
+#include "gen_cpp/Status_types.h"
 #include "olap/olap_engine.h"
 #include "olap/utils.h"
 #include "service/backend_options.h"
+#include "util/thrift_server.h"
 
 using std::fstream;
 using std::nothrow;

diff --git a/be/src/agent/heartbeat_server.h b/be/src/agent/heartbeat_server.h
@@ -31,6 +31,7 @@ namespace doris {
 const uint32_t HEARTBEAT_INTERVAL = 10;
 class OLAPEngine;
 class Status;
+class ThriftServer;
 
 class HeartbeatServer : public HeartbeatServiceIf {
 public:

diff --git a/be/src/agent/user_resource_listener.cpp b/be/src/agent/user_resource_listener.cpp
@@ -25,6 +25,7 @@
 #include <thrift/TApplicationException.h>
 #include "common/logging.h"
 #include "gen_cpp/FrontendService.h"
+#include "runtime/client_cache.h"
 
 namespace doris {
 
@@ -38,7 +39,7 @@ using apache::thrift::transport::TTransportException;
 UserResourceListener::UserResourceListener(ExecEnv* exec_env, 
                                            const TMasterInfo& master_info) 
     : _master_info(master_info), 
-      _master_client_cache(exec_env->frontend_client_cache()), 
+      _exec_env(exec_env),
       _cgroups_mgr(*(exec_env->cgroups_mgr())) {
 }
 
@@ -64,7 +65,7 @@ void UserResourceListener::update_users_resource(int64_t new_version) {
     // Call fe to get latest user resource
     Status master_status;
     // using 500ms as default timeout value    
-    FrontendServiceConnection client(_master_client_cache, 
+    FrontendServiceConnection client(_exec_env->frontend_client_cache(),
                                    _master_info.network_address,
                                    500, 
                                    &master_status);

diff --git a/be/src/agent/user_resource_listener.h b/be/src/agent/user_resource_listener.h
@@ -28,6 +28,8 @@
 
 namespace doris {
 
+class ExecEnv;
+
 class UserResourceListener : public TopicListener {
 
 public:
@@ -40,7 +42,7 @@ class UserResourceListener : public TopicListener {
                                const TTopicUpdate& topic_update);
 private:
     const TMasterInfo& _master_info;
-    FrontendServiceClientCache* _master_client_cache;
+    ExecEnv* _exec_env;
     CgroupsMgr& _cgroups_mgr;  
     // Call cgroups mgr to update user's cgroups resource share
     // Also refresh local user resource's cache

diff --git a/be/src/common/config.h b/be/src/common/config.h
@@ -337,42 +337,36 @@ namespace config {
     // cpu count
     CONF_Int32(flags_num_cores, "32");
 
-    CONF_Bool(FLAGS_thread_creation_fault_injection, "false");
+    CONF_Bool(thread_creation_fault_injection, "false");
 
     // Set this to encrypt and perform an integrity
     // check on all data spilled to disk during a query
-    CONF_Bool(FLAGS_disk_spill_encryption, "false");
+    CONF_Bool(disk_spill_encryption, "false");
 
     // Writable scratch directories
-    CONF_String(FLAGS_scratch_dirs, "/tmp");
+    CONF_String(scratch_dirs, "/tmp");
 
     // If false and --scratch_dirs contains multiple directories on the same device,
     // then only the first writable directory is used
-    CONF_Bool(FLAGS_allow_multiple_scratch_dirs_per_device, "false");
+    CONF_Bool(allow_multiple_scratch_dirs_per_device, "false");
 
     // linux transparent huge page
-    CONF_Bool(FLAGS_madvise_huge_pages, "false");
+    CONF_Bool(madvise_huge_pages, "false");
 
     // whether use mmap to allocate memory
-    CONF_Bool(FLAGS_mmap_buffers, "false");
-
-    // whether or not user mem pool
-    CONF_Bool(FLAGS_disable_mem_pools, "false");
+    CONF_Bool(mmap_buffers, "false");
 
     // max memory can be allocated by buffer pool
-    CONF_String(FLAGS_buffer_pool_limit, "80G");
+    CONF_String(buffer_pool_limit, "80G");
 
     // clean page can be hold by buffer pool
-    CONF_String(FLAGS_buffer_pool_clean_pages_limit, "20G");
-
-    // buffer pool can support min memory allocated
-    CONF_Int32(FLAGS_min_buffer_size, "1024");
+    CONF_String(buffer_pool_clean_pages_limit, "20G");
 
     // Sleep time in seconds between memory maintenance iterations
-    CONF_Int64(FLAGS_memory_maintenance_sleep_time_s, "10");
+    CONF_Int64(memory_maintenance_sleep_time_s, "10");
 
     // Aligement
-    CONF_Int32(FLAGS_MEMORY_MAX_ALIGNMENT, "16");
+    CONF_Int32(memory_max_alignment, "16");
 
     // write buffer size before flush
     CONF_Int32(write_buffer_size, "104857600");

diff --git a/be/src/common/daemon.cpp b/be/src/common/daemon.cpp
@@ -81,7 +81,7 @@ void* tcmalloc_gc_thread(void* dummy) {
 
 void* memory_maintenance_thread(void* dummy) {
     while (true) {
-        sleep(config::FLAGS_memory_maintenance_sleep_time_s);
+        sleep(config::memory_maintenance_sleep_time_s);
         ExecEnv* env = ExecEnv::GetInstance();
         // ExecEnv may not have been created yet or this may be the catalogd or statestored,
         // which don't have ExecEnvs.

diff --git a/be/src/common/status.cpp b/be/src/common/status.cpp
@@ -20,7 +20,6 @@
 #include <boost/algorithm/string/join.hpp>
 
 #include "common/logging.h"
-#include "util/debug_util.h"
 
 namespace doris {
 

diff --git a/be/src/common/status.h b/be/src/common/status.h
@@ -25,6 +25,7 @@
 #include "common/compiler_util.h"
 #include "gen_cpp/Status_types.h"  // for TStatus
 #include "gen_cpp/status.pb.h" // for PStatus
+#include "util/stack_util.h" // for PStatus
 
 namespace doris {
 

diff --git a/be/src/exec/aggregation_node.cpp b/be/src/exec/aggregation_node.cpp
@@ -40,7 +40,6 @@
 #include "runtime/string_value.hpp"
 #include "runtime/tuple.h"
 #include "runtime/tuple_row.h"
-#include "util/debug_util.h"
 #include "util/runtime_profile.h"
 
 using llvm::BasicBlock;
@@ -207,7 +206,7 @@ Status AggregationNode::open(RuntimeState* state) {
             for (int i = 0; i < batch.num_rows(); ++i) {
                 TupleRow* row = batch.get_row(i);
                 VLOG_ROW << "id=" << id() << " input row: "
-                        << print_row(row, _children[0]->row_desc());
+                        << row->to_string(_children[0]->row_desc());
             }
         }
 
@@ -291,7 +290,7 @@ Status AggregationNode::get_next(RuntimeState* state, RowBatch* row_batch, bool*
         row->set_tuple(0, output_tuple);
 
         if (ExecNode::eval_conjuncts(ctxs, num_ctxs, row)) {
-            VLOG_ROW << "output row: " << print_row(row, row_desc());
+            VLOG_ROW << "output row: " << row->to_string(row_desc());
             row_batch->commit_last_row();
             ++_num_rows_returned;
 

diff --git a/be/src/exec/blocking_join_node.cpp b/be/src/exec/blocking_join_node.cpp
@@ -22,7 +22,6 @@
 #include "exprs/expr.h"
 #include "runtime/row_batch.h"
 #include "runtime/runtime_state.h"
-#include "util/debug_util.h"
 #include "util/runtime_profile.h"
 #include "gen_cpp/PlanNodes_types.h"
 
@@ -191,9 +190,9 @@ std::string BlockingJoinNode::get_left_child_row_string(TupleRow* row) {
             std::find(_build_tuple_idx_ptr, _build_tuple_idx_ptr + _build_tuple_size, i);
 
         if (is_build_tuple != _build_tuple_idx_ptr + _build_tuple_size) {
-            out << print_tuple(NULL, *row_desc().tuple_descriptors()[i]);
+            out << Tuple::to_string(NULL, *row_desc().tuple_descriptors()[i]);
         } else {
-            out << print_tuple(row->get_tuple(i), *row_desc().tuple_descriptors()[i]);
+            out << Tuple::to_string(row->get_tuple(i), *row_desc().tuple_descriptors()[i]);
         }
     }
 

diff --git a/be/src/exec/broker_scan_node.cpp b/be/src/exec/broker_scan_node.cpp
@@ -26,7 +26,6 @@
 #include "runtime/dpp_sink_internal.h"
 #include "exec/broker_scanner.h"
 #include "exprs/expr.h"
-#include "util/debug_util.h"
 #include "util/runtime_profile.h"
 
 namespace doris {
@@ -213,7 +212,7 @@ Status BrokerScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool*
         for (int i = 0; i < row_batch->num_rows(); ++i) {
             TupleRow* row = row_batch->get_row(i);
             VLOG_ROW << "BrokerScanNode output row: "
-                << print_tuple(row->get_tuple(0), *_tuple_desc);
+                << Tuple::to_string(row->get_tuple(0), *_tuple_desc);
         }
     }
 
@@ -331,7 +330,7 @@ Status BrokerScanNode::scanner_scan(
 
                     std::stringstream error_msg;
                     error_msg << "No corresponding partition, partition id: " << partition_id;
-                    _runtime_state->append_error_msg_to_file(print_tuple(tuple, *_tuple_desc), 
+                    _runtime_state->append_error_msg_to_file(Tuple::to_string(tuple, *_tuple_desc), 
                                                              error_msg.str());
                     continue;
                 }

diff --git a/be/src/exec/exchange_node.cpp b/be/src/exec/exchange_node.cpp
@@ -24,7 +24,6 @@
 #include "runtime/exec_env.h"
 #include "runtime/runtime_state.h"
 #include "runtime/row_batch.h"
-#include "util/debug_util.h"
 #include "util/runtime_profile.h"
 #include "gen_cpp/PlanNodes_types.h"
 
@@ -172,7 +171,7 @@ Status ExchangeNode::get_next(RuntimeState* state, RowBatch* output_batch, bool*
             }
 
             if (VLOG_ROW_IS_ON) {
-                VLOG_ROW << "ExchangeNode output batch: " << print_batch(output_batch);
+                VLOG_ROW << "ExchangeNode output batch: " << output_batch->to_string();
             }
 
             COUNTER_SET(_rows_returned_counter, _num_rows_returned);

diff --git a/be/src/exec/hash_join_node.cpp b/be/src/exec/hash_join_node.cpp
@@ -26,7 +26,6 @@
 #include "exprs/slot_ref.h"
 #include "runtime/row_batch.h"
 #include "runtime/runtime_state.h"
-#include "util/debug_util.h"
 #include "util/runtime_profile.h"
 #include "gen_cpp/PlanNodes_types.h"
 
@@ -444,7 +443,7 @@ Status HashJoinNode::get_next(RuntimeState* state, RowBatch* out_batch, bool* eo
         VLOG_ROW << "probe row: " << get_probe_row_output_string(_current_probe_row);
         while (_hash_tbl_iterator.has_next()) {
             TupleRow* matched_build_row = _hash_tbl_iterator.get_row();
-            VLOG_ROW << "matched_build_row: " << print_row(matched_build_row, child(1)->row_desc());
+            VLOG_ROW << "matched_build_row: " << matched_build_row->to_string(child(1)->row_desc());
 
             if ((_join_op == TJoinOp::RIGHT_ANTI_JOIN || _join_op == TJoinOp::RIGHT_SEMI_JOIN)
                 && _hash_tbl_iterator.matched()) {
@@ -508,7 +507,7 @@ Status HashJoinNode::get_next(RuntimeState* state, RowBatch* out_batch, bool* eo
             _hash_tbl_iterator.next<true>();
             if (eval_conjuncts(conjunct_ctxs, num_conjunct_ctxs, out_row)) {
                 out_batch->commit_last_row();
-                VLOG_ROW << "match row: " << print_row(out_row, row_desc());
+                VLOG_ROW << "match row: " << out_row->to_string(row_desc());
                 ++_num_rows_returned;
                 COUNTER_SET(_rows_returned_counter, _num_rows_returned);
 
@@ -528,7 +527,7 @@ Status HashJoinNode::get_next(RuntimeState* state, RowBatch* out_batch, bool* eo
 
             if (eval_conjuncts(conjunct_ctxs, num_conjunct_ctxs, out_row)) {
                 out_batch->commit_last_row();
-                VLOG_ROW << "match row: " << print_row(out_row, row_desc());
+                VLOG_ROW << "match row: " << out_row->to_string(row_desc());
                 ++_num_rows_returned;
                 COUNTER_SET(_rows_returned_counter, _num_rows_returned);
                 _matched_probe = true;
@@ -628,7 +627,7 @@ Status HashJoinNode::get_next(RuntimeState* state, RowBatch* out_batch, bool* eo
             create_output_row(out_row, NULL, build_row);
             if (eval_conjuncts(conjunct_ctxs, num_conjunct_ctxs, out_row)) {
                 out_batch->commit_last_row();
-                VLOG_ROW << "match row: " << print_row(out_row, row_desc());
+                VLOG_ROW << "match row: " << out_row->to_string(row_desc());
                 ++_num_rows_returned;
                 COUNTER_SET(_rows_returned_counter, _num_rows_returned);
 
@@ -718,9 +717,9 @@ string HashJoinNode::get_probe_row_output_string(TupleRow* probe_row) {
             std::find(_build_tuple_idx_ptr, _build_tuple_idx_ptr + _build_tuple_size, i);
 
         if (is_build_tuple != _build_tuple_idx_ptr + _build_tuple_size) {
-            out << print_tuple(NULL, *row_desc().tuple_descriptors()[i]);
+            out << Tuple::to_string(NULL, *row_desc().tuple_descriptors()[i]);
         } else {
-            out << print_tuple(probe_row->get_tuple(i), *row_desc().tuple_descriptors()[i]);
+            out << Tuple::to_string(probe_row->get_tuple(i), *row_desc().tuple_descriptors()[i]);
         }
     }
 

diff --git a/be/src/exec/hash_table.cpp b/be/src/exec/hash_table.cpp
@@ -25,7 +25,6 @@
 #include "runtime/string_value.hpp"
 #include "runtime/mem_tracker.h"
 #include "runtime/runtime_state.h"
-#include "util/debug_util.h"
 #include "util/doris_metrics.h"
 
 using llvm::BasicBlock;
@@ -313,7 +312,7 @@ std::string HashTable::debug_string(bool skip_empty, const RowDescriptor* desc)
             if (desc == NULL) {
                 ss << node_idx << "(" << (void*)node->data() << ")";
             } else {
-                ss << (void*)node->data() << " " << print_row(node->data(), *desc);
+                ss << (void*)node->data() << " " << node->data()->to_string(*desc);
             }
 
             node_idx = node->_next_idx;

diff --git a/be/src/exec/new_partitioned_aggregation_node.cc b/be/src/exec/new_partitioned_aggregation_node.cc
@@ -44,7 +44,6 @@
 #include "runtime/tuple_row.h"
 #include "runtime/tuple.h"
 #include "udf/udf_internal.h"
-#include "util/debug_util.h"
 #include "util/runtime_profile.h"
 
 #include "gen_cpp/Exprs_types.h"
@@ -313,7 +312,7 @@ Status NewPartitionedAggregationNode::open(RuntimeState* state) {
     if (UNLIKELY(VLOG_ROW_IS_ON)) {
       for (int i = 0; i < batch.num_rows(); ++i) {
         TupleRow* row = batch.get_row(i);
-        VLOG_ROW << "input row: " << print_row(row, _children[0]->row_desc());
+        VLOG_ROW << "input row: " << row->to_string(_children[0]->row_desc());
       }
     }
 

diff --git a/be/src/exec/new_partitioned_hash_table.cc b/be/src/exec/new_partitioned_hash_table.cc
@@ -31,7 +31,6 @@
 #include "runtime/raw_value.h"
 #include "runtime/runtime_state.h"
 #include "runtime/string_value.h"
-#include "util/debug_util.h"
 #include "util/doris_metrics.h"
 
 #include "common/names.h"
@@ -566,7 +565,7 @@ void NewPartitionedHashTable::DebugStringTuple(std::stringstream& ss, HtData& ht
   }
   if (desc != NULL) {
     Tuple* row[num_build_tuples_];
-    ss << " " << print_row(GetRow(htdata, reinterpret_cast<TupleRow*>(row)), *desc);
+    ss << " " << GetRow(htdata, reinterpret_cast<TupleRow*>(row))->to_string(*desc);
   }
 }
 

diff --git a/be/src/exec/olap_rewrite_node.cpp b/be/src/exec/olap_rewrite_node.cpp
@@ -25,7 +25,6 @@
 #include "runtime/row_batch.h"
 #include "runtime/raw_value.h"
 #include "runtime/tuple.h"
-#include "util/debug_util.h"
 
 namespace doris {
 
@@ -232,7 +231,7 @@ bool OlapRewriteNode::copy_rows(RuntimeState* state, RowBatch* output_batch) {
     if (VLOG_ROW_IS_ON) {
         for (int i = 0; i < output_batch->num_rows(); ++i) {
             TupleRow* row = output_batch->get_row(i);
-            VLOG_ROW << "OlapRewriteNode input row: " << print_row(row, row_desc());
+            VLOG_ROW << "OlapRewriteNode input row: " << row->to_string(row_desc());
         }
     }
 

diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
@@ -38,6 +38,7 @@
 #include "util/runtime_profile.h"
 #include "util/thread_pool.hpp"
 #include "util/debug_util.h"
+#include "util/priority_thread_pool.hpp"
 #include "agent/cgroups_mgr.h"
 #include "common/resource_tls.h"
 #include <boost/variant.hpp>
@@ -289,7 +290,7 @@ Status OlapScanNode::get_next(RuntimeState* state, RowBatch* row_batch, bool* eo
             for (int i = 0; i < row_batch->num_rows(); ++i) {
                 TupleRow* row = row_batch->get_row(i);
                 VLOG_ROW << "OlapScanNode output row: "
-                    << print_tuple(row->get_tuple(0), *_tuple_desc);
+                    << Tuple::to_string(row->get_tuple(0), *_tuple_desc);
             }
         }
         __sync_fetch_and_sub(&_buffered_bytes,