[Bugfix](mem) Fix memory limit check may overflow (apache#12776)

This bug is because the result of subtracting signed and unsigned numbers may overflow if it is negative.

be/src/runtime/memory/mem_tracker_limiter.h

@@ -74,15 +74,16 @@ class MemTrackerLimiter final : public MemTracker {
         // but it may not actually alloc physical memory, which is not expected in mem hook fail.
         //
         // TODO: In order to ensure no OOM, currently reserve 200M, and then use the free mem in /proc/meminfo to ensure no OOM.
-        if (PerfCounters::get_vm_rss() - MemInfo::allocator_cache_mem() + bytes >=
+        if (PerfCounters::get_vm_rss() - static_cast<int64_t>(MemInfo::allocator_cache_mem()) +
+                            bytes >=
                     MemInfo::mem_limit() ||
             PerfCounters::get_vm_rss() + bytes >= MemInfo::hard_mem_limit()) {
             auto st = Status::MemoryLimitExceeded(
-                    "process memory used {}, tc/jemalloc cache {}, exceed limit {}, failed alloc "
-                    "size {}",
+                    "process memory used {}, tc/jemalloc cache {}, exceed limit {}, hard limit {}, "
+                    "failed alloc size {}",
                     print_bytes(PerfCounters::get_vm_rss()),
                     print_bytes(MemInfo::allocator_cache_mem()), print_bytes(MemInfo::mem_limit()),
-                    print_bytes(bytes));
+                    print_bytes(MemInfo::hard_mem_limit()), print_bytes(bytes));
             ExecEnv::GetInstance()->process_mem_tracker_raw()->print_log_usage(st.get_error_msg());
             return st;
         }

be/src/util/mem_info.cpp

@@ -94,7 +94,7 @@ void MemInfo::init() {
 
     bool is_percent = true;
     _s_mem_limit = ParseUtil::parse_mem_spec(config::mem_limit, -1, _s_physical_mem, &is_percent);
-    _s_hard_mem_limit = _s_physical_mem - std::min(209715200.0, _s_physical_mem * 0.1); // 200M
+    _s_hard_mem_limit = _s_physical_mem - std::min(209715200L, _s_physical_mem / 10); // 200M
 
     LOG(INFO) << "Physical Memory: " << PrettyPrinter::print(_s_physical_mem, TUnit::BYTES);
     _s_initialized = true;