From c31b6bf735b3c21244c9ff518fba0a87800cc119 Mon Sep 17 00:00:00 2001
From: haowen <19355821+wenhaocs@users.noreply.github.com>
Date: Mon, 21 Feb 2022 04:52:38 -0800
Subject: [PATCH] Expose rocksdb feature of disabling OS page cache (#3890)

* add support for direct io

* set for read only

Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>
---
 conf/nebula-storaged.conf.production | 3 +++
 src/kvstore/RocksEngineConfig.cpp    | 8 ++++++++
 2 files changed, 11 insertions(+)

diff --git a/conf/nebula-storaged.conf.production b/conf/nebula-storaged.conf.production
index d79c14c7b1..7856f34a40 100644
--- a/conf/nebula-storaged.conf.production
+++ b/conf/nebula-storaged.conf.production
@@ -62,6 +62,9 @@
 # The default block cache size used in BlockBasedTable. (MB)
 # recommend: 1/3 of all memory
 --rocksdb_block_cache=4096
+# Disable page cache to better control memory used by rocksdb.
+# Caution: Make sure to allocate enough block cache if disabling page cache!
+--disable_page_cache=false
 
 # Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd
 # For the sake of binary compatibility, the default value is snappy.
diff --git a/src/kvstore/RocksEngineConfig.cpp b/src/kvstore/RocksEngineConfig.cpp
index 952922fe37..5dbcc40532 100644
--- a/src/kvstore/RocksEngineConfig.cpp
+++ b/src/kvstore/RocksEngineConfig.cpp
@@ -50,6 +50,10 @@ DEFINE_int64(rocksdb_block_cache,
              1024,
              "The default block cache size used in BlockBasedTable. The unit is MB");
 
+DEFINE_bool(disable_page_cache,
+            false,
+            "Disable page cache to better control memory used by rocksdb.");
+
 DEFINE_int32(rocksdb_row_cache_num, 16 * 1000 * 1000, "Total keys inside the cache");
 
 DEFINE_int32(cache_bucket_exp, 8, "Total buckets number is 1 << cache_bucket_exp");
@@ -273,6 +277,10 @@ rocksdb::Status initRocksdbOptions(rocksdb::Options& baseOpts,
     return s;
   }
 
+  if (FLAGS_disable_page_cache) {
+    baseOpts.use_direct_reads = true;
+  }
+
   if (FLAGS_num_compaction_threads > 0) {
     static std::shared_ptr<rocksdb::ConcurrentTaskLimiter> compaction_thread_limiter{
         rocksdb::NewConcurrentTaskLimiter("compaction", FLAGS_num_compaction_threads)};