Expose rocksdb feature of disabling OS page cache (vesoft-inc#3890)

* add support for direct io

* set for read only

Co-authored-by: Sophie <84560950+Sophie-Xie@users.noreply.github.com>

conf/nebula-storaged.conf.production

@@ -62,6 +62,9 @@
 # The default block cache size used in BlockBasedTable. (MB)
 # recommend: 1/3 of all memory
 --rocksdb_block_cache=4096
+# Disable page cache to better control memory used by rocksdb.
+# Caution: Make sure to allocate enough block cache if disabling page cache!
+--disable_page_cache=false
 
 # Compression algorithm, options: no,snappy,lz4,lz4hc,zlib,bzip2,zstd
 # For the sake of binary compatibility, the default value is snappy.

src/kvstore/RocksEngineConfig.cpp

@@ -50,6 +50,10 @@ DEFINE_int64(rocksdb_block_cache,
              1024,
              "The default block cache size used in BlockBasedTable. The unit is MB");
 
+DEFINE_bool(disable_page_cache,
+            false,
+            "Disable page cache to better control memory used by rocksdb.");
+
 DEFINE_int32(rocksdb_row_cache_num, 16 * 1000 * 1000, "Total keys inside the cache");
 
 DEFINE_int32(cache_bucket_exp, 8, "Total buckets number is 1 << cache_bucket_exp");
@@ -273,6 +277,10 @@ rocksdb::Status initRocksdbOptions(rocksdb::Options& baseOpts,
     return s;
   }
 
+  if (FLAGS_disable_page_cache) {
+    baseOpts.use_direct_reads = true;
+  }
+
   if (FLAGS_num_compaction_threads > 0) {
     static std::shared_ptr<rocksdb::ConcurrentTaskLimiter> compaction_thread_limiter{
         rocksdb::NewConcurrentTaskLimiter("compaction", FLAGS_num_compaction_threads)};