change rocksDB config level_compaction_dynamic_level_bytes to CFOptio…

…ns (apache#3860)

After PR apache#3056 , Bookkeeper set `level_compaction_dynamic_level_bytes=true` as `TableOptions` in `entry_location_rocksdb.conf.default` , which will cause `level_compaction_dynamic_level_bytes` lose efficacy and will cause rocksDB .sst file compact sort chaos when update bookie release.
As RocksDB  conf, `level_compaction_dynamic_level_bytes` need set as `CFOptions` https://github.com/facebook/rocksdb/blob/master/examples/rocksdb_option_file_example.ini

<img width="703" alt="image" src="https://user-images.githubusercontent.com/84127069/224640399-d5481fe5-7b75-4229-ac06-3d280aa9ae6d.png">

<img width="240" alt="image" src="https://user-images.githubusercontent.com/84127069/224640621-737d0a42-4e01-4f38-bd5a-862a93bc4b32.png">

1. Change `level_compaction_dynamic_level_bytes=true` from `TableOptions` to `CFOptions`  in `entry_location_rocksdb.conf.default` ;

(cherry picked from commit 5542afe)

bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/storage/ldb/KeyValueStorageRocksDBTest.java

@@ -30,6 +30,8 @@
 import java.util.List;
 import org.apache.bookkeeper.conf.ServerConfiguration;
 import org.junit.Test;
+import org.rocksdb.BlockBasedTableConfig;
+import org.rocksdb.ChecksumType;
 import org.rocksdb.ColumnFamilyDescriptor;
 import org.rocksdb.ColumnFamilyOptions;
 import org.rocksdb.CompressionType;
@@ -55,6 +57,7 @@ public void testRocksDBInitiateWithBookieConfiguration() throws Exception {
         assertEquals(64 * 1024 * 1024, options.writeBufferSize());
         assertEquals(4, options.maxWriteBufferNumber());
         assertEquals(256 * 1024 * 1024, options.maxBytesForLevelBase());
+        assertEquals(true, options.levelCompactionDynamicLevelBytes());
         rocksDB.close();
     }
 
@@ -79,6 +82,56 @@ public void testRocksDBInitiateWithConfigurationFile() throws Exception {
         assertEquals(CompressionType.LZ4_COMPRESSION, familyOptions.compressionType());
         assertEquals(1024, familyOptions.writeBufferSize());
         assertEquals(1, familyOptions.maxWriteBufferNumber());
+        assertEquals(true, familyOptions.levelCompactionDynamicLevelBytes());
         rocksDB.close();
     }
+
+    @Test
+    public void testReadChecksumTypeFromBookieConfiguration() throws Exception {
+        ServerConfiguration configuration = new ServerConfiguration();
+        configuration.setEntryLocationRocksdbConf("entry_location_rocksdb.conf");
+        File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-conf").toFile();
+        Files.createDirectory(Paths.get(tmpDir.toString(), "subDir"));
+        KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir",
+            KeyValueStorageFactory.DbConfigType.EntryLocation, configuration);
+        assertNull(rocksDB.getColumnFamilyDescriptors());
+
+        Options options = (Options) rocksDB.getOptions();
+        assertEquals(ChecksumType.kxxHash, ((BlockBasedTableConfig) options.tableFormatConfig()).checksumType());
+    }
+
+    //@Test
+    public void testReadChecksumTypeFromConfigurationFile() throws Exception {
+        ServerConfiguration configuration = new ServerConfiguration();
+        URL url = getClass().getClassLoader().getResource("test_entry_location_rocksdb.conf");
+        configuration.setEntryLocationRocksdbConf(url.getPath());
+        File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-file").toFile();
+        Files.createDirectory(Paths.get(tmpDir.toString(), "subDir"));
+        KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir",
+            KeyValueStorageFactory.DbConfigType.EntryLocation, configuration);
+        assertNotNull(rocksDB.getColumnFamilyDescriptors());
+
+        List<ColumnFamilyDescriptor> columnFamilyDescriptorList = rocksDB.getColumnFamilyDescriptors();
+        ColumnFamilyOptions familyOptions = columnFamilyDescriptorList.get(0).getOptions();
+        // There is a bug in RocksDB, which can't load BlockedBasedTableConfig from Options file.
+        // https://github.com/facebook/rocksdb/issues/5297
+        // After the PR: https://github.com/facebook/rocksdb/pull/10826 merge, we can turn on this test.
+        assertEquals(ChecksumType.kxxHash, ((BlockBasedTableConfig) familyOptions.tableFormatConfig()).checksumType());
+    }
+
+    @Test
+    public void testLevelCompactionDynamicLevelBytesFromConfigurationFile() throws Exception {
+        ServerConfiguration configuration = new ServerConfiguration();
+        URL url = getClass().getClassLoader().getResource("conf/entry_location_rocksdb.conf");
+        configuration.setEntryLocationRocksdbConf(url.getPath());
+        File tmpDir = Files.createTempDirectory("bk-kv-rocksdbtest-file").toFile();
+        Files.createDirectory(Paths.get(tmpDir.toString(), "subDir"));
+        KeyValueStorageRocksDB rocksDB = new KeyValueStorageRocksDB(tmpDir.toString(), "subDir",
+                KeyValueStorageFactory.DbConfigType.EntryLocation, configuration);
+        assertNotNull(rocksDB.getColumnFamilyDescriptors());
+
+        List<ColumnFamilyDescriptor> columnFamilyDescriptorList = rocksDB.getColumnFamilyDescriptors();
+        ColumnFamilyOptions familyOptions = columnFamilyDescriptorList.get(0).getOptions();
+        assertEquals(true, familyOptions.levelCompactionDynamicLevelBytes());
+    }
 }

bookkeeper-server/src/test/resources/conf/entry_location_rocksdb.conf

@@ -51,6 +51,8 @@
  max_bytes_for_level_base=268435456
  # set by jni: options.setTargetFileSizeBase
  target_file_size_base=67108864
+ # set by jni: options.setLevelCompactionDynamicLevelBytes
+ level_compaction_dynamic_level_bytes=true
 
 [TableOptions/BlockBasedTable "default"]
  # set by jni: tableOptions.setBlockSize
@@ -64,6 +66,4 @@
  # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
  filter_policy=rocksdb.BloomFilter:10:false
  # set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
- # set by jni: options.setLevelCompactionDynamicLevelBytes
- level_compaction_dynamic_level_bytes=true
+ cache_index_and_filter_blocks=true

bookkeeper-server/src/test/resources/test_entry_location_rocksdb.conf

@@ -31,3 +31,19 @@
  write_buffer_size=1024
  # set by jni: options.setMaxWriteBufferNumber
  max_write_buffer_number=1
+ # set by jni: options.setLevelCompactionDynamicLevelBytes
+ level_compaction_dynamic_level_bytes=true
+
+[TableOptions/BlockBasedTable "default"]
+ # set by jni: tableOptions.setBlockSize
+ block_size=65536
+ # set by jni: tableOptions.setBlockCache
+ block_cache=206150041
+ # set by jni: tableOptions.setFormatVersion
+ format_version=2
+ # set by jni: tableOptions.setChecksumType
+ checksum=kxxHash
+ # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
+ filter_policy=rocksdb.BloomFilter:10:false
+ # set by jni: tableOptions.setCacheIndexAndFilterBlocks
+ cache_index_and_filter_blocks=true

conf/entry_location_rocksdb.conf

@@ -51,6 +51,8 @@
  max_bytes_for_level_base=268435456
  # set by jni: options.setTargetFileSizeBase
  target_file_size_base=67108864
+ # set by jni: options.setLevelCompactionDynamicLevelBytes
+ level_compaction_dynamic_level_bytes=true
 
 [TableOptions/BlockBasedTable "default"]
  # set by jni: tableOptions.setBlockSize
@@ -64,6 +66,4 @@
  # set by jni: tableOptions.setFilterPolicy, bloomfilter:[bits_per_key]:[use_block_based_builder]
  filter_policy=rocksdb.BloomFilter:10:false
  # set by jni: tableOptions.setCacheIndexAndFilterBlocks
- cache_index_and_filter_blocks=true
- # set by jni: options.setLevelCompactionDynamicLevelBytes
- level_compaction_dynamic_level_bytes=true
+ cache_index_and_filter_blocks=true