Merge pull request #12 from InCerryGit/ls_deleteOnClose

Add try recover
InCerryGit · Jun 19, 2023 · bfe696f · bfe696f
2 parents fb740a4 + 8a30565
commit bfe696f
Show file tree

Hide file tree

Showing 7 changed files with 326 additions and 10 deletions.
diff --git a/README.md b/README.md
@@ -177,8 +177,11 @@ public FasterKvCache(
 * PageSizeBit：FasterKv内存页的大小，配置为2的次方数。**默认为20，也就是2的20次方，每页大小为1MB内存。**
 * ReadCacheMemorySizeBit：FasterKv读缓存内存字节数，配置为2的次方数，缓存内的都是热点数据，最好设置为热点数据所占用的内存数量。**默认为20，也就是2的20次方，使用16MB内存。**
 * ReadCachePageSizeBit：FasterKv读缓存内存页的大小，配置为2的次方数。**默认为20，也就是2的20次方，每页大小为1MB内存。**
-* LogPath：FasterKv日志文件的目录，默认会创建两个日志文件，一个以`.log`结尾，一个以`obj.log`结尾，分别存放日志信息和Value序列化信息，**注意，不要让不同的FasterKvCache使用相同的日志文件，会出现不可预料异常**。**默认为{当前目录}/FasterKvCache/{进程Id}-HLog/{实例名称}.log**。
+* LogPath：FasterKv日志文件的目录，默认会创建两个日志文件，一个以`.log`结尾，一个以`obj.log`结尾，分别存放日志信息和Value序列化信息，如果开启了`DeleteFileOnClose`和`TryRecoverLatest`，也会创建一个`.checkpoint`来进行故障恢复，**注意，不要让不同的FasterKvCache使用相同的日志文件，会出现不可预料异常**。**默认为`{当前目录}/FasterKvCache/{进程Id}-HLog/{实例名称}.log`**。
 * SerializerName：Value序列化器名称，需要安装序列化Nuget包，如果没有单独指定`Name`的情况下，可以使用`MessagePack`和`SystemTextJson`。**默认无需指定**。
+* PreallocateFile: 是否预分配日志文件，如果开启，那么在创建日志文件的时候会预分配指定1GB大小的文件，如果有大量数据的话，预分配能提升性能。**默认为false**。
+* DeleteFileOnClose: 是否在关闭的时候删除日志文件，如果开启，那么在关闭的时候会删除日志文件，如果不开启，那么会保留日志文件，下次启动的时候会继续使用。**默认为true**。
+* TryRecoverLatest: 是否在启动的时候尝试恢复最新的日志文件，如果开启，那么在启动的时候会尝试恢复最新的日志文件，如果不开启，那么会重新开始，如果要使它生效，需关闭`DeleteFileOnClose`。**默认为false**。
 * ExpiryKeyScanInterval：由于FasterKv不支持过期删除功能，所以目前的实现是会定期扫描所有的key，将过期的key删除。这里配置的就是扫描间隔。**默认为5分钟**。
 * CustomStore：如果您不想使用自动生成的实例，那么可以自定义的FasterKv实例。**默认为null**。
 

diff --git a/src/FasterKv.Cache.Core/Configurations/FasterKvCacheOptions.cs b/src/FasterKv.Cache.Core/Configurations/FasterKvCacheOptions.cs
@@ -61,6 +61,21 @@ public sealed class FasterKvCacheOptions
     /// </summary>
     public string? SerializerName { get; set; }
 
+    /// <summary>
+    /// Preallocate file
+    /// </summary>
+    public bool PreallocateFile { get; set; } = false;
+
+    /// <summary>
+    /// Delete file on close
+    /// </summary>
+    public bool DeleteFileOnClose { get; set; } = true;
+
+    /// <summary>
+    /// Try recover latest
+    /// </summary>
+    public bool TryRecoverLatest { get; set; } = false;
+
     /// <summary>
     /// Expiry key scan thread interval
     /// </summary>
@@ -97,11 +112,11 @@ internal LogSettings GetLogSettings(string? name)
         return new LogSettings
         {
             LogDevice = Devices.CreateLogDevice(Path.Combine(LogPath, name) + ".log",
-                preallocateFile: true,
-                deleteOnClose: true),
+                preallocateFile: PreallocateFile,
+                deleteOnClose: DeleteFileOnClose),
             ObjectLogDevice = Devices.CreateLogDevice(Path.Combine(LogPath, name) + ".obj.log",
-                preallocateFile: true,
-                deleteOnClose: true),
+                preallocateFile: PreallocateFile,
+                deleteOnClose: DeleteFileOnClose),
             PageSizeBits = PageSizeBit,
             MemorySizeBits = MemorySizeBit,
             ReadCacheSettings = new ReadCacheSettings

diff --git a/src/FasterKv.Cache.Core/FasterKvCache.cs b/src/FasterKv.Cache.Core/FasterKvCache.cs
@@ -59,7 +59,13 @@ public FasterKvCache(string name,
             _fasterKv = new FasterKV<string, ValueWrapper>(
                 options.IndexCount,
                 _logSettings,
-                serializerSettings: serializer);
+                checkpointSettings: new CheckpointSettings()
+                {
+                    CheckpointDir = options.LogPath + ".checkpoint"
+                },
+                serializerSettings: serializer,
+                tryRecoverLatest: options.TryRecoverLatest
+            );
         }
         else
         {
@@ -312,10 +318,15 @@ private void Dispose(bool _)
 
         if (_options.CustomStore != _fasterKv)
         {
+            if (_options.DeleteFileOnClose == false)
+            {
+                _fasterKv.TakeFullCheckpointAsync(CheckpointType.FoldOver).AsTask().GetAwaiter().GetResult();
+            }
             _fasterKv.Dispose();
         }
-
+        
         _logSettings?.LogDevice.Dispose();
+        _logSettings?.ObjectLogDevice.Dispose();
     }
 
     public void Dispose()

diff --git a/src/FasterKv.Cache.Core/FasterKvStore.TValue.cs b/src/FasterKv.Cache.Core/FasterKvStore.TValue.cs
@@ -57,7 +57,13 @@ public FasterKvCache(string name,
             _fasterKv = new FasterKV<string, ValueWrapper<TValue>>(
                 options.IndexCount,
                 _logSettings,
-                serializerSettings: serializer);
+                checkpointSettings: new CheckpointSettings()
+                {
+                    CheckpointDir = options.LogPath + ".checkpoint"
+                },
+                serializerSettings: serializer,
+                tryRecoverLatest: options.TryRecoverLatest
+            );
         }
         else
         {
@@ -295,10 +301,15 @@ private void Dispose(bool _)
 
         if (_options.CustomStore != _fasterKv)
         {
+            if (_options.DeleteFileOnClose == false)
+            {
+                _fasterKv.TakeFullCheckpointAsync(CheckpointType.FoldOver).AsTask().GetAwaiter().GetResult();
+            }
             _fasterKv.Dispose();
         }
 
         _logSettings?.LogDevice.Dispose();
+        _logSettings?.ObjectLogDevice.Dispose();
     }
 
     public void Dispose()

diff --git a/tests/FasterKv.Cache.Core.Tests/KvStore/DeleteFileOnClose/DeleteOnCloseTest.cs b/tests/FasterKv.Cache.Core.Tests/KvStore/DeleteFileOnClose/DeleteOnCloseTest.cs
@@ -0,0 +1,138 @@
+using FasterKv.Cache.Core.Abstractions;
+using FasterKv.Cache.Core.Configurations;
+using FasterKv.Cache.MessagePack;
+
+namespace FasterKv.Cache.Core.Tests.KvStore.DeleteFileOnClose;
+
+public class DeleteOnCloseTest
+{
+    private string GetPath()
+    {
+        var guid = Guid.NewGuid().ToString("N");
+        return $"./unit-test/faster-kv-store-delete-on-close-test/{guid}/log";
+    }
+
+    [Fact]
+    public void Should_Not_Delete_On_Close()
+    {
+        var path = GetPath();
+        var fasterKv = new FasterKvCache<string>(null!,
+            new DefaultSystemClock(),
+            new FasterKvCacheOptions
+            {
+                SerializerName = "MessagePack",
+                ExpiryKeyScanInterval = TimeSpan.Zero,
+                IndexCount = 16384,
+                MemorySizeBit = 10,
+                PageSizeBit = 10,
+                ReadCacheMemorySizeBit = 10,
+                ReadCachePageSizeBit = 10,
+                PreallocateFile = false,
+                DeleteFileOnClose = false,
+                LogPath = path
+            },
+            new IFasterKvCacheSerializer[]
+            {
+                new MessagePackFasterKvCacheSerializer
+                {
+                    Name = "MessagePack"
+                }
+            },
+            null);
+
+        fasterKv.Set("key", "value");
+
+        Assert.Equal("value", fasterKv.Get("key"));
+
+        fasterKv.Dispose();
+
+        Assert.True(File.Exists($"{path}.log.0"));
+        Assert.True(File.Exists($"{path}.obj.log.0"));
+
+        Cleanup(path);
+    }
+
+    [Fact]
+    public void Should_Restore_The_Data()
+    {
+        var path = GetPath();
+        var fasterKv = new FasterKvCache<string>(null!,
+            new DefaultSystemClock(),
+            new FasterKvCacheOptions
+            {
+                SerializerName = "MessagePack",
+                ExpiryKeyScanInterval = TimeSpan.Zero,
+                IndexCount = 16384,
+                MemorySizeBit = 10,
+                PageSizeBit = 10,
+                ReadCacheMemorySizeBit = 10,
+                ReadCachePageSizeBit = 10,
+                PreallocateFile = false,
+                DeleteFileOnClose = false,
+                LogPath = path
+            },
+            new IFasterKvCacheSerializer[]
+            {
+                new MessagePackFasterKvCacheSerializer
+                {
+                    Name = "MessagePack"
+                }
+            },
+            null);
+
+        for (int i = 0; i < 100; i++)
+        {
+            fasterKv.Set($"key{i}", $"value{i}");   
+        }
+
+        Assert.Equal("value0", fasterKv.Get("key0"));
+
+        fasterKv.Dispose();
+
+        Assert.True(File.Exists($"{path}.log.0"));
+        Assert.True(File.Exists($"{path}.obj.log.0"));
+
+        fasterKv = new FasterKvCache<string>(null!,
+            new DefaultSystemClock(),
+            new FasterKvCacheOptions
+            {
+                SerializerName = "MessagePack",
+                ExpiryKeyScanInterval = TimeSpan.Zero,
+                IndexCount = 16384,
+                MemorySizeBit = 10,
+                PageSizeBit = 10,
+                ReadCacheMemorySizeBit = 10,
+                ReadCachePageSizeBit = 10,
+                PreallocateFile = false,
+                DeleteFileOnClose = false,
+                TryRecoverLatest = true,
+                LogPath = path
+            },
+            new IFasterKvCacheSerializer[]
+            {
+                new MessagePackFasterKvCacheSerializer
+                {
+                    Name = "MessagePack"
+                }
+            },
+            null);
+
+        for (int i = 0; i < 100; i++)
+        {
+            Assert.Equal($"value{i}", fasterKv.Get($"key{i}"));
+        }
+
+        fasterKv.Dispose();
+
+        Cleanup(path);
+    }
+
+    private static void Cleanup(string path)
+    {
+        var dir = Path.GetDirectoryName(path);
+        if (Directory.Exists(dir))
+        {
+            Directory.Delete(dir, true);
+        }
+    }
+}
diff --git a/tests/FasterKv.Cache.Core.Tests/KvStore/DeleteFileOnClose/DeleteOnCloseTestObject.cs b/tests/FasterKv.Cache.Core.Tests/KvStore/DeleteFileOnClose/DeleteOnCloseTestObject.cs
@@ -0,0 +1,138 @@
+using FasterKv.Cache.Core.Abstractions;
+using FasterKv.Cache.Core.Configurations;
+using FasterKv.Cache.MessagePack;
+
+namespace FasterKv.Cache.Core.Tests.KvStore.DeleteFileOnClose;
+
+public class DeleteOnCloseTestObject
+{
+    private string GetPath()
+    {
+        var guid = Guid.NewGuid().ToString("N");
+        return $"./unit-test/faster-kv-store-delete-on-close-object-test/{guid}/log";
+    }
+
+    [Fact]
+    public void Should_Not_Delete_On_Close()
+    {
+        var path = GetPath();
+        var fasterKv = new FasterKvCache(null!,
+            new DefaultSystemClock(),
+            new FasterKvCacheOptions
+            {
+                SerializerName = "MessagePack",
+                ExpiryKeyScanInterval = TimeSpan.Zero,
+                IndexCount = 16384,
+                MemorySizeBit = 10,
+                PageSizeBit = 10,
+                ReadCacheMemorySizeBit = 10,
+                ReadCachePageSizeBit = 10,
+                PreallocateFile = false,
+                DeleteFileOnClose = false,
+                LogPath = path
+            },
+            new IFasterKvCacheSerializer[]
+            {
+                new MessagePackFasterKvCacheSerializer
+                {
+                    Name = "MessagePack"
+                }
+            },
+            null);
+
+        fasterKv.Set("key", "value");
+
+        Assert.Equal("value", fasterKv.Get<string>("key"));
+
+        fasterKv.Dispose();
+
+        Assert.True(File.Exists($"{path}.log.0"));
+        Assert.True(File.Exists($"{path}.obj.log.0"));
+
+        Cleanup(path);
+    }
+
+    [Fact]
+    public void Should_Restore_The_Data()
+    {
+        var path = GetPath();
+        var fasterKv = new FasterKvCache(null!,
+            new DefaultSystemClock(),
+            new FasterKvCacheOptions
+            {
+                SerializerName = "MessagePack",
+                ExpiryKeyScanInterval = TimeSpan.Zero,
+                IndexCount = 16384,
+                MemorySizeBit = 10,
+                PageSizeBit = 10,
+                ReadCacheMemorySizeBit = 10,
+                ReadCachePageSizeBit = 10,
+                PreallocateFile = false,
+                DeleteFileOnClose = false,
+                LogPath = path
+            },
+            new IFasterKvCacheSerializer[]
+            {
+                new MessagePackFasterKvCacheSerializer
+                {
+                    Name = "MessagePack"
+                }
+            },
+            null);
+
+        for (int i = 0; i < 100; i++)
+        {
+            fasterKv.Set($"key{i}", $"value{i}");   
+        }
+
+        Assert.Equal("value0", fasterKv.Get<string>("key0"));
+
+        fasterKv.Dispose();
+
+        Assert.True(File.Exists($"{path}.log.0"));
+        Assert.True(File.Exists($"{path}.obj.log.0"));
+
+        fasterKv = new FasterKvCache(null!,
+            new DefaultSystemClock(),
+            new FasterKvCacheOptions
+            {
+                SerializerName = "MessagePack",
+                ExpiryKeyScanInterval = TimeSpan.Zero,
+                IndexCount = 16384,
+                MemorySizeBit = 10,
+                PageSizeBit = 10,
+                ReadCacheMemorySizeBit = 10,
+                ReadCachePageSizeBit = 10,
+                PreallocateFile = false,
+                DeleteFileOnClose = false,
+                TryRecoverLatest = true,
+                LogPath = path
+            },
+            new IFasterKvCacheSerializer[]
+            {
+                new MessagePackFasterKvCacheSerializer
+                {
+                    Name = "MessagePack"
+                }
+            },
+            null);
+
+        for (int i = 0; i < 100; i++)
+        {
+            Assert.Equal($"value{i}", fasterKv.Get<string>($"key{i}"));   
+        }
+
+        fasterKv.Dispose();
+
+        Cleanup(path);
+    }
+
+    private static void Cleanup(string path)
+    {
+        var dir = Path.GetDirectoryName(path);
+        if (Directory.Exists(dir))
+        {
+            Directory.Delete(dir, true);
+        }
+    }
+}