feat: support for file retention

#8 support retainedFileCountLimit

Author: cocowalla

src/Serilog.Sinks.File.Archive/ArchiveHooks.cs

@@ -1,6 +1,8 @@
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.IO.Compression;
+using System.Linq;
 using Serilog.Debugging;
 
 namespace Serilog.Sinks.File.Archive
@@ -13,6 +15,7 @@ namespace Serilog.Sinks.File.Archive
     public class ArchiveHooks : FileLifecycleHooks
     {
         private readonly CompressionLevel compressionLevel;
+        private readonly int retainedFileCountLimit;
         private readonly string targetDirectory;
 
         /// <summary>
@@ -27,12 +30,26 @@ public class ArchiveHooks : FileLifecycleHooks
         public ArchiveHooks(CompressionLevel compressionLevel = CompressionLevel.Fastest, string targetDirectory = null)
         {
             if (compressionLevel == CompressionLevel.NoCompression && targetDirectory == null)
-                throw new ArgumentException("Either compressionLevel or targetDirectory must be set");
+                throw new ArgumentException($"Either {nameof(compressionLevel)} or {nameof(targetDirectory)} must be set");
 
             this.compressionLevel = compressionLevel;
             this.targetDirectory = targetDirectory;
         }
 
+        public ArchiveHooks(int retainedFileCountLimit, CompressionLevel compressionLevel = CompressionLevel.Fastest, string targetDirectory = null)
+        {
+            if (retainedFileCountLimit <= 0)
+                throw new ArgumentException($"{nameof(retainedFileCountLimit)} must be greater than zero", nameof(retainedFileCountLimit));
+            if (targetDirectory is not null && TokenExpander.IsTokenised(targetDirectory))
+                throw new ArgumentException($"{nameof(targetDirectory)} must not be tokenised when using {nameof(retainedFileCountLimit)}", nameof(targetDirectory));
+            if (compressionLevel == CompressionLevel.NoCompression)
+                throw new ArgumentException($"{nameof(compressionLevel)} must not be 'NoCompression' when using {nameof(retainedFileCountLimit)}", nameof(compressionLevel));
+
+            this.compressionLevel = compressionLevel;
+            this.retainedFileCountLimit = retainedFileCountLimit;
+            this.targetDirectory = null;
+        }
+
         public override void OnFileDeleting(string path)
         {
             try
@@ -70,12 +87,62 @@ public override void OnFileDeleting(string path)
                         sourceStream.CopyTo(compressStream);
                     }
                 }
+                //only apply archive file limit if we are archiving to a non tokenised path (constant path)
+                if (this.retainedFileCountLimit > 0 && !this.IsArchivePathTokenised)
+                {
+                    RemoveExcessFiles(currentTargetDir);
+                }
             }
             catch (Exception ex)
             {
                 SelfLog.WriteLine("Error while archiving file {0}: {1}", path, ex);
                 throw;
             }
         }
+
+        private bool IsArchivePathTokenised => this.targetDirectory is not null && TokenExpander.IsTokenised(this.targetDirectory);
+
+        private void RemoveExcessFiles(string folder)
+        {
+            var searchPattern = this.compressionLevel != CompressionLevel.NoCompression
+                    ? "*.gz"
+                    : "*.*";
+
+            var filesToDelete = Directory.GetFiles(folder, searchPattern)
+                .Select(f => new FileInfo(f))
+                .OrderByDescending(f => f, LogFileComparer.Default)
+                .Skip(this.retainedFileCountLimit)
+                .ToList();
+            foreach (var file in filesToDelete)
+            {
+                try
+                {
+                    file.Delete();
+                }
+                catch (Exception ex)
+                {
+                    SelfLog.WriteLine("Error while deleting file {0}: {1}", file.FullName, ex);
+                }
+            }
+        }
+
+        private class LogFileComparer : IComparer<FileInfo>
+        {
+            public static IComparer<FileInfo> Default = new LogFileComparer();
+
+            //This will not work correctly when the file uses a date format where lexicographical order does not correspond to chronological order but frankly, if you
+            //are using non ISO 8601 date formats in your files you should be shot.
+            //It would be best if the file sink could expose a way to sort files chronologically because I think LastWriteTime is probably not determisitic enough.
+            public int Compare(FileInfo x, FileInfo y)
+            {
+                if (x is null && y is null)
+                    return 0;
+                if (x is null)
+                    return -1;
+                if (y is null)
+                    return 1;
+                return string.Compare(x.Name, y.Name, StringComparison.OrdinalIgnoreCase);
+            }
+        }
     }
 }

src/Serilog.Sinks.File.Archive/TokenExpander.cs

@@ -1,4 +1,4 @@
-using System;
+using System;
 using System.Collections.Generic;
 using System.Text.RegularExpressions;
 using Serilog.Debugging;
@@ -47,6 +47,10 @@ public static string Expand(string source)
 
             return source;
         }
+        public static bool IsTokenised(string source)
+        {
+            return TryFindNextToken(source, 0, out var _);
+        }
 
         private static bool TryFindNextToken(string source, int startIdx, out Token token)
         {

test/Serilog.Sinks.File.Archive.Test/RollingFileSinkTests.cs

@@ -17,6 +17,42 @@ public class RollingFileSinkTests
             Some.LogEvent()
         };
 
+        // Test for removing old archive files in the same directory
+        [Fact]
+        public void Should_remove_old_archives()
+        {
+            var retainedFiles = 1;
+            var archiveWrapper = new ArchiveHooks(retainedFiles);
+
+            using (var temp = TempFolder.ForCaller())
+            {
+                var path = temp.AllocateFilename("log");
+
+                // Write events, such that we end up with 2 deleted files and 1 retained file
+                WriteLogEvents(path, archiveWrapper, LogEvents);
+
+                // Get all the files in the test directory
+                var files = Directory.GetFiles(temp.Path)
+                    .OrderBy(p => p, StringComparer.OrdinalIgnoreCase)
+                    .ToArray();
+
+                // We should have a single log file, and 'retainedFiles' gz files
+                files.Count(x => x.EndsWith("log")).ShouldBe(1);
+                files.Count(x => x.EndsWith("gz")).ShouldBe(retainedFiles);
+
+                // Ensure the data was GZip compressed, by decompressing and comparing against what we wrote
+                int i = LogEvents.Length - retainedFiles - 1;
+                foreach (var gzipFile in files.Where(x => x.EndsWith("gz")))
+                {
+                    var lines = Utils.DecompressLines(gzipFile);
+
+                    lines.Count.ShouldBe(1);
+                    lines[0].ShouldEndWith(LogEvents[i].MessageTemplate.Text);
+                    i++;
+                }
+            }
+        }
+
         // Test for compressing log files in the same directory
         [Fact]
         public void Should_compress_deleting_log_files_in_place()
@@ -174,7 +210,7 @@ private static void WriteLogEvents(string path, ArchiveHooks hooks, LogEvent[] l
                 foreach (var logEvent in logEvents)
                 {
                     log.Write(logEvent);
-                }
+                }                
             }
         }
     }

test/Serilog.Sinks.File.Archive.Test/Serilog.Sinks.File.Archive.Test.csproj

@@ -1,7 +1,7 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
     <PropertyGroup>
-        <TargetFrameworks>net452;netcoreapp2.2;netcoreapp3.1</TargetFrameworks>
+        <TargetFrameworks>net452;netcoreapp3.1;net5.0</TargetFrameworks>
         <AssemblyName>Serilog.Sinks.File.Archive.Tests</AssemblyName>
         <RootNamespace>Serilog.Sinks.File.Archive.Tests</RootNamespace>
     </PropertyGroup>