Support postings list compression using varint #19 (#24)

* implementation of PostingListWriter for varint postings #19

* add IPostingsListBuilder

to represent PostingsList Builder functionality

* fixed issue when PostingLists reused between indices

* implement VarintPostingsListBuilder

to support Varint listings posts

* minor fix

Author: aligusnet

src/InformationRetrieval.App/Program.cs

@@ -100,7 +100,7 @@ static void BuildExternalIndex()
 
             var reader = new CorpusZipReader<IList<char>>(wikiPath, charDataSerializer);
             using var buildableIndex = new BlockedExternalBuildableIndex<int>(
-                DictonaryBasedExternalBuildableIndex<int>.GetCreateMethod(RangeThreshold), 
+                DictonaryBasedExternalBuildableIndex<int>.GetCreateMethodWithVarintPostingsLists(), 
                 externalIndexPath);
             var indexBuilder = new IndexBuilder<int, IEnumerable<int>>(buildableIndex);
             var processor = new WikitextProcessor();

src/InformationRetrieval.Test/Indexing/External/BlockedExternalBuildableIndexTest.cs

@@ -1,4 +1,6 @@
-using System.IO.Abstractions.TestingHelpers;
+using System;
+using System.IO;
+using System.IO.Abstractions.TestingHelpers;
 using System.Linq;
 using Xunit;
 
@@ -12,12 +14,29 @@ public class BlockedExternalBuildableIndexTest
         private const string path = @"C:\path";
 
         [Fact]
-        public void BuildBlockedExternalIndexTest()
+        public void BuildBlockedExternalIndexTest_SortBased()
+        {
+            BuildBlockedExternalIndexTest(SortBasedExternalBuildableIndex<string>.CreateMethod);
+        }
+
+        [Fact]
+        public void BuildBlockedExternalIndexTest_DictionaryBased_MixedPostingsList()
+        {
+            BuildBlockedExternalIndexTest(DictonaryBasedExternalBuildableIndex<string>.GetCreateMethodWithMixedPostingsLists(3));
+        }
+
+        [Fact]
+        public void BuildBlockedExternalIndexTest_DictionaryBased_VarintPostingsList()
+        {
+            BuildBlockedExternalIndexTest(DictonaryBasedExternalBuildableIndex<string>.GetCreateMethodWithVarintPostingsLists());
+        }
+
+        private void BuildBlockedExternalIndexTest(Func<Stream, IExternalBuildableIndex<string>> createIndex)
         {
             var docs = new (DocumentId Id, string[] Text)[]
             {
-                (new DocumentId(0, 1), "d e f a".Split()),
                 (new DocumentId(0, 0), "a b c d".Split()),
+                (new DocumentId(0, 1), "d e f a".Split()),
                 (new DocumentId(0, 2), "e e f d".Split()),
 
                 (new DocumentId(1, 0), "a b".Split()),
@@ -30,9 +49,7 @@ public void BuildBlockedExternalIndexTest()
             var fileSystem = new MockFileSystem();
             fileSystem.Directory.CreateDirectory(path);
 
-            var buildableIndex = new BlockedExternalBuildableIndex<string>(
-                SortBasedExternalBuildableIndex<string>.CreateMethod, 
-                path, fileSystem);
+            var buildableIndex = new BlockedExternalBuildableIndex<string>(createIndex, path, fileSystem);
 
             foreach (var doc in docs)
             {

src/InformationRetrieval.Test/Indexing/External/DictionaryBasedExternalBuildableIndexTest.cs

@@ -3,8 +3,9 @@
 
 using Xunit;
 
-using InformationRetrieval.Indexing.External;
 using Corpus;
+using InformationRetrieval.Indexing.External;
+using InformationRetrieval.Indexing.PostingsList;
 
 namespace InformationRetrieval.Test.Indexing.External
 {
@@ -16,7 +17,8 @@ public class DictionaryBasedExternalBuildableIndexTests : IndexUnitTestsBase<Ext
         public void ExternalIndexBuildTest()
         {
             var stream = new MemoryStream();
-            var buildableIndex = new DictonaryBasedExternalBuildableIndex<string>(RangeThreshold, stream);
+            var postingsListBuilder = new MixedPostingsListBuilder<string>(RangeThreshold);
+            var buildableIndex = new DictonaryBasedExternalBuildableIndex<string>(postingsListBuilder, stream);
 
             var docs = new (DocumentId Id, string[] Text)[]
             {
@@ -54,7 +56,8 @@ public void ExternalIndexBuildTest()
         protected override ExternalIndex<string> CreateIndex(string[][] corpus)
         {
             var stream = new MemoryStream();
-            var buildableIndex = new DictonaryBasedExternalBuildableIndex<string>(RangeThreshold, stream);
+            var postingsListBuilder = new MixedPostingsListBuilder<string>(RangeThreshold);
+            var buildableIndex = new DictonaryBasedExternalBuildableIndex<string>(postingsListBuilder, stream);
             IndexHelper.BuildIndex(buildableIndex, corpus);
             return buildableIndex.BuildExternalIndex();
         }

src/InformationRetrieval.Test/Indexing/PostingsList/MixedPostingsListBuilderTest.cs

@@ -54,7 +54,7 @@ public int GetCount()
 
             public void IndexTerm(DocumentId id, string term, int position)
             {
-                builder.Add(id, term);
+                builder.AddTerm(id, term);
             }
 
             public IReadOnlyCollection<DocumentId> Search(string word)

src/InformationRetrieval.Test/Indexing/PostingsList/PostingsListTest.cs

@@ -141,6 +141,38 @@ public void ChainedUncompressedReadWriteTest()
             Assert.Equal(chain, deserialized);
         }
 
+        [Fact]
+        public void ChainedVarintReadWriteTest()
+        {
+            var stream = new MemoryStream();
+
+            var chain = new ListChain<DocumentId>()
+            {
+                new VarintPostingsList()
+                {
+                    0, 1, 2
+                },
+                GetDocIds(10, 11),
+                new VarintPostingsList()
+                {
+                    12, 13, 14, 15, 100, 111
+                },
+            };
+
+            using var writer = new PostingsListWriter(stream);
+            writer.Write(chain);
+
+            stream.Seek(0, SeekOrigin.Begin);
+
+            using var reader = new PostingsListReader(stream, leaveOpen: false);
+            var count = reader.ReadCount(0);
+            var deserialized = reader.Read(0);
+
+            Assert.Equal(chain.Count, count);
+            Assert.True(deserialized is VarintPostingsList);
+            Assert.Equal(chain.ToArray(), deserialized.ToArray());
+        }
+
         public static DocumentId[] GetDocIds(params uint[] ids)
         {
             return ids.Select(id => new DocumentId(id)).ToArray();

src/InformationRetrieval.Test/Indexing/PostingsList/VarintPostingsListBuilderTest.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+
+using Corpus;
+using InformationRetrieval.Indexing;
+using InformationRetrieval.Indexing.PostingsList;
+
+namespace InformationRetrieval.Test.Indexing.PostingsList
+{
+    class VarintPostingsListBuilderTest : IndexUnitTestsBase<VarintPostingsListBuilderTest.Index>
+    {
+        protected override Index CreateIndex(string[][] corpus)
+        {
+            var index = new Index();
+            IndexHelper.BuildIndex(index, corpus);
+            return index;
+        }
+
+        public class Index : IBuildableIndex<string>, ISearchableIndex<string>
+        {
+            VarintPostingsListBuilder<string> builder = new VarintPostingsListBuilder<string>();
+
+            public ISearchableIndex<string> Build()
+            {
+                return this;
+            }
+
+            public IReadOnlyCollection<DocumentId> GetAll()
+            {
+                return builder.Documents;
+            }
+
+            public int GetCount(string word)
+            {
+                if (builder.VarintPostingsLists.TryGetValue(word, out var blockList))
+                {
+                    return blockList.Count;
+                }
+
+                return 0;
+            }
+
+            public int GetCount()
+            {
+                return builder.Documents.Count;
+            }
+
+            public void IndexTerm(DocumentId id, string term, int position)
+            {
+                builder.AddTerm(id, term);
+            }
+
+            public IReadOnlyCollection<DocumentId> Search(string word)
+            {
+                if (builder.VarintPostingsLists.TryGetValue(word, out var ps))
+                {
+                    return ps;
+                }
+
+                return Array.Empty<DocumentId>();
+            }
+        }
+    }
+}

src/InformationRetrieval/Indexing/External/BlockedExternalBuildableIndex.cs

@@ -198,7 +198,10 @@ public void Clear()
                 }
                 indices.Clear();
 
-                fileSystem.Directory.Delete(basePath, recursive: true);
+                if (fileSystem.Directory.Exists(basePath))
+                {
+                    fileSystem.Directory.Delete(basePath, recursive: true);
+                }
             }
 
             public void Dispose()

src/InformationRetrieval/Indexing/External/DictonaryBasedExternalBuildableIndex.cs

@@ -7,15 +7,18 @@ namespace InformationRetrieval.Indexing.External
 {
     public class DictonaryBasedExternalBuildableIndex<T> : IExternalBuildableIndex<T> where T : notnull
     {
-        public static Func<Stream, IExternalBuildableIndex<T>> GetCreateMethod(int rangeThreshold) 
-            => s => new DictonaryBasedExternalBuildableIndex<T>(rangeThreshold, s);
+        public static Func<Stream, IExternalBuildableIndex<T>> GetCreateMethodWithMixedPostingsLists(int rangeThreshold)
+            => s => new DictonaryBasedExternalBuildableIndex<T>(new MixedPostingsListBuilder<T>(rangeThreshold), s);
+
+        public static Func<Stream, IExternalBuildableIndex<T>> GetCreateMethodWithVarintPostingsLists()
+            => s => new DictonaryBasedExternalBuildableIndex<T>(new VarintPostingsListBuilder<T>(), s);
 
         private readonly Stream postingsStream;
-        private readonly MixedPostingsListBuilder<T> builder;
+        private readonly IPostingsListBuilder<T> builder;
 
-        public DictonaryBasedExternalBuildableIndex(int rangeThreshold, Stream postingsStream)
+        public DictonaryBasedExternalBuildableIndex(IPostingsListBuilder<T> postingListBuilder, Stream postingsStream)
         {
-            builder = new MixedPostingsListBuilder<T>(rangeThreshold);
+            builder = postingListBuilder;
             this.postingsStream = postingsStream;
         }
 
@@ -26,14 +29,9 @@ public ExternalIndex<T> BuildExternalIndex()
         {
             var composer = new ExternalIndexComposer<T>(postingsStream);
 
-            composer.AddAllDocuments(builder.AllDocuments);
-
-            foreach (var postings in builder.RangedPostingsLists)
-            {
-                composer.AddPostingsList(postings.Key, postings.Value);
-            }
+            composer.AddAllDocuments(builder.Documents);
 
-            foreach (var postings in builder.UncompressedPostingsLists)
+            foreach (var postings in builder.PostingsLists)
             {
                 composer.AddPostingsList(postings.Key, postings.Value);
             }
@@ -42,6 +40,6 @@ public ExternalIndex<T> BuildExternalIndex()
         }
 
         public void IndexTerm(DocumentId id, T term, int position) 
-            => builder.Add(id, term);
+            => builder.AddTerm(id, term);
     }
 }

src/InformationRetrieval/Indexing/PostingsList/IPostingsListBuilder.cs

@@ -0,0 +1,15 @@
+using Corpus;
+using System;
+using System.Collections.Generic;
+using System.Text;
+
+namespace InformationRetrieval.Indexing.PostingsList
+{
+    public interface IPostingsListBuilder<T> where T : notnull
+    {
+        void AddTerm(DocumentId id, T term);
+
+        IReadOnlyCollection<DocumentId> Documents { get; }
+        IEnumerable<KeyValuePair<T, IReadOnlyCollection<DocumentId>>> PostingsLists { get; }
+    }
+}

src/InformationRetrieval/Indexing/PostingsList/MixedPostingsListBuilder.cs

@@ -10,7 +10,7 @@ namespace InformationRetrieval.Indexing.PostingsList
     /// and uncompressed postings list otherwise.
     /// </summary>
     /// <typeparam name="T">Term's type</typeparam>
-    public class MixedPostingsListBuilder<T> where T : notnull
+    public class MixedPostingsListBuilder<T> : IPostingsListBuilder<T> where T : notnull
     {
         private readonly int rangeThreshold;
         public RangePostingsList AllDocuments;
@@ -25,7 +25,25 @@ public MixedPostingsListBuilder(int rangeThreshold)
             this.rangeThreshold = rangeThreshold;
         }
 
-        public void Add(DocumentId id, T word)
+        public IReadOnlyCollection<DocumentId> Documents => AllDocuments;
+
+        public IEnumerable<KeyValuePair<T, IReadOnlyCollection<DocumentId>>> PostingsLists
+        {
+            get
+            {
+                foreach (var postings in RangedPostingsLists)
+                {
+                    yield return new KeyValuePair<T, IReadOnlyCollection<DocumentId>>(postings.Key, postings.Value);
+                }
+
+                foreach (var postings in UncompressedPostingsLists)
+                {
+                    yield return new KeyValuePair<T, IReadOnlyCollection<DocumentId>>(postings.Key, postings.Value);
+                }
+            }
+        }
+
+        public void AddTerm(DocumentId id, T word)
         {
             if (RangedPostingsLists.TryGetValue(word, out var blockList))
             {