implement RangePostingsList serialization

Author: aligusnet

src/InformationRetrieval.Test/Indexing/PostingsList/PostingsListTest.cs

@@ -0,0 +1,63 @@
+using System.IO;
+
+using Xunit;
+
+using InformationRetrieval.Indexing.PostingsList;
+using Corpus;
+using System.Linq;
+
+namespace InformationRetrieval.Test.Indexing.PostingsList
+{
+    public class PostingsListTest
+    {
+        [Fact]
+        public void UncompressedReadWriteTest()
+        {
+            var stream = new MemoryStream();
+
+            var postings = GetDocIds(0, 1, 2, 10, 11, 12, 13, 14, 15, 100, 111);
+
+            using var writer = new PostingsListWriter(stream);
+            writer.Write(postings);
+
+            stream.Seek(0, SeekOrigin.Begin);
+
+            using var reader = new PostingsListReader(stream, leaveOpen: false);
+            var count = reader.ReadCount(0);
+            var deserialized = reader.Read(0);
+
+            Assert.Equal(postings.Length, count);
+            Assert.False(deserialized is RangePostingsList);
+            Assert.Equal(postings, deserialized);
+        }
+
+        [Fact]
+        public void RangeReadWriteTest()
+        {
+            var stream = new MemoryStream();
+
+            var rangePostings = new RangePostingsList()
+            {
+                0, 1, 2, 10, 11, 12, 13, 14, 15, 100, 111
+            };
+
+            using var writer = new PostingsListWriter(stream);
+            writer.Write(rangePostings);
+
+            stream.Seek(0, SeekOrigin.Begin);
+
+            using var reader = new PostingsListReader(stream, leaveOpen: false);
+            var count = reader.ReadCount(0);
+            var deserialized = reader.Read(0);
+
+            Assert.Equal(rangePostings.Count, count);
+            Assert.True(deserialized is RangePostingsList);
+            Assert.Equal(rangePostings, deserialized);
+        }
+
+        private DocumentId[] GetDocIds(params uint[] ids)
+        {
+            return ids.Select(id => new DocumentId(id)).ToArray();
+        }
+    }
+}

src/InformationRetrieval/Indexing/PostingsList/PostingsListReader.cs

@@ -22,12 +22,16 @@ public IReadOnlyCollection<DocumentId> Read(long position)
         {
             stream.Seek(position, SeekOrigin.Begin);
             var length = reader.ReadInt32();
-            var postings = new DocumentId[length];
-            for (int i = 0; i < length; ++i)
+            PostingsListType type = (PostingsListType)reader.ReadByte();
+
+            switch (type)
             {
-                postings[i] = new DocumentId(reader.ReadUInt32());
+                case PostingsListType.Ranged:
+                    return ReadRanged(length);
+
+                default:
+                    return ReadUmcompressed(length);
             }
-            return postings;
         }
 
         public int ReadCount(long position)
@@ -40,5 +44,41 @@ public void Dispose()
         {
             reader.Dispose();
         }
+
+        private IReadOnlyCollection<DocumentId> ReadUmcompressed(int length)
+        {
+            var postings = new DocumentId[length];
+            for (int i = 0; i < length; ++i)
+            {
+                postings[i] = new DocumentId(reader.ReadUInt32());
+            }
+
+            return postings;
+        }
+
+        private RangePostingsList ReadRanged(int count)
+        {
+            ushort length = reader.ReadUInt16();
+            var blocks = new List<DocumentIdRangeBlock>(length);
+            for (int i = 0; i < length; ++i)
+            {
+                blocks.Add(ReadBlock());
+            }
+
+            return new RangePostingsList(count, blocks);
+        }
+
+        private DocumentIdRangeBlock ReadBlock()
+        {
+            ushort blockId = reader.ReadUInt16();
+            ushort length = reader.ReadUInt16();
+            var ranges = new List<uint>(length);
+            for (int i = 0; i < length; ++i)
+            {
+                ranges.Add(reader.ReadUInt32());
+            }
+
+            return new DocumentIdRangeBlock(blockId, ranges);
+        }
     }
 }

src/InformationRetrieval/Indexing/PostingsList/PostingsListType.cs

@@ -0,0 +1,8 @@
+namespace InformationRetrieval.Indexing.PostingsList
+{
+    public enum PostingsListType : byte
+    {
+        Uncompressed,
+        Ranged
+    };
+}

src/InformationRetrieval/Indexing/PostingsList/PostingsListWriter.cs

@@ -22,10 +22,20 @@ public long Write(IReadOnlyCollection<DocumentId> postings)
         {
             var position = stream.Position;
             writer.Write(postings.Count);
-            foreach (var id in postings)
+
+            switch (postings)
             {
-                writer.Write(id.Id);
+                case RangePostingsList range:
+                    writer.Write((byte)PostingsListType.Ranged);
+                    WriteRanged(range.Blocks);
+                    break;
+
+                default:
+                    writer.Write((byte)PostingsListType.Uncompressed);
+                    WriteUncompressed(postings);
+                    break;
             }
+            
             writer.Flush();
 
             return position;
@@ -44,5 +54,32 @@ public void Dispose()
         {
             writer.Dispose();
         }
+
+        private void WriteUncompressed(IReadOnlyCollection<DocumentId> postings)
+        {
+            foreach (var id in postings)
+            {
+                writer.Write(id.Id);
+            }
+        }
+
+        private void WriteRanged(IList<DocumentIdRangeBlock> list)
+        {
+            writer.Write((ushort)list.Count);
+            foreach (var block in list)
+            {
+                WriteBlock(block);
+            }
+        }
+
+        private void WriteBlock(DocumentIdRangeBlock block)
+        {
+            writer.Write(block.BlockId);
+            writer.Write((ushort)block.Ranges.Count);
+            foreach (uint val in block.Ranges)
+            {
+                writer.Write(val);
+            }
+        }
     }
 }

src/InformationRetrieval/Indexing/PostingsList/RangePostingsList.cs

@@ -17,23 +17,31 @@ public class RangePostingsList : IReadOnlyCollection<DocumentId>
         private const int DefaultCapacity = 8;
 
         [ProtoMember(1)]
-        private readonly IList<DocumentIdRangeBlock> list = new List<DocumentIdRangeBlock>(DefaultCapacity);
+        public IList<DocumentIdRangeBlock> Blocks { get; }
 
         [ProtoMember(2)]
-        public int Count { get; private set; } = 0;
+        public int Count { get; private set; }
+
+        public RangePostingsList() : this(0, new List<DocumentIdRangeBlock>(DefaultCapacity)) { }
+
+        public RangePostingsList(int count, IList<DocumentIdRangeBlock> blocks)
+        {
+            this.Count = count;
+            this.Blocks = blocks;
+        }
 
         public void Add(DocumentId id)
         {
             DocumentIdRangeBlock block;
 
-            if (list.Count == 0 || list[^1].BlockId != id.BlockId)
+            if (Blocks.Count == 0 || Blocks[^1].BlockId != id.BlockId)
             {
                 block = new DocumentIdRangeBlock(id.BlockId);
-                list.Add(block);
+                Blocks.Add(block);
             }
             else
             {
-                block = list[^1];
+                block = Blocks[^1];
             }
 
             if (block.Add(id.LocalId)) Count++;
@@ -46,7 +54,7 @@ public void Add(uint id)
 
         public IEnumerator<DocumentId> GetEnumerator()
         {
-            foreach (var block in list)
+            foreach (var block in Blocks)
             {
                 foreach (var r in block.Ranges)
                 {
@@ -97,14 +105,14 @@ public class DocumentIdRangeBlock
         public IList<uint> Ranges { get; }
 
         // for protobuf deserialization
-        private DocumentIdRangeBlock() : this(0)
-        {
-        }
+        private DocumentIdRangeBlock() : this(0) { }
+
+        public DocumentIdRangeBlock(ushort id) : this(id, new List<uint>(DefaultCapacity)) { }
 
-        public DocumentIdRangeBlock(ushort id)
+        public DocumentIdRangeBlock(ushort id, IList<uint> ranges)
         {
             BlockId = id;
-            Ranges = new List<uint>(DefaultCapacity);
+            Ranges = ranges;
         }
 
         public bool Add(ushort localId)