add FindFirst/FindLast of VarintEncoder #19

Author: aligusnet

src/InformationRetrieval.Test/Utility/VarintEncoderTest.cs

@@ -23,5 +23,38 @@ public void VarintEncoderSmokeTest()
             Assert.Equal(new ulong[] { 1023, 5, ulong.MaxValue, ulong.MinValue }, encoded);
             Assert.Equal(4, VarintEncoder.GetIntegerCount(data.AsSpan()));
         }
+
+        [Fact]
+        public void FindFirstTest()
+        {
+            var data = new byte[30];
+            int pos = 0;
+            pos += VarintEncoder.Encode(1023, data.AsSpan(pos));
+            pos += VarintEncoder.Encode(5, data.AsSpan(pos));
+            pos += VarintEncoder.Encode(ulong.MaxValue, data.AsSpan(pos));
+            pos += VarintEncoder.Encode(ulong.MinValue, data.AsSpan(pos));
+
+            int posFirst = VarintEncoder.FindFirst(data.AsSpan());
+            ulong first = VarintEncoder.Decode(data.AsSpan(0, posFirst));
+
+            Assert.Equal(1023ul, first);
+        }
+
+        [Fact]
+        public void FindLastTest()
+        {
+            var data = new byte[30];
+            int pos = 0;
+            pos += VarintEncoder.Encode(1023, data.AsSpan(pos));
+            pos += VarintEncoder.Encode(5, data.AsSpan(pos));
+            pos += VarintEncoder.Encode(ulong.MaxValue, data.AsSpan(pos));
+            pos += VarintEncoder.Encode(ulong.MinValue, data.AsSpan(pos));
+            pos += VarintEncoder.Encode(719, data.AsSpan(pos));
+
+            int posLast = VarintEncoder.FindLast(data.AsSpan());
+            ulong last = VarintEncoder.Decode(data.AsSpan(posLast));
+
+            Assert.Equal(719ul, last);
+        }
     }
 }

src/InformationRetrieval/Indexing/PostingsList/VarintPostingsList.cs

@@ -34,7 +34,7 @@ public VarintPostingsList(byte[] buffer, int length)
             data = buffer;
             this.length = length;
             Count = VarintEncoder.GetIntegerCount(buffer.AsSpan(0, length));
-            prevInserted = this.LastOrDefault().Id;
+            prevInserted = this.LastOrDefault();
         }
 
         public int Count { get; private set; }
@@ -71,6 +71,19 @@ public IEnumerator<DocumentId> GetEnumerator()
             }
         }
 
+        private uint LastOrDefault()
+        {
+            int lastPos = VarintEncoder.FindLast(data.AsSpan(0, length));
+            if (lastPos >= 0)
+            {
+                return (uint)VarintEncoder.Decode(data.AsSpan(lastPos));
+            }
+            else
+            {
+                return 0;
+            }
+        }
+
         IEnumerator IEnumerable.GetEnumerator() 
             => GetEnumerator();
 

src/InformationRetrieval/Utility/VarintEncoder.cs

@@ -7,6 +7,8 @@ public static class VarintEncoder
     {
         public const int BufferLength = 10;
 
+        private const byte LastByteMask = 0x80;
+
         public static int Encode(ulong n, Span<byte> output)
         {
             Span<byte> buffer = stackalloc byte[BufferLength];
@@ -18,7 +20,7 @@ public static int Encode(ulong n, Span<byte> output)
             }
             while (n != 0);
 
-            buffer[BufferLength - 1] |= 0x80;
+            buffer[BufferLength - 1] |= LastByteMask;
 
             buffer.Slice(pos, BufferLength - pos).CopyTo(output);
 
@@ -31,7 +33,7 @@ public static IEnumerable<ulong> Decode(byte[] data, int start, int length)
             ulong n = 0;
             for (int i = start; i < end; ++i)
             {
-                if ((data[i] & 0x80) == 0x80)
+                if (IsLastByte(data[i]))
                 {
                     n = (n << 7) + (ulong)(data[i] & 0x7f);
                     yield return n;
@@ -44,19 +46,86 @@ public static IEnumerable<ulong> Decode(byte[] data, int start, int length)
             }
         }
 
+        /// <summary>
+        /// Decode teh first number from the buffer
+        /// </summary>
+        /// <param name="buffer">The buffer</param>
+        /// <returns>Decoded number</returns>
+        public static ulong Decode(ReadOnlySpan<byte> buffer)
+        {
+            ulong n = 0;
+            for (int i = 0; i < buffer.Length; ++i)
+            {
+                if (IsLastByte(buffer[i]))
+                {
+                    n = (n << 7) + (ulong)(buffer[i] & 0x7f);
+                    return n;
+                }
+                else
+                {
+                    n = (n << 7) + buffer[i];
+                }
+            }
+
+            throw new Exception("Failed to decode varint: last byte is missing");
+        }
+
         public static int GetIntegerCount(ReadOnlySpan<byte> buffer)
         {
             int count = 0;
 
             for (int i = 0; i < buffer.Length; ++i)
             {
-                if ((buffer[i] & 0x80) == 0x80)
+                if (IsLastByte(buffer[i]))
                 {
                     ++count;
                 }
             }
 
             return count;
         }
+
+        /// <summary>
+        /// Return length of the first number
+        /// </summary>
+        /// <param name="buffer">the buffer</param>
+        /// <returns>Length of the first byte if found, -1 otherwise</returns>
+        public static int FindFirst(ReadOnlySpan<byte> buffer)
+        {
+            for (int i = 0; i < buffer.Length; ++i)
+            {
+                if (IsLastByte(buffer[i]))
+                {
+                    return i + 1;
+                }
+            }
+
+            return -1;
+        }
+
+        /// <summary>
+        /// return positiong of the first byte of the last number
+        /// </summary>
+        /// <param name="buffer">the buffer</param>
+        /// <returns>Position of the last number if found, -1 otherwise</returns>
+        public static int FindLast(ReadOnlySpan<byte> buffer)
+        {
+            int lastBytesCount = 0;
+            for (int i = buffer.Length; i > 0; --i)
+            {
+                if (IsLastByte(buffer[i-1]))
+                {
+                    if(++lastBytesCount == 2)
+                    {
+                        return i;
+                    }
+                }
+            }
+
+            return -1;
+        }
+
+        private static bool IsLastByte(byte b) 
+            => (b & LastByteMask) == LastByteMask;
     }
 }