implement ExternalIndexSerializer #17

Author: aligusnet

src/NaturalLanguage.App/Program.cs

@@ -57,6 +57,7 @@ static void Main(string[] args)
                 ProcessAndIndexWikipedia,
                 PrintIndexStats,
                 BuildExternalIndex,
+                PrintExternalIndexStats,
             };
 
             foreach (var action in actions)
@@ -100,7 +101,17 @@ static void BuildExternalIndex()
             var processor = new WikitextProcessor();
             indexBuilder.IndexCorpus(processor.Transform(reader.Read()));
 
-            var index = buildableIndex.Build();
+            using var index = buildableIndex.Build();
+
+            var serializer = new ExternalIndexSerializer<int>();
+            serializer.Serialize(externalIndexPath, index);
+        }
+
+        static void PrintExternalIndexStats()
+        {
+            var serializer = new ExternalIndexSerializer<int>();
+            using var index = serializer.Deserialize(externalIndexPath);
+
             Console.WriteLine($"The: {index.Search(TextHasher.CalculateHashCode("the".AsSpan())).Count()}");
         }
 

src/NaturalLanguage.WPF/MainWindow.xaml.cs

@@ -30,10 +30,13 @@ public partial class MainWindow : Window
         static readonly string basePath = @"F:\wikipedia";
         static readonly string wikiPath = IO::Path.Combine(basePath, "enwiki");
         static readonly string indexPath = IO::Path.Combine(basePath, "index.bin");
+        static readonly string externalIndexPath = IO::Path.Combine(basePath, "external_index");
 
         private readonly Lazy<BooleanSearchEngine<int>> engine;
         private readonly Lazy<CorpusMetadata> metadata;
         private readonly CorpusZipReader<string> reader;
+
+        private bool useExternalIndex = true;
 
         Stopwatch timer = new Stopwatch();
 
@@ -102,12 +105,35 @@ public DocumentIdTemplate(DocumentId id, CorpusMetadata metadata)
             public override string ToString() => title;
         }
 
-        private BooleanSearchEngine<int> LoadSearchEngine()
+        private ISearchableIndex<int> LoadInMemoryIndex()
         {
             using var file = IO::File.OpenRead(indexPath);
+            return DictionaryIndex<int>.Deserialize(file);
+        }
+
+        private ISearchableIndex<int> LoadExternalIndex()
+        {
+            var serializer = new ExternalIndexSerializer<int>();
+            return serializer.Deserialize(externalIndexPath);
+        }
+
+        private ISearchableIndex<int> LoadIndex()
+        {
+            if (useExternalIndex)
+            {
+                return LoadExternalIndex();
+            }
+            else
+            {
+                return LoadInMemoryIndex();
+            }
+        }
+
+        private BooleanSearchEngine<int> LoadSearchEngine()
+        {
             var timer = new Stopwatch();
             timer.Start();
-            var index = DictionaryIndex<int>.Deserialize(file);
+            var index = LoadIndex();
             timer.Stop();
             Log($"Index loaded in {timer.Elapsed:g}");
 

src/NaturalLanguageTools/Indexing/BlockedExternalBuildableIndex.cs

@@ -17,16 +17,14 @@ public class BlockedExternalBuildableIndex<T> : IBuildableIndex<T>, IDisposable
         where T : IComparable<T>
     {
         private readonly BuildableIndexManager indexManager;
-        private readonly IFileSystem fileSystem;
         private readonly Stream stream;
 
         public BlockedExternalBuildableIndex(string basePath) : this(basePath, new FileSystem()) { }
 
         public BlockedExternalBuildableIndex(string basePath, IFileSystem fileSystem)
         {
             this.indexManager = new BuildableIndexManager(basePath, fileSystem);
-            this.fileSystem = fileSystem;
-            string indexPath = Path.Combine(basePath, "corpus.index");
+            string indexPath = Path.Combine(basePath, ExternalIndexSerializer<T>.IndexFileName);
             this.stream = fileSystem.File.Open(indexPath, FileMode.OpenOrCreate, FileAccess.ReadWrite);
         }
 
@@ -36,7 +34,7 @@ public void IndexTerm(DocumentId id, T term, int position)
             index.IndexTerm(id, term, position);
         }
 
-        public ISearchableIndex<T> Build()
+        public ExternalIndex<T> Build()
         {
             var composer = new ExternalIndexComposer<T>(stream);
 
@@ -132,6 +130,11 @@ public void Dispose()
             indexManager.Dispose();
         }
 
+        ISearchableIndex<T> IBuildableIndex<T>.Build()
+        {
+            return Build();
+        }
+
         private struct IndexInfo
         {
             public string Path;

src/NaturalLanguageTools/Indexing/ExternalIndexSerializer.cs

@@ -0,0 +1,47 @@
+using System.Collections.Generic;
+using System.IO;
+using System.IO.Abstractions;
+
+using ProtoBuf;
+
+namespace NaturalLanguageTools.Indexing
+{
+    public class ExternalIndexSerializer<T>
+    {
+        public const string IndexFileName = "corpus.index";
+        public const string OffsetsFileName = "corpus.offsets";
+
+        private readonly FileSystem fileSystem;
+
+        public ExternalIndexSerializer() : this(new FileSystem()) { }
+
+        public ExternalIndexSerializer(FileSystem fileSystem)
+        {
+            this.fileSystem = fileSystem;
+        }
+
+        public ExternalIndex<T> Deserialize(string basePath)
+        {
+            var postingStream = fileSystem.File.OpenRead(Path.Combine(basePath, IndexFileName));
+            using var offsetsStream = fileSystem.File.OpenRead(Path.Combine(basePath, OffsetsFileName));
+            var offsets = DeserializeOffsets(offsetsStream);
+            return new ExternalIndex<T>(offsets, postingStream);
+        }
+
+        public void Serialize(string basePath, ExternalIndex<T> index)
+        {
+            using var offsetsStream = fileSystem.File.OpenWrite(Path.Combine(basePath, OffsetsFileName));
+            SerializeOffsets(offsetsStream, index.Offsets);
+        }
+
+        public static void SerializeOffsets(Stream stream, IDictionary<T, long> offsets)
+        {
+            Serializer.Serialize(stream, offsets);
+        }
+
+        public static IDictionary<T, long> DeserializeOffsets(Stream stream)
+        {
+            return Serializer.Deserialize<Dictionary<T, long>>(stream);
+        }
+    }
+}