RavenDB-22534 Changed the handling of complex fields in Corax:

- Corax will throw an exception on attempt to index complex field by default (existing indexes will keep previous behavior) - added new configuration option that will allow to change the behavio - 'Throw' (default) or 'Skip' - avoid storing complex fields by default (user needs to mark it explicitly)
arekpalinski · Aug 12, 2024 · dc3d2f8 · dc3d2f8
1 parent 8270d29
commit dc3d2f8
Show file tree

Hide file tree

Showing 36 changed files with 493 additions and 93 deletions.
diff --git a/src/Corax/Mappings/IndexFieldBinding.cs b/src/Corax/Mappings/IndexFieldBinding.cs
@@ -16,10 +16,10 @@ public sealed class IndexFieldBinding
     private Analyzer _analyzer;
     public readonly bool HasSuggestions;
     public readonly bool HasSpatial;
-    public FieldIndexingMode FieldIndexingMode => _silentlyChangedIndexingMode ?? _fieldIndexingMode;
+    public FieldIndexingMode FieldIndexingMode => _silentlyChangedIndexingModeLegacy ?? _fieldIndexingMode;
     private readonly FieldIndexingMode _fieldIndexingMode;
     public readonly bool ShouldStore;
-    private FieldIndexingMode? _silentlyChangedIndexingMode;
+    private FieldIndexingMode? _silentlyChangedIndexingModeLegacy;
     private string _fieldName;
 
     private readonly bool _isFieldBindingForWriter;
@@ -75,7 +75,7 @@ public void OverrideFieldIndexingMode(FieldIndexingMode mode)
     {
         AssertBindingIsMadeForIndexing();
 
-        _silentlyChangedIndexingMode = mode;
+        _silentlyChangedIndexingModeLegacy = mode;
     }
 
     public void SetAnalyzer(Analyzer analyzer)

diff --git a/src/Raven.Server/Config/Categories/IndexingConfiguration.cs b/src/Raven.Server/Config/Categories/IndexingConfiguration.cs
@@ -580,6 +580,12 @@ public virtual bool RunInMemory
         [IndexUpdateType(IndexUpdateType.None)]
         public IndexResetMode ResetMode { get; set; }
 
+        [Description("The default complex field indexing behavior for static Corax indexes")]
+        [DefaultValue(CoraxComplexFieldIndexingBehavior.Throw)]
+        [IndexUpdateType(IndexUpdateType.Reset)]
+        [ConfigurationEntry("Indexing.Corax.Static.ComplexFieldIndexingBehavior", ConfigurationEntryScope.ServerWideOrPerDatabaseOrPerIndex)]
+        public CoraxComplexFieldIndexingBehavior CoraxStaticIndexComplexFieldIndexingBehavior { get; protected set; }
+
         protected override void ValidateProperty(PropertyInfo property)
         {
             base.ValidateProperty(property);
@@ -617,5 +623,12 @@ public enum IndexStartupBehaviorType
             Pause,
             Delay
         }
+
+        public enum CoraxComplexFieldIndexingBehavior
+        {
+            None,
+            Throw,
+            Skip
+        }
     }
 }
diff --git a/src/Raven.Server/Documents/Indexes/Index.cs b/src/Raven.Server/Documents/Indexes/Index.cs
@@ -959,6 +959,9 @@ protected virtual void LoadValues()
                 LastIndexingTime = _indexStorage.ReadLastIndexingTime(tx);
                 MaxNumberOfOutputsPerDocument = _indexStorage.ReadMaxNumberOfOutputsPerDocument(tx);
                 ArchivedDataProcessingBehavior = _indexStorage.ReadArchivedDataProcessingBehavior(tx);
+
+                if (SearchEngineType is SearchEngineType.Corax)
+                    CoraxComplexFieldIndexingBehavior = _indexStorage.ReadCoraxComplexFieldIndexingBehavior(tx);
             }
         }
 
@@ -4489,7 +4492,7 @@ private Size? TransactionSizeLimit
             }
         }
 
-
+        public IndexingConfiguration.CoraxComplexFieldIndexingBehavior CoraxComplexFieldIndexingBehavior { get; private set; }
 
         public ArchivedDataProcessingBehavior ArchivedDataProcessingBehavior { get; private set; }
 

diff --git a/src/Raven.Server/Documents/Indexes/IndexDefinitionBaseServerSide.cs b/src/Raven.Server/Documents/Indexes/IndexDefinitionBaseServerSide.cs
@@ -178,11 +178,12 @@ public static class IndexVersion
             public const long StoreOnlySupportInCoraxIndexes = 60_003; // RavenDB-22369
             public const long JavaScriptProperlyHandleDynamicFieldsIndexFields = 60_004; // RavenDB-22363
             public const long LoadDocumentWithDynamicCollectionNameShouldThrow = 61_000; // RavenDB-22359
+            public const long CoraxComplexFieldIndexingBehavior = 61_0001;
 
             /// <summary>
             /// Remember to bump this
             /// </summary>
-            public const long CurrentVersion = LoadDocumentWithDynamicCollectionNameShouldThrow;
+            public const long CurrentVersion = CoraxComplexFieldIndexingBehavior;
 
             public static bool IsTimeTicksInJavaScriptIndexesSupported(long indexVersion)
             {

diff --git a/src/Raven.Server/Documents/Indexes/IndexStorage.cs b/src/Raven.Server/Documents/Indexes/IndexStorage.cs
@@ -8,6 +8,7 @@
 using Raven.Client.Documents.Operations.DataArchival;
 using Raven.Client.Util;
 using Raven.Server.Config;
+using Raven.Server.Config.Categories;
 using Raven.Server.Documents.Indexes.Static;
 using Raven.Server.Exceptions;
 using Raven.Server.Indexing;
@@ -154,11 +155,14 @@ private unsafe void CreateSchema(DocumentDatabase documentDatabase)
                 void PersistConfiguration()
                 {
                     var configurationTree = tx.InnerTransaction.CreateTree(IndexSchema.ConfigurationTree);
-                    PersistSearchEngine(configurationTree);
+                    var searchEngine = PersistSearchEngine(configurationTree);
                     AssertAndPersistAnalyzer(configurationTree, RavenConfiguration.GetKey(x => x.Indexing.DefaultAnalyzer), _index.Configuration.DefaultAnalyzer, Raven.Client.Constants.Documents.Indexing.Analyzers.Default);
                     AssertAndPersistAnalyzer(configurationTree, RavenConfiguration.GetKey(x => x.Indexing.DefaultExactAnalyzer), _index.Configuration.DefaultExactAnalyzer, Raven.Client.Constants.Documents.Indexing.Analyzers.DefaultExact);
                     AssertAndPersistAnalyzer(configurationTree, RavenConfiguration.GetKey(x => x.Indexing.DefaultSearchAnalyzer), _index.Configuration.DefaultSearchAnalyzer, Raven.Client.Constants.Documents.Indexing.Analyzers.DefaultSearch);
                     PersistArchivedDataProcessingBehavior(configurationTree, _index.GetDefaultArchivedDataProcessingBehavior());
+
+                    if (searchEngine is SearchEngineType.Corax)
+                        PersistCoraxComplexFieldIndexingBehavior(configurationTree);
                 }
 
                 void AssertAndPersistAnalyzer(Tree configurationTree, string configurationKey, string expectedAnalyzer, string defaultAnalyzer)
@@ -181,7 +185,7 @@ void AssertAndPersistAnalyzer(Tree configurationTree, string configurationKey, s
                     configurationTree.Add(configurationKey, expectedAnalyzer);
                 }
 
-                void PersistSearchEngine(Tree configurationTree)
+                SearchEngineType PersistSearchEngine(Tree configurationTree)
                 {
                     string configurationKey = nameof(SearchEngineType);
                     string configurationName = _index.Type.IsAuto() ? RavenConfiguration.GetKey(x => x.Indexing.AutoIndexingEngineType) : RavenConfiguration.GetKey(x => x.Indexing.StaticIndexingEngineType);
@@ -198,13 +202,21 @@ void PersistSearchEngine(Tree configurationTree)
                         {
                             throw new InvalidDataException($"Invalid search engine for {_index.Name}  was saved previously or it's corrupted. Please reset the index.");
                         }
+
+                        return persistedSearchEngineType;
                     }
                     else
                     {
+                        SearchEngineType type;
+
                         if (_index.Definition.Version < IndexDefinitionBaseServerSide.IndexVersion.EngineTypeStored)
-                            configurationTree.Add(configurationKey, SearchEngineType.Lucene.ToString());
+                            type = SearchEngineType.Lucene;
                         else
-                            configurationTree.Add(configurationKey, defaultEngineType.ToString());
+                            type = defaultEngineType;
+
+                        configurationTree.Add(configurationKey, type.ToString());
+
+                        return type;
                     }
                 }
 
@@ -222,6 +234,29 @@ void PersistArchivedDataProcessingBehavior(Tree configurationTree, ArchivedDataP
                         configurationTree.Add(configurationKey, defaultBehavior.ToString());
                     }
                 }
+
+                void PersistCoraxComplexFieldIndexingBehavior(Tree configurationTree)
+                {
+                    if (_index.Definition.Version < IndexDefinitionBaseServerSide.IndexVersion.CoraxComplexFieldIndexingBehavior)
+                        return;
+
+                    const string configurationKey = nameof(IndexingConfiguration.CoraxComplexFieldIndexingBehavior);
+
+                    var configuredBehavior = _index.Configuration.CoraxStaticIndexComplexFieldIndexingBehavior;
+
+                    var result = configurationTree.Read(configurationKey);
+                    if (result != null)
+                    {
+                        if (Enum.TryParse(result.Reader.ToStringValue(), out IndexingConfiguration.CoraxComplexFieldIndexingBehavior persistedCoraxStaticIndexComplexFieldIndexingBehavior) == false)
+                        {
+                            throw new InvalidDataException($"Invalid indexing complex field behavior in '{_index.Name}' Corax index");
+                        }
+                    }
+                    else
+                    {
+                        configurationTree.Add(configurationKey, configuredBehavior.ToString());
+                    }
+                }
             }
         }
 
@@ -507,6 +542,31 @@ public ArchivedDataProcessingBehavior ReadArchivedDataProcessingBehavior(RavenTr
             return persistedArchivedDataProcessingBehavior;
         }
 
+        public IndexingConfiguration.CoraxComplexFieldIndexingBehavior ReadCoraxComplexFieldIndexingBehavior(RavenTransaction tx)
+        {
+            if (_index.Definition.Version < IndexDefinitionBaseServerSide.IndexVersion.CoraxComplexFieldIndexingBehavior)
+                return IndexingConfiguration.CoraxComplexFieldIndexingBehavior.None;
+
+            var configurationTree = tx.InnerTransaction.ReadTree(IndexSchema.ConfigurationTree);
+            if (configurationTree == null)
+            {
+                throw new InvalidOperationException($"Index does not contain {nameof(IndexSchema.ConfigurationTree)}' tree.");
+            }
+
+            var result = configurationTree.Read(IndexSchema.CoraxComplexFieldIndexingBehavior);
+            if (result == null)
+            {
+                throw new InvalidOperationException($"Index does not contain {nameof(IndexSchema.CoraxComplexFieldIndexingBehavior)}' key.");
+            }
+
+            if (Enum.TryParse(result.Reader.ToStringValue(), out IndexingConfiguration.CoraxComplexFieldIndexingBehavior persistedCoraxStaticIndexComplexFieldIndexingBehavior) == false)
+            {
+                throw new InvalidDataException($"Invalid indexing complex field behavior in '{_index.Name}' Corax index. It has: {result.Reader.ToStringValue()} defined.");
+            }
+
+            return persistedCoraxStaticIndexComplexFieldIndexingBehavior;
+        }
+
         public sealed class DocumentReferences : ReferencesBase
         {
             public DocumentReferences()
@@ -1187,6 +1247,8 @@ internal sealed class IndexSchema
 
             public static readonly Slice SearchEngineType;
 
+            public static readonly Slice CoraxComplexFieldIndexingBehavior;
+
             static IndexSchema()
             {
                 using (StorageEnvironment.GetStaticContext(out var ctx))
@@ -1213,6 +1275,7 @@ static IndexSchema()
                     Slice.From(ctx, "Time", ByteStringType.Immutable, out TimeSlice);
                     Slice.From(ctx, nameof(Client.Documents.Indexes.SearchEngineType), ByteStringType.Immutable, out SearchEngineType);
                     Slice.From(ctx, nameof(ArchivedDataProcessingBehavior), ByteStringType.Immutable, out ArchivedDataProcessingBehaviorSlice);
+                    Slice.From(ctx, nameof(CoraxComplexFieldIndexingBehavior), ByteStringType.Immutable, out CoraxComplexFieldIndexingBehavior);
                 }
             }
         }