Merge branch 'v6.0' of github.com:ravendb/ravendb into v6.1

src/Raven.Client/Documents/BulkInsert/BulkInsertOperation.cs

@@ -281,8 +281,12 @@ private static bool CheckServerVersion(string serverVersion)
             if (serverVersion != null && Version.TryParse(serverVersion, out var ver))
             {
                 // Version 6 only from 6.0.2
-                if (ver.Major == 6 && ver.Build < 2)
+                if (ver.Major == 6 && ver.Minor > 0)
+                    return true;
+
+                if (ver.Major == 6 && ver.Minor == 0 && ver.Build < 2)
                     return false;
+
                 // 5.4.108 or higher
                 if (ver.Major > 5 || (ver.Major == 5 && ver.Minor >= 4 && ver.Build >= 110))
                     return true;

src/Raven.Client/Documents/Operations/Backups/BackupProgress.cs

@@ -69,6 +69,7 @@ public void Fill(IOperationProgress progress, int shardNumber, string nodeTag)
             Counters = bp.Counters;
             TimeSeries = bp.TimeSeries;
             CompareExchangeTombstones = bp.CompareExchangeTombstones;
+            TimeSeriesDeletedRanges = bp.TimeSeriesDeletedRanges;
         }
 
         public override DynamicJsonValue ToJson()

src/Raven.Client/Documents/Smuggler/DatabaseItemType.cs

@@ -19,14 +19,13 @@ public enum DatabaseItemType
         LegacyAttachmentDeletions = 1 << 10,
         DatabaseRecord = 1 << 11,
         Unknown = 1 << 12,
-
         Attachments = 1 << 14,
         CounterGroups = 1 << 15,
         Subscriptions = 1 << 16,
         CompareExchangeTombstones = 1 << 17,
         TimeSeries = 1 << 18,
-
-        ReplicationHubCertificates = 1 << 19
+        ReplicationHubCertificates = 1 << 19,
+        TimeSeriesDeletedRanges = 1 << 20
     }
 
     [Flags]

src/Raven.Client/Documents/Smuggler/DatabaseSmugglerOptions.cs

@@ -16,7 +16,8 @@ public class DatabaseSmugglerOptions : IDatabaseSmugglerOptions
                                                               DatabaseItemType.Attachments |
                                                               DatabaseItemType.CounterGroups |
                                                               DatabaseItemType.Subscriptions |
-                                                              DatabaseItemType.TimeSeries;
+                                                              DatabaseItemType.TimeSeries | 
+                                                              DatabaseItemType.TimeSeriesDeletedRanges;
 
         public const DatabaseRecordItemType DefaultOperateOnDatabaseRecordTypes = DatabaseRecordItemType.Client |
                                                                                   DatabaseRecordItemType.ConflictSolverConfig |

src/Raven.Client/Documents/Smuggler/ShardedSmugglerProgress.cs

@@ -31,6 +31,7 @@ public void Fill(IOperationProgress progress, int shardNumber, string nodeTag)
         Counters = sp.Counters;
         TimeSeries = sp.TimeSeries;
         CompareExchangeTombstones = sp.CompareExchangeTombstones;
+        TimeSeriesDeletedRanges = sp.TimeSeriesDeletedRanges;
     }
 
     public override DynamicJsonValue ToJson()

src/Raven.Client/Documents/Smuggler/SmugglerProgressBase.cs

@@ -34,6 +34,8 @@ public abstract class SmugglerProgressBase : IOperationProgress
 
     public Counts CompareExchangeTombstones { get; set; }
 
+    public CountsWithSkippedCountAndLastEtag TimeSeriesDeletedRanges { get; set; }
+
     public virtual DynamicJsonValue ToJson()
     {
         return new DynamicJsonValue(GetType())
@@ -51,6 +53,7 @@ public virtual DynamicJsonValue ToJson()
             [nameof(CompareExchangeTombstones)] = CompareExchangeTombstones.ToJson(),
             [nameof(TimeSeries)] = TimeSeries.ToJson(),
             [nameof(ReplicationHubCertificates)] = ReplicationHubCertificates.ToJson(),
+            [nameof(TimeSeriesDeletedRanges)] = TimeSeriesDeletedRanges.ToJson()
         };
     }
 

src/Raven.Client/Documents/Smuggler/SmugglerResult.cs

@@ -43,6 +43,7 @@ public SmugglerResult()
             Subscriptions = new Counts();
             ReplicationHubCertificates = new Counts();
             TimeSeries = new CountsWithSkippedCountAndLastEtag();
+            TimeSeriesDeletedRanges = new CountsWithSkippedCountAndLastEtag();
             _progress = new SmugglerProgress(this);
         }
 
@@ -86,6 +87,11 @@ void IOperationResult.MergeWith(IOperationResult result)
 
             smugglerResult.TimeSeries.LastEtag = Math.Max(smugglerResult.TimeSeries.LastEtag, TimeSeries.LastEtag);
 
+            smugglerResult.TimeSeriesDeletedRanges.ReadCount += TimeSeriesDeletedRanges.ReadCount;
+            smugglerResult.TimeSeriesDeletedRanges.ErroredCount += TimeSeriesDeletedRanges.ErroredCount;
+            smugglerResult.TimeSeriesDeletedRanges.SizeInBytes += TimeSeriesDeletedRanges.SizeInBytes;
+            smugglerResult.TimeSeriesDeletedRanges.LastEtag = Math.Max(smugglerResult.TimeSeriesDeletedRanges.LastEtag, TimeSeriesDeletedRanges.LastEtag);
+
             smugglerResult.Identities.ReadCount += Identities.ReadCount;
             smugglerResult.Identities.ErroredCount += Identities.ErroredCount;
 
@@ -230,6 +236,7 @@ public SmugglerProgress(SmugglerResult result)
                 Subscriptions = _result?.Subscriptions;
                 TimeSeries = _result?.TimeSeries;
                 ReplicationHubCertificates = _result?.ReplicationHubCertificates;
+                TimeSeriesDeletedRanges = _result?.TimeSeriesDeletedRanges;
             }
 
             internal string Message { get; set; }
@@ -261,6 +268,9 @@ public long GetLastEtag()
             if (TimeSeries.LastEtag > lastEtag)
                 lastEtag = TimeSeries.LastEtag;
 
+            if (TimeSeriesDeletedRanges.LastEtag > lastEtag)
+                lastEtag = TimeSeriesDeletedRanges.LastEtag;
+
             return lastEtag;
         }
 

src/Raven.Server/Documents/BlittableMetadataModifier.cs

@@ -17,6 +17,8 @@ namespace Raven.Server.Documents
 {
     public sealed class BlittableMetadataModifier : IDisposable, IBlittableDocumentModifier
     {
+        private bool _disposed;
+
         private bool _readingMetadataObject;
         private int _depth;
         private State _state = State.None;
@@ -131,6 +133,8 @@ private enum State
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void StartObject()
         {
+            AssertNotDisposed();
+
             if (_readingMetadataObject == false)
                 return;
 
@@ -140,6 +144,8 @@ public void StartObject()
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public void EndObject()
         {
+            AssertNotDisposed();
+
             if (_readingMetadataObject == false)
                 return;
 
@@ -153,6 +159,8 @@ public void EndObject()
         [MethodImpl(MethodImplOptions.AggressiveInlining)]
         public bool AboutToReadPropertyName(IJsonParser reader, JsonParserState state)
         {
+            AssertNotDisposed();
+
             if (reader is UnmanagedJsonParser unmanagedParser)
                 return AboutToReadPropertyNameInternal(unmanagedParser, state);
             if (reader is ObjectJsonParser objectParser)
@@ -592,7 +600,7 @@ private unsafe bool AboutToReadPropertyNameInMetadataUnlikely(IJsonParser reader
                         aboutToReadPropertyName = true;
                         return true;
                     }
-                    
+
                     if ("Raven-Replication-Version"u8.IsEqualConstant(state.StringBuffer) == false ||
                         "Raven-Replication-History"u8.IsEqualConstant(state.StringBuffer) == false)
                     {
@@ -847,10 +855,14 @@ public void Dispose()
                 _ctx.ReturnMemory(_allocations[i]);
             }
             _allocations.Clear();
+
+            _disposed = true;
         }
 
         public void Reset(JsonOperationContext ctx)
         {
+            AssertNotDisposed();
+
             if (_ctx == null) // should never happen
             {
                 _ctx = ctx;
@@ -872,5 +884,12 @@ public void Reset(JsonOperationContext ctx)
             _metadataCollections = _ctx.GetLazyStringForFieldWithCaching(CollectionName.MetadataCollectionSegment);
             _metadataExpires = _ctx.GetLazyStringForFieldWithCaching(Constants.Documents.Metadata.Expires);
         }
+
+        [Conditional("DEBUG")]
+        private void AssertNotDisposed()
+        {
+            if (_disposed)
+                throw new ObjectDisposedException(nameof(BlittableMetadataModifier));
+        }
     }
 }

src/Raven.Server/Documents/Handlers/CountersHandler.cs

@@ -288,6 +288,9 @@ public sealed class SmugglerCounterBatchCommand : MergedTransactionCommand<Docum
 
             public long ErrorCount;
 
+            private BlittableJsonDocumentBuilder _builder;
+            private BlittableMetadataModifier _metadataModifier;
+
             public SmugglerCounterBatchCommand(DocumentDatabase database, SmugglerResult result)
             {
                 _database = database;
@@ -358,6 +361,21 @@ public void RegisterForDisposal(IDisposable data)
                 _toDispose.Add(data);
             }
 
+            public BlittableJsonDocumentBuilder GetOrCreateBuilder(UnmanagedJsonParser parser, JsonParserState state, string debugTag, BlittableMetadataModifier modifier = null)
+            {
+                return _builder ??= new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, debugTag, parser, state, modifier: modifier);
+            }
+
+            public BlittableMetadataModifier GetOrCreateMetadataModifier(string firstEtagOfLegacyRevision = null, long legacyRevisionsCount = 0, bool legacyImport = false,
+                bool readLegacyEtag = false, DatabaseItemType operateOnTypes = DatabaseItemType.None)
+            {
+                _metadataModifier ??= new BlittableMetadataModifier(_context, legacyImport, readLegacyEtag, operateOnTypes);
+                _metadataModifier.FirstEtagOfLegacyRevision = firstEtagOfLegacyRevision;
+                _metadataModifier.LegacyRevisionsCount = legacyRevisionsCount;
+
+                return _metadataModifier;
+            }
+
             protected override long ExecuteCmd(DocumentsOperationContext context)
             {
                 if (_legacyDictionary != null)
@@ -587,6 +605,9 @@ public void Dispose()
 
                 _legacyDictionary?.Clear();
 
+                _builder?.Dispose();
+                _metadataModifier?.Dispose();
+
                 _resetContext?.Dispose();
                 _resetContext = null;
 

src/Raven.Server/Documents/Handlers/Processors/Smuggler/AbstractSmugglerHandlerProcessor.cs

@@ -33,6 +33,13 @@ internal void ApplyBackwardCompatibility(DatabaseSmugglerOptionsServerSide optio
             if (RequestRouter.TryGetClientVersion(HttpContext, out var version) == false)
                 return;
 
+            if (version.Major == 5 && (version.Minor < 4 || version.Minor == 4 && version.Build < 200) &&
+                options.OperateOnTypes.HasFlag(DatabaseItemType.TimeSeries))
+            {
+                // version is older than 5.4.200
+                options.OperateOnTypes |= DatabaseItemType.TimeSeriesDeletedRanges;
+            }
+
             if (version.Major != RavenVersionAttribute.Instance.MajorVersion)
                 return;
 

src/Raven.Server/Documents/Handlers/TimeSeriesHandler.cs

@@ -13,6 +13,8 @@
 using Raven.Server.Smuggler.Documents;
 using Sparrow.Json;
 using System.Diagnostics.CodeAnalysis;
+using Raven.Client.Documents.Smuggler;
+using Sparrow.Json.Parsing;
 
 namespace Raven.Server.Documents.Handlers
 {
@@ -26,7 +28,7 @@ public async Task Stats()
                 await processor.ExecuteAsync();
             }
         }
-        
+
         [RavenAction("/databases/*/timeseries/ranges", "GET", AuthorizationStatus.ValidUser, EndpointType.Read)]
         public async Task ReadRanges()
         {
@@ -275,6 +277,8 @@ public sealed class SmugglerTimeSeriesBatchCommand : MergedTransactionCommand<Do
 
             private readonly Dictionary<string, List<TimeSeriesItem>> _dictionary;
 
+            private readonly Dictionary<string, List<TimeSeriesDeletedRangeItemForSmuggler>> _deletedRanges;
+
             private readonly DocumentsOperationContext _context;
 
             public DocumentsOperationContext Context => _context;
@@ -289,10 +293,14 @@ public sealed class SmugglerTimeSeriesBatchCommand : MergedTransactionCommand<Do
 
             public string LastChangeVector;
 
+            private BlittableJsonDocumentBuilder _builder;
+            private BlittableMetadataModifier _metadataModifier;
+
             public SmugglerTimeSeriesBatchCommand(DocumentDatabase database)
             {
                 _database = database;
-                _dictionary = new Dictionary<string, List<TimeSeriesItem>>();
+                _dictionary = new Dictionary<string, List<TimeSeriesItem>>(StringComparer.OrdinalIgnoreCase);
+                _deletedRanges = new Dictionary<string, List<TimeSeriesDeletedRangeItemForSmuggler>>(StringComparer.OrdinalIgnoreCase);
                 _toDispose = new();
                 _toReturn = new();
                 _releaseContext = _database.DocumentsStorage.ContextPool.AllocateOperationContext(out _context);
@@ -304,6 +312,27 @@ protected override long ExecuteCmd(DocumentsOperationContext context)
 
                 var changes = 0L;
 
+                foreach (var (docId, items) in _deletedRanges)
+                {
+                    foreach (var item in items)
+                    {
+                        using (item)
+                        {
+                            var deletionRangeRequest = new TimeSeriesStorage.DeletionRangeRequest
+                            {
+                                DocumentId = docId,
+                                Collection = item.Collection,
+                                Name = item.Name,
+                                From = item.From,
+                                To = item.To
+                            };
+                            tss.DeleteTimestampRange(context, deletionRangeRequest, remoteChangeVector: null, updateMetadata: false);
+                        }
+                    }
+
+                    changes += items.Count;
+                }
+
                 foreach (var (docId, items) in _dictionary)
                 {
                     var collectionName = _database.DocumentsStorage.ExtractCollectionName(context, items[0].Collection);
@@ -346,6 +375,21 @@ public bool AddToDictionary(TimeSeriesItem item)
                 return newItem;
             }
 
+            public bool AddToDeletedRanges(TimeSeriesDeletedRangeItemForSmuggler item)
+            {
+                bool newItem = false;
+
+                if (_deletedRanges.TryGetValue(item.DocId, out var deletedRangesList) == false)
+                {
+                    _deletedRanges[item.DocId] = deletedRangesList = [];
+                    newItem = true;
+                }
+
+                deletedRangesList.Add(item);
+                return newItem;
+            }
+
+
             public void AddToDisposal(IDisposable disposable)
             {
                 _toDispose.Add(disposable);
@@ -356,6 +400,21 @@ public void AddToReturn(AllocatedMemoryData allocatedMemoryData)
                 _toReturn.Add(allocatedMemoryData);
             }
 
+            public BlittableJsonDocumentBuilder GetOrCreateBuilder(UnmanagedJsonParser parser, JsonParserState state, string debugTag, BlittableMetadataModifier modifier = null)
+            {
+                return _builder ??= new BlittableJsonDocumentBuilder(_context, BlittableJsonDocumentBuilder.UsageMode.ToDisk, debugTag, parser, state, modifier: modifier);
+            }
+
+            public BlittableMetadataModifier GetOrCreateMetadataModifier(string firstEtagOfLegacyRevision = null, long legacyRevisionsCount = 0, bool legacyImport = false,
+                bool readLegacyEtag = false, DatabaseItemType operateOnTypes = DatabaseItemType.None)
+            {
+                _metadataModifier ??= new BlittableMetadataModifier(_context, legacyImport, readLegacyEtag, operateOnTypes);
+                _metadataModifier.FirstEtagOfLegacyRevision = firstEtagOfLegacyRevision;
+                _metadataModifier.LegacyRevisionsCount = legacyRevisionsCount;
+
+                return _metadataModifier;
+            }
+
             public void Dispose()
             {
                 if (_isDisposed)
@@ -373,6 +432,9 @@ public void Dispose()
                     _context.ReturnMemory(returnable);
                 _toReturn.Clear();
 
+                _builder?.Dispose();
+                _metadataModifier?.Dispose();
+
                 _releaseContext?.Dispose();
                 _releaseContext = null;
             }

src/Raven.Server/Documents/PeriodicBackup/Restore/Sharding/ShardedRestoreResult.cs

@@ -84,6 +84,7 @@ public void Fill(IOperationProgress progress, int shardNumber, string nodeTag)
             Counters = rp.Counters;
             TimeSeries = rp.TimeSeries;
             CompareExchangeTombstones = rp.CompareExchangeTombstones;
+            TimeSeriesDeletedRanges = rp.TimeSeriesDeletedRanges;
         }
 
         public override DynamicJsonValue ToJson()