RavenDB-22973 Added failing (only in Debug) tests which reproduce the…

… real issues that was happening on live system with AssertNoPagesAllocatedInTransactionOlderThan() assertion running in Release

src/Voron/Impl/Journal/WriteAheadJournal.cs

@@ -772,7 +772,12 @@ private void ApplyJournalStateAfterFlush(CancellationToken token,
             private void WaitForJournalStateToBeUpdated(CancellationToken token, TransactionPersistentContext transactionPersistentContext,
                 Action<LowLevelTransaction> currentAction, ByteStringContext byteStringContext)
             {
+                _forTestingPurposes?.OnWaitForJournalStateToBeUpdated_BeforeAssigning_updateJournalStateAfterFlush?.Invoke();
+
                 Interlocked.Exchange(ref _updateJournalStateAfterFlush, currentAction);
+
+                _forTestingPurposes?.OnWaitForJournalStateToBeUpdated_AfterAssigning_updateJournalStateAfterFlush?.Invoke();
+
                 do
                 {
                     LowLevelTransaction txw = null;
@@ -972,6 +977,9 @@ internal sealed class TestingStuff
 
                 internal Action OnApplyLogsToDataFileUnderFlushingLock;
 
+                internal Action OnWaitForJournalStateToBeUpdated_BeforeAssigning_updateJournalStateAfterFlush;
+
+                internal Action OnWaitForJournalStateToBeUpdated_AfterAssigning_updateJournalStateAfterFlush;
             }
 
             // This can take a LONG time, and it needs to run concurrently with the

src/Voron/StorageEnvironment.cs

@@ -874,7 +874,9 @@ private void WriteTransactionCompleted(LowLevelTransaction tx)
                 GlobalFlushingBehavior.GlobalFlusher.Value.MaybeFlushEnvironment(this);
             }
 
-
+#if DEBUG
+            _forTestingPurposes?.OnWriteTransactionCompleted?.Invoke(tx);
+#endif
             // this must occur when we are holding the transaction lock
             Journal.Applicator.OnTransactionCompleted(tx);
 
@@ -1625,6 +1627,7 @@ internal TestingStuff ForTestingPurposesOnly()
         internal sealed class TestingStuff
         {
             internal Action ActionToCallDuringFullBackupRighAfterCopyHeaders;
+            public Action<LowLevelTransaction> OnWriteTransactionCompleted { get; set; }
         }
 
 

test/SlowTests/Voron/Issues/RavenDB_22973.cs

@@ -1,9 +1,11 @@
 using System;
 using System.Linq;
+using System.Threading;
 using FastTests.Voron;
 using Tests.Infrastructure;
 using Voron;
 using Voron.Global;
+using Voron.Impl;
 using Voron.Impl.Scratch;
 using Xunit;
 using Xunit.Abstractions;
@@ -286,4 +288,103 @@ public void Overflow_should_reuse_shrinked_page_that_got_freed()
             Assert.Equal(shrinkedPageInScratch.PositionInScratchBuffer, overflowScratchPage.PositionInScratchBuffer);
         }
     }
+
+    [RavenFact(RavenTestCategory.Voron)]
+    public void Must_not_omit_UpdateJournalStateUnderWriteTransactionLock_during_flush()
+    {
+        using (var txw1 = Env.WriteTransaction())
+        {
+            txw1.LowLevelTransaction.AllocatePage(1);
+
+            txw1.Commit();
+        }
+
+        using (var txw2 = Env.WriteTransaction())
+        {
+            txw2.LowLevelTransaction.AllocatePage(1);
+
+            txw2.Commit();
+        }
+
+        using (var txr = Env.ReadTransaction())
+            Env.FlushLogToDataFile(); // flush and free scratch pages up to txw1
+
+        Transaction writeTransactionExecutedDuringWaitForJournalStateToBeUpdated = null;
+
+        Env.Journal.Applicator.ForTestingPurposesOnly().OnWaitForJournalStateToBeUpdated_BeforeAssigning_updateJournalStateAfterFlush += () =>
+        {
+            writeTransactionExecutedDuringWaitForJournalStateToBeUpdated = Env.WriteTransaction();
+
+            writeTransactionExecutedDuringWaitForJournalStateToBeUpdated.Commit();
+        };
+
+        Env.Journal.Applicator.ForTestingPurposesOnly().OnWaitForJournalStateToBeUpdated_AfterAssigning_updateJournalStateAfterFlush += () =>
+        {
+            writeTransactionExecutedDuringWaitForJournalStateToBeUpdated.Dispose();
+        };
+
+        Env.FlushLogToDataFile(); // flush and free scratch pages up to txw2
+
+        using (var txw3 = Env.WriteTransaction())
+        {
+            txw3.LowLevelTransaction.AllocatePage(1);
+
+            txw3.Commit();
+        }
+
+        Env.FlushLogToDataFile(); // next flush will detect unreleased scratch pages via AssertNoPagesAllocatedInTransactionOlderThan() - only in DEBUG
+    }
+
+    [RavenFact(RavenTestCategory.Voron)]
+    public void Must_not_omit_UpdateJournalStateUnderWriteTransactionLock_during_flush_2()
+    {
+        using (var txw1 = Env.WriteTransaction())
+        {
+            txw1.LowLevelTransaction.AllocatePage(1);
+
+            txw1.Commit();
+        }
+
+        Env.FlushLogToDataFile();
+
+        var txw = Env.WriteTransaction();
+        txw.LowLevelTransaction.AllocatePage(1);
+
+        txw.Commit();
+
+        var mre = new ManualResetEvent(false);
+
+        Env.Journal.Applicator.ForTestingPurposesOnly().OnWaitForJournalStateToBeUpdated_AfterAssigning_updateJournalStateAfterFlush += () =>
+        {
+            mre.Set();
+        };
+
+        Env.ForTestingPurposesOnly().OnWriteTransactionCompleted += tx =>
+        {
+            if (tx == txw.LowLevelTransaction)
+            {
+                var t = new Thread(() =>
+                {
+                    Env.FlushLogToDataFile();
+                });
+                t.Start();
+
+                mre.WaitOne();
+            }
+        };
+
+        txw.Dispose();
+
+        Env.Journal.Applicator.ForTestingPurposesOnly().OnWaitForJournalStateToBeUpdated_AfterAssigning_updateJournalStateAfterFlush = null;
+        Env.ForTestingPurposesOnly().OnWriteTransactionCompleted = null;
+
+        using (var txw2 = Env.WriteTransaction())
+        {
+            txw2.LowLevelTransaction.AllocatePage(1);
+
+            txw2.Commit();
+        }
+
+        Env.FlushLogToDataFile(); // next flush will detect unreleased scratch pages via AssertNoPagesAllocatedInTransactionOlderThan() - only in DEBUG
+    }
 }