Remove capture groups from negative lookarounds (#118084)

Any captures performed inside of negative lookarounds do not persist to outside of the lookaround. As such, as long as there are no backreferences inside of the lookaround that would read on those captures, we can eliminate the capturing.

Author: stephentoub

src/libraries/System.Text.RegularExpressions/src/System/Text/RegularExpressions/RegexNode.cs

@@ -2044,6 +2044,43 @@ private RegexNode ReduceLookaround()
             Debug.Assert(Kind is RegexNodeKind.PositiveLookaround or RegexNodeKind.NegativeLookaround);
             Debug.Assert(ChildCount() == 1);
 
+            // Captures inside of negative lookarounds are undone after the lookaround. Thus, if there's nothing
+            // inside of the negative lookaround that needs that capture group (namely a backreference), we can
+            // remove the capture.
+            if (Kind is RegexNodeKind.NegativeLookaround && ContainsBackreference(Child(0)) is false)
+            {
+                if (RemoveCaptures(this, 0))
+                {
+                    // If we removed captures, we may have changed the structure of the tree in a way that exposed more
+                    // optimization possibility, so re-reduce the children.
+                    ReplaceChild(0, Child(0));
+                }
+
+                static bool RemoveCaptures(RegexNode parent, int nodeIndex)
+                {
+                    RegexNode node = parent.Child(nodeIndex);
+
+                    if (node.Kind is RegexNodeKind.Capture)
+                    {
+                        parent.ReplaceChild(nodeIndex, node.Child(0));
+                        RemoveCaptures(parent, nodeIndex);
+                        return true;
+                    }
+
+                    bool changesMade = false;
+                    if (StackHelper.TryEnsureSufficientExecutionStack())
+                    {
+                        int childCount = node.ChildCount();
+                        for (int i = 0; i < childCount; i++)
+                        {
+                            changesMade |= RemoveCaptures(node, i);
+                        }
+                    }
+
+                    return changesMade;
+                }
+            }
+
             // A lookaround is a zero-width atomic assertion.
             // As it's atomic, nothing will backtrack into it, and we can
             // eliminate any ending backtracking from it.
@@ -2066,6 +2103,32 @@ private RegexNode ReduceLookaround()
             return this;
         }
 
+        /// <summary>Gets whether the node contains a backreference anywhere in its tree.</summary>
+        private static bool? ContainsBackreference(RegexNode node)
+        {
+            if (node.Kind is RegexNodeKind.Backreference or RegexNodeKind.BackreferenceConditional)
+            {
+                return true;
+            }
+
+            if (!StackHelper.TryEnsureSufficientExecutionStack())
+            {
+                // If we can't recur further, just stop optimizing.
+                return null;
+            }
+
+            int childCount = node.ChildCount();
+            for (int i = 0; i < childCount; i++)
+            {
+                if (ContainsBackreference(node.Child(i)) is true)
+                {
+                    return true;
+                }
+            }
+
+            return false;
+        }
+
         /// <summary>Optimizations for backreference conditionals.</summary>
         private RegexNode ReduceBackreferenceConditional()
         {

src/libraries/System.Text.RegularExpressions/tests/UnitTests/RegexReductionTests.cs

@@ -269,6 +269,10 @@ public class RegexReductionTests
         [InlineData("(?>(?>(?>(?>))))", "")]
         [InlineData("(?>(?>(?>(?>(?!)))))", "(?!)")]
         [InlineData("(?=(?>))", "")]
+        // Lookaround reduction
+        [InlineData("(?!(abc))", "(?!abc)")]
+        [InlineData("(?!a(b*)c)", "(?!ab*c)")]
+        [InlineData("(?!a((((b))))c)", "(?!abc)")]
         // Alternation reduction
         [InlineData("a|b", "[ab]")]
         [InlineData("a|b|c|d|e|g|h|z", "[a-eghz]")]
@@ -541,6 +545,10 @@ public void PatternsReduceIdentically(string actual, string expected)
         [InlineData("(abc?)*?d", "(?>(ab(?>c?))*)d")]
         [InlineData("(aba)+d", "(?>(aba)+)d")]
         [InlineData("(abc*)*d", "(?>(ab(?>c*))*)d")]
+        // Lookaround reduction
+        [InlineData("(?=(abc))", "(?=abc)")]
+        [InlineData("(?=a(b*)c)", "(?=ab*c)")]
+        [InlineData("(?=a((((b))))c)", "(?=abc)")]
         // Loops inside alternation constructs
         [InlineData("(abc*|def)chi", "(ab(?>c*)|def)chi")]
         [InlineData("(abc|def*)fhi", "(abc|de(?>f*))fhi")]