Update regex parsing to latest .Net core parsing (and diagnostic messages). (#76269)

Author: CyrusNajmabadi

src/EditorFeatures/CSharpTest2/EmbeddedLanguages/Json/CSharpJsonParserTests.cs

@@ -92,7 +92,7 @@ private static string RemoveMessagesInNonSupportedLanguage(string value)
 
         var diagnosticsElement = XElement.Parse(value);
         foreach (var diagnosticElement in diagnosticsElement.Elements("Diagnostic"))
-            diagnosticElement.Attribute("Message").Remove();
+            diagnosticElement.Attribute("Message")!.Remove();
 
         return diagnosticsElement.ToString();
     }

src/EditorFeatures/CSharpTest2/EmbeddedLanguages/RegularExpressions/CSharpRegexParserTests.cs

@@ -40,46 +40,26 @@ private static SyntaxToken GetStringToken(string text)
             return token;
         }
 
-        private void Test(string stringText, string expected, RegexOptions options,
-            bool runSubTreeTests = true,
-            bool allowIndexOutOfRange = false,
-            bool allowNullReference = false,
-            bool allowOutOfMemory = false,
-            bool allowDiagnosticsMismatch = false)
+        private void Test(
+            string stringText, string expected, RegexOptions options)
         {
-            var (tree, sourceText) = TryParseTree(stringText, options, conversionFailureOk: false,
-                allowIndexOutOfRange, allowNullReference, allowOutOfMemory, allowDiagnosticsMismatch);
+            var (tree, sourceText) = TryParseTree(stringText, options, conversionFailureOk: false);
 
-            // Tests are allowed to not run the subtree tests.  This is because some
-            // subtrees can cause the native regex parser to exhibit very bad behavior
-            // (like not ever actually finishing compiling).
-            if (runSubTreeTests)
-            {
-                TryParseSubTrees(stringText, options,
-                    allowIndexOutOfRange, allowNullReference, allowOutOfMemory, allowDiagnosticsMismatch);
-            }
+            TryParseSubTrees(stringText, options);
 
-            const string DoubleQuoteEscaping = "\"\"";
             var actual = TreeToText(sourceText, tree)
-                .Replace("\"", DoubleQuoteEscaping)
-                .Replace(""", DoubleQuoteEscaping);
-            AssertEx.Equal(expected.Replace("\"", DoubleQuoteEscaping), actual);
+                .Replace(""", "\"");
+            AssertEx.Equal(expected, actual);
         }
 
-        private void TryParseSubTrees(
-            string stringText, RegexOptions options,
-            bool allowIndexOutOfRange,
-            bool allowNullReference,
-            bool allowOutOfMemory,
-            bool allowDiagnosticsMismatch)
+        private void TryParseSubTrees(string stringText, RegexOptions options)
         {
             // Trim the input from the right and make sure tree invariants hold
             var current = stringText;
             while (current is not "@\"\"" and not "\"\"")
             {
                 current = current[..^2] + "\"";
-                TryParseTree(current, options, conversionFailureOk: true,
-                    allowIndexOutOfRange, allowNullReference, allowOutOfMemory, allowDiagnosticsMismatch);
+                TryParseTree(current, options, conversionFailureOk: true);
             }
 
             // Trim the input from the left and make sure tree invariants hold
@@ -95,17 +75,16 @@ private void TryParseSubTrees(
                     current = "\"" + current[2..];
                 }
 
-                TryParseTree(current, options, conversionFailureOk: true,
-                    allowIndexOutOfRange, allowNullReference, allowOutOfMemory, allowDiagnosticsMismatch);
+                TryParseTree(current, options, conversionFailureOk: true);
             }
 
             for (var start = stringText[0] == '@' ? 2 : 1; start < stringText.Length - 1; start++)
             {
                 TryParseTree(
                     stringText[..start] +
                     stringText[(start + 1)..],
-                    options, conversionFailureOk: true,
-                    allowIndexOutOfRange, allowNullReference, allowOutOfMemory, allowDiagnosticsMismatch);
+                    options,
+                    conversionFailureOk: true);
             }
         }
 
@@ -125,12 +104,7 @@ private void TryParseSubTrees(
         }
 
         private (RegexTree, SourceText) TryParseTree(
-            string stringText, RegexOptions options,
-            bool conversionFailureOk,
-            bool allowIndexOutOfRange,
-            bool allowNullReference,
-            bool allowOutOfMemory,
-            bool allowDiagnosticsMismatch = false)
+            string stringText, RegexOptions options, bool conversionFailureOk)
         {
             var (token, tree, allChars) = JustParseTree(stringText, options, conversionFailureOk);
             if (tree == null)
@@ -148,42 +122,24 @@ private void TryParseSubTrees(
             {
                 regex = new Regex(token.ValueText, options);
             }
-            catch (IndexOutOfRangeException) when (allowIndexOutOfRange)
-            {
-                // bug with .NET regex parser.  Can happen with patterns like: (?<-0
-                Assert.NotEmpty(tree.Diagnostics);
-                return treeAndText;
-            }
-            catch (NullReferenceException) when (allowNullReference)
-            {
-                // bug with .NET regex parser.  can happen with patterns like: (?(?S))
-                return treeAndText;
-            }
-            catch (OutOfMemoryException) when (allowOutOfMemory)
-            {
-                // bug with .NET regex parser.  can happen with patterns like: a{2147483647,}
-                return treeAndText;
-            }
             catch (ArgumentException ex)
             {
-                if (!allowDiagnosticsMismatch)
+                Assert.NotEmpty(tree.Diagnostics);
+
+                // Ensure the diagnostic we emit is the same as the .NET one. Note: we can only
+                // do this in en-US as that's the only culture where we control the text exactly
+                // and can ensure it exactly matches Regex.  We depend on localization to do a
+                // good enough job here for other languages.
+                if (Thread.CurrentThread.CurrentCulture.Name == "en-US")
                 {
-                    Assert.NotEmpty(tree.Diagnostics);
-
-                    // Ensure the diagnostic we emit is the same as the .NET one. Note: we can only
-                    // do this in en-US as that's the only culture where we control the text exactly
-                    // and can ensure it exactly matches Regex.  We depend on localization to do a
-                    // good enough job here for other languages.
-                    if (Thread.CurrentThread.CurrentCulture.Name == "en-US")
-                    {
-                        Assert.True(tree.Diagnostics.Any(d => ex.Message.Contains(d.Message)));
-                    }
+                    var result = tree.Diagnostics.Any(d => ex.Message.Contains(d.Message));
+                    Assert.True(result);
                 }
 
                 return treeAndText;
             }
 
-            if (!tree.Diagnostics.IsEmpty && !allowDiagnosticsMismatch)
+            if (!tree.Diagnostics.IsEmpty)
             {
                 var expectedDiagnostics = CreateDiagnosticsElement(sourceText, tree);
                 Assert.False(true, "Expected diagnostics: \r\n" + expectedDiagnostics.ToString().Replace(@"""", @""""""));