Fixes for parsing escaped / quoted strings

src/System.Linq.Dynamic.Core/Parser/ExpressionParser.cs

@@ -1,4 +1,5 @@
-using System.Collections;
+using JetBrains.Annotations;
+using System.Collections;
 using System.Collections.Generic;
 using System.ComponentModel;
 using System.Globalization;
@@ -10,7 +11,6 @@
 using System.Linq.Dynamic.Core.Validation;
 using System.Linq.Expressions;
 using System.Reflection;
-using JetBrains.Annotations;
 
 namespace System.Linq.Dynamic.Core.Parser
 {
@@ -781,37 +781,23 @@ Expression ParsePrimaryStart()
         Expression ParseStringLiteral()
         {
             _textParser.ValidateToken(TokenId.StringLiteral);
-            char quote = _textParser.CurrentToken.Text[0];
-            string s = _textParser.CurrentToken.Text.Substring(1, _textParser.CurrentToken.Text.Length - 2);
-            int index1 = 0;
-            while (true)
-            {
-                int index2 = s.IndexOf(quote, index1);
-                if (index2 < 0)
-                {
-                    break;
-                }
 
-                if (index2 + 1 < s.Length && s[index2 + 1] == quote)
-                {
-                    s = s.Remove(index2, 1);
-                }
-                index1 = index2 + 1;
-            }
+            string result = StringParser.ParseString(_textParser.CurrentToken.Text);
 
-            if (quote == '\'')
+            if (_textParser.CurrentToken.Text[0] == '\'')
             {
-                if (s.Length != 1)
+                if (result.Length > 1)
                 {
                     throw ParseError(Res.InvalidCharacterLiteral);
                 }
+
                 _textParser.NextToken();
-                return ConstantExpressionHelper.CreateLiteral(s[0], s);
+                return ConstantExpressionHelper.CreateLiteral(result[0], result);
             }
+
             _textParser.NextToken();
-            return ConstantExpressionHelper.CreateLiteral(s, s);
+            return ConstantExpressionHelper.CreateLiteral(result, result);
         }
-
         Expression ParseIntegerLiteral()
         {
             _textParser.ValidateToken(TokenId.IntegerLiteral);
@@ -1522,7 +1508,7 @@ Expression ParseTypeAccess(Type type)
 
                 // If only 1 argument, and the arg is ConstantExpression, return the conversion
                 // If only 1 argument, and the arg is null, return the conversion (Can't use constructor)
-                if (args.Length == 1 
+                if (args.Length == 1
                     && (args[0] == null || args[0] is ConstantExpression))
                 {
                     return GenerateConversion(args[0], type, errorPos);

src/System.Linq.Dynamic.Core/Parser/StringParser.cs

@@ -0,0 +1,81 @@
+using System.Globalization;
+using System.Linq.Dynamic.Core.Exceptions;
+using System.Text;
+using System.Text.RegularExpressions;
+
+namespace System.Linq.Dynamic.Core.Parser
+{
+    /// <summary>
+    /// Parse a Double and Single Quoted string.
+    /// Some parts of the code is based on https://github.com/zzzprojects/Eval-Expression.NET
+    /// </summary>
+    internal static class StringParser
+    {
+        public static string ParseString(string s)
+        {
+            var inputStringBuilder = new StringBuilder(s);
+            var tempStringBuilder = new StringBuilder();
+            string found = null;
+
+            char quote = inputStringBuilder[0];
+            int pos = 1;
+
+            while (pos < inputStringBuilder.Length)
+            {
+                char ch = inputStringBuilder[pos];
+
+                if (ch == '\\' && pos + 1 < inputStringBuilder.Length && (inputStringBuilder[pos + 1] == '\\' || inputStringBuilder[pos + 1] == quote))
+                {
+                    tempStringBuilder.Append(inputStringBuilder[pos + 1]);
+                    pos++; // Treat as escape character for \\ or \'
+                }
+                else if (ch == '\\' && pos + 1 < inputStringBuilder.Length && inputStringBuilder[pos + 1] == 'u')
+                {
+                    if (pos + 5 >= inputStringBuilder.Length)
+                    {
+                        throw new ParseException(string.Format(CultureInfo.CurrentCulture, Res.UnexpectedUnrecognizedEscapeSequence, pos, inputStringBuilder.ToString(pos, inputStringBuilder.Length - pos - 1)), pos);
+                    }
+
+                    string unicode = inputStringBuilder.ToString(pos, 6);
+                    tempStringBuilder.Append(Regex.Unescape(unicode));
+                    pos += 5;
+                }
+                else if (ch == quote)
+                {
+                    found = Replace(tempStringBuilder);
+                    break;
+                }
+                else
+                {
+                    tempStringBuilder.Append(ch);
+                }
+
+                pos++;
+            }
+
+            if (found == null)
+            {
+                throw new ParseException(string.Format(CultureInfo.CurrentCulture, Res.UnexpectedUnclosedString, pos, inputStringBuilder.ToString()), pos);
+            }
+
+            return found;
+        }
+
+        private static string Replace(StringBuilder inputStringBuilder)
+        {
+            var sb = new StringBuilder(inputStringBuilder.ToString())
+                .Replace(@"\\", "\\") // \\ – backslash
+                .Replace(@"\0", "\0") // Unicode character 0
+                .Replace(@"\a", "\a") // Alert(character 7)
+                .Replace(@"\b", "\b") // Backspace(character 8)
+                .Replace(@"\f", "\f") // Form feed(character 12)
+                .Replace(@"\n", "\n") // New line(character 10)
+                .Replace(@"\r", "\r") // Carriage return (character 13)
+                .Replace(@"\t", "\t") // Horizontal tab(character 9)
+                .Replace(@"\v", "\v") // Vertical quote(character 11)
+            ; 
+
+            return sb.ToString();
+        }
+    }
+}

src/System.Linq.Dynamic.Core/Res.cs

@@ -62,6 +62,8 @@ internal static class Res
         public const string TokenExpected = "{0} expected";
         public const string TypeHasNoNullableForm = "Type '{0}' has no nullable form";
         public const string TypeNotFound = "Type '{0}' not found";
+        public const string UnexpectedUnclosedString = "Unexpected end of string with unclosed string at position {0} near '{1}'.";
+        public const string UnexpectedUnrecognizedEscapeSequence = "Unexpected unrecognized escape sequence at position {0} near '{1}'.";
         public const string UnknownIdentifier = "Unknown identifier '{0}'";
         public const string UnknownPropertyOrField = "No property or field '{0}' exists in type '{1}'";
         public const string UnterminatedStringLiteral = "Unterminated string literal";

test/System.Linq.Dynamic.Core.Tests/DynamicExpressionParserTests.cs

@@ -600,17 +600,15 @@ public void DynamicExpressionParser_ParseLambda_Config_StringLiteralEmpty_Return
         [Fact]
         public void DynamicExpressionParser_ParseLambda_StringLiteralEmbeddedQuote_ReturnsBooleanLambdaExpression()
         {
-            string expectedRightValue = "\"test \\\"string\"";
-
             // Act
             var expression = DynamicExpressionParser.ParseLambda(
                 new[] { Expression.Parameter(typeof(string), "Property1") },
                 typeof(bool),
-                string.Format("Property1 == {0}", expectedRightValue));
+                string.Format("Property1 == {0}", "\"test \\\"string\""));
 
             string rightValue = ((BinaryExpression)expression.Body).Right.ToString();
             Assert.Equal(typeof(bool), expression.Body.Type);
-            Assert.Equal(expectedRightValue, rightValue);
+            Assert.Equal("\"test \"string\"", rightValue);
         }
 
         /// <summary>
@@ -650,17 +648,15 @@ public void DynamicExpressionParser_ParseLambda_MultipleLambdas()
         [Fact]
         public void DynamicExpressionParser_ParseLambda_StringLiteralStartEmbeddedQuote_ReturnsBooleanLambdaExpression()
         {
-            // Assign
-            string expectedRightValue = "\"\\\"test\"";
-
+            // Act
             var expression = DynamicExpressionParser.ParseLambda(
                 new[] { Expression.Parameter(typeof(string), "Property1") },
                 typeof(bool),
-                string.Format("Property1 == {0}", expectedRightValue));
+                string.Format("Property1 == {0}", "\"\\\"test\""));
 
             string rightValue = ((BinaryExpression)expression.Body).Right.ToString();
             Assert.Equal(typeof(bool), expression.Body.Type);
-            Assert.Equal(expectedRightValue, rightValue);
+            Assert.Equal("\"\"test\"", rightValue);
         }
 
         [Fact]
@@ -677,51 +673,51 @@ public void DynamicExpressionParser_ParseLambda_StringLiteral_MissingClosingQuot
         [Fact]
         public void DynamicExpressionParser_ParseLambda_StringLiteralEscapedBackslash_ReturnsBooleanLambdaExpression()
         {
-            // Assign
-            string expectedRightValue = "\"test\\string\"";
-
             // Act
             var expression = DynamicExpressionParser.ParseLambda(
                 new[] { Expression.Parameter(typeof(string), "Property1") },
                 typeof(bool),
-                string.Format("Property1 == {0}", expectedRightValue));
+                string.Format("Property1 == {0}", "\"test\\\\string\""));
 
             string rightValue = ((BinaryExpression)expression.Body).Right.ToString();
             Assert.Equal(typeof(Boolean), expression.Body.Type);
-            Assert.Equal(expectedRightValue, rightValue);
+            Assert.Equal("\"test\\string\"", rightValue);
         }
 
         [Fact]
         public void DynamicExpressionParser_ParseLambda_StringLiteral_Backslash()
         {
-            string expectedLeftValue = "Property1.IndexOf(\"\\\\\")";
+            // Assign
             string expectedRightValue = "0";
+
+            //Act
             var expression = DynamicExpressionParser.ParseLambda(
                 new[] { Expression.Parameter(typeof(string), "Property1") },
                 typeof(Boolean),
-                string.Format("{0} >= {1}", expectedLeftValue, expectedRightValue));
+                string.Format("{0} >= {1}", "Property1.IndexOf(\"\\\\\")", expectedRightValue));
 
             string leftValue = ((BinaryExpression)expression.Body).Left.ToString();
             string rightValue = ((BinaryExpression)expression.Body).Right.ToString();
+
+            // Assert
             Assert.Equal(typeof(Boolean), expression.Body.Type);
-            Assert.Equal(expectedLeftValue, leftValue);
+            Assert.Equal("Property1.IndexOf(\"\\\")", leftValue);
             Assert.Equal(expectedRightValue, rightValue);
         }
 
         [Fact]
         public void DynamicExpressionParser_ParseLambda_StringLiteral_QuotationMark()
         {
-            string expectedLeftValue = "Property1.IndexOf(\"\\\"\")";
             string expectedRightValue = "0";
             var expression = DynamicExpressionParser.ParseLambda(
                 new[] { Expression.Parameter(typeof(string), "Property1") },
                 typeof(Boolean),
-                string.Format("{0} >= {1}", expectedLeftValue, expectedRightValue));
+                string.Format("{0} >= {1}", "Property1.IndexOf(\"\\\"\")", expectedRightValue));
 
             string leftValue = ((BinaryExpression)expression.Body).Left.ToString();
             string rightValue = ((BinaryExpression)expression.Body).Right.ToString();
             Assert.Equal(typeof(Boolean), expression.Body.Type);
-            Assert.Equal(expectedLeftValue, leftValue);
+            Assert.Equal("Property1.IndexOf(\"\"\")", leftValue);
             Assert.Equal(expectedRightValue, rightValue);
         }
 

test/System.Linq.Dynamic.Core.Tests/ExpressionTests.cs

@@ -1730,6 +1730,24 @@ public void ExpressionTests_StringConcatenation()
             Assert.Equal("FirstNamex y", resultAddWithAmpAndParams.First());
         }
 
+        [Fact]
+        public void ExpressionTests_StringEscaping()
+        {
+            // Arrange
+            var baseQuery = new[] { new { Value = "ab\"cd" }, new { Value = "a \\ b" } }.AsQueryable();
+
+            // Act
+            var result1 = baseQuery.Where("it.Value == \"ab\\\"cd\"").ToList();
+            var result2 = baseQuery.Where("it.Value.IndexOf('\\\\') != -1").ToList();
+
+            // Assert
+            Assert.Single(result1);
+            Assert.Equal("ab\"cd", result1[0].Value);
+
+            Assert.Single(result2);
+            Assert.Equal("a \\ b", result2[0].Value);
+        }
+
         [Fact]
         public void ExpressionTests_BinaryAnd()
         {

test/System.Linq.Dynamic.Core.Tests/Parser/StringParserTests.cs

@@ -0,0 +1,71 @@
+using System.Linq.Dynamic.Core.Exceptions;
+using System.Linq.Dynamic.Core.Parser;
+using FluentAssertions;
+using Xunit;
+
+namespace System.Linq.Dynamic.Core.Tests.Parser
+{
+    public class StringParserTests
+    {
+        [Theory]
+        [InlineData("'s")]
+        [InlineData("\"s")]
+        public void StringParser_WithUnexpectedUnclosedString_ThrowsException(string input)
+        {
+            // Act
+            var exception = Assert.Throws<ParseException>(() => StringParser.ParseString(input));
+
+            // Assert
+            Assert.Equal($"Unexpected end of string with unclosed string at position 2 near '{input}'.", exception.Message);
+        }
+
+        [Fact]
+        public void StringParser_With_UnexpectedUnrecognizedEscapeSequence_ThrowsException()
+        {
+            // Arrange
+            string input = new string(new[] { '"', '\\', 'u', '?', '"' });
+
+            // Act
+            var exception = Assert.Throws<ParseException>(() => StringParser.ParseString(input));
+
+            // Assert
+            Assert.Equal("Unexpected unrecognized escape sequence at position 1 near '\\u?'.", exception.Message);
+        }
+
+        [Theory]
+        [InlineData("'s'", "s")]
+        [InlineData("'\\r'", "\r")]
+        [InlineData("'\\\\'", "\\")]
+        public void StringParser_Parse_SingleQuotedString(string input, string expectedResult)
+        {
+            // Act
+            string result = StringParser.ParseString(input);
+
+            // Assert
+            result.Should().Be(expectedResult);
+        }
+
+        [Theory]
+        [InlineData("\"\"", "")]
+        [InlineData("\"[]\"", "[]")]
+        [InlineData("\"()\"", "()")]
+        [InlineData("\"(\\\"\\\")\"", "(\"\")")]
+        [InlineData("\"/\"", "/")]
+        [InlineData("\"a\"", "a")]
+        [InlineData("\"This \\\"is\\\" a test.\"", "This \"is\" a test.")]
+        [InlineData(@"""This \""is\"" b test.""", @"This ""is"" b test.")]
+        [InlineData("\"ab\\\"cd\"", "ab\"cd")]
+        [InlineData("\"\\\"\"", "\"")]
+        [InlineData("\"\\\"\\\"\"", "\"\"")]
+        [InlineData("\"\\\\\"", "\\")]
+        [InlineData("\"AB YZ 19 \uD800\udc05 \u00e4\"", "AB YZ 19 \uD800\udc05 \u00e4")]
+        public void StringParser_Parse_DoubleQuotedString(string input, string expectedResult)
+        {
+            // Act
+            string result = StringParser.ParseString(input);
+
+            // Assert
+            result.Should().Be(expectedResult);
+        }
+    }
+}

test/System.Linq.Dynamic.Core.Tests/QueryableTests.Where.cs

@@ -124,22 +124,25 @@ public void Where_Dynamic_Exceptions()
         [Fact]
         public void Where_Dynamic_StringQuoted()
         {
-            //Arrange
+            // Arrange
             var testList = User.GenerateSampleModels(2, allowNullableProfiles: true);
             testList[0].UserName = @"This \""is\"" a test.";
             var qry = testList.AsQueryable();
 
-            //Act
-            var result1a = qry.Where(@"UserName == ""This \""is\"" a test.""").ToArray();
-            var result1b = qry.Where("UserName == \"This \\\"is\\\" a test.\"").ToArray();
-            var result2 = qry.Where("UserName == @0", @"This \""is\"" a test.").ToArray();
+            // Act
+            // var result1a = qry.Where(@"UserName == ""This \\""is\\"" a test.""").ToArray();
+            var result1b = qry.Where("UserName == \"This \\\\\\\"is\\\\\\\" a test.\"").ToArray();
+            var result2a = qry.Where("UserName == @0", @"This \""is\"" a test.").ToArray();
+            var result2b = qry.Where("UserName == @0", "This \\\"is\\\" a test.").ToArray();
+
             var expected = qry.Where(x => x.UserName == @"This \""is\"" a test.").ToArray();
 
-            //Assert
+            // Assert
             Assert.Single(expected);
-            Assert.Equal(expected, result1a);
+            // Assert.Equal(expected, result1a);
             Assert.Equal(expected, result1b);
-            Assert.Equal(expected, result2);
+            Assert.Equal(expected, result2a);
+            Assert.Equal(expected, result2b);
         }
 
         [Fact]