Skip to content

Commit d87c2ca

Browse files
matrivmatriv
authored
EQL: Replace ?"..." with """...""" for unescaped strings (#62539)
Use triple double quotes enclosing a string literal to interpret it as unescaped, in order to use `?` for marking query params and avoid user confusion. `?` also usually implies regex expressions. Any character inside the `"""` beginning-closing markings is considered raw and the only thing that is not permitted is the `"""` sequence itself. If a user wants to use that, needs to resort to the normal `"` string literal and use proper escaping. Relates to #61659
1 parent ec3388d commit d87c2ca

File tree

8 files changed

+245
-158
lines changed

8 files changed

+245
-158
lines changed

‎x-pack/plugin/eql/qa/common/src/main/resources/additional_test_queries.toml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -138,24 +138,24 @@ expected_event_ids = [98]
138138
notes = "regexp doesn't support character classes"
139139
query = '''
140140
//
141-
// ?".*?net1\s+localgroup.*?")
142-
process where match(command_line, ?".*?net1[ ]+localgroup.*?")
141+
// """.*?net1\s+localgroup.*?""")
142+
process where match(command_line, """.*?net1[ ]+localgroup.*?""")
143143
'''
144144

145145
[[queries]]
146146
name = "matchLiteAdditional"
147147
expected_event_ids = [98]
148148
query = '''
149-
process where matchLite(command_line, ?".*?net1.*?")
149+
process where matchLite(command_line, """.*?net1.*?""")
150150
'''
151151

152152
[[queries]]
153153
name = "matchWithCharacterClasses2"
154154
expected_event_ids = [98]
155155
notes = "regexp doesn't support predefined character classes (like \\s)"
156156
query = '''
157-
// ?".*?net1\s+\w{4,15}\s+.*?"
158-
process where match(command_line, ?".*?net1[ ]+[a-z]{4,15}[ ]+.*?")
157+
// """.*?net1\s+\w{4,15}\s+.*?"""
158+
process where match(command_line, """.*?net1[ ]+[a-z]{4,15}[ ]+.*?""")
159159
'''
160160

161161

‎x-pack/plugin/eql/qa/common/src/main/resources/test_queries.toml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1399,35 +1399,35 @@ registry where bytes_written_string_list[1] : "en"
13991399
[[queries]]
14001400
name = "matchLite1"
14011401
query = '''
1402-
process where matchLite(command_line, ?".*?net1\s+localgroup\s+.*?")
1402+
process where matchLite(command_line, """.*?net1\s+localgroup\s+.*?""")
14031403
'''
14041404
expected_event_ids = [98]
14051405

14061406
[[queries]]
14071407
name = "matchLite2"
14081408
query = '''
1409-
process where matchLite(command_line, ?".*?net1\s+\w+\s+.*?")
1409+
process where matchLite(command_line, """.*?net1\s+\w+\s+.*?""")
14101410
'''
14111411
expected_event_ids = [98]
14121412

14131413
[[queries]]
14141414
name = "matchLite3"
14151415
query = '''
1416-
process where matchLite(command_line, ?".*?net1\s+\w{4,15}\s+.*?")
1416+
process where matchLite(command_line, """.*?net1\s+\w{4,15}\s+.*?""")
14171417
'''
14181418
expected_event_ids = [98]
14191419

14201420
[[queries]]
14211421
name = "match1"
14221422
expected_event_ids = [98]
14231423
query = '''
1424-
process where match(command_line, ?".*?net1\s+\w{4,15}\s+.*?")
1424+
process where match(command_line, """.*?net1\s+\w{4,15}\s+.*?""")
14251425
'''
14261426

14271427
[[queries]]
14281428
name = "matchLite4"
14291429
query = '''
1430-
process where matchLite(command_line, ?".*?net1\s+[localgrup]{4,15}\s+.*?")
1430+
process where matchLite(command_line, """.*?net1\s+[localgrup]{4,15}\s+.*?""")
14311431
'''
14321432
expected_event_ids = [98]
14331433

‎x-pack/plugin/eql/qa/common/src/main/resources/test_queries_unsupported.toml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -797,35 +797,35 @@ registry where bytes_written_string_list[1] : "en"
797797
[[queries]]
798798
name = "matchLite1"
799799
query = '''
800-
process where matchLite(command_line, ?".*?net1\s+localgroup\s+.*?")
800+
process where matchLite(command_line, """.*?net1\s+localgroup\s+.*?""")
801801
'''
802802
expected_event_ids = [98]
803803

804804
[[queries]]
805805
name = "matchLite2"
806806
query = '''
807-
process where matchLite(command_line, ?".*?net1\s+\w+\s+.*?")
807+
process where matchLite(command_line, """.*?net1\s+\w+\s+.*?""")
808808
'''
809809
expected_event_ids = [98]
810810

811811
[[queries]]
812812
name = "matchLite3"
813813
query = '''
814-
process where matchLite(command_line, ?".*?net1\s+\w{4,15}\s+.*?")
814+
process where matchLite(command_line, """.*?net1\s+\w{4,15}\s+.*?""")
815815
'''
816816
expected_event_ids = [98]
817817

818818
[[queries]]
819819
name = "match1"
820820
expected_event_ids = [98]
821821
query = '''
822-
process where match(command_line, ?".*?net1\s+\w{4,15}\s+.*?")
822+
process where match(command_line, """.*?net1\s+\w{4,15}\s+.*?""")
823823
'''
824824

825825
[[queries]]
826826
name = "matchLite4"
827827
query = '''
828-
process where matchLite(command_line, ?".*?net1\s+[localgrup]{4,15}\s+.*?")
828+
process where matchLite(command_line, """.*?net1\s+[localgrup]{4,15}\s+.*?""")
829829
'''
830830
expected_event_ids = [98]
831831

‎x-pack/plugin/eql/src/main/antlr/EqlBase.g4

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -202,6 +202,7 @@ STRING
202202
| '"' ('\\' [btnfr"'\\] | ~[\r\n"\\])* '"'
203203
| '?"' ('\\"' |~["\r\n])* '"'
204204
| '?\'' ('\\\'' |~['\r\n])* '\''
205+
| '"""' (~[\r\n])*? '"""' '"'? '"'?
205206
;
206207
207208
INTEGER_VALUE

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/AbstractBuilder.java

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -122,10 +122,15 @@ public static String unquoteString(Source source) {
122122
return null;
123123
}
124124

125-
// unescaped strings can be interpreted directly
125+
// catch old method of ?" and ?' to define unescaped strings
126126
if (text.startsWith("?")) {
127-
checkForSingleQuotedString(source, text, 1);
128-
return text.substring(2, text.length() - 1);
127+
throw new ParsingException(source,
128+
"Use triple double quotes [\"\"\"] to define unescaped string literals, not [?{}]", text.charAt(1));
129+
}
130+
131+
// unescaped strings can be interpreted directly
132+
if (text.startsWith("\"\"\"")) {
133+
return text.substring(3, text.length() - 3);
129134
}
130135

131136
checkForSingleQuotedString(source, text, 0);

‎x-pack/plugin/eql/src/main/java/org/elasticsearch/xpack/eql/parser/EqlBaseLexer.java

Lines changed: 128 additions & 119 deletions
Large diffs are not rendered by default.

‎x-pack/plugin/eql/src/test/java/org/elasticsearch/xpack/eql/parser/ExpressionTests.java

Lines changed: 88 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,24 @@ public void testStrings() {
6666
assertEquals("hello\\\nworld", unquoteString(source("\"hello\\\\\\nworld\"")));
6767
assertEquals("hello\\\"world", unquoteString(source("\"hello\\\\\\\"world\"")));
6868

69-
// test for unescaped strings: ?"...."
70-
assertEquals("hello\"world", unquoteString(source("?\"hello\"world\"")));
71-
assertEquals("hello\\\"world", unquoteString(source("?\"hello\\\"world\"")));
72-
assertEquals("hello'world", unquoteString(source("?\"hello'world\"")));
73-
assertEquals("hello\\nworld", unquoteString(source("?\"hello\\nworld\"")));
74-
assertEquals("hello\\\\nworld", unquoteString(source("?\"hello\\\\nworld\"")));
75-
assertEquals("hello\\\\\\nworld", unquoteString(source("?\"hello\\\\\\nworld\"")));
76-
assertEquals("hello\\\\\\\"world", unquoteString(source("?\"hello\\\\\\\"world\"")));
69+
// test for unescaped strings: """...."""
70+
assertEquals("hello\"world", unquoteString(source("\"\"\"hello\"world\"\"\"")));
71+
assertEquals("hello\\\"world", unquoteString(source("\"\"\"hello\\\"world\"\"\"")));
72+
assertEquals("\"\"hello\"\\\"world\"\"\"", unquoteString(source("\"\"\"\"\"hello\"\\\"world\"\"\"\"\"\"")));
73+
assertEquals("hello'world", unquoteString(source("\"\"\"hello'world\"\"\"")));
74+
assertEquals("hello'world", unquoteString(source("\"\"\"hello\'world\"\"\"")));
75+
assertEquals("hello\\'world", unquoteString(source("\"\"\"hello\\\'world\"\"\"")));
76+
assertEquals("hello\\nworld", unquoteString(source("\"\"\"hello\\nworld\"\"\"")));
77+
assertEquals("hello\\\\nworld", unquoteString(source("\"\"\"hello\\\\nworld\"\"\"")));
78+
assertEquals("hello\\\\\\nworld", unquoteString(source("\"\"\"hello\\\\\\nworld\"\"\"")));
79+
assertEquals("hello\\\\\\\"world", unquoteString(source("\"\"\"hello\\\\\\\"world\"\"\"")));
80+
assertEquals("\"\\\"", unquoteString(source("\"\"\"\"\\\"\"\"\"")));
81+
assertEquals("\\\"\"\"", unquoteString(source("\"\"\"\\\"\"\"\"\"\"")));
82+
assertEquals("\"\\\"\"", unquoteString(source("\"\"\"\"\\\"\"\"\"\"")));
83+
assertEquals("\"\"\\\"", unquoteString(source("\"\"\"\"\"\\\"\"\"\"")));
84+
assertEquals("\"\"", unquoteString(source("\"\"\"\"\"\"\"\"")));
85+
assertEquals("\"\" \"\"", unquoteString(source("\"\"\"\"\" \"\"\"\"\"")));
86+
assertEquals("", unquoteString(source("\"\"\"\"\"\"")));
7787
}
7888

7989
public void testLiterals() {
@@ -100,18 +110,80 @@ public void testDoubleQuotedString() {
100110

101111
public void testSingleQuotedUnescapedStringDisallowed() {
102112
ParsingException e = expectThrows(ParsingException.class, () -> expr("?'hello world'"));
103-
assertEquals("line 1:2: Use double quotes [\"] to define string literals, not single quotes [']",
113+
assertEquals("line 1:2: Use triple double quotes [\"\"\"] to define unescaped string literals, not [?']",
104114
e.getMessage());
105-
e = expectThrows(ParsingException.class, () -> parser.createStatement("process where name==?'hello world'"));
106-
assertEquals("line 1:22: Use double quotes [\"] to define string literals, not single quotes [']",
115+
e = expectThrows(ParsingException.class, () -> parser.createStatement("process where name == ?'hello world'"));
116+
assertEquals("line 1:24: Use triple double quotes [\"\"\"] to define unescaped string literals, not [?']",
107117
e.getMessage());
108118
}
109119

110-
public void testDoubleQuotedUnescapedString() {
111-
// "hello \" world"
112-
Expression parsed = expr("?\"hello \\\" world!\"");
113-
Expression expected = new Literal(null, "hello \\\" world!", DataTypes.KEYWORD);
114-
assertEquals(expected, parsed);
120+
public void testDoubleQuotedUnescapedStringForbidden() {
121+
ParsingException e = expectThrows(ParsingException.class, () -> expr("?\"hello world\""));
122+
assertEquals("line 1:2: Use triple double quotes [\"\"\"] to define unescaped string literals, not [?\"]",
123+
e.getMessage());
124+
e = expectThrows(ParsingException.class, () -> parser.createStatement("process where name == ?\"hello world\""));
125+
assertEquals("line 1:24: Use triple double quotes [\"\"\"] to define unescaped string literals, not [?\"]",
126+
e.getMessage());
127+
}
128+
129+
public void testTripleDoubleQuotedUnescapedString() {
130+
// """hello world!"""" == """foobar""" => hello world! = foobar
131+
String str = "\"\"\"hello world!\"\"\" == \"\"\"foobar\"\"\"";
132+
String expectedStrLeft = "hello world!";
133+
String expectedStrRight = "foobar";
134+
Expression parsed = expr(str);
135+
assertEquals(Equals.class, parsed.getClass());
136+
Equals eq = (Equals) parsed;
137+
assertEquals(Literal.class, eq.left().getClass());
138+
assertEquals(expectedStrLeft, ((Literal) eq.left()).value());
139+
assertEquals(Literal.class, eq.right().getClass());
140+
assertEquals(expectedStrRight, ((Literal) eq.right()).value());
141+
142+
// """""hello""world!"""" == """"foo"bar""""" => ""hello""world!" = "foo""bar""
143+
str = " \"\"\"\"\"hello\"\"world!\"\"\"\" == \"\"\"\"foo\"bar\"\"\"\"\" ";
144+
expectedStrLeft = "\"\"hello\"\"world!\"";
145+
expectedStrRight = "\"foo\"bar\"\"";
146+
parsed = expr(str);
147+
assertEquals(Equals.class, parsed.getClass());
148+
eq = (Equals) parsed;
149+
assertEquals(Literal.class, eq.left().getClass());
150+
assertEquals(expectedStrLeft, ((Literal) eq.left()).value());
151+
assertEquals(Literal.class, eq.right().getClass());
152+
assertEquals(expectedStrRight, ((Literal) eq.right()).value());
153+
154+
// """""\""hello\\""\""world!\\""""" == """\\""\""foo""\\""\"bar""\\""\"""" =>
155+
// ""\""hello\\""\""world!\\"" == \\""\""foo""\\""\"bar""\\""\"
156+
str = " \"\"\"\"\"\\\"\"hello\\\\\"\"\\\"\"world!\\\\\"\"\"\"\" == " +
157+
" \"\"\"\\\\\"\"\\\"\"foo\"\"\\\\\"\"\\\"bar\"\"\\\\\"\"\\\"\"\"\" ";
158+
expectedStrLeft = "\"\"\\\"\"hello\\\\\"\"\\\"\"world!\\\\\"\"";
159+
expectedStrRight = "\\\\\"\"\\\"\"foo\"\"\\\\\"\"\\\"bar\"\"\\\\\"\"\\\"";
160+
parsed = expr(str);
161+
assertEquals(Equals.class, parsed.getClass());
162+
eq = (Equals) parsed;
163+
assertEquals(Literal.class, eq.left().getClass());
164+
assertEquals(expectedStrLeft, ((Literal) eq.left()).value());
165+
assertEquals(Literal.class, eq.right().getClass());
166+
assertEquals(expectedStrRight, ((Literal) eq.right()).value());
167+
168+
// """"""hello world!""" == """foobar"""
169+
ParsingException e = expectThrows(ParsingException.class, "Expected syntax error",
170+
() -> expr("\"\"\"\"\"\"hello world!\"\"\" == \"\"\"foobar\"\"\""));
171+
assertThat(e.getMessage(), startsWith("line 1:7: mismatched input 'hello' expecting {<EOF>,"));
172+
173+
// """""\"hello world!"""""" == """foobar"""
174+
e = expectThrows(ParsingException.class, "Expected syntax error",
175+
() -> expr("\"\"\"\"\"\\\"hello world!\"\"\"\"\"\" == \"\"\"foobar\"\"\""));
176+
assertThat(e.getMessage(), startsWith("line 1:25: mismatched input '\" == \"' expecting {<EOF>,"));
177+
178+
// """""\"hello world!""\"""" == """"""foobar"""
179+
e = expectThrows(ParsingException.class, "Expected syntax error",
180+
() -> expr("\"\"\"\"\"\\\"hello world!\"\"\\\"\"\"\" == \"\"\"\"\"\"foobar\"\"\""));
181+
assertThat(e.getMessage(), startsWith("line 1:37: mismatched input 'foobar' expecting {<EOF>,"));
182+
183+
// """""\"hello world!""\"""" == """""\"foobar\"\""""""
184+
e = expectThrows(ParsingException.class, "Expected syntax error",
185+
() -> expr("\"\"\"\"\"\\\"hello world!\"\"\\\"\"\"\" == \"\"\"\"\"\\\"foobar\\\"\\\"\"\"\"\"\""));
186+
assertEquals("line 1:52: token recognition error at: '\"'", e.getMessage());
115187
}
116188

117189
public void testNumbers() {

‎x-pack/plugin/eql/src/test/resources/queries-supported.eql

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -178,19 +178,19 @@ process where command_line : "*%*%*" ;
178178
process where command_line : "%*%*" ;
179179

180180

181-
process where match(?".*?net1\s+localgroup\s+.*?", command_line)
181+
process where match(""".*?net1\s+localgroup\s+.*?""", command_line)
182182
;
183183

184-
process where match(?".*?net1\s+\w+\s+.*?", command_line)
184+
process where match(""".*?net1\s+\w+\s+.*?""", command_line)
185185
;
186186

187-
process where match(?".*?net1\s+\w{4,15}\s+.*?", command_line)
187+
process where match(""".*?net1\s+\w{4,15}\s+.*?""", command_line)
188188
;
189189

190-
process where match(?".*?net1\s+\w{4,15}\s+.*?", command_line)
190+
process where match(""".*?net1\s+\w{4,15}\s+.*?""", command_line)
191191
;
192192

193-
process where match(?".*?net1\s+[localgrup]{4,15}\s+.*?", command_line)
193+
process where match(""".*?net1\s+[localgrup]{4,15}\s+.*?""", command_line)
194194
;
195195

196196
file where opcode:0 and startsWith(file_name, "exploRER.")

0 commit comments

Comments
 (0)