Skip to content

24.8 Backport of #79969 - Fix match(col, '^…') index analysis with escaped metacharacters to avoid wrong results and crashes #850

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: customizations/24.8.14
Choose a base branch
from
Open

24.8 Backport of #79969 - Fix match(col, '^…') index analysis with escaped metacharacters to avoid wrong results and crashes #850

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
6cc60dc
MergeTree: improve extractFixedPrefixFromRegularExpression()
filimonov May 8, 2025
b9e6fa3
Update tests/queries/0_stateless/03456_match_index_prefix_extraction.sql
filimonov Jun 5, 2025
8aee8d5
Merge branch 'customizations/24.8.14' into backports/24.8/79969_fix_i…
arthurpassos Jun 10, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 8 additions & 5 deletions src/Storages/MergeTree/KeyCondition.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -101,7 +101,7 @@ static String extractFixedPrefixFromRegularExpression(const String & regexp)
const char * pos = begin;
const char * end = regexp.data() + regexp.size();

while (pos != end)
while (pos < end)
{
switch (*pos)
{
Expand All @@ -124,19 +124,22 @@ static String extractFixedPrefixFromRegularExpression(const String & regexp)
case '$':
case '.':
case '[':
case ']':
case '?':
case '*':
case '+':
case '\\':
case '{':
case '}':
case '-':
fixed_prefix += *pos;
break;
++pos;
break;
default:
/// all other escape sequences are not supported
pos = end;
break;
pos = end;
}

++pos;
break;
}

Expand Down
52 changes: 52 additions & 0 deletions tests/queries/0_stateless/03456_match_index_prefix_extraction.reference
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
Condition: (path in [\'xxx(zzz\', \'xxx(zz{\'))
1
Condition: (path in [\'xxx)zzz\', \'xxx)zz{\'))
1
Condition: (path in [\'xxx^zzz\', \'xxx^zz{\'))
1
Condition: (path in [\'xxx$zzz\', \'xxx$zz{\'))
1
Condition: (path in [\'xxx.zzz\', \'xxx.zz{\'))
1
Condition: (path in [\'xxx[zzz\', \'xxx[zz{\'))
1
Condition: (path in [\'xxx]zzz\', \'xxx]zz{\'))
1
Condition: (path in [\'xxx?zzz\', \'xxx?zz{\'))
1
Condition: (path in [\'xxx*zzz\', \'xxx*zz{\'))
1
Condition: (path in [\'xxx+zzz\', \'xxx+zz{\'))
1
Condition: (path in [\'xxx\\\\zzz\', \'xxx\\\\zz{\'))
1
Condition: (path in [\'xxx{zzz\', \'xxx{zz{\'))
1
Condition: (path in [\'xxx}zzz\', \'xxx}zz{\'))
1
Condition: (path in [\'xxx-zzz\', \'xxx-zz{\'))
1
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
15
Condition: (path in [\'xxx\', \'xxy\'))
15
Condition: (path in [\'xxx\', \'xxy\'))
15
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: (path in [\'xxx\', \'xxy\'))
0
Condition: true
Condition: true
111 changes: 111 additions & 0 deletions tests/queries/0_stateless/03456_match_index_prefix_extraction.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
SET parallel_replicas_local_plan=1;

drop table if exists foo;

CREATE TABLE foo (id UInt8, path String) engine = MergeTree ORDER BY (path) SETTINGS index_granularity=1;

INSERT INTO foo VALUES (1, 'xxx|yyy'),
(2, 'xxx(zzz'),
(3, 'xxx)zzz'),
(4, 'xxx^zzz'),
(5, 'xxx$zzz'),
(6, 'xxx.zzz'),
(7, 'xxx[zzz'),
(8, 'xxx]zzz'),
(9, 'xxx?zzz'),
(10, 'xxx*zzz'),
(11, 'xxx+zzz'),
(12, 'xxx\\zzz'),
(13, 'xxx{zzz'),
(14, 'xxx}zzz'),
(15, 'xxx-zzz');


-- check if also escaped sequence are properly extracted
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\(zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\(zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\)zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\)zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\^zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\^zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\$zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\$zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\.zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\.zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\[zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\[zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\]zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\]zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\?zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\?zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\*zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\*zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\+zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\+zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\\\zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\\\zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\{zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\{zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\}zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\}zzz') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\-zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\-zzz') SETTINGS force_primary_key = 1;


-- those regex chars prevent the index use (only 3 first chars used during index scan)
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\0bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\0bla') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx(bla)')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx(bla)') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx[bla]')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx[bla]') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx^bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx^bla') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx.bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx.bla') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx+bla')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx+bla') SETTINGS force_primary_key = 1;


-- here the forth char is not used during index, because it has 0+ quantifier
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx{0,1}')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx{0,1}') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx?')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx?') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx*')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx*') SETTINGS force_primary_key = 1;

-- some unsupported regex chars - only 3 first chars used during index scan
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\d+')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\d+') SETTINGS force_primary_key = 1;

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\w+')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\w+') SETTINGS force_primary_key = 1;


-- fully disabled for pipes - see https://github.com/ClickHouse/ClickHouse/pull/54696
SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxx\\|zzz')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxx\\|zzz') SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED }

SELECT trimLeft(explain) FROM (EXPLAIN PLAN indexes=1 SELECT id FROM foo WHERE match(path, '^xxxx|foo')) WHERE explain like '%Condition%';
SELECT count() FROM foo WHERE match(path, '^xxxx|foo') SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED }
Loading