Skip to content

Commit dcae073

Browse files
ThinkRedstoneThinkRedstone
committed
parser: attach a span to most common errors describing the problematic token
1 parent a00d5cd commit dcae073

14 files changed

+694
-328
lines changed

‎src/ast/spans.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2516,7 +2516,7 @@ pub mod tests {
25162516
span,
25172517
})) => {
25182518
assert_eq!(":fooBar", s);
2519-
assert_eq!(&Span::new((3, 3).into(), (3, 10).into()), span);
2519+
assert_eq!(&Span::new((3, 3), (3, 10)), span);
25202520
}
25212521
_ => panic!("expected unnamed expression; got {col:?}"),
25222522
}

‎src/parser/mod.rs‎

Lines changed: 102 additions & 91 deletions
Large diffs are not rendered by default.

‎src/tokenizer.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -574,8 +574,11 @@ impl Span {
574574
const EMPTY: Span = Self::empty();
575575

576576
/// Create a new span from a start and end [`Location`]
577-
pub fn new(start: Location, end: Location) -> Span {
578-
Span { start, end }
577+
pub fn new(start: impl Into<Location>, end: impl Into<Location>) -> Span {
578+
Span {
579+
start: start.into(),
580+
end: end.into(),
581+
}
579582
}
580583

581584
/// Returns an empty span `(0, 0) -> (0, 0)`

‎tests/sqlparser_bigquery.rs‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -295,13 +295,13 @@ fn parse_begin() {
295295
bigquery()
296296
.parse_sql_statements("BEGIN SELECT 1; SELECT 2 END")
297297
.unwrap_err(),
298-
ParserError::ParserError("Expected: ;, found: END".to_string())
298+
ParserError::SpannedParserError("Expected: ;, found: END".to_string(), Span::empty())
299299
);
300300
assert_eq!(
301301
bigquery()
302302
.parse_sql_statements("BEGIN SELECT 1; EXCEPTION WHEN ERROR THEN SELECT 2 END")
303303
.unwrap_err(),
304-
ParserError::ParserError("Expected: ;, found: END".to_string())
304+
ParserError::SpannedParserError("Expected: ;, found: END".to_string(), Span::empty())
305305
);
306306
}
307307

@@ -2011,7 +2011,7 @@ fn parse_merge_invalid_statements() {
20112011
] {
20122012
let res = dialects.parse_sql_statements(sql);
20132013
assert_eq!(
2014-
ParserError::ParserError(err_msg.to_string()),
2014+
ParserError::SpannedParserError(err_msg.to_string(), Span::empty()),
20152015
res.unwrap_err()
20162016
);
20172017
}
@@ -2135,13 +2135,19 @@ fn parse_big_query_declare() {
21352135

21362136
let error_sql = "DECLARE x";
21372137
assert_eq!(
2138-
ParserError::ParserError("Expected: a data type name, found: EOF".to_owned()),
2138+
ParserError::SpannedParserError(
2139+
"Expected: a data type name, found: EOF".to_owned(),
2140+
Span::empty()
2141+
),
21392142
bigquery().parse_sql_statements(error_sql).unwrap_err()
21402143
);
21412144

21422145
let error_sql = "DECLARE x 42";
21432146
assert_eq!(
2144-
ParserError::ParserError("Expected: a data type name, found: 42".to_owned()),
2147+
ParserError::SpannedParserError(
2148+
"Expected: a data type name, found: 42".to_owned(),
2149+
Span::empty()
2150+
),
21452151
bigquery().parse_sql_statements(error_sql).unwrap_err()
21462152
);
21472153
}
@@ -2345,7 +2351,7 @@ fn test_bigquery_create_function() {
23452351
];
23462352
for (sql, error) in error_sqls {
23472353
assert_eq!(
2348-
ParserError::ParserError(error.to_owned()),
2354+
ParserError::SpannedParserError(error.to_owned(), Span::empty()),
23492355
bigquery().parse_sql_statements(sql).unwrap_err()
23502356
);
23512357
}
@@ -2375,7 +2381,7 @@ fn test_bigquery_trim() {
23752381
// missing comma separation
23762382
let error_sql = "SELECT TRIM('xyz' 'a')";
23772383
assert_eq!(
2378-
ParserError::ParserError("Expected: ), found: 'a'".to_owned()),
2384+
ParserError::SpannedParserError("Expected: ), found: 'a'".to_owned(), Span::empty()),
23792385
bigquery().parse_sql_statements(error_sql).unwrap_err()
23802386
);
23812387
}
@@ -2538,16 +2544,17 @@ fn test_struct_trailing_and_nested_bracket() {
25382544

25392545
// Bad case with missing closing bracket
25402546
assert_eq!(
2541-
ParserError::ParserError("Expected: >, found: )".to_owned()),
2547+
ParserError::SpannedParserError("Expected: >, found: )".to_owned(), Span::empty()),
25422548
bigquery()
25432549
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRING, b INT64)")
25442550
.unwrap_err()
25452551
);
25462552

25472553
// Bad case with redundant closing bracket
25482554
assert_eq!(
2549-
ParserError::ParserError(
2550-
"unmatched > after parsing data type STRUCT<a STRING, b INT64>)".to_owned()
2555+
ParserError::SpannedParserError(
2556+
"unmatched > after parsing data type STRUCT<a STRING, b INT64>".to_owned(),
2557+
Span::empty()
25512558
),
25522559
bigquery()
25532560
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRING, b INT64>>)")
@@ -2556,8 +2563,9 @@ fn test_struct_trailing_and_nested_bracket() {
25562563

25572564
// Base case with redundant closing bracket in nested struct
25582565
assert_eq!(
2559-
ParserError::ParserError(
2560-
"Expected: ',' or ')' after column definition, found: >".to_owned()
2566+
ParserError::SpannedParserError(
2567+
"Expected: ',' or ')' after column definition, found: >".to_owned(),
2568+
Span::empty()
25612569
),
25622570
bigquery()
25632571
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRUCT<b INT>>>, c INT64)")
@@ -2569,24 +2577,25 @@ fn test_struct_trailing_and_nested_bracket() {
25692577
bigquery_and_generic()
25702578
.parse_sql_statements(sql)
25712579
.unwrap_err(),
2572-
ParserError::ParserError("unmatched > in STRUCT literal".to_string())
2580+
ParserError::SpannedParserError("unmatched > in STRUCT literal".to_string(), Span::empty())
25732581
);
25742582

25752583
let sql = "SELECT STRUCT<STRUCT<INT64>>>(NULL)";
25762584
assert_eq!(
25772585
bigquery_and_generic()
25782586
.parse_sql_statements(sql)
25792587
.unwrap_err(),
2580-
ParserError::ParserError("Expected: (, found: >".to_string())
2588+
ParserError::SpannedParserError("Expected: (, found: >".to_string(), Span::empty())
25812589
);
25822590

25832591
let sql = "CREATE TABLE table (x STRUCT<STRUCT<INT64>>>)";
25842592
assert_eq!(
25852593
bigquery_and_generic()
25862594
.parse_sql_statements(sql)
25872595
.unwrap_err(),
2588-
ParserError::ParserError(
2589-
"Expected: ',' or ')' after column definition, found: >".to_string()
2596+
ParserError::SpannedParserError(
2597+
"Expected: ',' or ')' after column definition, found: >".to_string(),
2598+
Span::empty()
25902599
)
25912600
);
25922601
}

‎tests/sqlparser_clickhouse.rs‎

Lines changed: 56 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ use sqlparser::ast::Value::Boolean;
3232
use sqlparser::ast::*;
3333
use sqlparser::dialect::ClickHouseDialect;
3434
use sqlparser::dialect::GenericDialect;
35-
use sqlparser::parser::ParserError::ParserError;
35+
use sqlparser::parser::ParserError;
3636

3737
#[test]
3838
fn parse_map_access_expr() {
@@ -289,13 +289,19 @@ fn parse_alter_table_attach_and_detach_partition() {
289289
clickhouse_and_generic()
290290
.parse_sql_statements(format!("ALTER TABLE t0 {operation} PARTITION").as_str())
291291
.unwrap_err(),
292-
ParserError("Expected: an expression, found: EOF".to_string())
292+
ParserError::SpannedParserError(
293+
"Expected: an expression, found: EOF".to_string(),
294+
Span::empty()
295+
)
293296
);
294297
assert_eq!(
295298
clickhouse_and_generic()
296299
.parse_sql_statements(format!("ALTER TABLE t0 {operation} PART").as_str())
297300
.unwrap_err(),
298-
ParserError("Expected: an expression, found: EOF".to_string())
301+
ParserError::SpannedParserError(
302+
"Expected: an expression, found: EOF".to_string(),
303+
Span::empty()
304+
)
299305
);
300306
}
301307
}
@@ -358,19 +364,22 @@ fn parse_alter_table_add_projection() {
358364
clickhouse_and_generic()
359365
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name")
360366
.unwrap_err(),
361-
ParserError("Expected: (, found: EOF".to_string())
367+
ParserError::SpannedParserError("Expected: (, found: EOF".to_string(), Span::empty())
362368
);
363369
assert_eq!(
364370
clickhouse_and_generic()
365371
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name ()")
366372
.unwrap_err(),
367-
ParserError("Expected: SELECT, found: )".to_string())
373+
ParserError::SpannedParserError("Expected: SELECT, found: )".to_string(), Span::empty())
368374
);
369375
assert_eq!(
370376
clickhouse_and_generic()
371377
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name (SELECT)")
372378
.unwrap_err(),
373-
ParserError("Expected: an expression, found: )".to_string())
379+
ParserError::SpannedParserError(
380+
"Expected: an expression, found: )".to_string(),
381+
Span::empty()
382+
)
374383
);
375384
}
376385

@@ -400,7 +409,10 @@ fn parse_alter_table_drop_projection() {
400409
clickhouse_and_generic()
401410
.parse_sql_statements("ALTER TABLE t0 DROP PROJECTION")
402411
.unwrap_err(),
403-
ParserError("Expected: identifier, found: EOF".to_string())
412+
ParserError::SpannedParserError(
413+
"Expected: identifier, found: EOF".to_string(),
414+
Span::empty()
415+
)
404416
);
405417
}
406418

@@ -447,7 +459,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
447459
clickhouse_and_generic()
448460
.parse_sql_statements(format!("ALTER TABLE t0 {keyword} PROJECTION",).as_str())
449461
.unwrap_err(),
450-
ParserError("Expected: identifier, found: EOF".to_string())
462+
ParserError::SpannedParserError(
463+
"Expected: identifier, found: EOF".to_string(),
464+
Span::empty()
465+
)
451466
);
452467

453468
assert_eq!(
@@ -456,7 +471,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
456471
format!("ALTER TABLE t0 {keyword} PROJECTION my_name IN PARTITION",).as_str()
457472
)
458473
.unwrap_err(),
459-
ParserError("Expected: identifier, found: EOF".to_string())
474+
ParserError::SpannedParserError(
475+
"Expected: identifier, found: EOF".to_string(),
476+
Span::empty()
477+
)
460478
);
461479

462480
assert_eq!(
@@ -465,7 +483,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
465483
format!("ALTER TABLE t0 {keyword} PROJECTION my_name IN",).as_str()
466484
)
467485
.unwrap_err(),
468-
ParserError("Expected: end of statement, found: IN".to_string())
486+
ParserError::SpannedParserError(
487+
"Expected: end of statement, found: IN".to_string(),
488+
Span::empty()
489+
)
469490
);
470491
}
471492
}
@@ -513,19 +534,28 @@ fn parse_optimize_table() {
513534
clickhouse_and_generic()
514535
.parse_sql_statements("OPTIMIZE TABLE t0 DEDUPLICATE BY")
515536
.unwrap_err(),
516-
ParserError("Expected: an expression, found: EOF".to_string())
537+
ParserError::SpannedParserError(
538+
"Expected: an expression, found: EOF".to_string(),
539+
Span::empty()
540+
)
517541
);
518542
assert_eq!(
519543
clickhouse_and_generic()
520544
.parse_sql_statements("OPTIMIZE TABLE t0 PARTITION")
521545
.unwrap_err(),
522-
ParserError("Expected: an expression, found: EOF".to_string())
546+
ParserError::SpannedParserError(
547+
"Expected: an expression, found: EOF".to_string(),
548+
Span::empty()
549+
)
523550
);
524551
assert_eq!(
525552
clickhouse_and_generic()
526553
.parse_sql_statements("OPTIMIZE TABLE t0 PARTITION ID")
527554
.unwrap_err(),
528-
ParserError("Expected: identifier, found: EOF".to_string())
555+
ParserError::SpannedParserError(
556+
"Expected: identifier, found: EOF".to_string(),
557+
Span::empty()
558+
)
529559
);
530560
}
531561

@@ -1063,7 +1093,7 @@ fn parse_settings_in_query() {
10631093
clickhouse_and_generic()
10641094
.parse_sql_statements(sql)
10651095
.unwrap_err(),
1066-
ParserError(error_msg.to_string())
1096+
ParserError::SpannedParserError(error_msg.to_string(), Span::empty())
10671097
);
10681098
}
10691099
}
@@ -1568,23 +1598,32 @@ fn parse_freeze_and_unfreeze_partition() {
15681598
clickhouse_and_generic()
15691599
.parse_sql_statements(format!("ALTER TABLE t0 {operation_name} PARTITION").as_str())
15701600
.unwrap_err(),
1571-
ParserError("Expected: an expression, found: EOF".to_string())
1601+
ParserError::SpannedParserError(
1602+
"Expected: an expression, found: EOF".to_string(),
1603+
Span::empty()
1604+
)
15721605
);
15731606
assert_eq!(
15741607
clickhouse_and_generic()
15751608
.parse_sql_statements(
15761609
format!("ALTER TABLE t0 {operation_name} PARTITION p0 WITH").as_str()
15771610
)
15781611
.unwrap_err(),
1579-
ParserError("Expected: NAME, found: EOF".to_string())
1612+
ParserError::SpannedParserError(
1613+
"Expected: NAME, found: EOF".to_string(),
1614+
Span::empty()
1615+
)
15801616
);
15811617
assert_eq!(
15821618
clickhouse_and_generic()
15831619
.parse_sql_statements(
15841620
format!("ALTER TABLE t0 {operation_name} PARTITION p0 WITH NAME").as_str()
15851621
)
15861622
.unwrap_err(),
1587-
ParserError("Expected: identifier, found: EOF".to_string())
1623+
ParserError::SpannedParserError(
1624+
"Expected: identifier, found: EOF".to_string(),
1625+
Span::empty()
1626+
)
15881627
);
15891628
}
15901629
}

0 commit comments

Comments
 (0)