Skip to content

Commit 73349c9

Browse files
ThinkRedstoneThinkRedstone
committed
parser: attach a span to most common errors describing the problematic token
1 parent 308a723 commit 73349c9

14 files changed

+694
-328
lines changed

‎src/ast/spans.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2511,7 +2511,7 @@ pub mod tests {
25112511
span,
25122512
})) => {
25132513
assert_eq!(":fooBar", s);
2514-
assert_eq!(&Span::new((3, 3).into(), (3, 10).into()), span);
2514+
assert_eq!(&Span::new((3, 3), (3, 10)), span);
25152515
}
25162516
_ => panic!("expected unnamed expression; got {col:?}"),
25172517
}

‎src/parser/mod.rs‎

Lines changed: 102 additions & 91 deletions
Large diffs are not rendered by default.

‎src/tokenizer.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -574,8 +574,11 @@ impl Span {
574574
const EMPTY: Span = Self::empty();
575575

576576
/// Create a new span from a start and end [`Location`]
577-
pub fn new(start: Location, end: Location) -> Span {
578-
Span { start, end }
577+
pub fn new(start: impl Into<Location>, end: impl Into<Location>) -> Span {
578+
Span {
579+
start: start.into(),
580+
end: end.into(),
581+
}
579582
}
580583

581584
/// Returns an empty span `(0, 0) -> (0, 0)`

‎tests/sqlparser_bigquery.rs‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -295,13 +295,13 @@ fn parse_begin() {
295295
bigquery()
296296
.parse_sql_statements("BEGIN SELECT 1; SELECT 2 END")
297297
.unwrap_err(),
298-
ParserError::ParserError("Expected: ;, found: END".to_string())
298+
ParserError::SpannedParserError("Expected: ;, found: END".to_string(), Span::empty())
299299
);
300300
assert_eq!(
301301
bigquery()
302302
.parse_sql_statements("BEGIN SELECT 1; EXCEPTION WHEN ERROR THEN SELECT 2 END")
303303
.unwrap_err(),
304-
ParserError::ParserError("Expected: ;, found: END".to_string())
304+
ParserError::SpannedParserError("Expected: ;, found: END".to_string(), Span::empty())
305305
);
306306
}
307307

@@ -2008,7 +2008,7 @@ fn parse_merge_invalid_statements() {
20082008
] {
20092009
let res = dialects.parse_sql_statements(sql);
20102010
assert_eq!(
2011-
ParserError::ParserError(err_msg.to_string()),
2011+
ParserError::SpannedParserError(err_msg.to_string(), Span::empty()),
20122012
res.unwrap_err()
20132013
);
20142014
}
@@ -2132,13 +2132,19 @@ fn parse_big_query_declare() {
21322132

21332133
let error_sql = "DECLARE x";
21342134
assert_eq!(
2135-
ParserError::ParserError("Expected: a data type name, found: EOF".to_owned()),
2135+
ParserError::SpannedParserError(
2136+
"Expected: a data type name, found: EOF".to_owned(),
2137+
Span::empty()
2138+
),
21362139
bigquery().parse_sql_statements(error_sql).unwrap_err()
21372140
);
21382141

21392142
let error_sql = "DECLARE x 42";
21402143
assert_eq!(
2141-
ParserError::ParserError("Expected: a data type name, found: 42".to_owned()),
2144+
ParserError::SpannedParserError(
2145+
"Expected: a data type name, found: 42".to_owned(),
2146+
Span::empty()
2147+
),
21422148
bigquery().parse_sql_statements(error_sql).unwrap_err()
21432149
);
21442150
}
@@ -2342,7 +2348,7 @@ fn test_bigquery_create_function() {
23422348
];
23432349
for (sql, error) in error_sqls {
23442350
assert_eq!(
2345-
ParserError::ParserError(error.to_owned()),
2351+
ParserError::SpannedParserError(error.to_owned(), Span::empty()),
23462352
bigquery().parse_sql_statements(sql).unwrap_err()
23472353
);
23482354
}
@@ -2372,7 +2378,7 @@ fn test_bigquery_trim() {
23722378
// missing comma separation
23732379
let error_sql = "SELECT TRIM('xyz' 'a')";
23742380
assert_eq!(
2375-
ParserError::ParserError("Expected: ), found: 'a'".to_owned()),
2381+
ParserError::SpannedParserError("Expected: ), found: 'a'".to_owned(), Span::empty()),
23762382
bigquery().parse_sql_statements(error_sql).unwrap_err()
23772383
);
23782384
}
@@ -2535,16 +2541,17 @@ fn test_struct_trailing_and_nested_bracket() {
25352541

25362542
// Bad case with missing closing bracket
25372543
assert_eq!(
2538-
ParserError::ParserError("Expected: >, found: )".to_owned()),
2544+
ParserError::SpannedParserError("Expected: >, found: )".to_owned(), Span::empty()),
25392545
bigquery()
25402546
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRING, b INT64)")
25412547
.unwrap_err()
25422548
);
25432549

25442550
// Bad case with redundant closing bracket
25452551
assert_eq!(
2546-
ParserError::ParserError(
2547-
"unmatched > after parsing data type STRUCT<a STRING, b INT64>)".to_owned()
2552+
ParserError::SpannedParserError(
2553+
"unmatched > after parsing data type STRUCT<a STRING, b INT64>".to_owned(),
2554+
Span::empty()
25482555
),
25492556
bigquery()
25502557
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRING, b INT64>>)")
@@ -2553,8 +2560,9 @@ fn test_struct_trailing_and_nested_bracket() {
25532560

25542561
// Base case with redundant closing bracket in nested struct
25552562
assert_eq!(
2556-
ParserError::ParserError(
2557-
"Expected: ',' or ')' after column definition, found: >".to_owned()
2563+
ParserError::SpannedParserError(
2564+
"Expected: ',' or ')' after column definition, found: >".to_owned(),
2565+
Span::empty()
25582566
),
25592567
bigquery()
25602568
.parse_sql_statements("CREATE TABLE my_table(f1 STRUCT<a STRUCT<b INT>>>, c INT64)")
@@ -2566,24 +2574,25 @@ fn test_struct_trailing_and_nested_bracket() {
25662574
bigquery_and_generic()
25672575
.parse_sql_statements(sql)
25682576
.unwrap_err(),
2569-
ParserError::ParserError("unmatched > in STRUCT literal".to_string())
2577+
ParserError::SpannedParserError("unmatched > in STRUCT literal".to_string(), Span::empty())
25702578
);
25712579

25722580
let sql = "SELECT STRUCT<STRUCT<INT64>>>(NULL)";
25732581
assert_eq!(
25742582
bigquery_and_generic()
25752583
.parse_sql_statements(sql)
25762584
.unwrap_err(),
2577-
ParserError::ParserError("Expected: (, found: >".to_string())
2585+
ParserError::SpannedParserError("Expected: (, found: >".to_string(), Span::empty())
25782586
);
25792587

25802588
let sql = "CREATE TABLE table (x STRUCT<STRUCT<INT64>>>)";
25812589
assert_eq!(
25822590
bigquery_and_generic()
25832591
.parse_sql_statements(sql)
25842592
.unwrap_err(),
2585-
ParserError::ParserError(
2586-
"Expected: ',' or ')' after column definition, found: >".to_string()
2593+
ParserError::SpannedParserError(
2594+
"Expected: ',' or ')' after column definition, found: >".to_string(),
2595+
Span::empty()
25872596
)
25882597
);
25892598
}

‎tests/sqlparser_clickhouse.rs‎

Lines changed: 56 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ use sqlparser::ast::Value::Boolean;
3232
use sqlparser::ast::*;
3333
use sqlparser::dialect::ClickHouseDialect;
3434
use sqlparser::dialect::GenericDialect;
35-
use sqlparser::parser::ParserError::ParserError;
35+
use sqlparser::parser::ParserError;
3636

3737
#[test]
3838
fn parse_map_access_expr() {
@@ -289,13 +289,19 @@ fn parse_alter_table_attach_and_detach_partition() {
289289
clickhouse_and_generic()
290290
.parse_sql_statements(format!("ALTER TABLE t0 {operation} PARTITION").as_str())
291291
.unwrap_err(),
292-
ParserError("Expected: an expression, found: EOF".to_string())
292+
ParserError::SpannedParserError(
293+
"Expected: an expression, found: EOF".to_string(),
294+
Span::empty()
295+
)
293296
);
294297
assert_eq!(
295298
clickhouse_and_generic()
296299
.parse_sql_statements(format!("ALTER TABLE t0 {operation} PART").as_str())
297300
.unwrap_err(),
298-
ParserError("Expected: an expression, found: EOF".to_string())
301+
ParserError::SpannedParserError(
302+
"Expected: an expression, found: EOF".to_string(),
303+
Span::empty()
304+
)
299305
);
300306
}
301307
}
@@ -358,19 +364,22 @@ fn parse_alter_table_add_projection() {
358364
clickhouse_and_generic()
359365
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name")
360366
.unwrap_err(),
361-
ParserError("Expected: (, found: EOF".to_string())
367+
ParserError::SpannedParserError("Expected: (, found: EOF".to_string(), Span::empty())
362368
);
363369
assert_eq!(
364370
clickhouse_and_generic()
365371
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name ()")
366372
.unwrap_err(),
367-
ParserError("Expected: SELECT, found: )".to_string())
373+
ParserError::SpannedParserError("Expected: SELECT, found: )".to_string(), Span::empty())
368374
);
369375
assert_eq!(
370376
clickhouse_and_generic()
371377
.parse_sql_statements("ALTER TABLE t0 ADD PROJECTION my_name (SELECT)")
372378
.unwrap_err(),
373-
ParserError("Expected: an expression, found: )".to_string())
379+
ParserError::SpannedParserError(
380+
"Expected: an expression, found: )".to_string(),
381+
Span::empty()
382+
)
374383
);
375384
}
376385

@@ -400,7 +409,10 @@ fn parse_alter_table_drop_projection() {
400409
clickhouse_and_generic()
401410
.parse_sql_statements("ALTER TABLE t0 DROP PROJECTION")
402411
.unwrap_err(),
403-
ParserError("Expected: identifier, found: EOF".to_string())
412+
ParserError::SpannedParserError(
413+
"Expected: identifier, found: EOF".to_string(),
414+
Span::empty()
415+
)
404416
);
405417
}
406418

@@ -447,7 +459,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
447459
clickhouse_and_generic()
448460
.parse_sql_statements(format!("ALTER TABLE t0 {keyword} PROJECTION",).as_str())
449461
.unwrap_err(),
450-
ParserError("Expected: identifier, found: EOF".to_string())
462+
ParserError::SpannedParserError(
463+
"Expected: identifier, found: EOF".to_string(),
464+
Span::empty()
465+
)
451466
);
452467

453468
assert_eq!(
@@ -456,7 +471,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
456471
format!("ALTER TABLE t0 {keyword} PROJECTION my_name IN PARTITION",).as_str()
457472
)
458473
.unwrap_err(),
459-
ParserError("Expected: identifier, found: EOF".to_string())
474+
ParserError::SpannedParserError(
475+
"Expected: identifier, found: EOF".to_string(),
476+
Span::empty()
477+
)
460478
);
461479

462480
assert_eq!(
@@ -465,7 +483,10 @@ fn parse_alter_table_clear_and_materialize_projection() {
465483
format!("ALTER TABLE t0 {keyword} PROJECTION my_name IN",).as_str()
466484
)
467485
.unwrap_err(),
468-
ParserError("Expected: end of statement, found: IN".to_string())
486+
ParserError::SpannedParserError(
487+
"Expected: end of statement, found: IN".to_string(),
488+
Span::empty()
489+
)
469490
);
470491
}
471492
}
@@ -513,19 +534,28 @@ fn parse_optimize_table() {
513534
clickhouse_and_generic()
514535
.parse_sql_statements("OPTIMIZE TABLE t0 DEDUPLICATE BY")
515536
.unwrap_err(),
516-
ParserError("Expected: an expression, found: EOF".to_string())
537+
ParserError::SpannedParserError(
538+
"Expected: an expression, found: EOF".to_string(),
539+
Span::empty()
540+
)
517541
);
518542
assert_eq!(
519543
clickhouse_and_generic()
520544
.parse_sql_statements("OPTIMIZE TABLE t0 PARTITION")
521545
.unwrap_err(),
522-
ParserError("Expected: an expression, found: EOF".to_string())
546+
ParserError::SpannedParserError(
547+
"Expected: an expression, found: EOF".to_string(),
548+
Span::empty()
549+
)
523550
);
524551
assert_eq!(
525552
clickhouse_and_generic()
526553
.parse_sql_statements("OPTIMIZE TABLE t0 PARTITION ID")
527554
.unwrap_err(),
528-
ParserError("Expected: identifier, found: EOF".to_string())
555+
ParserError::SpannedParserError(
556+
"Expected: identifier, found: EOF".to_string(),
557+
Span::empty()
558+
)
529559
);
530560
}
531561

@@ -1063,7 +1093,7 @@ fn parse_settings_in_query() {
10631093
clickhouse_and_generic()
10641094
.parse_sql_statements(sql)
10651095
.unwrap_err(),
1066-
ParserError(error_msg.to_string())
1096+
ParserError::SpannedParserError(error_msg.to_string(), Span::empty())
10671097
);
10681098
}
10691099
}
@@ -1568,23 +1598,32 @@ fn parse_freeze_and_unfreeze_partition() {
15681598
clickhouse_and_generic()
15691599
.parse_sql_statements(format!("ALTER TABLE t0 {operation_name} PARTITION").as_str())
15701600
.unwrap_err(),
1571-
ParserError("Expected: an expression, found: EOF".to_string())
1601+
ParserError::SpannedParserError(
1602+
"Expected: an expression, found: EOF".to_string(),
1603+
Span::empty()
1604+
)
15721605
);
15731606
assert_eq!(
15741607
clickhouse_and_generic()
15751608
.parse_sql_statements(
15761609
format!("ALTER TABLE t0 {operation_name} PARTITION p0 WITH").as_str()
15771610
)
15781611
.unwrap_err(),
1579-
ParserError("Expected: NAME, found: EOF".to_string())
1612+
ParserError::SpannedParserError(
1613+
"Expected: NAME, found: EOF".to_string(),
1614+
Span::empty()
1615+
)
15801616
);
15811617
assert_eq!(
15821618
clickhouse_and_generic()
15831619
.parse_sql_statements(
15841620
format!("ALTER TABLE t0 {operation_name} PARTITION p0 WITH NAME").as_str()
15851621
)
15861622
.unwrap_err(),
1587-
ParserError("Expected: identifier, found: EOF".to_string())
1623+
ParserError::SpannedParserError(
1624+
"Expected: identifier, found: EOF".to_string(),
1625+
Span::empty()
1626+
)
15881627
);
15891628
}
15901629
}

0 commit comments

Comments
 (0)