Skip to content

Commit 439386f

Browse files
authored
Revert "National strings: check if dialect supports backslash escape (apache#1672)"
This reverts commit bbabf35.
1 parent 66c5e07 commit 439386f

File tree

2 files changed

+2
-44
lines changed

2 files changed

+2
-44
lines changed

src/test_utils.rs

Lines changed: 1 addition & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use core::fmt::Debug;
3333

3434
use crate::dialect::*;
3535
use crate::parser::{Parser, ParserError};
36-
use crate::tokenizer::{Token, Tokenizer};
36+
use crate::tokenizer::Tokenizer;
3737
use crate::{ast::*, parser::ParserOptions};
3838

3939
#[cfg(test)]
@@ -237,18 +237,6 @@ impl TestedDialects {
237237
pub fn verified_expr(&self, sql: &str) -> Expr {
238238
self.expr_parses_to(sql, sql)
239239
}
240-
241-
/// Check that the tokenizer returns the expected tokens for the given SQL.
242-
pub fn tokenizes_to(&self, sql: &str, expected: Vec<Token>) {
243-
self.dialects.iter().for_each(|dialect| {
244-
let mut tokenizer = Tokenizer::new(&**dialect, sql);
245-
if let Some(options) = &self.options {
246-
tokenizer = tokenizer.with_unescape(options.unescape);
247-
}
248-
let tokens = tokenizer.tokenize().unwrap();
249-
assert_eq!(expected, tokens);
250-
});
251-
}
252240
}
253241

254242
/// Returns all available dialects.

src/tokenizer.rs

Lines changed: 1 addition & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -971,10 +971,7 @@ impl<'a> Tokenizer<'a> {
971971
match chars.peek() {
972972
Some('\'') => {
973973
// N'...' - a <national character string literal>
974-
let backslash_escape =
975-
self.dialect.supports_string_literal_backslash_escape();
976-
let s =
977-
self.tokenize_single_quoted_string(chars, '\'', backslash_escape)?;
974+
let s = self.tokenize_single_quoted_string(chars, '\'', true)?;
978975
Ok(Some(Token::NationalStringLiteral(s)))
979976
}
980977
_ => {
@@ -2158,7 +2155,6 @@ mod tests {
21582155
use crate::dialect::{
21592156
BigQueryDialect, ClickHouseDialect, HiveDialect, MsSqlDialect, MySqlDialect, SQLiteDialect,
21602157
};
2161-
use crate::test_utils::all_dialects_where;
21622158
use core::fmt::Debug;
21632159

21642160
#[test]
@@ -3547,30 +3543,4 @@ mod tests {
35473543
];
35483544
compare(expected, tokens);
35493545
}
3550-
3551-
#[test]
3552-
fn test_national_strings_backslash_escape_not_supported() {
3553-
all_dialects_where(|dialect| !dialect.supports_string_literal_backslash_escape())
3554-
.tokenizes_to(
3555-
"select n'''''\\'",
3556-
vec![
3557-
Token::make_keyword("select"),
3558-
Token::Whitespace(Whitespace::Space),
3559-
Token::NationalStringLiteral("''\\".to_string()),
3560-
],
3561-
);
3562-
}
3563-
3564-
#[test]
3565-
fn test_national_strings_backslash_escape_supported() {
3566-
all_dialects_where(|dialect| dialect.supports_string_literal_backslash_escape())
3567-
.tokenizes_to(
3568-
"select n'''''\\''",
3569-
vec![
3570-
Token::make_keyword("select"),
3571-
Token::Whitespace(Whitespace::Space),
3572-
Token::NationalStringLiteral("'''".to_string()),
3573-
],
3574-
);
3575-
}
35763546
}

0 commit comments

Comments
 (0)