From bc4ce4abdae948ea1fa2b5c54f1e09b1c490ad06 Mon Sep 17 00:00:00 2001 From: Nicholas Nethercote Date: Wed, 24 Jan 2024 09:35:07 +1100 Subject: [PATCH] XXX: gate --- compiler/rustc_ast/src/util/literal.rs | 10 +-- compiler/rustc_lexer/src/unescape.rs | 93 +++++++++++++++----------- compiler/rustc_parse/src/lexer/mod.rs | 42 ++++++------ compiler/rustc_parse_format/src/lib.rs | 3 +- 4 files changed, 82 insertions(+), 66 deletions(-) diff --git a/compiler/rustc_ast/src/util/literal.rs b/compiler/rustc_ast/src/util/literal.rs index cfd5c5aa8fb3c..f0035a0ccf209 100644 --- a/compiler/rustc_ast/src/util/literal.rs +++ b/compiler/rustc_ast/src/util/literal.rs @@ -49,7 +49,8 @@ impl LitKind { // For byte/char/string literals, chars and escapes have already been // checked in the lexer (in `cook_lexer_literal`). So we can assume all - // chars and escapes are valid here. + // chars and escapes are valid here, we can also ignore Rfc3349 return + // values. Ok(match kind { token::Bool => { assert!(symbol.is_bool_lit()); @@ -84,7 +85,7 @@ impl LitKind { // Force-inlining here is aggressive but the closure is // called on every char in the string, so it can be hot in // programs with many long strings containing escapes. - unescape_non_mixed( + _ = unescape_non_mixed( s, Mode::Str, &mut #[inline(always)] @@ -111,8 +112,7 @@ impl LitKind { // We can just use `rfc3349 = true` here, which is more // permissive than `rfc3349 = false`, because escapes and // chars were checked by the lexer. - let rfc3349 = true; - unescape_mixed(s, Mode::ByteStr { rfc3349 }, &mut |_, c| match c { + _ = unescape_mixed(s, Mode::ByteStr { rfc3349: true }, &mut |_, c| match c { Ok(MixedUnit::Char(c)) => { buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()) } @@ -132,7 +132,7 @@ impl LitKind { token::CStr => { let s = symbol.as_str(); let mut buf = Vec::with_capacity(s.len()); - unescape_mixed(s, Mode::CStr, &mut |_span, c| match c { + _ = unescape_mixed(s, Mode::CStr, &mut |_span, c| match c { Ok(MixedUnit::Char(c)) => { buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes()) } diff --git a/compiler/rustc_lexer/src/unescape.rs b/compiler/rustc_lexer/src/unescape.rs index 1b0496646f4ff..9ae2fa367d21e 100644 --- a/compiler/rustc_lexer/src/unescape.rs +++ b/compiler/rustc_lexer/src/unescape.rs @@ -88,7 +88,7 @@ impl EscapeError { /// /// Values are returned by invoking `callback`. For `Char` and `Byte` modes, /// the callback will be called exactly once. -pub fn unescape_non_mixed(src: &str, mode: Mode, callback: &mut F) +pub fn unescape_non_mixed(src: &str, mode: Mode, callback: &mut F) -> Rfc3349 where F: FnMut(Range, Result), { @@ -97,6 +97,7 @@ where let mut chars = src.chars(); let res = unescape_char_or_byte(&mut chars, mode); callback(0..(src.len() - chars.as_str().len()), res); + Rfc3349::Unused // rfc3349 never triggered by char or byte literals } Str => unescape_non_raw_common(src, mode, callback), RawStr => check_raw_common(src, mode, callback), @@ -107,7 +108,7 @@ where result = Err(EscapeError::NulInCStr); } callback(r, result) - }); + }) } ByteStr { .. } | CStr => unreachable!(), } @@ -148,7 +149,7 @@ impl From for MixedUnit { /// a sequence of escaped characters or errors. /// /// Values are returned by invoking `callback`. -pub fn unescape_mixed(src: &str, mode: Mode, callback: &mut F) +pub fn unescape_mixed(src: &str, mode: Mode, callback: &mut F) -> Rfc3349 where F: FnMut(Range, Result), { @@ -160,7 +161,7 @@ where result = Err(EscapeError::NulInCStr); } callback(r, result) - }); + }) } Char | Byte | Str | RawStr | RawByteStr { .. } | RawCStr => unreachable!(), } @@ -178,6 +179,15 @@ pub fn unescape_byte(src: &str) -> Result { unescape_char_or_byte(&mut src.chars(), Byte).map(byte_from_char) } +/// Used to indicate if rfc3349 (mixed-utf8-literals) was required for the +/// literal to be valid. +#[derive(Debug, PartialEq)] +#[must_use] +pub enum Rfc3349 { + Used, + Unused, +} + /// What kind of literal do we parse. #[derive(Debug, Clone, Copy, PartialEq)] pub enum Mode { @@ -214,24 +224,24 @@ impl Mode { /// Are unicode (non-ASCII) chars allowed? #[inline] - fn allow_unicode_chars(self) -> bool { + fn allow_unicode_chars(self, rfc3349: &mut Rfc3349) -> bool { match self { - Byte | ByteStr { rfc3349: false } | RawByteStr { rfc3349: false } => false, + Byte => false, + ByteStr { .. } | RawByteStr { .. } => { *rfc3349 = Rfc3349::Used; true } Char | Str | RawStr - | ByteStr { rfc3349: true } - | RawByteStr { rfc3349: true } | CStr | RawCStr => true, } } /// Are unicode escapes (`\u`) allowed? - fn allow_unicode_escapes(self) -> bool { + fn allow_unicode_escapes(self, rfc3349: &mut Rfc3349) -> bool { match self { - Byte | ByteStr { rfc3349: false } => false, - Char | Str | ByteStr { rfc3349: true } | CStr => true, + Byte => false, + ByteStr { .. } => { *rfc3349 = Rfc3349::Used; true } + Char | Str | CStr => true, RawByteStr { .. } | RawStr | RawCStr => unreachable!(), } } @@ -245,9 +255,12 @@ impl Mode { } } +// The bool in the return value indicates if rfc3349 must be enabled for the +// escape to be accepted. fn scan_escape + From>( chars: &mut Chars<'_>, mode: Mode, + rfc3349: &mut Rfc3349, ) -> Result { // Previous character was '\\', unescape what follows. let res: char = match chars.next().ok_or(EscapeError::LoneSlash)? { @@ -277,15 +290,17 @@ fn scan_escape + From>( Ok(T::from(value as u8)) }; } - // njn: gate: is it a ByteStr? - 'u' => return scan_unicode(chars, mode.allow_unicode_escapes()).map(T::from), + 'u' => { + // njn: convert all mode matches back to equality checks + return scan_unicode(chars, mode, rfc3349).map(T::from); + } _ => return Err(EscapeError::InvalidEscape), }; Ok(T::from(res)) } // njn: change arg to mode in precursor? -fn scan_unicode(chars: &mut Chars<'_>, allow_unicode_escapes: bool) -> Result { +fn scan_unicode(chars: &mut Chars<'_>, mode: Mode, rfc3349: &mut Rfc3349) -> Result { // We've parsed '\u', now we have to parse '{..}'. if chars.next() != Some('{') { @@ -313,7 +328,7 @@ fn scan_unicode(chars: &mut Chars<'_>, allow_unicode_escapes: bool) -> Result, allow_unicode_escapes: bool) -> Result Result { - if allow_unicode_chars || c.is_ascii() { Ok(c) } else { Err(EscapeError::NonAsciiCharInByte) } +fn ascii_check(c: char, mode: Mode, rfc3349: &mut Rfc3349) -> Result { + // Note: we must check `is_ascii` first, to avoid setting `rfc3349` unnecessarily. + if c.is_ascii() || mode.allow_unicode_chars(rfc3349) { + Ok(c) + } else { + Err(EscapeError::NonAsciiCharInByte) + } } fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result { let c = chars.next().ok_or(EscapeError::ZeroChars)?; + let mut rfc3349 = Rfc3349::Unused; let res = match c { - '\\' => scan_escape(chars, mode), + '\\' => scan_escape(chars, mode, &mut rfc3349), '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar), '\r' => Err(EscapeError::BareCarriageReturn), - // njn: this is the only ascii_check that will remain - _ => ascii_check(c, mode.allow_unicode_chars()), + _ => ascii_check(c, mode, &mut rfc3349), }?; + + // rfc3349 cannot be triggered for char or byte literals. + assert_eq!(rfc3349, Rfc3349::Unused); + if chars.next().is_some() { return Err(EscapeError::MoreThanOneChar); } @@ -360,12 +384,12 @@ fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result + From>(src: &str, mode: Mode, callback: &mut F) +fn unescape_non_raw_common + From>(src: &str, mode: Mode, callback: &mut F) -> Rfc3349 where F: FnMut(Range, Result), { let mut chars = src.chars(); - let allow_unicode_chars = mode.allow_unicode_chars(); // get this outside the loop + let mut rfc3349 = Rfc3349::Unused; // The `start` and `end` computation here is complicated because // `skip_ascii_whitespace` makes us to skip over chars without counting @@ -385,20 +409,17 @@ where }); continue; } - _ => scan_escape::(&mut chars, mode), + _ => scan_escape::(&mut chars, mode, &mut rfc3349), } } '"' => Err(EscapeError::EscapeOnlyChar), '\r' => Err(EscapeError::BareCarriageReturn), - - // njn: gate, similar to check_raw_common, check: - // - is it a ByteStr AND does it contain a unicode char - - _ => ascii_check(c, allow_unicode_chars).map(T::from), + _ => ascii_check(c, mode, &mut rfc3349).map(T::from), }; let end = src.len() - chars.as_str().len(); callback(start..end, res); } + rfc3349 } fn skip_ascii_whitespace(chars: &mut Chars<'_>, start: usize, callback: &mut F) @@ -431,12 +452,12 @@ where /// sequence of characters or errors. /// NOTE: Raw strings do not perform any explicit character escaping, here we /// only produce errors on bare CR. -fn check_raw_common(src: &str, mode: Mode, callback: &mut F) +fn check_raw_common(src: &str, mode: Mode, callback: &mut F) -> Rfc3349 where F: FnMut(Range, Result), { let mut chars = src.chars(); - let allow_unicode_chars = mode.allow_unicode_chars(); // get this outside the loop + let mut rfc3349 = Rfc3349::Unused; // The `start` and `end` computation here matches the one in // `unescape_non_raw_common` for consistency, even though this function @@ -445,20 +466,12 @@ where let start = src.len() - chars.as_str().len() - c.len_utf8(); let res = match c { '\r' => Err(EscapeError::BareCarriageReturnInRawString), - - // njn: gate: need to somehow return an indication of whether - // rfc3349 unicode char allowance was required for this literal, - // i.e. check - // - is it a RawByteStr AND does it contain a unicode char - // - // njn: but the ascii_check itself isn't necessary - // - or make it return three values? ok, ok-with-3349, bad? - - _ => ascii_check(c, allow_unicode_chars), + _ => ascii_check(c, mode, &mut rfc3349), }; let end = src.len() - chars.as_str().len(); callback(start..end, res); } + rfc3349 } #[inline] diff --git a/compiler/rustc_parse/src/lexer/mod.rs b/compiler/rustc_parse/src/lexer/mod.rs index 02c7b0b052795..7c3bd91304432 100644 --- a/compiler/rustc_parse/src/lexer/mod.rs +++ b/compiler/rustc_parse/src/lexer/mod.rs @@ -8,9 +8,8 @@ use rustc_ast::token::{self, CommentKind, Delimiter, Token, TokenKind}; use rustc_ast::tokenstream::TokenStream; use rustc_ast::util::unicode::contains_text_flow_control_chars; use rustc_errors::{error_code, Applicability, DiagCtxt, DiagnosticBuilder, StashKey}; -use rustc_lexer::unescape::{self, EscapeError, Mode}; -use rustc_lexer::{Base, DocStyle, RawStrError}; -use rustc_lexer::{Cursor, LiteralKind}; +use rustc_lexer::unescape::{self, EscapeError, Mode, Rfc3349}; +use rustc_lexer::{Base, DocStyle, RawStrError, Cursor, LiteralKind}; use rustc_session::lint::builtin::{ RUST_2021_PREFIXES_INCOMPATIBLE_SYNTAX, TEXT_DIRECTION_CODEPOINT_IN_COMMENT, }; @@ -226,7 +225,10 @@ impl<'sess, 'src> StringReader<'sess, 'src> { } rustc_lexer::TokenKind::Literal { kind, suffix_start } => { let suffix_start = start + BytePos(suffix_start); - let (kind, symbol) = self.cook_lexer_literal(start, suffix_start, kind); + let (kind, symbol, rfc3349) = self.cook_lexer_literal(start, suffix_start, kind); + if rfc3349 == Rfc3349::Used { + eprintln!("rfc3349!"); + } let suffix = if suffix_start < self.pos { let string = self.str_from(suffix_start); if string == "_" { @@ -391,10 +393,7 @@ impl<'sess, 'src> StringReader<'sess, 'src> { start: BytePos, end: BytePos, kind: rustc_lexer::LiteralKind, - ) -> (token::LitKind, Symbol) { - // njn: gate - let rfc3349 = true; - + ) -> (token::LitKind, Symbol, Rfc3349) { match kind { rustc_lexer::LiteralKind::Char { terminated } => { if !terminated { @@ -444,7 +443,7 @@ impl<'sess, 'src> StringReader<'sess, 'src> { // here if the new features were required, and updated // gated_spans - let mode = Mode::ByteStr { rfc3349 }; + let mode = Mode::ByteStr { rfc3349: true }; self.cook_mixed(token::ByteStr, mode, start, end, 2, 1) // b" " } rustc_lexer::LiteralKind::CStr { terminated } => { @@ -472,7 +471,7 @@ impl<'sess, 'src> StringReader<'sess, 'src> { if let Some(n_hashes) = n_hashes { let n = u32::from(n_hashes); let kind = token::ByteStrRaw(n_hashes); - let mode = Mode::RawByteStr { rfc3349 }; + let mode = Mode::RawByteStr { rfc3349: true }; // njn: update gated_spans here if new features required @@ -494,7 +493,7 @@ impl<'sess, 'src> StringReader<'sess, 'src> { if empty_int { let span = self.mk_sp(start, end); self.dcx().emit_err(errors::NoDigitsLiteral { span }); - (token::Integer, sym::integer(0)) + (token::Integer, sym::integer(0), Rfc3349::Unused) } else { if matches!(base, Base::Binary | Base::Octal) { let base = base as u32; @@ -509,7 +508,7 @@ impl<'sess, 'src> StringReader<'sess, 'src> { } } } - (token::Integer, self.symbol_from_to(start, end)) + (token::Integer, self.symbol_from_to(start, end), Rfc3349::Unused) } } rustc_lexer::LiteralKind::Float { base, empty_exponent } => { @@ -527,7 +526,8 @@ impl<'sess, 'src> StringReader<'sess, 'src> { let span = self.mk_sp(start, end); self.dcx().emit_err(errors::FloatLiteralUnsupportedBase { span, base }); } - (token::Float, self.symbol_from_to(start, end)) + // njn: hmm, using Rfc3349 for numeric literals is a bit weird + (token::Float, self.symbol_from_to(start, end), Rfc3349::Unused) } } } @@ -710,13 +710,13 @@ impl<'sess, 'src> StringReader<'sess, 'src> { end: BytePos, prefix_len: u32, postfix_len: u32, - unescape: fn(&str, Mode, &mut dyn FnMut(Range, Result<(), EscapeError>)), - ) -> (token::LitKind, Symbol) { + unescape: fn(&str, Mode, &mut dyn FnMut(Range, Result<(), EscapeError>)) -> Rfc3349, + ) -> (token::LitKind, Symbol, Rfc3349) { let mut has_fatal_err = false; let content_start = start + BytePos(prefix_len); let content_end = end - BytePos(postfix_len); let lit_content = self.str_from_to(content_start, content_end); - unescape(lit_content, mode, &mut |range, result| { + let rfc3349 = unescape(lit_content, mode, &mut |range, result| { // Here we only check for errors. The actual unescaping is done later. if let Err(err) = result { let span_with_quotes = self.mk_sp(start, end); @@ -739,12 +739,14 @@ impl<'sess, 'src> StringReader<'sess, 'src> { } }); + // njn: maybe this is the right place to record gated_spans + // We normally exclude the quotes for the symbol, but for errors we // include it because it results in clearer error messages. if !has_fatal_err { - (kind, Symbol::intern(lit_content)) + (kind, Symbol::intern(lit_content), rfc3349) } else { - (token::Err, self.symbol_from_to(start, end)) + (token::Err, self.symbol_from_to(start, end), rfc3349) } } @@ -756,7 +758,7 @@ impl<'sess, 'src> StringReader<'sess, 'src> { end: BytePos, prefix_len: u32, postfix_len: u32, - ) -> (token::LitKind, Symbol) { + ) -> (token::LitKind, Symbol, Rfc3349) { self.cook_common(kind, mode, start, end, prefix_len, postfix_len, |src, mode, callback| { unescape::unescape_non_mixed(src, mode, &mut |span, result| { callback(span, result.map(drop)) @@ -772,7 +774,7 @@ impl<'sess, 'src> StringReader<'sess, 'src> { end: BytePos, prefix_len: u32, postfix_len: u32, - ) -> (token::LitKind, Symbol) { + ) -> (token::LitKind, Symbol, Rfc3349) { self.cook_common(kind, mode, start, end, prefix_len, postfix_len, |src, mode, callback| { unescape::unescape_mixed(src, mode, &mut |span, result| { callback(span, result.map(drop)) diff --git a/compiler/rustc_parse_format/src/lib.rs b/compiler/rustc_parse_format/src/lib.rs index a2030ed83ce40..257594a5712c5 100644 --- a/compiler/rustc_parse_format/src/lib.rs +++ b/compiler/rustc_parse_format/src/lib.rs @@ -1056,7 +1056,8 @@ fn find_width_map_from_snippet( fn unescape_string(string: &str) -> Option { let mut buf = string::String::new(); let mut ok = true; - unescape::unescape_non_mixed(string, unescape::Mode::Str, &mut |_, unescaped_char| { + // njn: argh, need to use the Rfc3349 return value + _ = unescape::unescape_non_mixed(string, unescape::Mode::Str, &mut |_, unescaped_char| { match unescaped_char { Ok(c) => buf.push(c), Err(_) => ok = false,