XXX: gate

compiler/rustc_ast/src/util/literal.rs

@@ -49,7 +49,8 @@ impl LitKind {
 
         // For byte/char/string literals, chars and escapes have already been
         // checked in the lexer (in `cook_lexer_literal`). So we can assume all
-        // chars and escapes are valid here.
+        // chars and escapes are valid here, we can also ignore Rfc3349 return
+        // values.
         Ok(match kind {
             token::Bool => {
                 assert!(symbol.is_bool_lit());
@@ -84,7 +85,7 @@ impl LitKind {
                     // Force-inlining here is aggressive but the closure is
                     // called on every char in the string, so it can be hot in
                     // programs with many long strings containing escapes.
-                    unescape_non_mixed(
+                    _ = unescape_non_mixed(
                         s,
                         Mode::Str,
                         &mut #[inline(always)]
@@ -111,8 +112,7 @@ impl LitKind {
                 // We can just use `rfc3349 = true` here, which is more
                 // permissive than `rfc3349 = false`,  because escapes and
                 // chars were checked by the lexer.
-                let rfc3349 = true;
-                unescape_mixed(s, Mode::ByteStr { rfc3349 }, &mut |_, c| match c {
+                _ = unescape_mixed(s, Mode::ByteStr { rfc3349: true }, &mut |_, c| match c {
                     Ok(MixedUnit::Char(c)) => {
                         buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
                     }
@@ -132,7 +132,7 @@ impl LitKind {
             token::CStr => {
                 let s = symbol.as_str();
                 let mut buf = Vec::with_capacity(s.len());
-                unescape_mixed(s, Mode::CStr, &mut |_span, c| match c {
+                _ = unescape_mixed(s, Mode::CStr, &mut |_span, c| match c {
                     Ok(MixedUnit::Char(c)) => {
                         buf.extend_from_slice(c.encode_utf8(&mut [0; 4]).as_bytes())
                     }

compiler/rustc_lexer/src/unescape.rs

@@ -88,7 +88,7 @@ impl EscapeError {
 ///
 /// Values are returned by invoking `callback`. For `Char` and `Byte` modes,
 /// the callback will be called exactly once.
-pub fn unescape_non_mixed<F>(src: &str, mode: Mode, callback: &mut F)
+pub fn unescape_non_mixed<F>(src: &str, mode: Mode, callback: &mut F) -> Rfc3349
 where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
 {
@@ -97,6 +97,7 @@ where
             let mut chars = src.chars();
             let res = unescape_char_or_byte(&mut chars, mode);
             callback(0..(src.len() - chars.as_str().len()), res);
+            Rfc3349::Unused // rfc3349 never triggered by char or byte literals
         }
         Str => unescape_non_raw_common(src, mode, callback),
         RawStr => check_raw_common(src, mode, callback),
@@ -107,7 +108,7 @@ where
                     result = Err(EscapeError::NulInCStr);
                 }
                 callback(r, result)
-            });
+            })
         }
         ByteStr { .. } | CStr => unreachable!(),
     }
@@ -148,7 +149,7 @@ impl From<u8> for MixedUnit {
 /// a sequence of escaped characters or errors.
 ///
 /// Values are returned by invoking `callback`.
-pub fn unescape_mixed<F>(src: &str, mode: Mode, callback: &mut F)
+pub fn unescape_mixed<F>(src: &str, mode: Mode, callback: &mut F) -> Rfc3349
 where
     F: FnMut(Range<usize>, Result<MixedUnit, EscapeError>),
 {
@@ -160,7 +161,7 @@ where
                     result = Err(EscapeError::NulInCStr);
                 }
                 callback(r, result)
-            });
+            })
         }
         Char | Byte | Str | RawStr | RawByteStr { .. } | RawCStr => unreachable!(),
     }
@@ -178,6 +179,15 @@ pub fn unescape_byte(src: &str) -> Result<u8, EscapeError> {
     unescape_char_or_byte(&mut src.chars(), Byte).map(byte_from_char)
 }
 
+/// Used to indicate if rfc3349 (mixed-utf8-literals) was required for the
+/// literal to be valid.
+#[derive(Debug, PartialEq)]
+#[must_use]
+pub enum Rfc3349 {
+    Used,
+    Unused,
+}
+
 /// What kind of literal do we parse.
 #[derive(Debug, Clone, Copy, PartialEq)]
 pub enum Mode {
@@ -214,24 +224,24 @@ impl Mode {
 
     /// Are unicode (non-ASCII) chars allowed?
     #[inline]
-    fn allow_unicode_chars(self) -> bool {
+    fn allow_unicode_chars(self, rfc3349: &mut Rfc3349) -> bool {
         match self {
-            Byte | ByteStr { rfc3349: false } | RawByteStr { rfc3349: false } => false,
+            Byte => false,
+            ByteStr { .. } | RawByteStr { .. } => { *rfc3349 = Rfc3349::Used; true }
             Char
             | Str
             | RawStr
-            | ByteStr { rfc3349: true }
-            | RawByteStr { rfc3349: true }
             | CStr
             | RawCStr => true,
         }
     }
 
     /// Are unicode escapes (`\u`) allowed?
-    fn allow_unicode_escapes(self) -> bool {
+    fn allow_unicode_escapes(self, rfc3349: &mut Rfc3349) -> bool {
         match self {
-            Byte | ByteStr { rfc3349: false } => false,
-            Char | Str | ByteStr { rfc3349: true } | CStr => true,
+            Byte => false,
+            ByteStr { .. } => { *rfc3349 = Rfc3349::Used; true }
+            Char | Str | CStr => true,
             RawByteStr { .. } | RawStr | RawCStr => unreachable!(),
         }
     }
@@ -245,9 +255,12 @@ impl Mode {
     }
 }
 
+// The bool in the return value indicates if rfc3349 must be enabled for the
+// escape to be accepted.
 fn scan_escape<T: From<char> + From<u8>>(
     chars: &mut Chars<'_>,
     mode: Mode,
+    rfc3349: &mut Rfc3349,
 ) -> Result<T, EscapeError> {
     // Previous character was '\\', unescape what follows.
     let res: char = match chars.next().ok_or(EscapeError::LoneSlash)? {
@@ -277,15 +290,17 @@ fn scan_escape<T: From<char> + From<u8>>(
                 Ok(T::from(value as u8))
             };
         }
-        // njn: gate: is it a ByteStr?
-        'u' => return scan_unicode(chars, mode.allow_unicode_escapes()).map(T::from),
+        'u' => {
+            // njn: convert all mode matches back to equality checks
+            return scan_unicode(chars, mode, rfc3349).map(T::from);
+        }
         _ => return Err(EscapeError::InvalidEscape),
     };
     Ok(T::from(res))
 }
 
 // njn: change arg to mode in precursor?
-fn scan_unicode(chars: &mut Chars<'_>, allow_unicode_escapes: bool) -> Result<char, EscapeError> {
+fn scan_unicode(chars: &mut Chars<'_>, mode: Mode, rfc3349: &mut Rfc3349) -> Result<char, EscapeError> {
     // We've parsed '\u', now we have to parse '{..}'.
 
     if chars.next() != Some('{') {
@@ -313,7 +328,7 @@ fn scan_unicode(chars: &mut Chars<'_>, allow_unicode_escapes: bool) -> Result<ch
 
                 // Incorrect syntax has higher priority for error reporting
                 // than unallowed value for a literal.
-                if !allow_unicode_escapes {
+                if !mode.allow_unicode_escapes(rfc3349) {
                     return Err(EscapeError::UnicodeEscapeInByte);
                 }
 
@@ -339,19 +354,28 @@ fn scan_unicode(chars: &mut Chars<'_>, allow_unicode_escapes: bool) -> Result<ch
 }
 
 #[inline]
-fn ascii_check(c: char, allow_unicode_chars: bool) -> Result<char, EscapeError> {
-    if allow_unicode_chars || c.is_ascii() { Ok(c) } else { Err(EscapeError::NonAsciiCharInByte) }
+fn ascii_check(c: char, mode: Mode, rfc3349: &mut Rfc3349) -> Result<char, EscapeError> {
+    // Note: we must check `is_ascii` first, to avoid setting `rfc3349` unnecessarily.
+    if c.is_ascii() || mode.allow_unicode_chars(rfc3349) {
+        Ok(c)
+    } else {
+        Err(EscapeError::NonAsciiCharInByte)
+    }
 }
 
 fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, EscapeError> {
     let c = chars.next().ok_or(EscapeError::ZeroChars)?;
+    let mut rfc3349 = Rfc3349::Unused;
     let res = match c {
-        '\\' => scan_escape(chars, mode),
+        '\\' => scan_escape(chars, mode, &mut rfc3349),
         '\n' | '\t' | '\'' => Err(EscapeError::EscapeOnlyChar),
         '\r' => Err(EscapeError::BareCarriageReturn),
-        // njn: this is the only ascii_check that will remain
-        _ => ascii_check(c, mode.allow_unicode_chars()),
+        _ => ascii_check(c, mode, &mut rfc3349),
     }?;
+
+    // rfc3349 cannot be triggered for char or byte literals.
+    assert_eq!(rfc3349, Rfc3349::Unused);
+
     if chars.next().is_some() {
         return Err(EscapeError::MoreThanOneChar);
     }
@@ -360,12 +384,12 @@ fn unescape_char_or_byte(chars: &mut Chars<'_>, mode: Mode) -> Result<char, Esca
 
 /// Takes a contents of a string literal (without quotes) and produces a
 /// sequence of escaped characters or errors.
-fn unescape_non_raw_common<F, T: From<char> + From<u8>>(src: &str, mode: Mode, callback: &mut F)
+fn unescape_non_raw_common<F, T: From<char> + From<u8>>(src: &str, mode: Mode, callback: &mut F) -> Rfc3349
 where
     F: FnMut(Range<usize>, Result<T, EscapeError>),
 {
     let mut chars = src.chars();
-    let allow_unicode_chars = mode.allow_unicode_chars(); // get this outside the loop
+    let mut rfc3349 = Rfc3349::Unused;
 
     // The `start` and `end` computation here is complicated because
     // `skip_ascii_whitespace` makes us to skip over chars without counting
@@ -385,20 +409,17 @@ where
                         });
                         continue;
                     }
-                    _ => scan_escape::<T>(&mut chars, mode),
+                    _ => scan_escape::<T>(&mut chars, mode, &mut rfc3349),
                 }
             }
             '"' => Err(EscapeError::EscapeOnlyChar),
             '\r' => Err(EscapeError::BareCarriageReturn),
-
-            // njn: gate, similar to check_raw_common, check:
-            // - is it a ByteStr AND does it contain a unicode char
-
-            _ => ascii_check(c, allow_unicode_chars).map(T::from),
+            _ => ascii_check(c, mode, &mut rfc3349).map(T::from),
         };
         let end = src.len() - chars.as_str().len();
         callback(start..end, res);
     }
+    rfc3349
 }
 
 fn skip_ascii_whitespace<F>(chars: &mut Chars<'_>, start: usize, callback: &mut F)
@@ -431,12 +452,12 @@ where
 /// sequence of characters or errors.
 /// NOTE: Raw strings do not perform any explicit character escaping, here we
 /// only produce errors on bare CR.
-fn check_raw_common<F>(src: &str, mode: Mode, callback: &mut F)
+fn check_raw_common<F>(src: &str, mode: Mode, callback: &mut F) -> Rfc3349
 where
     F: FnMut(Range<usize>, Result<char, EscapeError>),
 {
     let mut chars = src.chars();
-    let allow_unicode_chars = mode.allow_unicode_chars(); // get this outside the loop
+    let mut rfc3349 = Rfc3349::Unused;
 
     // The `start` and `end` computation here matches the one in
     // `unescape_non_raw_common` for consistency, even though this function
@@ -445,20 +466,12 @@ where
         let start = src.len() - chars.as_str().len() - c.len_utf8();
         let res = match c {
             '\r' => Err(EscapeError::BareCarriageReturnInRawString),
-
-            // njn: gate: need to somehow return an indication of whether
-            // rfc3349 unicode char allowance was required for this literal,
-            // i.e. check
-            // - is it a RawByteStr AND does it contain a unicode char
-            //
-            // njn: but the ascii_check itself isn't necessary
-            // - or make it return three values? ok, ok-with-3349, bad?
-
-            _ => ascii_check(c, allow_unicode_chars),
+            _ => ascii_check(c, mode, &mut rfc3349),
         };
         let end = src.len() - chars.as_str().len();
         callback(start..end, res);
     }
+    rfc3349
 }
 
 #[inline]