fix!: support other charset labels by fallback to encoding_rs

Breaking change to public DecoderFnc that is now a struct.

Author: sftse

src/decoders/charsets/map.rs

@@ -8,10 +8,10 @@ use super::{
     multi_byte::*,
     single_byte::*,
     utf::{decoder_utf16, decoder_utf16_be, decoder_utf16_le, decoder_utf7},
-    DecoderFnc,
+    Decoder, DecoderFnc,
 };
 
-pub fn charset_decoder(charset: &[u8]) -> Option<DecoderFnc> {
+fn primary_charset_decoder(charset: &[u8]) -> Option<fn(&[u8]) -> String> {
     let mut l_charset = [0u8; 45];
 
     for (dest, src) in l_charset.iter_mut().zip(charset.iter()) {
@@ -175,6 +175,14 @@ pub fn charset_decoder(charset: &[u8]) -> Option<DecoderFnc> {
     )
 }
 
+pub fn charset_decoder(charset: &[u8]) -> Option<DecoderFnc> {
+    primary_charset_decoder(charset)
+        .map(Decoder::Fn)
+        // fallback labels
+        .or_else(|| encoding_rs::Encoding::for_label(charset).map(Decoder::Encoding))
+        .map(DecoderFnc)
+}
+
 #[cfg(test)]
 mod tests {
     use super::charset_decoder;

src/decoders/charsets/mod.rs

@@ -9,7 +9,21 @@ pub mod multi_byte;
 pub mod single_byte;
 pub mod utf;
 
-pub type DecoderFnc = fn(&[u8]) -> String;
+pub struct DecoderFnc(Decoder);
+
+enum Decoder {
+    Encoding(&'static encoding_rs::Encoding),
+    Fn(fn(&[u8]) -> String),
+}
+
+impl DecoderFnc {
+    pub(crate) fn decode(&self, bytes: &[u8]) -> String {
+        match self.0 {
+            Decoder::Encoding(enc) => enc.decode(bytes).0.into_owned(),
+            Decoder::Fn(decoder) => decoder(bytes),
+        }
+    }
+}
 
 #[cfg(test)]
 mod tests {
@@ -63,7 +77,7 @@ mod tests {
             let decoder = charset_decoder(input.0.as_bytes())
                 .expect(&("Failed to find decoder for ".to_owned() + input.0));
 
-            assert_eq!(decoder(&input.1), input.2);
+            assert_eq!(decoder.decode(&input.1), input.2);
         }
     }
 }

src/decoders/encoded_word.rs

@@ -77,7 +77,7 @@ impl MessageStream<'_> {
 
         if let Some(bytes) = decode_fnc.and_then(|fnc| fnc(self)) {
             if let Some(decoder) = charset_decoder(self.bytes(charset_start..charset_end)) {
-                decoder(&bytes).into()
+                decoder.decode(&bytes).into()
             } else {
                 String::from_utf8(bytes)
                     .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
@@ -149,11 +149,6 @@ mod tests {
                 "مرحبا بالعالم",
                 true,
             ),
-            (
-                "?iso-8859-8-i?Q?=E4=F9=E1:_Septier_-_cooperation?=",
-                "השב: Septier - cooperation",
-                true,
-            ),
             #[cfg(feature = "full_encoding")]
             (
                 "?shift_jis?B?g26DjYFbgUWDj4Fbg4uDaA==?=",
@@ -166,6 +161,12 @@ mod tests {
                 "ハロー・ワールド",
                 true,
             ),
+            #[cfg(feature = "full_encoding")]
+            (
+                "?iso-8859-8-i?Q?=E4=F9=E1:_Septier_-_cooperation?=",
+                "השב: Septier - cooperation",
+                true,
+            ),
         ] {
             match MessageStream::new(input.as_bytes()).decode_rfc2047() {
                 Some(result) => {

src/parsers/fields/content_type.rs

@@ -188,7 +188,7 @@ impl<'x> ContentTypeParser<'x> {
                         .as_ref()
                         .and_then(|c| charset_decoder(c.as_bytes()))
                     {
-                        decoder(&decoded_bytes).into()
+                        decoder.decode(&decoded_bytes).into()
                     } else {
                         String::from_utf8(decoded_bytes)
                             .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())

src/parsers/message.rs

@@ -320,12 +320,14 @@ impl MessageParser {
                                 .and_then(|c| charset_decoder(c.as_bytes()))
                         }),
                     ) {
-                        (Cow::Owned(vec), Some(charset_decoder)) => charset_decoder(&vec).into(),
+                        (Cow::Owned(vec), Some(charset_decoder)) => {
+                            charset_decoder.decode(&vec).into()
+                        }
                         (Cow::Owned(vec), None) => String::from_utf8(vec)
                             .unwrap_or_else(|e| String::from_utf8_lossy(e.as_bytes()).into_owned())
                             .into(),
                         (Cow::Borrowed(bytes), Some(charset_decoder)) => {
-                            charset_decoder(bytes).into()
+                            charset_decoder.decode(bytes).into()
                         }
                         (Cow::Borrowed(bytes), None) => String::from_utf8_lossy(bytes),
                     };