Transcoding replaces invalid chars with "�" instead of discarding them

Make it clear that the source string had invalid characters instead of
silently dropping them. � U+FFFD is the Unicode replacement character.

Only supported on Ruby 1.9+. Ruby 1.8 continues to discard unrecognized
and invalid characters.

Author: kjg

CHANGELOG.rdoc

@@ -4,6 +4,7 @@ Features:
 * #853 - `Mail::Message#set_sort_order` overrides the default message part sort order. (rafbm)
 * #650 - UTF-7 charset support. (johngrimes)
 * #1065 - Require STARTTLS using :enable_starttls. (bk2204)
+* #1002 - Transcoding replaces invalid chars with "�" instead of discarding them. (kjg)
 
 Performance:
 * #1059 - Switch from mime-types to mini_mime for a much smaller memory footprint. (SamSaffron)

lib/mail/version_specific/ruby_1_9.rb

@@ -88,7 +88,9 @@ def Ruby19.get_constant(klass, string)
     end
 
     def Ruby19.transcode_charset(str, from_encoding, to_encoding = Encoding::UTF_8)
-      charset_encoder.encode(str.dup, from_encoding).encode(to_encoding, :undef => :replace, :invalid => :replace, :replace => '')
+      to_encoding = Encoding.find(to_encoding)
+      replacement_char = to_encoding == Encoding::UTF_8 ? '�' : '?'
+      charset_encoder.encode(str.dup, from_encoding).encode(to_encoding, :undef => :replace, :invalid => :replace, :replace => replacement_char)
     end
 
     # From Ruby stdlib Net::IMAP
@@ -125,8 +127,7 @@ def Ruby19.b_value_decode(str)
         str = Ruby19.decode_base64(match[2])
         str = charset_encoder.encode(str, charset)
       end
-      decoded = str.encode(Encoding::UTF_8, :undef => :replace, :invalid => :replace, :replace => "")
-      decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "").encode(Encoding::UTF_8)
+      transcode_to_scrubbed_utf8(str)
     rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
       warn "Encoding conversion failed #{$!}"
       str.dup.force_encoding(Encoding::UTF_8)
@@ -150,8 +151,7 @@ def Ruby19.q_value_decode(str)
         # jruby/jruby#829 which subtly changes String#encode semantics.
         str.force_encoding(Encoding::UTF_8) if str.encoding == Encoding::ASCII_8BIT
       end
-      decoded = str.encode(Encoding::UTF_8, :invalid => :replace, :replace => "")
-      decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "").encode(Encoding::UTF_8)
+      transcode_to_scrubbed_utf8(str)
     rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
       warn "Encoding conversion failed #{$!}"
       str.dup.force_encoding(Encoding::UTF_8)
@@ -253,6 +253,11 @@ def convert_to_encoding(encoding)
           end
         end
       end
+
+      def transcode_to_scrubbed_utf8(str)
+        decoded = str.encode(Encoding::UTF_8, :undef => :replace, :invalid => :replace, :replace => "�")
+        decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "�").encode(Encoding::UTF_8)
+      end
     end
   end
 end

spec/mail/encoding_spec.rb

@@ -186,20 +186,18 @@
     expect(mail.parts[0].content_type).to eq "text/html; charset=ISO-8859-1"
   end
 
-  it "should skip invalid characters" do
+  it "should replace invalid characters" do
     m = Mail.new
     m['Subject'] = Mail::SubjectField.new("=?utf-8?Q?Hello_=96_World?=")
-    if RUBY_VERSION > '1.9'
-      expect { expect(m.subject).to be_valid_encoding }.not_to raise_error
-    else
-      expect(m.subject).to eq "Hello  World"
-    end
+    replace = '�' if RUBY_VERSION > '1.9'
+    expect(m.subject).to eq "Hello #{replace} World"
   end
 
-  it "should skip characters of unknown and invalid encoding" do
+  it "should replace characters of unknown and invalid encoding" do
     m = Mail.new
     m['Subject'] = Mail::SubjectField.new("Hello=?UNKNOWN?B?4g==?=")
-    expect(m.subject).to eq "Hello"
+    replace = '�' if RUBY_VERSION > '1.9'
+    expect(m.subject).to eq "Hello#{replace}"
   end
 
   if RUBY_VERSION > '1.9'

spec/mail/encodings_spec.rb

@@ -316,7 +316,7 @@
 
     it "should treat unrecognized charsets as binary" do
       if RUBY_VERSION >= "1.9"
-        expect(Mail::Encodings.value_decode("=?ISO-FOOO?Q?Morten_R=F8verdatt=E9r?=")).to eq "Morten Rverdattr"
+        expect(Mail::Encodings.value_decode("=?ISO-FOOO?Q?Morten_R=F8verdatt=E9r?=")).to eq "Morten R�verdatt�r"
       end
     end
   end