Use the unicode replacement character

instead of an empty string so that it is clear that there was some sort of
encoding issue as opposed silently dropping data

This is only easily accomplished on ruby 1.9 and above, so ruby 1.8.x will
keep the empty string behavior.

Author: kjg

CHANGELOG.rdoc

@@ -2,6 +2,7 @@
 
 Features:
 * #853 - `Mail::Message#set_sort_order` overrides the default message part sort order. (rafbm)
+* #1002 - Replace invalid encoded characters with the unicode replacement char instead of empty string. (kjg)
 
 Compatibility:
 * #655 - Sort attachments to the end of the parts list to work around email clients that may mistake a text attachment for the message body. (npickens)

lib/mail/version_specific/ruby_1_9.rb

@@ -78,7 +78,12 @@ def Ruby19.get_constant(klass, string)
     end
 
     def Ruby19.transcode_charset(str, from_encoding, to_encoding = Encoding::UTF_8)
-      charset_encoder.encode(str.dup, from_encoding).encode(to_encoding, :undef => :replace, :invalid => :replace, :replace => '')
+      if to_encoding == Encoding::UTF_8
+        replacement_char = '�'
+      else
+        replacement_char = '?'
+      end
+      charset_encoder.encode(str.dup, from_encoding).encode(to_encoding, :undef => :replace, :invalid => :replace, :replace => replacement_char)
     end
 
     def Ruby19.b_value_encode(str, encoding = nil)
@@ -93,11 +98,7 @@ def Ruby19.b_value_decode(str)
         str = Ruby19.decode_base64(match[2])
         str = charset_encoder.encode(str, charset)
       end
-      decoded = str.encode(Encoding::UTF_8, :undef => :replace, :invalid => :replace, :replace => "")
-      decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "").encode(Encoding::UTF_8)
-    rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
-      warn "Encoding conversion failed #{$!}"
-      str.dup.force_encoding(Encoding::UTF_8)
+      encode_to_utf8(str)
     end
 
     def Ruby19.q_value_encode(str, encoding = nil)
@@ -118,11 +119,7 @@ def Ruby19.q_value_decode(str)
         # jruby/jruby#829 which subtly changes String#encode semantics.
         str.force_encoding(Encoding::UTF_8) if str.encoding == Encoding::ASCII_8BIT
       end
-      decoded = str.encode(Encoding::UTF_8, :invalid => :replace, :replace => "")
-      decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "").encode(Encoding::UTF_8)
-    rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
-      warn "Encoding conversion failed #{$!}"
-      str.dup.force_encoding(Encoding::UTF_8)
+      encode_to_utf8(str)
     end
 
     def Ruby19.param_decode(str, encoding)
@@ -221,6 +218,14 @@ def convert_to_encoding(encoding)
           end
         end
       end
+
+      def encode_to_utf8(str)
+        decoded = str.encode(Encoding::UTF_8, :undef => :replace, :invalid => :replace, :replace => "�")
+        decoded.valid_encoding? ? decoded : decoded.encode(Encoding::UTF_16LE, :invalid => :replace, :replace => "�").encode(Encoding::UTF_8)
+      rescue Encoding::UndefinedConversionError, ArgumentError, Encoding::ConverterNotFoundError
+        warn "Encoding conversion failed #{$!}"
+        str.dup.force_encoding(Encoding::UTF_8)
+      end
     end
   end
 end

spec/mail/encoding_spec.rb

@@ -186,7 +186,7 @@
     expect(mail.parts[0].content_type).to eq "text/html; charset=ISO-8859-1"
   end
 
-  it "should skip invalid characters" do
+  it "should handle invalid characters" do
     m = Mail.new
     m['Subject'] = Mail::SubjectField.new("=?utf-8?Q?Hello_=96_World?=")
     if RUBY_VERSION > '1.9'
@@ -196,10 +196,14 @@
     end
   end
 
-  it "should skip characters of unknown and invalid encoding" do
+  it "should replace characters of unknown and invalid encoding" do
     m = Mail.new
     m['Subject'] = Mail::SubjectField.new("Hello=?UNKNOWN?B?4g==?=")
-    expect(m.subject).to eq "Hello"
+    if RUBY_VERSION > '1.9'
+      expect(m.subject).to eq "Hello�"
+    else
+      expect(m.subject).to eq "Hello"
+    end
   end
 
   if RUBY_VERSION > '1.9'

spec/mail/encodings_spec.rb

@@ -316,7 +316,7 @@
 
     it "should treat unrecognized charsets as binary" do
       if RUBY_VERSION >= "1.9"
-        expect(Mail::Encodings.value_decode("=?ISO-FOOO?Q?Morten_R=F8verdatt=E9r?=")).to eq "Morten Rverdattr"
+        expect(Mail::Encodings.value_decode("=?ISO-FOOO?Q?Morten_R=F8verdatt=E9r?=")).to eq "Morten R�verdatt�r"
       end
     end
   end