Support non-ASCII addresses (IDN)

.github/workflows/test.yml

@@ -9,7 +9,7 @@ on:
 
 jobs:
   ruby:
-    name: ${{ matrix.ruby }} (timeout ${{ matrix.timeout }})
+    name: ${{ matrix.ruby }} ${{ matrix.env }} (timeout ${{ matrix.timeout }})
     runs-on: ubuntu-latest
     timeout-minutes: ${{ matrix.timeout }}
     strategy:
@@ -18,6 +18,9 @@ jobs:
        include:
          - ruby: 2.5
            timeout: 5
+         - ruby: 2.5
+           timeout: 5
+           env: SKIP_SIMPLEIDN=true
          - ruby: 2.6
            timeout: 5
          - ruby: 2.7
@@ -28,6 +31,9 @@ jobs:
            timeout: 5
          - ruby: 3.2
            timeout: 5
+         - ruby: 3.2
+           timeout: 5
+           env: SKIP_SIMPLEIDN=true
          - ruby: truffleruby
            timeout: 50
          - ruby: truffleruby-head
@@ -55,7 +61,7 @@ jobs:
         bundler-cache: true
         cache-version: 4
     - name: Run tests
-      run: bundle exec rake spec
+      run: ${{ matrix.env }} bundle exec rake spec
       continue-on-error: ${{ matrix.ruby == 'truffleruby-head' }}
   actionmailer:
     runs-on: ubuntu-latest

CHANGELOG.rdoc

@@ -1,7 +1,8 @@
 == Version 2.9.0 (unreleased)
 
 Breaking changes:
-* Mail::Field::FIELDS_MAP now contains class names, not Class instances (c960657)
+
+* Mail::Field::FIELDS_MAP now contains class names, not Class instances @c960657
 
 Compatibility:
 
@@ -11,6 +12,7 @@ Features:
 
 * Updated README to improve around sending multipart mail @kapfenho
 * Add delivery_interceptors method to Mail class to fetch registered interceptors @ghousemohamed
+* Support for non-ASCII addresses (IDN) @c960657
 
 Code Improvements:
 

Gemfile

@@ -13,5 +13,6 @@ end
 gem 'jruby-openssl', :platforms => :jruby
 
 gem 'mini_mime'
+gem 'simpleidn' unless ENV.key?('SKIP_SIMPLEIDN')
 
 gem 'byebug', :platforms => :mri

README.md

@@ -129,6 +129,10 @@ Installation is fairly simple, I host mail on rubygems, so you can just do:
 
     # gem install mail
 
+If you need support for email addresses with non-ASCII domains (IDNs), and your mail server lacks native support for these, you can install this additional optional dependency:
+
+    # gem install simpleidn
+
 ## Encodings
 
 If you didn't know, handling encodings in Emails is not as straight forward as you

lib/mail/elements/address.rb

@@ -71,6 +71,18 @@ def address(output_type = :decode)
       end
     end
 
+    # Returns the address that is in the address itself with domain IDNA-encoded.
+    #
+    #  a = Address.new('Mikel Lindsaar (My email address) <mikel@tést.lindsaar.net>')
+    #  a.address_idna #=> 'mikel@xn--tst-bma.lindsaar.net'
+    def address_idna
+      if d = domain_idna
+        "#{local}@#{d}"
+      else
+        local
+      end
+    end
+
     # Provides a way to assign an address to an already made Mail::Address object.
     #
     #  a = Address.new
@@ -120,6 +132,17 @@ def domain(output_type = :decode)
       Encodings.decode_encode(strip_all_comments(get_domain), output_type) if get_domain
     end
 
+    # Returns the domain part (the right hand side of the @ sign in the email address) of
+    # the address in IDNA encoding.
+    #
+    #  a = Address.new('Mikel Lindsaar (My email address) <mikel@tést.lindsaar.net>')
+    #  a.domain_idna #=> 'xn--tst-bma.lindsaar.net'
+    def domain_idna
+      if d = domain
+        Encodings.idna_encode(d)
+      end
+    end
+
     # Returns an array of comments that are in the email, or nil if there
     # are no comments
     #

lib/mail/encodings.rb

@@ -310,5 +310,22 @@ def Encodings.each_chunk_byterange(str, max_bytesize_per_chunk)
 
       yield Utilities.string_byteslice(str, offset, chunksize)
     end
+
+    def Encodings.idna_supported?
+      return @idna_supported unless @idna_supported.nil?
+      @idna_supported ||= begin
+        require 'simpleidn'
+        true
+      rescue LoadError => e
+        false
+      end
+    end
+
+    def Encodings.idna_encode(str)
+      return str if str.ascii_only?
+      raise 'Must install simpleidn gem' unless Encodings.idna_supported?
+
+      SimpleIDN.to_ascii(str)
+    end
   end
 end

lib/mail/network/delivery_methods/smtp_connection.rb

@@ -49,7 +49,9 @@ def initialize(values)
     # Send the message via SMTP.
     # The from and to attributes are optional. If not set, they are retrieve from the Message.
     def deliver!(mail)
-      envelope = Mail::SmtpEnvelope.new(mail)
+      # Net::SMTP#cabable? was added in net-smtp 0.3.1 (included in Ruby 3.1).
+      smtputf8 = smtp.respond_to?(:capable?) && smtp.capable?('SMTPUTF8')
+      envelope = Mail::SmtpEnvelope.new(mail, smtputf8_supported: smtputf8)
       response = smtp.sendmail(envelope.message, envelope.from, envelope.to)
       settings[:return_response] ? response : self
     end

lib/mail/smtp_envelope.rb

@@ -8,9 +8,11 @@ class SmtpEnvelope #:nodoc:
 
     attr_reader :from, :to, :message
 
-    def initialize(mail)
-      self.from = mail.smtp_envelope_from
+    def initialize(mail, smtputf8_supported: false)
+      # Net::SMTP::Address was added in net-smtp 0.3.1 (included in Ruby 3.1).
+      @smtputf8_supported = smtputf8_supported && defined?(Net::SMTP::Address)
       self.to = mail.smtp_envelope_to
+      self.from = mail.smtp_envelope_from
       self.message = mail.encoded
     end
 
@@ -19,7 +21,10 @@ def from=(addr)
         raise ArgumentError, "SMTP From address may not be blank: #{addr.inspect}"
       end
 
-      @from = validate_addr 'From', addr
+      addr = validate_addr 'From', addr
+      addr = Net::SMTP::Address.new(addr, 'SMTPUTF8') if @smtputf8
+
+      @from = addr
     end
 
     def to=(addr)
@@ -43,14 +48,31 @@ def message=(message)
 
     private
       def validate_addr(addr_name, addr)
-        if addr.bytesize > MAX_ADDRESS_BYTESIZE
-          raise ArgumentError, "SMTP #{addr_name} address may not exceed #{MAX_ADDRESS_BYTESIZE} bytes: #{addr.inspect}"
-        end
-
         if /[\r\n]/.match?(addr)
           raise ArgumentError, "SMTP #{addr_name} address may not contain CR or LF line breaks: #{addr.inspect}"
         end
 
+        if !addr.ascii_only?
+          if @smtputf8_supported
+            # The SMTP server supports the SMTPUTF8 extension, so we can legally pass
+            # non-ASCII addresses, if we specify the SMTPUTF8 parameter for MAIL FROM.
+            @smtputf8 = true
+          elsif Encodings.idna_supported?
+            # The SMTP server does not announce support for the SMTPUTF8 extension, so do the
+            # IDNa encoding of the domain part client-side.
+            addr = Address.new(addr).address_idna
+          end
+
+          # If we cannot IDNa-encode the domain part, of if the local part contains
+          # non-ASCII characters, there is no standards-complaint way to send the
+          # mail via a server without SMTPUTF8 support. Our best chance is to just
+          # pass the UTF8-encoded address to the server.
+        end
+
+        if addr.to_s.bytesize > MAX_ADDRESS_BYTESIZE
+          raise ArgumentError, "SMTP #{addr_name} address may not exceed #{MAX_ADDRESS_BYTESIZE} bytes: #{addr.inspect}"
+        end
+
         addr
       end
   end

spec/mail/elements/address_spec.rb

@@ -26,6 +26,12 @@
       end
     end
 
+    it "should allow us to instantiate an empty address object and call address_idna" do
+      [nil, '', ' '].each do |input|
+        expect(Mail::Address.new(input).address_idna).to be_nil
+      end
+    end
+
     it "should allow us to instantiate an empty address object and call local" do
       [nil, '', ' '].each do |input|
         expect(Mail::Address.new(input).local).to be_nil
@@ -38,6 +44,12 @@
       end
     end
 
+    it "should allow us to instantiate an empty address object and call domain_idna" do
+      [nil, '', ' '].each do |input|
+        expect(Mail::Address.new(input).domain_idna).to be_nil
+      end
+    end
+
     it "should allow us to instantiate an empty address object and call name" do
       [nil, '', ' '].each do |input|
         expect(Mail::Address.new(input).name).to be_nil
@@ -119,7 +131,18 @@
       expect(a.domain).to eq result
     end
 
-    it "should give back the formated address" do
+    it "should give back the IDNA-encoded domain" do
+      parse_text = 'Mikel Lindsaar <test@lindsäär.net>'
+      result     = 'xn--lindsr-fuaa.net'
+      a          = Mail::Address.new(parse_text)
+      if Mail::Encodings.idna_supported?
+        expect(a.domain_idna).to eq result
+      else
+        expect {a.domain_idna}.to raise_error("Must install simpleidn gem")
+      end
+    end
+
+    it "should give back the formatted address" do
       parse_text = 'Mikel Lindsaar <test@lindsaar.net>'
       result     = 'Mikel Lindsaar <test@lindsaar.net>'
       a          = Mail::Address.new(parse_text)

spec/mail/encodings_spec.rb

@@ -932,4 +932,35 @@ def convert(from, to)
       expect(Mail::Utilities.pick_encoding("ISO-Foo")).to eq Encoding::BINARY
     end
   end
+
+  describe "IDNA encoding" do
+    after do
+      Mail::Encodings.instance_variable_set(:@idna_supported, nil)
+    end
+
+    it "should report on IDNA support" do
+      expect(Mail::Encodings.idna_supported?).to be !ENV.key?('SKIP_SIMPLEIDN')
+    end
+
+    it "should encode a string correctly" do
+      skip "simpleidn gem not installed" unless Mail::Encodings.idna_supported?
+
+      raw = 'tést.example.com'
+      encoded = 'xn--tst-bma.example.com'
+      expect(Mail::Encodings.idna_encode(raw)).to eq encoded
+    end
+
+    it "should raise on non-ASCII string if simpleidn gem is missing" do
+      Mail::Encodings.instance_variable_set(:@idna_supported, false)
+      raw = 'tést.example.com'
+      expect {Mail::Encodings.idna_encode(raw)}.to raise_error("Must install simpleidn gem")
+    end
+
+    it "should not raise on ASCII string if simpleidn gem is missing" do
+      Mail::Encodings.instance_variable_set(:@idna_supported, false)
+      raw = 'example.com'
+      encoded = Mail::Encodings.idna_encode(raw)
+      expect(encoded).to eq raw
+    end
+  end
 end

spec/mail/network/delivery_methods/smtp_connection_spec.rb

@@ -9,6 +9,8 @@
       smtp = Net::SMTP.start('127.0.0.1', 25)
       delivery_method :smtp_connection, :connection => smtp
     end
+
+    MockSMTP.reset
   end
 
   after(:each) do

spec/mail/network/delivery_methods/smtp_spec.rb

@@ -285,6 +285,97 @@ def redefine_verify_none(new_value)
       expect(MockSMTP.deliveries[0][2]).to eq %w(smtp_to@someemail.com)
     end
 
+    it "IDNA-encodes non-ASCII From and To addresses without SMTPUTF8 support" do
+      Mail.defaults do
+        delivery_method :smtp
+      end
+
+      Mail.deliver do
+        from "fröm@soméemail.com"
+        to "tö@soméemail.com"
+      end
+
+      if Mail::Encodings.idna_supported?
+        expect(MockSMTP.deliveries[0][1]).to eq 'fröm@xn--somemail-d1a.com'
+        expect(MockSMTP.deliveries[0][2]).to eq %w(tö@xn--somemail-d1a.com)
+      else
+        expect(MockSMTP.deliveries[0][1]).to eq 'fröm@soméemail.com'
+        expect(MockSMTP.deliveries[0][2]).to eq %w(tö@soméemail.com)
+      end
+    end
+
+    it "does not pass SMTPUTF8 parameter for ASCII From and To addresses with SMTPUTF8 support" do
+      Mail.defaults do
+        delivery_method :smtp
+      end
+
+      MockSMTP.capabilities = ['SMTPUTF8']
+
+      Mail.deliver do
+        to "to@someemail.com"
+        from "from@someemail.com"
+      end
+
+      expect(MockSMTP.deliveries[0][1]).to eq 'from@someemail.com'
+      expect(MockSMTP.deliveries[0][2]).to eq %w(to@someemail.com)
+    end
+
+    it "passes SMTPUTF8 parameter for non-ASCII From address with SMTPUTF8 support" do
+      Mail.defaults do
+        delivery_method :smtp
+      end
+
+      MockSMTP.capabilities = ['SMTPUTF8']
+
+      Mail.deliver do
+        from "fröm@soméemail.com"
+        to "to@someemail.com"
+      end
+
+      if defined?(Net::SMTP::Address)
+        from_address = MockSMTP.deliveries[0][1]
+        expect(from_address).to be_instance_of Net::SMTP::Address
+        expect(from_address.to_s).to eq 'fröm@soméemail.com'
+        expect(from_address.parameters).to eq ['SMTPUTF8']
+      elsif Mail::Encodings.idna_supported?
+        expect(MockSMTP.deliveries[0][1]).to eq 'fröm@xn--somemail-d1a.com'
+      else
+        expect(MockSMTP.deliveries[0][1]).to eq 'fröm@soméemail.com'
+      end
+
+      expect(MockSMTP.deliveries[0][2]).to eq %w(to@someemail.com)
+    end
+
+    it "passes SMTPUTF8 parameter for non-ASCII To address with SMTPUTF8 support" do
+      Mail.defaults do
+        delivery_method :smtp
+      end
+
+      MockSMTP.capabilities = ['SMTPUTF8']
+
+      Mail.deliver do
+        to "tö@soméemail.com"
+        from "from@somemail.com"
+      end
+
+      if defined?(Net::SMTP::Address)
+        from_address = MockSMTP.deliveries[0][1]
+        expect(from_address).to be_instance_of Net::SMTP::Address
+        expect(from_address.to_s).to eq 'from@somemail.com'
+        expect(from_address.parameters).to eq ['SMTPUTF8']
+
+        expect(MockSMTP.deliveries[0][2]).to eq %w(tö@soméemail.com)
+      else
+        expect(MockSMTP.deliveries[0][1]).to eq 'from@somemail.com'
+
+        if Mail::Encodings.idna_supported?
+          expect(MockSMTP.deliveries[0][2]).to eq %w(tö@xn--somemail-d1a.com)
+        else
+          expect(MockSMTP.deliveries[0][2]).to eq %w(tö@soméemail.com)
+        end
+      end
+    end
+
     it "supports the null sender in the envelope from address" do
       Mail.deliver do
         to "to@someemail.com"