Enhanced verify_email rake task

.ruby-version

@@ -1 +1 @@
-3.3.0
+3.0.3

Gemfile

@@ -88,12 +88,12 @@ end
 group :test do
   gem 'capybara'
   gem 'database_cleaner'
-  gem 'minitest', '~> 5.17'
+  gem 'minitest'
   gem 'minitest-stub_any_instance'
   gem 'selenium-webdriver'
-  # gem 'webdrivers'
   gem 'simplecov', '0.17.1', require: false # CC last supported v0.17
   gem 'spy'
+  # gem 'webdrivers'
   gem 'webmock'
 end
 

Gemfile.lock

@@ -139,7 +139,7 @@ GEM
       minitest (>= 5.1)
       tzinfo (~> 2.0)
       zeitwerk (~> 2.3)
-    addressable (2.8.1)
+    addressable (2.8.6)
       public_suffix (>= 2.0.2, < 6.0)
     aes_key_wrap (1.1.0)
     airbrake (11.0.3)
@@ -167,6 +167,7 @@ GEM
       aws-sigv4 (~> 1.1)
     aws-sigv4 (1.2.4)
       aws-eventstream (~> 1, >= 1.0.2)
+    base64 (0.2.0)
     bcrypt (3.1.16)
     bindata (2.4.14)
     bootsnap (1.17.1)
@@ -176,15 +177,15 @@ GEM
       sassc (>= 2.0.0)
     builder (3.2.4)
     cancancan (3.3.0)
-    capybara (3.35.3)
+    capybara (3.40.0)
       addressable
+      matrix
       mini_mime (>= 0.1.3)
-      nokogiri (~> 1.8)
+      nokogiri (~> 1.11)
       rack (>= 1.6.0)
       rack-test (>= 0.6.3)
       regexp_parser (>= 1.5, < 3.0)
       xpath (~> 3.2)
-    childprocess (3.0.0)
     chronic (0.10.2)
     coderay (1.1.3)
     coffee-rails (5.0.0)
@@ -235,13 +236,9 @@ GEM
       thor (>= 0.14.0, < 2)
     globalid (1.0.1)
       activesupport (>= 5.0)
-    google-protobuf (3.25.2)
     google-protobuf (3.25.2-x86_64-linux)
     googleapis-common-protos-types (1.3.0)
       google-protobuf (~> 3.14)
-    grpc (1.60.0)
-      google-protobuf (~> 3.25)
-      googleapis-common-protos-types (~> 1.0)
     grpc (1.60.0-x86_64-linux)
       google-protobuf (~> 3.25)
       googleapis-common-protos-types (~> 1.0)
@@ -301,6 +298,7 @@ GEM
     mail (2.7.1)
       mini_mime (>= 0.1.1)
     marcel (1.0.2)
+    matrix (0.4.2)
     method_source (1.0.0)
     mime-types (3.3.1)
       mime-types-data (~> 3.2015)
@@ -309,7 +307,6 @@ GEM
       nokogiri (~> 1)
       rake
     mini_mime (1.1.5)
-    mini_portile2 (2.8.5)
     minitest (5.18.1)
     minitest-stub_any_instance (1.0.3)
     monetize (1.9.4)
@@ -332,10 +329,7 @@ GEM
       newrelic_rpm (= 8.1.0)
     newrelic_rpm (8.1.0)
     nio4r (2.5.9)
-    nokogiri (1.16.2)
-      mini_portile2 (~> 2.8.2)
-      racc (~> 1.4)
-    nokogiri (1.16.2-x86_64-linux)
+    nokogiri (1.16.4-x86_64-linux)
       racc (~> 1.4)
     nori (2.6.0)
     omniauth (2.1.0)
@@ -369,11 +363,11 @@ GEM
     pry (0.14.2)
       coderay (~> 1.1)
       method_source (~> 1.0)
-    public_suffix (5.0.0)
+    public_suffix (5.0.5)
     puma (5.6.8)
       nio4r (~> 2.0)
     racc (1.7.3)
-    rack (2.2.8.1)
+    rack (2.2.9)
     rack-oauth2 (1.21.3)
       activesupport
       attr_required
@@ -421,7 +415,7 @@ GEM
       redis-client (>= 0.9.0)
     redis-client (0.14.1)
       connection_pool
-    regexp_parser (2.1.1)
+    regexp_parser (2.9.0)
     request_store (1.5.1)
       rack (>= 1.4)
     responders (3.0.1)
@@ -455,9 +449,11 @@ GEM
       wasabi (~> 3.4)
     select2-rails (4.0.13)
     selectize-rails (0.12.6)
-    selenium-webdriver (3.142.7)
-      childprocess (>= 0.5, < 4.0)
-      rubyzip (>= 1.2.2)
+    selenium-webdriver (4.20.1)
+      base64 (~> 0.2)
+      rexml (~> 3.2, >= 3.2.5)
+      rubyzip (>= 1.2.2, < 3.0)
+      websocket (~> 1.0)
     sidekiq (7.1.4)
       concurrent-ruby (< 2)
       connection_pool (>= 2.3.0)
@@ -520,7 +516,8 @@ GEM
       addressable (>= 2.8.0)
       crack (>= 0.3.2)
       hashdiff (>= 0.4.0, < 2.0.0)
-    websocket-driver (0.7.6)
+    websocket (1.2.10)
+    websocket-driver (0.7.5)
       websocket-extensions (>= 0.1.0)
     websocket-extensions (0.1.5)
     whenever (1.0.0)
@@ -531,7 +528,6 @@ GEM
     zeitwerk (2.6.13)
 
 PLATFORMS
-  ruby
   x86_64-linux
 
 DEPENDENCIES
@@ -567,7 +563,7 @@ DEPENDENCIES
   lhv!
   mime-types-data
   mimemagic (= 0.4.3)
-  minitest (~> 5.17)
+  minitest
   minitest-stub_any_instance
   money-rails
   newrelic-infinite_tracing
@@ -604,4 +600,4 @@ DEPENDENCIES
   wkhtmltopdf-binary (~> 0.12.6.1)
 
 BUNDLED WITH
-   2.5.4
+   2.5.10

app/interactions/actions/a_and_aaaa_email_validation.rb

@@ -10,11 +10,13 @@ def call(email:, value:)
 
     def check_for_records_value(email:, value:)
       email_domain = Mail::Address.new(email).domain
+
       dns_servers = ENV['dnssec_resolver_ips'].to_s.split(',').map(&:strip)
 
       resolve_a_and_aaaa_records(dns_servers: dns_servers, email_domain: email_domain, value: value)
-    rescue Mail::Field::IncompleteParseError => e
-      Rails.logger.info "Failed to parse email #{email}."
+    rescue Mail::Field::ParseError => e
+      Rails.logger.info "Mail parsing error: #{e.message}"
+      []
     end
 
     def resolve_a_and_aaaa_records(dns_servers:, email_domain:, value:)

app/jobs/verify_emails_job.rb

@@ -1,31 +1,37 @@
 class VerifyEmailsJob < ApplicationJob
   discard_on StandardError
 
-  def perform(email:, check_level: 'mx')
-    contact = Contact.find_by(email: email)
-
-    return logger.info "Contact #{email} not found!" if contact.nil?
+  def perform(email:, check_level: 'mx', force: false)
+    contact = fetch_contact(email)
+    return unless contact && need_to_verify?(contact, force)
 
-    return unless need_to_verify?(contact)
-
-    validate_check_level(check_level)
-
-    logger.info "Trying to verify contact email #{email} with check_level #{check_level}"
-    contact.verify_email(check_level: check_level)
+    verify_contact_email(contact, check_level)
   rescue StandardError => e
     handle_error(e)
   end
 
   private
 
+  def fetch_contact(email)
+    contact = Contact.find_by(email: email)
+    logger.info "Contact #{email} not found!" unless contact
+    contact
+  end
+
+  def verify_contact_email(contact, check_level)
+    validate_check_level(check_level)
+    logger.info "Trying to verify contact email #{contact.email} with check_level #{check_level}"
+    contact.verify_email(check_level: check_level)
+  end
+
   def validate_check_level(check_level)
     return if valid_check_levels.include? check_level
 
     raise StandardError, "Check level #{check_level} is invalid"
   end
 
-  def need_to_verify?(contact)
-    return true if contact.validation_events.empty?
+  def need_to_verify?(contact, force)
+    return true if contact.validation_events.empty? || force
 
     last_validation = contact.validation_events.last
     expired_last_validation = last_validation.successful? && last_validation.created_at < validation_expiry_date

app/models/concerns/email_verifable.rb

@@ -79,9 +79,7 @@ def verify(email:, check_level: 'regex')
     action.call
   end
 
-  # rubocop:disable Metrics/LineLength
   def process_error(field)
     errors.add(field, I18n.t('activerecord.errors.models.contact.attributes.email.email_regex_check_error'))
   end
-  # rubocop:enable Metrics/LineLength
 end

config/initializers/truemail.rb

@@ -62,18 +62,25 @@
   # domain only, i.e. if domain whitelisted, validation will passed to Regex, MX or SMTP validators.
   # Validation of email which not contains whitelisted domain always will return false.
   # It is equal false by default.
-  #config.whitelist_validation = true
+  # config.whitelist_validation = true
 
   # Optional parameter. Validation of email which contains blacklisted domain always will
   # return false. Other validations will not processed even if it was defined in validation_type_for
   # It is equal to empty array by default.
-  #config.blacklisted_domains = []
+  # config.blacklisted_domains = []
 
   # Optional parameter. This option will provide to use not RFC MX lookup flow.
   # It means that MX and Null MX records will be cheked on the DNS validation layer only.
   # By default this option is disabled.
   # config.not_rfc_mx_lookup_flow = true
 
+  # Optional parameter. This option will provide to use smtp fail fast behavior. When
+  # smtp_fail_fast = true it means that Truemail ends smtp validation session after first
+  # attempt on the first mx server in any fail cases (network connection/timeout error,
+  # smtp validation error). This feature helps to reduce total time of SMTP validation
+  # session up to 1 second. By default this option is disabled.
+  # config.smtp_fail_fast = true
+
   # Optional parameter. This option will be parse bodies of SMTP errors. It will be helpful
   # if SMTP server does not return an exact answer that the email does not exist
   # By default this option is disabled, available for SMTP validation only.

lib/tasks/verify_email.rake

@@ -4,17 +4,25 @@ require 'syslog/logger'
 require 'active_record'
 
 SPAM_PROTECT_TIMEOUT = 30.seconds
+PATCH_SIZE = 10
+PATCH_INTERVAL = 10.minutes
 
 namespace :verify_email do
   # bundle exec rake verify_email:check_all -- --check_level=mx --spam_protect=true
-  # bundle exec rake verify_email:check_all -- -dshop.test -cmx -strue
+  # bundle exec rake verify_email:check_all -- -d shop.test -c mx -s true
+  # bunlde exec rake verify_email:check_all -- -e email1@example.com,email2@example.com -c mx
+  # bundle exec rake verify_email:check_all -- --email_regex='^test\d*@example\.com$' --check_level=mx --spam_protect=true
   desc 'Starts verifying email jobs with optional check level and spam protection'
   task check_all: :environment do
     options = {
       domain_name: nil,
       check_level: 'mx',
       spam_protect: false,
+      emails: [],
+      email_regex: nil,
+      force: false
     }
+
     banner = 'Usage: rake verify_email:check_all -- [options]'
     options = RakeOptionParserBoilerplate.process_args(options: options,
                                                        banner: banner,
@@ -27,9 +35,11 @@ namespace :verify_email do
 end
 
 def enqueue_email_verification(email_contacts, options)
-  email_contacts.each do |email|
-    VerifyEmailsJob.set(wait_until: spam_protect_timeout(options))
-                   .perform_later(email: email, check_level: options[:check_level])
+  email_contacts.each_slice(PATCH_SIZE).with_index do |slice, index|
+    slice.each do |email|
+      VerifyEmailsJob.set(wait: spam_protect_timeout(options) + index * PATCH_INTERVAL)
+                     .perform_later(email: email, check_level: options[:check_level], force: options[:force])
+    end
   end
 end
 
@@ -38,7 +48,11 @@ def spam_protect_timeout(options)
 end
 
 def prepare_contacts(options)
-  if options[:domain_name].present?
+  if options[:emails].any?
+    options[:emails]
+  elsif options[:email_regex].present?
+    contacts_by_regex(options[:email_regex])
+  elsif options[:domain_name].present?
     contacts_by_domain(options[:domain_name])
   else
     unvalidated_and_failed_contacts_emails
@@ -70,10 +84,17 @@ def contacts_by_domain(domain_name)
   domain.contacts.pluck(:email).uniq
 end
 
+def contacts_by_regex(regex)
+  Contact.where('email ~ ?', regex).pluck(:email).uniq
+end
+
 def opts_hash
   {
     domain_name: ['-d [DOMAIN_NAME]', '--domain_name [DOMAIN_NAME]', String],
     check_level: ['-c [CHECK_LEVEL]', '--check_level [CHECK_LEVEL]', String],
     spam_protect: ['-s [SPAM_PROTECT]', '--spam_protect [SPAM_PROTECT]', FalseClass],
+    emails: ['-e [EMAILS]', '--emails [EMAILS]', Array],
+    email_regex: ['-r [EMAIL_REGEX]', '--email_regex [EMAIL_REGEX]', String],
+    force: ['-f', '--force', FalseClass]
   }
 end

renovate.json

@@ -14,7 +14,8 @@
     },
     {
       "matchDepTypes": [".ruby-version"],
-      "addLabels": ["ruby-version"]
+      "addLabels": ["ruby-version"],
+      "automerge": false
     }
   ],
   "docker": {

test/interactions/email_check_test.rb

@@ -97,7 +97,7 @@ def test_should_remove_old_record_if_multiple_contacts_has_the_same_email
   end
 
   def test_should_test_email_with_punnycode
-    email = "info@xn--energiathus-mfb.ee"
+    email = 'info@xn--energiathus-mfb.ee'
     result = Actions::SimpleMailValidator.run(email: email, level: :mx)
 
     assert result