FEATURE: Add more columns to outbound EmailLog (discourse#13449)

This adds the following columns to EmailLog:

* cc_addresses
* cc_user_ids
* topic_id
* raw

This is to bring the EmailLog table closer in parity to
IncomingEmail so it can be better utilized for Group SMTP
and IMAP mailing.

The raw column contains the full content of the outbound email,
but _only_ if the new hidden site setting
enable_raw_outbound_email_logging is enabled. Most sites do not
need it, and it's mostly required for IMAP and SMTP sending.

In the next pull request, there will be a migration to backfill
topic_id on the EmailLog table, at which point we can remove the
topic fallback method on EmailLog.

app/models/email_log.rb

@@ -18,8 +18,6 @@ class EmailLog < ActiveRecord::Base
   belongs_to :post
   belongs_to :smtp_group, class_name: 'Group'
 
-  has_one :topic, through: :post
-
   validates :email_type, :to_address, presence: true
 
   scope :bounced, -> { where(bounced: true) }
@@ -40,6 +38,10 @@ class EmailLog < ActiveRecord::Base
     User.where(id: user_id).update_all("last_emailed_at = CURRENT_TIMESTAMP") if user_id.present?
   end
 
+  def topic
+    @topic ||= self.topic_id.present? ? Topic.find_by(id: self.topic_id) : self.post&.topic
+  end
+
   def self.unique_email_per_post(post, user)
     return yield unless post && user
 
@@ -85,6 +87,14 @@ def bounce_key
     super&.delete('-')
   end
 
+  def cc_users
+    return [] if !self.cc_user_ids
+    @cc_users ||= User.where(id: self.cc_user_ids)
+  end
+
+  def cc_addresses_split
+    @cc_addresses_split ||= self.cc_addresses&.split(";") || []
+  end
 end
 
 # == Schema Information
@@ -102,14 +112,18 @@ def bounce_key
 #  bounced       :boolean          default(FALSE), not null
 #  message_id    :string
 #  smtp_group_id :integer
+#  cc_addresses  :text
+#  cc_user_ids   :integer          is an Array
+#  raw           :text
+#  topic_id      :integer
 #
 # Indexes
 #
-#  idx_email_logs_on_smtp_group_id  (smtp_group_id)
-#  index_email_logs_on_bounce_key   (bounce_key) UNIQUE WHERE (bounce_key IS NOT NULL)
-#  index_email_logs_on_bounced      (bounced)
-#  index_email_logs_on_created_at   (created_at)
-#  index_email_logs_on_message_id   (message_id)
-#  index_email_logs_on_post_id      (post_id)
-#  index_email_logs_on_user_id      (user_id)
+#  index_email_logs_on_bounce_key  (bounce_key) UNIQUE WHERE (bounce_key IS NOT NULL)
+#  index_email_logs_on_bounced     (bounced)
+#  index_email_logs_on_created_at  (created_at)
+#  index_email_logs_on_message_id  (message_id)
+#  index_email_logs_on_post_id     (post_id)
+#  index_email_logs_on_topic_id    (topic_id) WHERE (topic_id IS NOT NULL)
+#  index_email_logs_on_user_id     (user_id)
 #

db/migrate/20210621002201_add_columns_to_email_log_to_match_incoming_for_smtp_imap.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+class AddColumnsToEmailLogToMatchIncomingForSmtpImap < ActiveRecord::Migration[6.1]
+  def up
+    add_column :email_logs, :cc_addresses, :text, null: true
+    add_column :email_logs, :cc_user_ids, :integer, array: true, null: true
+    add_column :email_logs, :raw, :text, null: true
+    add_column :email_logs, :topic_id, :integer, null: true
+
+    add_index :email_logs, :topic_id, where: 'topic_id IS NOT NULL'
+  end
+
+  def down
+    remove_column :email_logs, :cc_addresses if column_exists?(:email_logs, :cc_addresses)
+    remove_column :email_logs, :cc_user_ids if column_exists?(:email_logs, :cc_user_ids)
+    remove_column :email_logs, :raw if column_exists?(:email_logs, :raw)
+    remove_column :email_logs, :topic_id if column_exists?(:email_logs, :topic_id)
+
+    remove_index :email_logs, :topic_id if index_exists?(:email_logs, [:topic_id])
+  end
+end

lib/email/sender.rb

@@ -92,6 +92,11 @@ def send
         user_id: user_id
       )
 
+      if cc_addresses.any?
+        email_log.cc_addresses = cc_addresses.join(";")
+        email_log.cc_user_ids = User.with_email(cc_addresses).pluck(:id)
+      end
+
       host = Email::Sender.host_for(Discourse.base_url)
 
       post_id   = header_value('X-Discourse-Post-Id')
@@ -201,6 +206,7 @@ def send
       end
 
       email_log.post_id = post_id if post_id.present?
+      email_log.topic_id = topic_id if topic_id.present?
 
       # Remove headers we don't need anymore
       @message.header['X-Discourse-Topic-Id'] = nil if topic_id.present?
@@ -243,7 +249,16 @@ def send
 
       # Log when a message is being sent from a group SMTP address, so we
       # can debug deliverability issues.
-      email_log.smtp_group_id = smtp_group_id
+      if smtp_group_id
+        email_log.smtp_group_id = smtp_group_id
+
+        # Store contents of all outgoing emails using group SMTP
+        # for greater visibility and debugging. If the size of this
+        # gets out of hand, we should look into a group-level setting
+        # to enable this; size should be kept in check by regular purging
+        # of EmailLog though.
+        email_log.raw = Email::Cleaner.new(@message).execute
+      end
 
       DiscourseEvent.trigger(:before_email_send, @message, @email_type)
 
@@ -270,6 +285,12 @@ def to_address
       end
     end
 
+    def cc_addresses
+      @cc_addresses ||= begin
+        @message.try(:cc) || []
+      end
+    end
+
     def self.host_for(base_url)
       host = "localhost"
       if base_url.present?

spec/components/email/sender_spec.rb

@@ -336,6 +336,7 @@
         expect(email_log.email_type).to eq('valid_type')
         expect(email_log.to_address).to eq('eviltrout@test.domain')
         expect(email_log.user_id).to be_blank
+        expect(email_log.raw).to eq(nil)
       end
 
       context 'when the email is sent using group SMTP credentials' do
@@ -355,7 +356,7 @@
           SiteSetting.enable_smtp = true
         end
 
-        it 'adds the group id to the email log' do
+        it 'adds the group id and raw content to the email log' do
           TopicAllowedGroup.create(topic: post.topic, group: group)
 
           email_sender.send
@@ -365,6 +366,7 @@
           expect(email_log.to_address).to eq(post.user.email)
           expect(email_log.user_id).to be_blank
           expect(email_log.smtp_group_id).to eq(group.id)
+          expect(email_log.raw).to include("Hello world")
         end
 
         it "does not add any of the mailing list headers" do
@@ -393,6 +395,7 @@
       it 'should create the right log' do
         email_sender.send
         expect(email_log.post_id).to eq(post.id)
+        expect(email_log.topic_id).to eq(topic.id)
         expect(email_log.topic.id).to eq(topic.id)
       end
     end
@@ -687,4 +690,30 @@
     end
   end
 
+  context "with cc addresses" do
+    let(:message) do
+      message = Mail::Message.new to: 'eviltrout@test.domain', body: 'test body', cc: 'someguy@test.com;otherguy@xyz.com'
+      message.stubs(:deliver_now)
+      message
+    end
+
+    fab!(:user) { Fabricate(:user) }
+    let(:email_sender) { Email::Sender.new(message, :valid_type, user) }
+
+    it "logs the cc addresses in the email log (but not users if they do not match the emails)" do
+      email_sender.send
+      email_log = EmailLog.last
+      expect(email_log.cc_addresses).to eq("someguy@test.com;otherguy@xyz.com")
+      expect(email_log.cc_users).to eq([])
+    end
+
+    it "logs the cc users if they match the emails" do
+      user1 = Fabricate(:user, email: "someguy@test.com")
+      user2 = Fabricate(:user, email: "otherguy@xyz.com")
+      email_sender.send
+      email_log = EmailLog.last
+      expect(email_log.cc_addresses).to eq("someguy@test.com;otherguy@xyz.com")
+      expect(email_log.cc_users).to match_array([user1, user2])
+    end
+  end
 end

spec/models/email_log_spec.rb

@@ -110,4 +110,32 @@
       expect(EmailLog.find(email_log.id).bounce_key).to eq(hex)
     end
   end
+
+  describe "cc addresses handling" do
+    let!(:email_log) { Fabricate(:email_log, user: user) }
+
+    describe "#cc_addresses_split" do
+      it "returns empty array if there are no cc addresses" do
+        expect(email_log.cc_addresses_split).to eq([])
+      end
+
+      it "returns array of cc addresses if there are any" do
+        email_log.update(cc_addresses: "test@test.com;test@test2.com")
+        expect(email_log.cc_addresses_split).to eq(["test@test.com", "test@test2.com"])
+      end
+    end
+
+    describe "#cc_users" do
+      it "returns empty array if there are no cc users" do
+        expect(email_log.cc_users).to eq([])
+      end
+
+      it "returns array of users if cc_user_ids is present" do
+        cc_user = Fabricate(:user, email: "test@test.com")
+        cc_user2 = Fabricate(:user, email: "test@test2.com")
+        email_log.update(cc_addresses: "test@test.com;test@test2.com", cc_user_ids: [cc_user.id, cc_user2.id])
+        expect(email_log.cc_users).to match_array([cc_user, cc_user2])
+      end
+    end
+  end
 end