fix: Order of messages if Sentbox is synced before Inbox

This fixes a Gmail-like scenario when outgoing messages are saved to Sentbox as well and it is
fetched before Inbox by chance, by adding a new `OutRcvd` message state for received outgoing
messages so that they can mingle with fresh incoming ones. As for `OutPending`, `OutDelivered`
etc. messages, they are sent locally by the user and there's no need to make them more noticeable
even if they are newer.

All APIs still return `OutDelivered` instead of `OutRcvd` for compatibility.

Author: iequidoo

deltachat-ffi/src/lib.rs

@@ -29,7 +29,7 @@ use deltachat::context::{Context, ContextBuilder};
 use deltachat::ephemeral::Timer as EphemeralTimer;
 use deltachat::imex::BackupProvider;
 use deltachat::key::preconfigure_keypair;
-use deltachat::message::MsgId;
+use deltachat::message::{MessageState, MsgId};
 use deltachat::qr_code_generator::{create_qr_svg, generate_backup_qr, get_securejoin_qr_svg};
 use deltachat::stock_str::StockMessage;
 use deltachat::webxdc::StatusUpdateSerial;
@@ -3344,7 +3344,11 @@ pub unsafe extern "C" fn dc_msg_get_state(msg: *mut dc_msg_t) -> libc::c_int {
         return 0;
     }
     let ffi_msg = &*msg;
-    ffi_msg.message.get_state() as libc::c_int
+    let state = match ffi_msg.message.get_state() {
+        MessageState::OutRcvd => MessageState::OutDelivered,
+        s => s,
+    };
+    state as libc::c_int
 }
 
 #[no_mangle]

deltachat-ffi/src/lot.rs

@@ -240,7 +240,7 @@ impl From<MessageState> for LotState {
             OutDraft => LotState::MsgOutDraft,
             OutPending => LotState::MsgOutPending,
             OutFailed => LotState::MsgOutFailed,
-            OutDelivered => LotState::MsgOutDelivered,
+            OutDelivered | OutRcvd => LotState::MsgOutDelivered,
             OutMdnRcvd => LotState::MsgOutMdnRcvd,
         }
     }

deltachat-jsonrpc/src/api/types/message.rs

@@ -9,6 +9,7 @@ use deltachat::contact::Contact;
 use deltachat::context::Context;
 use deltachat::download;
 use deltachat::message::Message;
+use deltachat::message::MessageState;
 use deltachat::message::MsgId;
 use deltachat::message::Viewtype;
 use deltachat::reaction::get_msg_reactions;
@@ -149,6 +150,12 @@ impl MessageObject {
 
         let parent_id = message.parent(context).await?.map(|m| m.get_id().to_u32());
 
+        let state = match message.get_state() {
+            MessageState::OutRcvd => MessageState::OutDelivered,
+            s => s,
+        }
+        .to_u32()
+        .context("state conversion to number failed")?;
         let download_state = message.download_state().into();
 
         let quote = if let Some(quoted_text) = message.quoted_text() {
@@ -212,10 +219,7 @@ impl MessageObject {
             has_location: message.has_location(),
             has_html: message.has_html(),
             view_type: message.get_viewtype().into(),
-            state: message
-                .get_state()
-                .to_u32()
-                .context("state conversion to number failed")?,
+            state,
             error: message.error(),
 
             timestamp: message.get_timestamp(),

src/chat.rs

@@ -1427,17 +1427,21 @@ impl ChatId {
             // received message purely by timestamp. We could place it just before that seen
             // message, but anyway the user may not notice it.
             //
-            // NB: Received outgoing messages may break sorting of fresh incoming ones, but this
-            // shouldn't happen frequently. Seen incoming messages don't really break sorting of
-            // fresh ones, they rather mean that older incoming messages are actually seen as well.
+            // NB: Seen incoming messages don't really break sorting of fresh ones, they rather mean
+            // that older incoming messages are actually seen as well.
             context
                 .sql
                 .query_row_optional(
                     "SELECT MAX(timestamp), MAX(IIF(state=?,timestamp_sent,0))
                      FROM msgs
-                     WHERE chat_id=? AND hidden=0 AND state>?
+                     WHERE chat_id=? AND hidden=0 AND state>? AND state!=?
                      HAVING COUNT(*) > 0",
-                    (MessageState::InSeen, self, MessageState::InFresh),
+                    (
+                        MessageState::InSeen,
+                        self,
+                        MessageState::InFresh,
+                        MessageState::OutRcvd,
+                    ),
                     |row| {
                         let ts: i64 = row.get(0)?;
                         let ts_sent_seen: i64 = row.get(1)?;
@@ -4444,6 +4448,7 @@ pub async fn resend_msgs(context: &Context, msg_ids: &[MsgId]) -> Result<()> {
             MessageState::OutPending
             | MessageState::OutFailed
             | MessageState::OutDelivered
+            | MessageState::OutRcvd
             | MessageState::OutMdnRcvd => {
                 message::update_msg_state(context, msg.id, MessageState::OutPending).await?
             }

src/message.rs

@@ -1453,6 +1453,9 @@ pub enum MessageState {
     /// the OutFailed state if we get such a hint from the server.
     OutDelivered = 26,
 
+    /// Received outgoing message sent by another MUA or device.
+    OutRcvd = 27,
+
     /// Outgoing message read by the recipient (two checkmarks; this
     /// requires goodwill on the receiver's side). Not used in the db for new messages.
     OutMdnRcvd = 28,
@@ -1473,6 +1476,7 @@ impl std::fmt::Display for MessageState {
                 Self::OutPending => "Pending",
                 Self::OutFailed => "Failed",
                 Self::OutDelivered => "Delivered",
+                Self::OutRcvd => "Other MUA",
                 Self::OutMdnRcvd => "Read",
             }
         )
@@ -1485,7 +1489,9 @@ impl MessageState {
         use MessageState::*;
         matches!(
             self,
-            OutPreparing | OutPending | OutDelivered | OutMdnRcvd // OutMdnRcvd can still fail because it could be a group message and only some recipients failed.
+            // OutMdnRcvd can still fail because it could be a group message and only some
+            // recipients failed.
+            OutPreparing | OutPending | OutDelivered | OutRcvd | OutMdnRcvd
         )
     }
 
@@ -1494,13 +1500,13 @@ impl MessageState {
         use MessageState::*;
         matches!(
             self,
-            OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutMdnRcvd
+            OutPreparing | OutDraft | OutPending | OutFailed | OutDelivered | OutRcvd | OutMdnRcvd
         )
     }
 
     /// Returns adjusted message state if the message has MDNs.
     pub(crate) fn with_mdns(self, has_mdns: bool) -> Self {
-        if self == MessageState::OutDelivered && has_mdns {
+        if has_mdns && self >= MessageState::OutDelivered {
             return MessageState::OutMdnRcvd;
         }
         self

src/mimeparser/mimeparser_tests.rs

@@ -1522,7 +1522,7 @@ async fn test_ignore_read_receipt_to_self() -> Result<()> {
     )
     .await?;
     let msg = alice.get_last_msg().await;
-    assert_eq!(msg.state, MessageState::OutDelivered);
+    assert_eq!(msg.state, MessageState::OutRcvd);
 
     // Due to a bug in the old version running on the other device, Alice receives a read
     // receipt from self.
@@ -1561,7 +1561,7 @@ async fn test_ignore_read_receipt_to_self() -> Result<()> {
 
     // Check that the state has not changed to `MessageState::OutMdnRcvd`.
     let msg = Message::load_from_db(&alice, msg.id).await?;
-    assert_eq!(msg.state, MessageState::OutDelivered);
+    assert_eq!(msg.state, MessageState::OutRcvd);
 
     Ok(())
 }

src/reaction.rs

@@ -476,7 +476,7 @@ Can we chat at 1pm pacific, today?"
         )
         .await?;
         let msg = alice.get_last_msg().await;
-        assert_eq!(msg.state, MessageState::OutDelivered);
+        assert_eq!(msg.state, MessageState::OutRcvd);
         let reactions = get_msg_reactions(&alice, msg.id).await?;
         let contacts = reactions.contacts();
         assert_eq!(contacts.len(), 0);

src/receive_imf.rs

@@ -1708,7 +1708,7 @@ async fn add_parts(
     };
 
     let state = if !mime_parser.incoming {
-        MessageState::OutDelivered
+        MessageState::OutRcvd
     } else if seen || is_mdn || chat_id_blocked == Blocked::Yes || group_changes.silent
     // No check for `hidden` because only reactions are such and they should be `InFresh`.
     {

src/receive_imf/receive_imf_tests.rs

@@ -684,7 +684,7 @@ async fn test_parse_ndn(
         if error_msg.is_some() {
             MessageState::OutFailed
         } else {
-            MessageState::OutDelivered
+            MessageState::OutRcvd
         }
     );
 
@@ -4312,7 +4312,7 @@ async fn test_outgoing_msg_forgery() -> Result<()> {
 
     let sent_msg = malice.send_text(malice_chat_id, "hi from malice").await;
     let msg = alice.recv_msg(&sent_msg).await;
-    assert_eq!(msg.state, MessageState::OutDelivered);
+    assert_eq!(msg.state, MessageState::OutRcvd);
     assert!(!msg.get_showpadlock());
 
     Ok(())

src/tests/verified_chats.rs

@@ -351,7 +351,7 @@ async fn test_old_message_5() -> Result<()> {
     .await?
     .unwrap();
 
-    assert!(msg_sent.sort_timestamp == msg_incoming.sort_timestamp);
+    assert!(msg_incoming.sort_timestamp < msg_sent.sort_timestamp);
     alice
         .golden_test_chat(msg_sent.chat_id, "test_old_message_5")
         .await;