Truncate messages on load and add an option to disable it

deltachat-jsonrpc/src/api.rs

@@ -23,8 +23,9 @@ use deltachat::ephemeral::Timer;
 use deltachat::imex;
 use deltachat::location;
 use deltachat::message::{
-    self, delete_msgs_ex, get_existing_msg_ids, get_msg_read_receipt_count, get_msg_read_receipts,
-    markseen_msgs, Message, MessageState, MsgId, Viewtype,
+    self, delete_msgs_ex, dont_truncate_long_messages, get_existing_msg_ids,
+    get_msg_read_receipt_count, get_msg_read_receipts, markseen_msgs, Message, MessageState, MsgId,
+    Viewtype,
 };
 use deltachat::peer_channels::{
     leave_webxdc_realtime, send_webxdc_realtime_advertisement, send_webxdc_realtime_data,
@@ -1373,6 +1374,15 @@ impl CommandApi {
         MsgId::new(message_id).get_html(&ctx).await
     }
 
+    /// Out out of truncating long messages when loading.
+    ///
+    /// Should be used by the UIs that can handle long text messages.
+    async fn dont_truncate_long_messages(&self, account_id: u32) -> Result<()> {
+        let ctx = self.get_context(account_id).await?;
+        dont_truncate_long_messages(&ctx);
+        Ok(())
+    }
+
     /// get multiple messages in one call,
     /// if loading one message fails the error is stored in the result object in it's place.
     ///

src/chat.rs

@@ -49,7 +49,7 @@ use crate::sync::{self, Sync::*, SyncData};
 use crate::tools::{
     IsNoneOrEmpty, SystemTime, buf_compress, create_broadcast_secret, create_id,
     create_outgoing_rfc724_mid, create_smeared_timestamp, create_smeared_timestamps, get_abs_path,
-    gm2local_offset, normalize_text, smeared_time, time, truncate_msg_text,
+    gm2local_offset, normalize_text, smeared_time, time,
 };
 use crate::webxdc::StatusUpdateSerial;
 
@@ -1884,7 +1884,8 @@ impl Chat {
             EphemeralTimer::Enabled { duration } => time().saturating_add(duration.into()),
         };
 
-        let (msg_text, was_truncated) = truncate_msg_text(context, msg.text.clone()).await?;
+        let msg_text = msg.text.clone();
+
         let new_mime_headers = if msg.has_html() {
             if msg.param.exists(Param::Forwarded) {
                 msg.get_id().get_html(context).await?
@@ -1901,13 +1902,6 @@ impl Chat {
             html_part.write_part(cursor).ok();
             String::from_utf8_lossy(&buffer).to_string()
         });
-        let new_mime_headers = new_mime_headers.or_else(|| match was_truncated {
-            // We need to add some headers so that they are stripped before formatting HTML by
-            // `MsgId::get_html()`, not a part of the actual text. Let's add "Content-Type", it's
-            // anyway a useful metadata about the stored text.
-            true => Some("Content-Type: text/plain; charset=utf-8\r\n\r\n".to_string() + &msg.text),
-            false => None,
-        });
         let new_mime_headers = match new_mime_headers {
             Some(h) => Some(tokio::task::block_in_place(move || {
                 buf_compress(h.as_bytes())

src/context.rs

@@ -226,7 +226,18 @@ impl WeakContext {
 pub struct InnerContext {
     /// Blob directory path
     pub(crate) blobdir: PathBuf,
+
     pub(crate) sql: Sql,
+
+    /// True if long text messages should be truncated
+    /// and full message HTML added.
+    ///
+    /// This should be set by the UIs that cannot handle
+    /// long messages but can display HTML messages.
+    ///
+    /// Ignored for bots, bots never get truncated messages.
+    pub(crate) truncate_long_messages: AtomicBool,
+
     pub(crate) smeared_timestamp: SmearedTimestamp,
     /// The global "ongoing" process state.
     ///
@@ -473,6 +484,7 @@ impl Context {
             blobdir,
             running_state: RwLock::new(Default::default()),
             sql: Sql::new(dbfile),
+            truncate_long_messages: AtomicBool::new(true),
             smeared_timestamp: SmearedTimestamp::new(),
             generating_key_mutex: Mutex::new(()),
             oauth2_mutex: Mutex::new(()),

src/html.rs

@@ -30,7 +30,7 @@ impl Message {
     /// The corresponding ffi-function is `dc_msg_has_html()`.
     /// To get the HTML-code of the message, use `MsgId.get_html()`.
     pub fn has_html(&self) -> bool {
-        self.mime_modified
+        self.mime_modified || self.full_text.is_some()
     }
 
     /// Set HTML-part part of a message that is about to be sent.
@@ -270,8 +270,19 @@ impl MsgId {
                 Ok((parser, _)) => Ok(Some(parser.html)),
             }
         } else {
-            warn!(context, "get_html: no mime for {}", self);
-            Ok(None)
+            let msg = Message::load_from_db(context, self).await?;
+            if let Some(full_text) = &msg.full_text {
+                let html = PlainText {
+                    text: full_text.clone(),
+                    flowed: false,
+                    delsp: false,
+                }
+                .to_html();
+                Ok(Some(html))
+            } else {
+                warn!(context, "get_html: no mime for {}", self);
+                Ok(None)
+            }
         }
     }
 }

src/message.rs

@@ -4,6 +4,7 @@ use std::collections::BTreeSet;
 use std::collections::HashSet;
 use std::path::{Path, PathBuf};
 use std::str;
+use std::sync::atomic::Ordering;
 
 use anyhow::{Context as _, Result, ensure, format_err};
 use deltachat_contact_tools::{VcardContact, parse_vcard};
@@ -35,10 +36,9 @@ use crate::reaction::get_msg_reactions;
 use crate::sql;
 use crate::summary::Summary;
 use crate::sync::SyncData;
-use crate::tools::create_outgoing_rfc724_mid;
 use crate::tools::{
-    buf_compress, buf_decompress, get_filebytes, get_filemeta, gm2local_offset, read_file,
-    sanitize_filename, time, timestamp_to_str,
+    buf_compress, buf_decompress, create_outgoing_rfc724_mid, get_filebytes, get_filemeta,
+    gm2local_offset, read_file, sanitize_filename, time, timestamp_to_str, truncate_msg_text,
 };
 
 /// Message ID, including reserved IDs.
@@ -431,7 +431,13 @@ pub struct Message {
     pub(crate) timestamp_rcvd: i64,
     pub(crate) ephemeral_timer: EphemeralTimer,
     pub(crate) ephemeral_timestamp: i64,
+
+    /// Message text, possibly truncated if the message is large.
     pub(crate) text: String,
+
+    /// Full text if the message text is truncated.
+    pub(crate) full_text: Option<String>,
+
     /// Text that is added to the end of Message.text
     ///
     /// Currently used for adding the download information on pre-messages
@@ -556,6 +562,7 @@ impl Message {
                         }
                         _ => String::new(),
                     };
+
                     let msg = Message {
                         id: row.get("id")?,
                         rfc724_mid: row.get::<_, String>("rfc724mid")?,
@@ -580,6 +587,7 @@ impl Message {
                         original_msg_id: row.get("original_msg_id")?,
                         mime_modified: row.get("mime_modified")?,
                         text,
+                        full_text: None,
                         additional_text: String::new(),
                         subject: row.get("subject")?,
                         param: row.get::<_, String>("param")?.parse().unwrap_or_default(),
@@ -597,6 +605,15 @@ impl Message {
             .with_context(|| format!("failed to load message {id} from the database"))?;
 
         if let Some(msg) = &mut msg {
+            if !msg.mime_modified {
+                let (truncated_text, was_truncated) =
+                    truncate_msg_text(context, msg.text.clone()).await?;
+                if was_truncated {
+                    msg.full_text = Some(msg.text.clone());
+                    msg.text = truncated_text;
+                }
+            }
+
             msg.additional_text =
                 Self::get_additional_text(context, msg.download_state, &msg.param).await?;
         }
@@ -2378,5 +2395,22 @@ impl Viewtype {
     }
 }
 
+/// Opt out of truncating long messages.
+///
+/// After calling this function, long messages
+/// will not be truncated during loading.
+///
+/// UIs should call this function if they
+/// can handle long messages by cutting them
+/// and displaying "Show full message" option.
+///
+/// Has no effect for bots which never
+/// truncate messages when loading.
+pub fn dont_truncate_long_messages(context: &Context) {
+    context
+        .truncate_long_messages
+        .store(false, Ordering::Relaxed);
+}
+
 #[cfg(test)]
 mod message_tests;

src/mimeparser.rs

@@ -32,9 +32,7 @@ use crate::message::{self, Message, MsgId, Viewtype, get_vcard_summary, set_msg_
 use crate::param::{Param, Params};
 use crate::simplify::{SimplifiedText, simplify};
 use crate::sync::SyncItems;
-use crate::tools::{
-    get_filemeta, parse_receive_headers, smeared_time, time, truncate_msg_text, validate_id,
-};
+use crate::tools::{get_filemeta, parse_receive_headers, smeared_time, time, validate_id};
 use crate::{chatlist_events, location, tools};
 
 /// Public key extracted from `Autocrypt-Gossip`
@@ -1445,12 +1443,6 @@ impl MimeMessage {
                             (simplified_txt, top_quote)
                         };
 
-                        let (simplified_txt, was_truncated) =
-                            truncate_msg_text(context, simplified_txt).await?;
-                        if was_truncated {
-                            self.is_mime_modified = was_truncated;
-                        }
-
                         if !simplified_txt.is_empty() || simplified_quote.is_some() {
                             let mut part = Part {
                                 dehtml_failed,

src/mimeparser/mimeparser_tests.rs

@@ -1283,12 +1283,12 @@ async fn test_mime_modified_large_plain() -> Result<()> {
 
     {
         let mimemsg = MimeMessage::from_bytes(&t, long_txt.as_ref()).await?;
-        assert!(mimemsg.is_mime_modified);
-        assert!(
-            mimemsg.parts[0].msg.matches("just repeated").count()
-                <= DC_DESIRED_TEXT_LEN / REPEAT_TXT.len()
+        assert!(!mimemsg.is_mime_modified);
+        assert!(mimemsg.parts[0].msg.matches("just repeated").count() == REPEAT_CNT);
+        assert_eq!(
+            mimemsg.parts[0].msg.len() + 1,
+            REPEAT_TXT.len() * REPEAT_CNT
         );
-        assert!(mimemsg.parts[0].msg.len() <= DC_DESIRED_TEXT_LEN + DC_ELLIPSIS.len());
     }
 
     for draft in [false, true] {

src/receive_imf/receive_imf_tests.rs

@@ -3583,39 +3583,17 @@ async fn test_big_forwarded_with_big_attachment() -> Result<()> {
             .starts_with("this text with 42 chars is just repeated.")
     );
     assert!(msg.get_text().ends_with("[...]"));
-    assert!(!msg.has_html());
-
-    let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?;
-    assert_eq!(msg.get_viewtype(), Viewtype::File);
     assert!(msg.has_html());
     let html = msg.id.get_html(t).await?.unwrap();
-    let tail = html
-        .split_once("Hello!")
-        .unwrap()
-        .1
-        .split_once("From: AAA")
-        .unwrap()
-        .1
-        .split_once("aaa@example.org")
-        .unwrap()
-        .1
-        .split_once("To: Alice")
-        .unwrap()
-        .1
-        .split_once("alice@example.org")
-        .unwrap()
-        .1
-        .split_once("Subject: Some subject")
-        .unwrap()
-        .1
-        .split_once("Date: Fri, 2 Jun 2023 12:29:17 +0000")
-        .unwrap()
-        .1;
     assert_eq!(
-        tail.matches("this text with 42 chars is just repeated.")
+        html.matches("this text with 42 chars is just repeated.")
             .count(),
         128
     );
+
+    let msg = Message::load_from_db(t, rcvd.msg_ids[2]).await?;
+    assert_eq!(msg.get_viewtype(), Viewtype::File);
+    assert!(!msg.has_html());
     Ok(())
 }
 

src/tools.rs

@@ -9,6 +9,7 @@ use std::mem;
 use std::ops::{AddAssign, Deref};
 use std::path::{Path, PathBuf};
 use std::str::from_utf8;
+use std::sync::atomic::Ordering;
 // If a time value doesn't need to be sent to another host, saved to the db or otherwise used across
 // program restarts, a monotonically nondecreasing clock (`Instant`) should be used. But as
 // `Instant` may use `libc::clock_gettime(CLOCK_MONOTONIC)`, e.g. on Android, and does not advance
@@ -137,7 +138,9 @@ pub(crate) fn truncate_by_lines(
 ///
 /// Returns the resulting text and a bool telling whether a truncation was done.
 pub(crate) async fn truncate_msg_text(context: &Context, text: String) -> Result<(String, bool)> {
-    if context.get_config_bool(Config::Bot).await? {
+    if !context.truncate_long_messages.load(Ordering::Relaxed)
+        || context.get_config_bool(Config::Bot).await?
+    {
         return Ok((text, false));
     }
     // Truncate text if it has too many lines