feat: improve email date parsing for imap ingestion

packages/backend/src/api/controllers/storage.controller.ts

@@ -40,7 +40,8 @@ export class StorageController {
 
 			const fileStream = await this.storageService.get(safePath);
 			const fileName = path.basename(safePath);
-			res.setHeader('Content-Disposition', `attachment; filename="${fileName}"`);
+			const encodedFileName = encodeURIComponent(fileName);
+			res.setHeader('Content-Disposition', `attachment; filename*=UTF-8''${encodedFileName}`);
 			fileStream.pipe(res);
 		} catch (error) {
 			console.error('Error downloading file:', error);

packages/backend/src/services/ingestion-connectors/ImapConnector.ts

@@ -6,10 +6,10 @@ import type {
 	MailboxUser,
 } from '@open-archiver/types';
 import type { IEmailConnector } from '../EmailProviderFactory';
-import { ImapFlow } from 'imapflow';
+import { FetchMessageObject, ImapFlow } from 'imapflow';
 import { simpleParser, ParsedMail, Attachment, AddressObject, Headers } from 'mailparser';
 import { logger } from '../../config/logger';
-import { getThreadId } from './helpers/utils';
+import { getMailDate, getThreadId } from './helpers/utils';
 
 export class ImapConnector implements IEmailConnector {
 	private client: ImapFlow;
@@ -188,6 +188,7 @@ export class ImapConnector implements IEmailConnector {
 					);
 					const lastUid = syncState?.imap?.[mailboxPath]?.maxUid;
 					let currentMaxUid = lastUid || 0;
+					let minUid = 1;
 
 					if (mailbox.exists > 0) {
 						const lastMessage = await this.client.fetchOne(String(mailbox.exists), {
@@ -196,6 +197,11 @@ export class ImapConnector implements IEmailConnector {
 						if (lastMessage && lastMessage.uid > currentMaxUid) {
 							currentMaxUid = lastMessage.uid;
 						}
+
+						const firstMessage = await this.client.fetchOne('1', { uid: true });
+						if (firstMessage) {
+							minUid = firstMessage.uid;
+						}
 					}
 
 					// Initialize with last synced UID, not the maximum UID in mailbox
@@ -204,7 +210,7 @@ export class ImapConnector implements IEmailConnector {
 					// Only fetch if the mailbox has messages, to avoid errors on empty mailboxes with some IMAP servers.
 					if (mailbox.exists > 0) {
 						const BATCH_SIZE = 250; // A configurable batch size
-						let startUid = (lastUid || 0) + 1;
+						let startUid = lastUid ? lastUid + 1 : minUid;
 						const maxUidToFetch = currentMaxUid;
 
 						while (startUid <= maxUidToFetch) {
@@ -215,6 +221,7 @@ export class ImapConnector implements IEmailConnector {
 								envelope: true,
 								source: true,
 								bodyStructure: true,
+								internalDate: true,
 								uid: true,
 							})) {
 								if (lastUid && msg.uid <= lastUid) {
@@ -258,8 +265,8 @@ export class ImapConnector implements IEmailConnector {
 		}
 	}
 
-	private async parseMessage(msg: any, mailboxPath: string): Promise<EmailObject> {
-		const parsedEmail: ParsedMail = await simpleParser(msg.source);
+	private async parseMessage(msg: FetchMessageObject, mailboxPath: string): Promise<EmailObject> {
+		const parsedEmail: ParsedMail = await simpleParser(msg.source!);
 		const attachments = parsedEmail.attachments.map((attachment: Attachment) => ({
 			filename: attachment.filename || 'untitled',
 			contentType: attachment.contentType,
@@ -291,7 +298,7 @@ export class ImapConnector implements IEmailConnector {
 			html: parsedEmail.html || '',
 			headers: parsedEmail.headers,
 			attachments,
-			receivedAt: parsedEmail.date || new Date(),
+			receivedAt: getMailDate(parsedEmail, msg),
 			eml: msg.source,
 			path: mailboxPath,
 		};

packages/backend/src/services/ingestion-connectors/helpers/utils.ts

@@ -1,4 +1,6 @@
-import type { Headers } from 'mailparser';
+import type { FetchMessageObject } from 'imapflow';
+import type { Headers, ParsedMail } from 'mailparser';
+import { logger } from '../../../config/logger';
 
 function getHeaderValue(header: any): string | undefined {
 	if (typeof header === 'string') {
@@ -52,3 +54,34 @@ export function getThreadId(headers: Headers): string | undefined {
 	console.warn('No thread ID found, returning undefined');
 	return undefined;
 }
+
+export function getMailDate(mail: ParsedMail, msg: FetchMessageObject): Date {
+	// First we try to get the date from the email headers.
+	const dateFromHeader = mail.headers.get('date');
+	const headerDate = getHeaderValue(dateFromHeader);
+
+	// Some emails might have an invalid date header that cannot be parsed by mailparser.
+	// (e.g. "Date: [date", "date: Wed, 10 Apr 2019 18:01:01 Asia/Shanghai")
+	// In that case, mail parser will fallback to current date, which is not what we want.
+	// See: https://github.com/nodemailer/mailparser/blob/v3.7.5/lib/mail-parser.js#L333
+	const isHeaderDateValid = headerDate && !isNaN(new Date(headerDate).getTime());
+
+	// So if the header date is valid, we use it. Otherwise we fallback to internalDate.
+	if (isHeaderDateValid && mail.date) {
+		return mail.date;
+	}
+
+	// INTERNALDATE: the date and time when the message was received by the server.
+	// See: https://datatracker.ietf.org/doc/html/rfc3501#section-2.3.3
+	const internalDate = msg.internalDate;
+
+	if (internalDate) {
+		const date = internalDate instanceof Date ? internalDate : new Date(internalDate);
+		if (!isNaN(date.getTime())) {
+			return date;
+		}
+	}
+
+	logger.warn({ mail, msg }, 'Email date is missing or invalid');
+	return new Date();
+}