Added work-around for Yandex IMAP GetBodyPart() response not includin…

…g content

Fixes issue #1708

MailKit/Net/Imap/ImapFolderFetch.cs

@@ -2088,6 +2088,14 @@ void FetchStream (ImapEngine engine, ImapCommand ic, int index)
 						stream = CreateStream (uid, name, offset, 0);
 						length = 0;
 						break;
+					case ImapTokenType.CloseParen:
+						// Note: Yandex IMAP servers sometimes do not include the BODY[<section>] content value in the FETCH response.
+						//
+						// See https://github.com/jstedfast/MailKit/issues/1708 for details.
+						//
+						// Unget the ')' token and pretend we got a NIL token.
+						engine.Stream.UngetToken (token);
+						goto case ImapTokenType.Nil;
 					default:
 						throw ImapEngine.UnexpectedToken (ImapEngine.GenericItemSyntaxErrorFormat, atom, token);
 					}
@@ -2338,6 +2346,14 @@ async Task FetchStreamAsync (ImapEngine engine, ImapCommand ic, int index)
 						stream = CreateStream (uid, name, offset, 0);
 						length = 0;
 						break;
+					case ImapTokenType.CloseParen:
+						// Note: Yandex IMAP servers sometimes do not include the BODY[<section>] content value in the FETCH response.
+						//
+						// See https://github.com/jstedfast/MailKit/issues/1708 for details.
+						//
+						// Unget the ')' token and pretend we got a NIL token.
+						engine.Stream.UngetToken (token);
+						goto case ImapTokenType.Nil;
 					default:
 						throw ImapEngine.UnexpectedToken (ImapEngine.GenericItemSyntaxErrorFormat, atom, token);
 					}

UnitTests/Net/Imap/ImapFolderFetchTests.cs

@@ -2473,6 +2473,123 @@ public async Task TestFetchNegativeModSeqResponseValuesAsync ()
 			}
 		}
 
+		static IList<ImapReplayCommand> CreateYandexGetBodyPartMissingContentCommands ()
+		{
+			return new List<ImapReplayCommand> {
+				new ImapReplayCommand ("", "yandex.greeting.txt"),
+				new ImapReplayCommand ("A00000000 CAPABILITY\r\n", "yandex.capability.txt"),
+				new ImapReplayCommand ("A00000001 AUTHENTICATE PLAIN\r\n", ImapReplayCommandResponse.Plus),
+				new ImapReplayCommand ("A00000001", "AHVzZXJuYW1lAHBhc3N3b3Jk\r\n", "yandex.authenticate.txt"),
+				new ImapReplayCommand ("A00000002 NAMESPACE\r\n", "yandex.namespace.txt"),
+				new ImapReplayCommand ("A00000003 LIST \"\" \"INBOX\"\r\n", "yandex.list-inbox.txt"),
+				new ImapReplayCommand ("A00000004 XLIST \"\" \"*\"\r\n", "yandex.xlist.txt"),
+				new ImapReplayCommand ("A00000005 SELECT INBOX\r\n", "yandex.select-inbox.txt"),
+				new ImapReplayCommand ("A00000006 UID FETCH 3016 (BODY.PEEK[2.MIME] BODY.PEEK[2])\r\n", "yandex.getbodypart-missing-content.txt")
+			};
+		}
+
+		[Test]
+		public void TestYandexGetBodyPartMissingContent ()
+		{
+			// IMAP4rev1 CHILDREN UNSELECT LITERAL+ NAMESPACE XLIST UIDPLUS ENABLE ID AUTH=PLAIN AUTH=XOAUTH2 IDLE MOVE
+			const ImapCapabilities YandexGreetingCapabilities = ImapCapabilities.IMAP4rev1 | ImapCapabilities.Children | ImapCapabilities.Unselect |
+				ImapCapabilities.LiteralPlus | ImapCapabilities.Namespace | ImapCapabilities.XList | ImapCapabilities.UidPlus | ImapCapabilities.Enable |
+				ImapCapabilities.Id | ImapCapabilities.Idle | ImapCapabilities.Move | ImapCapabilities.Status;
+			var commands = CreateYandexGetBodyPartMissingContentCommands ();
+
+			using (var client = new ImapClient () { TagPrefix = 'A' }) {
+				try {
+					client.Connect (new ImapReplayStream (commands, false), "localhost", 143, SecureSocketOptions.None);
+				} catch (Exception ex) {
+					Assert.Fail ($"Did not expect an exception in Connect: {ex}");
+				}
+
+				Assert.That (client.Capabilities, Is.EqualTo (YandexGreetingCapabilities), "Greeting Capabilities");
+				Assert.That (client.AuthenticationMechanisms, Has.Count.EqualTo (2));
+				Assert.That (client.AuthenticationMechanisms, Does.Contain ("PLAIN"), "Expected SASL PLAIN auth mechanism");
+				Assert.That (client.AuthenticationMechanisms, Does.Contain ("XOAUTH2"), "Expected SASL XOAUTH2 auth mechanism");
+
+				try {
+					client.Authenticate ("username", "password");
+				} catch (Exception ex) {
+					Assert.Fail ($"Did not expect an exception in Authenticate: {ex}");
+				}
+
+				Assert.That (client.Capabilities, Is.EqualTo (YandexGreetingCapabilities), "Greeting Capabilities");
+
+				client.Inbox.Open (FolderAccess.ReadWrite);
+
+				//var messages = client.Inbox.Fetch (0, -1, MessageSummaryItems.UniqueId | MessageSummaryItems.Flags | MessageSummaryItems.ModSeq);
+				//Assert.That (messages, Has.Count.EqualTo (74), "Count");
+
+				var bodyPart = new BodyPartBasic {
+					PartSpecifier = "2",
+				};
+
+				var body = client.Inbox.GetBodyPart (new UniqueId (3016), bodyPart);
+				Assert.That (body, Is.Not.Null);
+				Assert.That (body, Is.InstanceOf<MimePart> ());
+				var part = (MimePart) body;
+				Assert.That (part.ContentType.MimeType, Is.EqualTo ("application/pdf"), "Content-Type");
+				Assert.That (part.ContentType.Name, Is.EqualTo ("empty.pdf"), "name");
+				Assert.That (part.ContentDisposition.Disposition, Is.EqualTo (ContentDisposition.Attachment), "Content-Disposition");
+				Assert.That (part.ContentDisposition.FileName, Is.EqualTo ("empty.pdf"), "filename");
+				Assert.That (part.ContentTransferEncoding, Is.EqualTo (ContentEncoding.Base64), "Content-Transfer-Encoding");
+				Assert.That (part.Content, Is.Null);
+			}
+		}
+
+		[Test]
+		public async Task TestYandexGetBodyPartMissingContentAsync ()
+		{
+			// IMAP4rev1 CHILDREN UNSELECT LITERAL+ NAMESPACE XLIST UIDPLUS ENABLE ID AUTH=PLAIN AUTH=XOAUTH2 IDLE MOVE
+			const ImapCapabilities YandexGreetingCapabilities = ImapCapabilities.IMAP4rev1 | ImapCapabilities.Children | ImapCapabilities.Unselect |
+				ImapCapabilities.LiteralPlus | ImapCapabilities.Namespace | ImapCapabilities.XList | ImapCapabilities.UidPlus | ImapCapabilities.Enable |
+				ImapCapabilities.Id | ImapCapabilities.Idle | ImapCapabilities.Move | ImapCapabilities.Status;
+			var commands = CreateYandexGetBodyPartMissingContentCommands ();
+
+			using (var client = new ImapClient () { TagPrefix = 'A' }) {
+				try {
+					await client.ConnectAsync (new ImapReplayStream (commands, true), "localhost", 143, SecureSocketOptions.None);
+				} catch (Exception ex) {
+					Assert.Fail ($"Did not expect an exception in Connect: {ex}");
+				}
+
+				Assert.That (client.Capabilities, Is.EqualTo (YandexGreetingCapabilities), "Greeting Capabilities");
+				Assert.That (client.AuthenticationMechanisms, Has.Count.EqualTo (2));
+				Assert.That (client.AuthenticationMechanisms, Does.Contain ("PLAIN"), "Expected SASL PLAIN auth mechanism");
+				Assert.That (client.AuthenticationMechanisms, Does.Contain ("XOAUTH2"), "Expected SASL XOAUTH2 auth mechanism");
+
+				try {
+					await client.AuthenticateAsync ("username", "password");
+				} catch (Exception ex) {
+					Assert.Fail ($"Did not expect an exception in Authenticate: {ex}");
+				}
+
+				Assert.That (client.Capabilities, Is.EqualTo (YandexGreetingCapabilities), "Greeting Capabilities");
+
+				await client.Inbox.OpenAsync (FolderAccess.ReadWrite);
+
+				//var messages = client.Inbox.Fetch (0, -1, MessageSummaryItems.UniqueId | MessageSummaryItems.Flags | MessageSummaryItems.ModSeq);
+				//Assert.That (messages, Has.Count.EqualTo (74), "Count");
+
+				var bodyPart = new BodyPartBasic {
+					PartSpecifier = "2",
+				};
+
+				var body = await client.Inbox.GetBodyPartAsync (new UniqueId (3016), bodyPart);
+				Assert.That (body, Is.Not.Null);
+				Assert.That (body, Is.InstanceOf<MimePart> ());
+				var part = (MimePart) body;
+				Assert.That (part.ContentType.MimeType, Is.EqualTo ("application/pdf"), "Content-Type");
+				Assert.That (part.ContentType.Name, Is.EqualTo ("empty.pdf"), "name");
+				Assert.That (part.ContentDisposition.Disposition, Is.EqualTo (ContentDisposition.Attachment), "Content-Disposition");
+				Assert.That (part.ContentDisposition.FileName, Is.EqualTo ("empty.pdf"), "filename");
+				Assert.That (part.ContentTransferEncoding, Is.EqualTo (ContentEncoding.Base64), "Content-Transfer-Encoding");
+				Assert.That (part.Content, Is.Null);
+			}
+		}
+
 		[TestCase ("ALL", MessageSummaryItems.All)]
 		[TestCase ("FAST", MessageSummaryItems.Fast)]
 		[TestCase ("FULL", MessageSummaryItems.Full)]

UnitTests/Net/Imap/Resources/yandex/authenticate.txt

@@ -0,0 +1,2 @@
+* CAPABILITY IMAP4rev1 CHILDREN UNSELECT LITERAL+ NAMESPACE XLIST UIDPLUS ENABLE ID IDLE MOVE
+A######## OK AUTHENTICATE Completed.

UnitTests/Net/Imap/Resources/yandex/capability.txt

@@ -0,0 +1,2 @@
+* CAPABILITY IMAP4rev1 CHILDREN UNSELECT LITERAL+ NAMESPACE XLIST UIDPLUS ENABLE ID AUTH=PLAIN AUTH=XOAUTH2 IDLE MOVE
+A######## OK CAPABILITY Completed.

UnitTests/Net/Imap/Resources/yandex/getbodypart-missing-content.txt

@@ -0,0 +1,9 @@
+* 1 FETCH (UID 3016 BODY[2.MIME] {145}
+Content-Disposition: attachment;
+	filename="empty.pdf"
+Content-Transfer-Encoding: base64
+Content-Type: application/pdf;
+	name="empty.pdf"
+
+ BODY[2] )
+A######## OK FETCH Completed.

UnitTests/Net/Imap/Resources/yandex/greeting.txt

@@ -0,0 +1 @@
+* OK Yandex IMAP4rev1 at mail-imap-production-171.sas.yp-c.yandex.net:993 ready to talk with ::ffff:91.77.169.5:50580, 2024-Feb-13 08:17:42, gHfsEJKZ3Gk0

UnitTests/Net/Imap/Resources/yandex/list-inbox.txt

@@ -0,0 +1,2 @@
+* LIST (\HasNoChildren \Marked \NoInferiors) "|" INBOX
+A######## OK LIST Completed.

UnitTests/Net/Imap/Resources/yandex/namespace.txt

@@ -0,0 +1,2 @@
+* NAMESPACE (("" "|")) NIL NIL
+A######## OK NAMESPACE Completed.

UnitTests/Net/Imap/Resources/yandex/select-inbox.txt

@@ -0,0 +1,8 @@
+* FLAGS (\Answered \Seen \Draft \Deleted $Forwarded)
+* 49 EXISTS
+* 1 RECENT
+* OK [UNSEEN 8]
+* OK [PERMANENTFLAGS (\Answered \Seen \Draft \Flagged \Deleted $Forwarded \*)] Limited
+* OK [UIDNEXT 3065] Ok
+* OK [UIDVALIDITY 1679853140] Ok
+A######## OK [READ-WRITE] SELECT Completed.

UnitTests/Net/Imap/Resources/yandex/xlist.txt

@@ -0,0 +1,8 @@
+* XLIST (\HasChildren \Unmarked \Drafts \Drafts) "|" Drafts
+* XLIST (\HasNoChildren \Unmarked \Templates) "|" "Drafts|template"
+* XLIST (\HasNoChildren \Marked \NoInferiors \Inbox) "|" INBOX
+* XLIST (\HasNoChildren \Unmarked) "|" Outbox
+* XLIST (\HasNoChildren \Unmarked \Sent \Sent) "|" Sent
+* XLIST (\HasNoChildren \Unmarked \Junk \Spam) "|" Spam
+* XLIST (\HasNoChildren \Unmarked \Trash \Trash) "|" Trash
+A######## OK XLIST Completed.