Replace illegal multibyte sequences when encoding emails

martinburchell · martinburchell · commit dc92a17bdec3 · 2025-05-13T10:04:11.000+01:00
diff --git a/cardinal_pythonlib/extract_text.py b/cardinal_pythonlib/extract_text.py
@@ -1299,7 +1299,7 @@ def _get_email_content(
             content_type_header = message.get("Content-Type")
             if content_type_header:
                 charset = content_type_header.params.get("charset", "utf-8")
-            blob = content.encode(charset)
+            blob = content.encode(charset, "replace")
         elif isinstance(content, EmailMessage):
             blob = content.as_bytes()
             if message.get("Content-Transfer-Encoding") == "base64":
diff --git a/cardinal_pythonlib/tests/extract_text_tests.py b/cardinal_pythonlib/tests/extract_text_tests.py
@@ -502,6 +502,35 @@ def test_eml_with_empty_body_converted(self) -> None:
 
         self.assertEqual("", text)
 
+    def test_eml_with_illegal_multibyte_sequence_replaced(self) -> None:
+        content = """From: bar@example.org
+Subject: Illegal multibyte sequence
+To: foo@example.org
+Mime-Version: 1.0
+Content-Type: multipart/mixed;boundary="==="
+
+--===
+Content-Type: text/html; charset="big5"
+Content-Transfer-Encoding: quoted-printable
+
+<html><head>
+<meta http-equiv=3D"Content-Type" content=3D"text/html; charset=3Dbig5">
+</head>
+<body>
+=F9=F9
+</body>
+</html>
+--===--
+"""
+        message = message_from_string(content, policy=policy.default)
+        blob = message.as_bytes()
+
+        text = document_to_text(
+            blob=blob, extension=".eml", config=self.config
+        )
+
+        self.assertEqual(text.strip(), "??")
+
     def test_unsupported_converted(self) -> None:
         with mock.patch.multiple(
             "cardinal_pythonlib.extract_text.subprocess",
