fix(normalization): preserve blank-line runs and trailing spaces inside fenced code blocks (#2528)

packages/markitdown/src/markitdown/_markitdown.py

@@ -678,11 +678,33 @@ def _convert(
                         file_stream.seek(cur_pos)
 
                 if res is not None:
-                    # Normalize the content
-                    res.text_content = "\n".join(
-                        [line.rstrip() for line in re.split(r"\r?\n", res.text_content)]
+                    # Normalize the content while preserving whitespace and blank-line runs inside fenced code blocks
+                    fence_pattern = re.compile(
+                        r"(^[ \t]*(?P<fence>`{3,}|~{3,})[^\n]*\n[\s\S]*?\n[ \t]*(?P=fence)[ \t]*$)",
+                        re.MULTILINE,
                     )
-                    res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)
+                    normalized_parts = []
+                    last_end = 0
+                    for match in fence_pattern.finditer(res.text_content):
+                        start, end = match.span()
+                        outside = res.text_content[last_end:start]
+                        outside_norm = "\n".join(
+                            [line.rstrip() for line in re.split(r"\r?\n", outside)]
+                        )
+                        outside_norm = re.sub(r"\n{3,}", "\n\n", outside_norm)
+                        normalized_parts.append(outside_norm)
+                        # Inside fenced code blocks, normalize CRLF to LF without stripping spaces or blank lines
+                        normalized_parts.append(match.group(1).replace("\r\n", "\n"))
+                        last_end = end
+
+                    outside = res.text_content[last_end:]
+                    outside_norm = "\n".join(
+                        [line.rstrip() for line in re.split(r"\r?\n", outside)]
+                    )
+                    outside_norm = re.sub(r"\n{3,}", "\n\n", outside_norm)
+                    normalized_parts.append(outside_norm)
+
+                    res.text_content = "".join(normalized_parts)
                     return res
 
         # If we got this far without success, report any exceptions

packages/markitdown/tests/test_module_misc.py

@@ -876,5 +876,27 @@ def test_a_declared_charset_still_wins() -> None:
     assert "浦发银行" in result.markdown
 
 
+def test_fenced_code_block_preserves_whitespace_and_blank_lines() -> None:
+    code = (
+        "Here is text.\n\n"
+        "```python\n"
+        "def a():\n"
+        "    pass   \n"
+        "\n"
+        "\n"
+        "def b():\n"
+        "    pass\n"
+        "```\n\n"
+        "Trailing prose   \n"
+    )
+    markitdown = MarkItDown()
+    result = markitdown.convert_stream(io.BytesIO(code.encode("utf-8")), file_extension=".md")
+
+    # Inside the fence, trailing space and consecutive blank lines are preserved
+    assert "def a():\n    pass   \n\n\ndef b():" in result.markdown
+    # Outside the fence, trailing spaces are trimmed
+    assert "Trailing prose\n" in result.markdown
+
+
 if __name__ == "__main__":
     raise SystemExit(pytest.main([__file__]))