fix: keep whitespace inside fenced code blocks when normalizing results

packages/markitdown/src/markitdown/_markitdown.py

@@ -52,6 +52,186 @@
     FailedConversionAttempt,
 )
 
+# A fence may follow block-quote markers. Its indentation is measured inside
+# the container, rather than from the start of the Markdown line.
+_CONTAINER_PREFIX_RE = re.compile(r"^(?P<quotes>(?: {0,3}>[ \t]?)*)(?P<rest>.*)$")
+_CODE_FENCE_RE = re.compile(
+    r"^(?P<quotes>(?: {0,3}>[ \t]?)*)(?P<indent> *)(?P<marker>`{3,}|~{3,})"
+)
+_LIST_ITEM_RE = re.compile(
+    r"^(?P<indent> *)(?P<marker>[-+*]|\d{1,9}[.)])(?P<space> {1,4}|\t)"
+)
+
+
+@dataclass(frozen=True)
+class _FenceMarker:
+    marker: str
+    end: int
+    quote_depth: int
+
+
+def _is_code_fence(
+    line: str, list_indent: Optional[int] = None, *, allow_list_marker: bool = False
+) -> Optional[_FenceMarker]:
+    """Match a fence with at most three spaces inside its container."""
+    container = _CONTAINER_PREFIX_RE.match(line)
+    assert container is not None
+    quote_depth = container.group("quotes").count(">")
+    rest = container.group("rest")
+    candidates = [rest]
+    if list_indent is not None and rest.startswith(" " * list_indent):
+        candidates.append(rest[list_indent:])
+    if allow_list_marker and list_indent is not None:
+        list_item = _LIST_ITEM_RE.match(rest)
+        if list_item is not None and list_item.end() == list_indent:
+            # A fenced block may be the first block of a list item, directly
+            # following its marker. A closing fence cannot carry that marker.
+            candidates.append(rest[list_item.end() :])
+
+    for candidate in candidates:
+        match = _CODE_FENCE_RE.match(candidate)
+        if match is None or len(match.group("indent")) > 3:
+            continue
+        return _FenceMarker(
+            marker=match.group("marker"),
+            end=len(line) - len(candidate) + match.end(),
+            quote_depth=quote_depth + match.group("quotes").count(">"),
+        )
+    return None
+
+
+def _closes_fence(
+    line: str,
+    fence_char: str,
+    fence_len: int,
+    quote_depth: int,
+    list_indent: Optional[int],
+) -> bool:
+    """Whether a line inside a fenced code block closes the fence."""
+    match = _is_code_fence(line, list_indent)
+    if match is None or match.quote_depth != quote_depth:
+        return False
+    marker = match.marker
+    # The closing fence uses the same character, is at least as long, and
+    # carries no info string.
+    return (
+        marker[0] == fence_char
+        and len(marker) >= fence_len
+        and not line[match.end :].strip()
+    )
+
+
+def _normalize_whitespace_outside_code_fences(text: str) -> str:
+    """Normalize whitespace while leaving fenced code blocks untouched.
+
+    Outside code fences, the previous normalization is applied: trailing
+    whitespace is stripped from each line and runs of three or more newlines
+    are collapsed to two. Content inside a fenced code block is
+    whitespace-significant, so it is passed through verbatim apart from
+    CRLF newline normalization.
+    """
+    lines = re.split(r"\r?\n", text)
+    out: list[str] = []
+    segment: list[str] = []
+    fence_char = ""
+    fence_len = 0
+    fence_quote_depth = 0
+    fence_list_indent: Optional[int] = None
+    list_contexts: list[tuple[int, int]] = []  # (quote depth, content indentation)
+
+    def flush_segment(fence_follows: bool, nested_fence: bool = False) -> None:
+        if segment:
+            normalized = re.sub(
+                r"\n{3,}", "\n\n", "\n".join(line.rstrip() for line in segment)
+            )
+            if fence_follows:
+                # The final "\n".join(out) adds one newline before the fence
+                # line. Preserve an existing blank line inside a container,
+                # but do not insert one where the source had none.
+                keep_blank_line = nested_fence and normalized.endswith("\n")
+                normalized = normalized.rstrip("\n")
+                if keep_blank_line:
+                    normalized += "\n"
+            out.append(normalized)
+            segment.clear()
+
+    for line in lines:
+        container = _CONTAINER_PREFIX_RE.match(line)
+        assert container is not None
+        quote_depth = container.group("quotes").count(">")
+        rest = container.group("rest")
+        indent = len(rest) - len(rest.lstrip(" "))
+
+        if fence_char:
+            # A fence also ends when its block quote or list item ends.
+            nested_quote_depth = quote_depth
+            if fence_list_indent is not None and rest.startswith(
+                " " * fence_list_indent
+            ):
+                nested = _CONTAINER_PREFIX_RE.match(rest[fence_list_indent:])
+                assert nested is not None
+                nested_quote_depth += nested.group("quotes").count(">")
+            if nested_quote_depth < fence_quote_depth or (
+                fence_list_indent is not None
+                and rest.strip()
+                and indent < fence_list_indent
+            ):
+                fence_char = ""
+            else:
+                out.append(line)
+                if _closes_fence(
+                    line, fence_char, fence_len, fence_quote_depth, fence_list_indent
+                ):
+                    fence_char = ""
+                continue
+
+        if rest.strip():
+            while list_contexts and (
+                quote_depth < list_contexts[-1][0]
+                or (
+                    quote_depth == list_contexts[-1][0]
+                    and indent < list_contexts[-1][1]
+                )
+            ):
+                list_contexts.pop()
+            list_item = _LIST_ITEM_RE.match(rest)
+            if list_item is not None and (list_contexts or indent <= 3):
+                content_indent = sum(
+                    len(list_item.group(name)) for name in ("indent", "marker", "space")
+                )
+                list_contexts.append((quote_depth, content_indent))
+
+        active_list_indent = next(
+            (
+                content_indent
+                for depth, content_indent in reversed(list_contexts)
+                if depth == quote_depth
+            ),
+            None,
+        )
+        match = _is_code_fence(line, active_list_indent, allow_list_marker=True)
+        if match is not None:
+            flushed_prose = bool(segment)
+            nested_fence = match.quote_depth > 0 or active_list_indent is not None
+            flush_segment(fence_follows=True, nested_fence=nested_fence)
+            # Keep the existing top-level separator behavior. Inside a quote
+            # or list, an extra blank line would change the container layout.
+            if flushed_prose and not nested_fence:
+                out.append("")
+            # An opening fence may carry an info string (e.g. ```python).
+            marker = match.marker
+            fence_char = marker[0]
+            fence_len = len(marker)
+            fence_quote_depth = match.quote_depth
+            fence_list_indent = active_list_indent
+            out.append(line.rstrip())
+            continue
+
+        segment.append(line)
+
+    flush_segment(fence_follows=False)
+    return "\n".join(out)
+
 
 def _get_content_disposition_filename(content_disposition: str) -> Optional[str]:
     message = Message()
@@ -679,10 +859,9 @@ def _convert(
 
                 if res is not None:
                     # Normalize the content
-                    res.text_content = "\n".join(
-                        [line.rstrip() for line in re.split(r"\r?\n", res.text_content)]
+                    res.text_content = _normalize_whitespace_outside_code_fences(
+                        res.text_content
                     )
-                    res.text_content = re.sub(r"\n{3,}", "\n\n", res.text_content)
                     return res
 
         # If we got this far without success, report any exceptions

packages/markitdown/tests/test_code_fence_whitespace.py

@@ -0,0 +1,103 @@
+"""Regression test: whitespace inside fenced code blocks must survive conversion.
+
+MarkItDown._convert used to rstrip every line and collapse runs of 3+ newlines
+across the whole result, which corrupts code blocks: two consecutive blank
+lines inside a fence collapsed to one, and trailing spaces were stripped.
+"""
+
+import io
+
+from markitdown import MarkItDown, StreamInfo
+
+
+def _convert_markdown(content: bytes) -> str:
+    return (
+        MarkItDown()
+        .convert(
+            io.BytesIO(content),
+            stream_info=StreamInfo(mimetype="text/markdown", extension=".md"),
+        )
+        .markdown
+    )
+
+
+def test_blank_line_runs_inside_code_fence_are_preserved() -> None:
+    content = b"# Title\n\n```python\ndef a():\n    pass\n\n\ndef b():\n    pass\n```\n\nEnd\n"
+    result = _convert_markdown(content)
+    assert "pass\n\n\ndef b" in result
+
+
+def test_trailing_spaces_inside_code_fence_are_preserved() -> None:
+    content = b"```text\nline with trailing spaces   \n```\n"
+    result = _convert_markdown(content)
+    assert "line with trailing spaces   \n" in result
+
+
+def test_normalization_still_applies_outside_code_fence() -> None:
+    content = b"para   \n\n\n\nother\n"
+    result = _convert_markdown(content)
+    assert result == "para\n\nother\n"
+
+
+def test_tilde_fence_content_is_preserved() -> None:
+    # An info string ("text") keeps the sample away from charset misdetection
+    # of bare "~~~" lines (charset_normalizer reads them as shift_jis_2004).
+    content = b"~~~text\n\n\nkept\n~~~\n"
+    result = _convert_markdown(content)
+    assert "~~~text\n\n\nkept\n~~~" in result
+
+
+def test_block_quote_fence_preserves_code_whitespace() -> None:
+    content = b"> ```text\n> first  \n> \n> \n> second\n> ```\n"
+    assert _convert_markdown(content) == content.decode()
+
+
+def test_list_fence_preserves_code_whitespace() -> None:
+    content = b"- item\n\n    ```text\n    first  \n    \n    \n    second\n    ```\n"
+    assert _convert_markdown(content) == content.decode()
+
+
+def test_block_quote_fence_ends_with_its_container() -> None:
+    content = b"> ```text\n> code  \noutside  \n"
+    assert _convert_markdown(content) == "> ```text\n> code  \noutside\n"
+
+
+def test_list_fence_ends_with_its_container() -> None:
+    content = b"- item\n\n    ```text\n    code  \noutside  \n"
+    assert _convert_markdown(content) == "- item\n\n    ```text\n    code  \noutside\n"
+
+
+def test_parent_list_fence_after_nested_list_preserves_whitespace() -> None:
+    content = b"- outer\n  - inner\n\n  after\n\n    ```text\n    first  \n    ```\n"
+    assert _convert_markdown(content) == content.decode()
+
+
+def test_block_quote_inside_list_fence_preserves_whitespace() -> None:
+    content = b"- item\n\n    > ```text\n    > first  \n    > ```\n"
+    assert _convert_markdown(content) == content.decode()
+
+
+def test_block_quote_fence_adjacent_to_quote_text_keeps_layout() -> None:
+    content = b"> intro\n> ```text\n> code  \n> ```\n"
+    assert _convert_markdown(content) == content.decode()
+
+
+def test_list_fence_adjacent_to_item_text_keeps_layout() -> None:
+    content = b"- item\n  ```text\n  code  \n  ```\n"
+    assert _convert_markdown(content) == content.decode()
+
+
+def test_indented_literal_fence_does_not_hide_following_prose() -> None:
+    content = b"    ```\noutside  \n"
+    assert _convert_markdown(content) == "    ```\noutside\n"
+
+
+def test_blank_line_run_before_fence_collapses_to_one_blank_line() -> None:
+    """A blank-line run immediately before a fence must collapse like prose.
+
+    flush_segment used to keep the trailing newlines and the final join added
+    one more, so "para\n\n\n```" kept three newlines instead of two.
+    """
+    content = b"para\n\n\n```python\nx = 1\n```\n"
+    result = _convert_markdown(content)
+    assert "para\n\n```python" in result

packages/markitdown/tests/test_list_marker_code_fences.py

@@ -0,0 +1,41 @@
+"""Fences may open on the same line as a Markdown list marker."""
+
+import io
+
+import pytest
+
+from markitdown import MarkItDown, StreamInfo
+
+
+@pytest.mark.parametrize(
+    "marker,indent", [("- ", "  "), ("1. ", "   "), ("> - ", ">   ")]
+)
+@pytest.mark.parametrize("fence", ["```", "~~~"])
+def test_fence_after_list_marker_preserves_whitespace(marker, indent, fence):
+    content = f"{marker}{fence}text\n{indent}first  \n{indent}\n{indent}\n{indent}second\n{indent}{fence}\noutside  \n"
+    result = (
+        MarkItDown()
+        .convert(
+            io.BytesIO(content.encode()),
+            stream_info=StreamInfo(
+                mimetype="text/markdown", extension=".md", charset="utf-8"
+            ),
+        )
+        .markdown
+    )
+    assert result == content.replace("outside  \n", "outside\n")
+
+
+def test_list_marker_inside_fence_does_not_close_fence():
+    content = "```text\n- ```\nkept  \n\n\nlast\n```\n"
+    result = (
+        MarkItDown()
+        .convert(
+            io.BytesIO(content.encode()),
+            stream_info=StreamInfo(
+                mimetype="text/markdown", extension=".md", charset="utf-8"
+            ),
+        )
+        .markdown
+    )
+    assert result == content