fix(kb): add_document chunks, partial-success ingest, stable upload rejection codes (#17830)

autobot-backend/api/kb_ingest_status_17530_test.py

@@ -0,0 +1,41 @@
+# Copyright 2025-2026 mrveiss
+# SPDX-License-Identifier: Apache-2.0
+# AutoBot - AI-Powered Automation Platform
+# Author: mrveiss
+"""``_store_fact_in_kb`` reports what ``add_document`` actually did (#17530)."""
+
+from unittest.mock import AsyncMock, MagicMock
+
+import pytest
+
+from api import knowledge as mod
+from knowledge.upload_rejections import UploadRejection, UploadRejectionCode
+
+
+def _kb(result):
+    return MagicMock(add_document=AsyncMock(return_value=result))
+
+
+@pytest.mark.asyncio
+async def test_partial_is_returned_with_its_chunk_count() -> None:
+    result = {"status": "partial", "fact_id": "d1", "chunks_stored": 3, "chunks_total": 9}
+    assert await mod._store_fact_in_kb(_kb(result), "t", {}) == "d1"
+    note = mod.partial_note(result)
+    assert (note["status"], note["chunks_stored"], note["chunks_total"]) == ("partial", 3, 9)
+    assert "3 of 9" in note["message"]
+    assert mod.partial_note({"status": "success"}) == {}
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "result,code,http",
+    [
+        ({"status": "error", "message": "boom"}, UploadRejectionCode.INGEST_FAILED, 500),
+        ({"status": "timeout"}, UploadRejectionCode.INGEST_TIMEOUT, 504),
+        ({"status": "success", "fact_id": None}, UploadRejectionCode.INGEST_FAILED, 500),
+    ],
+)
+async def test_error_and_timeout_are_rejected_not_reported_stored(result, code, http) -> None:
+    with pytest.raises(UploadRejection) as exc:
+        await mod._store_fact_in_kb(_kb(result), "t", {})
+    assert (exc.value.detail_code, exc.value.status_code) == (code.value, http)

autobot-backend/api/kb_ingestion_sites_system_16693_test.py

@@ -39,7 +39,7 @@ async def test_the_system_configuration_document_is_written_system():
     from api import knowledge_population as pop
 
     kb = MagicMock()
-    kb.store_fact = AsyncMock(return_value={"fact_id": "f1"})
+    kb.store_fact = AsyncMock(return_value={"status": "success", "fact_id": "f1"})
     with patch.object(pop, "_build_system_config_info", return_value="config"):
         assert await pop._store_system_config(kb)
     assert _is_system_document(kb.store_fact.await_args.kwargs["metadata"], "documentation")
@@ -107,17 +107,16 @@ async def test_an_admin_file_upload_is_written_system():
     req = MagicMock()
     req.form = AsyncMock(return_value={"file": upload, "title": "Guide"})
     kb = MagicMock()
-    kb.store_fact = AsyncMock(return_value={"fact_id": "f1"})
+    kb.add_document = AsyncMock(return_value={"status": "success", "fact_id": "f1"})
     with (
         patch.object(mod, "get_or_create_knowledge_base", AsyncMock(return_value=kb)),
         patch.object(mod, "_extract_file_content", return_value=("hello world", None)),
         patch.object(mod, "_has_usable_content", return_value=True),
-        patch.object(mod, "_page_provenance_metadata", return_value={}),
         patch.object(mod, "get_auth_middleware", MagicMock()),
         patch.object(mod, "audit_record", MagicMock()),
     ):
         await mod.upload_file_to_knowledge(admin_check=True, req=req)
-    assert _is_system_document(kb.store_fact.await_args.kwargs["metadata"], "file_upload")
+    assert _is_system_document(kb.add_document.await_args.args[1], "file_upload")
 
 
 @pytest.mark.asyncio
@@ -126,7 +125,7 @@ async def test_only_an_upload_is_stamped_by_the_shared_store_helper():
     from api import knowledge as mod
 
     kb = MagicMock()
-    kb.store_fact = AsyncMock(return_value={"fact_id": "f1"})
+    kb.add_document = AsyncMock(return_value={"status": "success", "fact_id": "f1"})
     for metadata in ({"source": "autobot_docs_population"}, {"type": "url", "source": "https://example.invalid"}):
         await mod._store_fact_in_kb(kb, "text", dict(metadata))
-        assert "visibility" not in kb.store_fact.await_args.kwargs["metadata"]
+        assert "visibility" not in kb.add_document.await_args.args[1]

autobot-backend/api/kb_upload_chunking_18407_test.py

@@ -0,0 +1,123 @@
+# Copyright 2025-2026 mrveiss
+# SPDX-License-Identifier: Apache-2.0
+# AutoBot - AI-Powered Automation Platform
+# Author: mrveiss
+"""Upload, /facts and /url store through add_document, so long input is chunked (#18407)."""
+
+import re
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, MagicMock, patch
+
+import pytest
+
+import knowledge.document_ingest as ingest_module
+from api.knowledge_upload_redaction_test import _SECRET
+from knowledge.documents import DocumentsMixin
+
+LONG_TEXT = "\n\n".join("Section %d. " % i + "lorem ipsum dolor sit amet " * 40 for i in range(30))
+
+
+class _KB(DocumentsMixin):
+    """Every path ends in ``store_fact``, so the fact count is the same measure before and after (#18407)."""
+
+    def __init__(self):
+        self.redis_client = MagicMock(hgetall=MagicMock(return_value={}))
+        self.stored = {}
+
+    async def store_fact(self, content, metadata=None, fact_id=None):
+        fact_id = fact_id or "single-fact"
+        self.stored[fact_id] = (content, dict(metadata or {}))
+        return {"status": "success", "fact_id": fact_id}
+
+    async def _add_document_internal(self, content, metadata=None, doc_id=None):
+        return await self.store_fact(content, metadata, doc_id)
+
+
+@pytest.fixture(autouse=True)
+def _small_chunks():
+    with patch.object(ingest_module, "KB_CHUNK_MAX_TOKENS", 200):
+        yield
+
+
+def _patched(mod, kb):
+    return (
+        patch.object(mod, "get_or_create_knowledge_base", AsyncMock(return_value=kb)),
+        patch.object(mod, "get_auth_middleware", MagicMock()),
+        patch.object(mod, "audit_record", MagicMock()),
+    )
+
+
+def _assert_chunked_under_one_document(kb, response_id):
+    assert len(kb.stored) > 1, "a long input was stored as a single fact"
+    assert {m["document_id"] for _c, m in kb.stored.values()} == {response_id}
+    assert response_id in kb.stored
+
+
+@pytest.mark.asyncio
+async def test_a_long_upload_is_stored_as_several_chunks_under_the_returned_document_id():
+    from api import knowledge as mod
+
+    upload = SimpleNamespace(filename="notes.txt", read=AsyncMock(return_value=LONG_TEXT.encode("utf-8")))
+    req = MagicMock()
+    req.form = AsyncMock(return_value={"file": upload, "title": "Notes"})
+    kb = _KB()
+    p1, p2, p3 = _patched(mod, kb)
+    with p1, p2, p3:
+        response = await mod.upload_file_to_knowledge(admin_check=True, req=req)
+    _assert_chunked_under_one_document(kb, response.document_id)
+
+
+@pytest.mark.asyncio
+async def test_pdf_chunks_name_the_right_pages_even_when_redaction_changes_the_length():
+    """The secret is redacted to a shorter placeholder on page 1; spans taken before cleaning would drift."""
+    from api import knowledge as mod
+    from media.document.extraction import ExtractedDocument, PageText, render_plain
+
+    secret = _SECRET
+    raw = [
+        PageText(
+            n,
+            "Page %d marker. " % n
+            + ("token %s " % secret if n == 1 else "")
+            + " ".join("w%d_%d" % (n, i) for i in range(400)),
+        )
+        for n in range(1, 7)
+    ]
+    extracted = ExtractedDocument(format="pdf", text="", pages=tuple(raw), page_count=6)
+    upload = SimpleNamespace(filename="doc.pdf", read=AsyncMock(return_value=b"x"))
+    req = MagicMock()
+    req.form = AsyncMock(return_value={"file": upload, "title": "Doc"})
+    kb = _KB()
+    p1, p2, p3 = _patched(mod, kb)
+    extraction = AsyncMock(return_value=(render_plain(raw)[0], extracted))
+    with (
+        p1,
+        p2,
+        p3,
+        patch.object(mod, "extract_within_deadline", extraction),
+        patch.object(mod, "_has_usable_content", return_value=True),
+    ):
+        response = await mod.upload_file_to_knowledge(admin_check=True, req=req)
+    _assert_chunked_under_one_document(kb, response.document_id)
+    assert all(secret not in chunk for chunk, _m in kb.stored.values()), "a secret reached the KB"
+    marker = re.compile(r"Page (\d) marker")
+    for chunk, meta in kb.stored.values():
+        spans = meta["page_spans"]
+        for span in spans:
+            text = chunk[span["start"] : span["end"]]
+            assert 0 <= span["start"] < span["end"] <= len(chunk)
+            assert not (
+                marker.search(text) and int(marker.search(text).group(1)) != span["page"]
+            ), "span names the wrong page"
+            assert {int(n) for n in re.findall(r"w(\d)_\d+", text)} <= {span["page"]}
+        assert {s["page"] for s in spans} == {int(n) for n in re.findall(r"(?:Page |\bw)(\d)(?: marker|_\d+)", chunk)}
+
+
+@pytest.mark.asyncio
+async def test_add_fact_and_url_content_are_chunked_through_the_same_helper():
+    from api import knowledge as mod
+
+    for metadata in ({"source": "s"}, {"type": "url", "source": "https://example.invalid"}):
+        kb = _KB()
+        doc_id = await mod._store_fact_in_kb(kb, LONG_TEXT, dict(metadata))
+        _assert_chunked_under_one_document(kb, doc_id)