fix(tasks): resolve the man-page indexer from the repo root, not the cwd (#15853)

autobot-backend/tasks/knowledge_tasks.py

@@ -13,13 +13,12 @@
 import asyncio
 import fnmatch
 import os
-import subprocess  # nosec B404  # used for internal script execution only
-import sys
 import time
 from pathlib import Path
 
 from autobot_shared.logging_manager import get_logger
 from celery_app import celery_app
+from tasks.man_page_indexing import run_indexing_subprocess
 from type_defs.common import Metadata
 
 logger = get_logger(__name__)
@@ -59,63 +58,14 @@ def _is_excluded(path: Path) -> bool:
     return False
 
 
-def _parse_indexing_output(output: str) -> tuple:
-    """Parse indexing script output for statistics (Issue #315: extracted helper).
-
-    Args:
-        output: Raw stdout from indexing script
-
-    Returns:
-        Tuple of (indexed_count, total_facts)
-    """
-    indexed_count = 0
-    total_facts = 0
-    for line in output.split("\n"):
-        if "Successfully indexed:" in line:
-            indexed_count = int(line.split(":")[1].strip())
-        elif "Total facts in KB:" in line:
-            total_facts = int(line.split(":")[1].strip())
-    return indexed_count, total_facts
-
-
-def _run_indexing_subprocess() -> dict:
-    """Helper for refresh_system_knowledge. Ref: #1088.
-
-    Runs the index_all_man_pages.py script as a subprocess and returns a result
-    dict.  On non-zero exit returns a 'failed' dict; on success returns a
-    'success' dict with commands_indexed and total_facts.
-    """
-    result = subprocess.run(  # nosec B603  # uses sys.executable with fixed internal script path
-        [sys.executable, "scripts/utilities/index_all_man_pages.py"],
-        capture_output=True,
-        text=True,
-    )
-    if result.returncode != 0:
-        error_msg = result.stderr[:500] if result.stderr else "Unknown error"
-        logger.error("System knowledge refresh failed: %s", error_msg)
-        return {
-            "status": "failed",
-            "error": error_msg,
-            "message": "Knowledge refresh failed",
-        }
-    indexed_count, total_facts = _parse_indexing_output(result.stdout)
-    logger.info(f"System knowledge refresh complete: {indexed_count} commands indexed, " f"{total_facts} total facts")
-    return {
-        "status": "success",
-        "commands_indexed": indexed_count,
-        "total_facts": total_facts,
-        "message": "System knowledge refreshed successfully",
-    }
-
-
 @celery_app.task(bind=True, name="tasks.refresh_system_knowledge")
 def refresh_system_knowledge(self) -> Metadata:
     """
     Refresh ALL system knowledge (man pages + AutoBot docs) in background.
 
     This is a long-running operation (can take up to 10 minutes) that indexes
     all system man pages and AutoBot documentation into the knowledge base.
-    Issue #1088: Subprocess execution extracted to _run_indexing_subprocess.
+    Issue #1088: subprocess execution extracted; #15853 moved it to tasks.man_page_indexing.
 
     Args:
         self: Celery task instance (bound for progress updates)
@@ -136,7 +86,7 @@ def refresh_system_knowledge(self) -> Metadata:
             },
         )
         logger.info("Starting comprehensive system knowledge refresh (background task)...")
-        return _run_indexing_subprocess()
+        return run_indexing_subprocess()
 
     except Exception as e:
         logger.exception("System knowledge refresh task failed: %s", e)

autobot-backend/tasks/man_page_indexing.py

@@ -0,0 +1,86 @@
+# Copyright 2025-2026 mrveiss
+# SPDX-License-Identifier: Apache-2.0
+# AutoBot - AI-Powered Automation Platform
+# Author: mrveiss
+"""Running the man-page indexer and reading its output (#15853).
+
+Extracted from ``knowledge_tasks`` rather than added to it. That module sits at
+its recorded file-size ceiling (#14236), and the exemption freezes the size the
+file was granted -- it does not license more. Fixing the indexer path made the
+module grow, so the fix and its two helpers moved here instead, and the ceiling
+came down.
+
+The two functions belong together in any case: one runs the indexer, the other
+reads the numbers back out of its stdout. Neither has another caller.
+"""
+
+import subprocess  # nosec B404  # used for internal script execution only
+import sys
+
+from autobot_shared.logging_manager import get_logger
+from autobot_shared.ssot_constants import PATH
+
+logger = get_logger(__name__)
+
+#: The man-page indexer, anchored to the repository root.
+#:
+#: This was addressed relative to the process working directory, which is not
+#: the repo root under Celery, systemd or a test runner -- and no such directory
+#: exists at the root under any cwd. The script is real; only the path was not.
+#: So the refresh returned its failed dict on every run.
+MAN_PAGE_INDEXER = (
+    PATH.PROJECT_ROOT / "autobot-infrastructure" / "shared" / "scripts" / "utilities" / "index_all_man_pages.py"
+)
+
+
+def parse_indexing_output(output: str) -> tuple:
+    """Parse indexing script output for statistics (Issue #315: extracted helper).
+
+    Args:
+        output: Raw stdout from indexing script
+
+    Returns:
+        Tuple of (indexed_count, total_facts)
+    """
+    indexed_count = 0
+    total_facts = 0
+    for line in output.split("\n"):
+        if "Successfully indexed:" in line:
+            indexed_count = int(line.split(":")[1].strip())
+        elif "Total facts in KB:" in line:
+            total_facts = int(line.split(":")[1].strip())
+    return indexed_count, total_facts
+
+
+def run_indexing_subprocess() -> dict:
+    """Run the man-page indexer and return a result dict. Ref: #1088.
+
+    On a missing indexer or a non-zero exit returns a ``failed`` dict; on
+    success returns a ``success`` dict with commands_indexed and total_facts.
+    """
+    if not MAN_PAGE_INDEXER.is_file():
+        # Name the path. The previous failure mode truncated the subprocess's
+        # stderr into a generic message, so "the indexer is not where we looked"
+        # was indistinguishable from "the indexer ran and failed".
+        message = f"man-page indexer not found at {MAN_PAGE_INDEXER}"
+        logger.error("System knowledge refresh cannot run: %s", message)
+        return {"status": "failed", "error": message, "message": "Knowledge refresh failed"}
+
+    result = subprocess.run(  # nosec B603  # uses sys.executable with fixed internal script path
+        [sys.executable, str(MAN_PAGE_INDEXER)],
+        capture_output=True,
+        text=True,
+    )
+    if result.returncode != 0:
+        error_msg = result.stderr[:500] if result.stderr else "Unknown error"
+        logger.error("System knowledge refresh failed: %s", error_msg)
+        return {"status": "failed", "error": error_msg, "message": "Knowledge refresh failed"}
+
+    indexed_count, total_facts = parse_indexing_output(result.stdout)
+    logger.info("System knowledge refresh complete: %d commands indexed, %d total facts", indexed_count, total_facts)
+    return {
+        "status": "success",
+        "commands_indexed": indexed_count,
+        "total_facts": total_facts,
+        "message": "System knowledge refreshed successfully",
+    }

docs/system-state.md

@@ -830,7 +830,7 @@ export class KnowledgeController {
    - `get_compatible_os_list()` - Maps OS families (Kali → Debian, Ubuntu)
    - Tested and verified on Kali 2025.2
 
-2. **Enhanced Man Page Indexer** (`scripts/utilities/index_all_man_pages.py`):
+2. **Enhanced Man Page Indexer** (`autobot-infrastructure/shared/scripts/utilities/index_all_man_pages.py`):
    - Added OS/machine context to all man page metadata
    - Unique key generation for deduplication
    - Applicability lists (compatible OSes)

repo_tests/man_page_indexer_path_resolves_15853_test.py

@@ -0,0 +1,134 @@
+# Copyright 2025-2026 mrveiss
+# SPDX-License-Identifier: Apache-2.0
+# AutoBot - AI-Powered Automation Platform
+# Author: mrveiss
+"""The man-page indexer is where the refresh task looks for it (#15853).
+
+`knowledge_tasks._run_indexing_subprocess` -- now
+`tasks.man_page_indexing.run_indexing_subprocess` -- shelled out to
+`"scripts/utilities/index_all_man_pages.py"` -- relative to the process working
+directory, and there is no `scripts/utilities/` at the repository root at all.
+The script is real; it lives under `autobot-infrastructure/shared/`. So the
+refresh returned its `failed` dict on every run, with the subprocess's stderr
+truncated into a generic message, and system knowledge was presumably never
+refreshed by this task.
+
+Same defect class as #15845, whose hook pointed at an indexer that was never
+created: a wired path that resolves nowhere, degrading quietly instead of
+failing loudly.
+
+**The path is read out of the module under test, never restated here.** A test
+carrying its own copy of the path passes when the module's copy moves alone,
+which is precisely the failure it exists to catch.
+"""
+
+import ast
+import re
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+_MODULE = REPO_ROOT / "autobot-backend/tasks/man_page_indexing.py"
+_CALLER = REPO_ROOT / "autobot-backend/tasks/knowledge_tasks.py"
+
+
+def _module_source() -> str:
+    """The module's code, with comment lines removed.
+
+    The module documents the old cwd-relative path in a comment directly above
+    the constant that replaced it. Matching raw text, "the bad literal is gone"
+    is then false while the code is correct -- the explanation of the fix sits
+    nearer to the fix than anything else does, and it is made of the very string
+    the check looks for. Same reason the #15724 publish-contract guard strips
+    comments before matching.
+    """
+    text = _MODULE.read_text(encoding="utf-8")
+    return "\n".join(line for line in text.splitlines() if not line.lstrip().startswith("#"))
+
+
+def _indexer_path_expression() -> str:
+    """The right-hand side of MAN_PAGE_INDEXER, as written in the module."""
+    tree = ast.parse(_module_source())
+    for node in tree.body:
+        if isinstance(node, ast.Assign) and any(
+            isinstance(t, ast.Name) and t.id == "MAN_PAGE_INDEXER" for t in node.targets
+        ):
+            return ast.unparse(node.value)
+    raise AssertionError("MAN_PAGE_INDEXER is not defined -- this test would prove nothing")
+
+
+def _resolved_indexer() -> Path:
+    """Resolve the module's expression to a real path, without importing the backend.
+
+    Importing `knowledge_tasks` pulls in Celery and the whole task graph. The
+    expression is a chain of `/` operands, so the string literals in it are the
+    path segments -- read them from the AST instead.
+    """
+    expression = _indexer_path_expression()
+    assert "PATH.PROJECT_ROOT" in expression, f"MAN_PAGE_INDEXER is not anchored to the project root: {expression}"
+    segments = re.findall(r"'([^']+)'|\"([^\"]+)\"", expression)
+    parts = [a or b for a, b in segments]
+    assert parts, f"no path segments found in {expression}"
+    return REPO_ROOT.joinpath(*parts)
+
+
+def test_the_indexer_path_is_anchored_to_the_repo_root_not_the_cwd():
+    """A cwd-relative path resolves differently under Celery, systemd and pytest."""
+    expression = _indexer_path_expression()
+
+    assert "PATH.PROJECT_ROOT" in expression, expression
+    assert not re.search(
+        r"^['\"]scripts/", expression
+    ), "the indexer is addressed relative to the process working directory"
+
+
+def test_the_indexer_exists_where_the_task_looks_for_it():
+    """The defect itself, with the path read from the caller rather than restated."""
+    indexer = _resolved_indexer()
+
+    assert indexer.is_file(), (
+        f"knowledge_tasks resolves the man-page indexer to {indexer.relative_to(REPO_ROOT)}, " "which does not exist"
+    )
+
+
+def test_a_missing_indexer_is_reported_by_path():
+    """Absence must be distinguishable from a run that failed.
+
+    The old code returned the subprocess's truncated stderr, so "the indexer is
+    not where we looked" and "the indexer ran and failed" produced the same
+    message.
+    """
+    source = _module_source()
+
+    assert "MAN_PAGE_INDEXER.is_file()" in source, "nothing checks whether the indexer exists before shelling out to it"
+    assert re.search(
+        r"not found at \{MAN_PAGE_INDEXER\}", source
+    ), "the not-found message does not name the path it looked for"
+
+
+def test_the_subprocess_invokes_the_anchored_path():
+    """The constant must be what is executed.
+
+    Defining an anchored constant and then passing a literal to `subprocess.run`
+    would satisfy every assertion above while changing nothing.
+    """
+    source = _module_source()
+
+    assert "[sys.executable, str(MAN_PAGE_INDEXER)]" in source, "the subprocess does not execute MAN_PAGE_INDEXER"
+    assert (
+        "scripts/utilities/index_all_man_pages.py" not in source
+    ), "the cwd-relative literal is still present in the module"
+
+
+def test_the_caller_delegates_rather_than_carrying_its_own_path():
+    """`knowledge_tasks` must not regrow its own copy of the invocation.
+
+    The helpers moved out because that module sits at its file-size ceiling and
+    a grandfathered file may not grow (#14236). A second copy of the path here
+    would be both the drift this guard exists to catch and a ceiling breach.
+    """
+    caller = "\n".join(
+        line for line in _CALLER.read_text(encoding="utf-8").splitlines() if not line.lstrip().startswith("#")
+    )
+
+    assert "from tasks.man_page_indexing import run_indexing_subprocess" in caller
+    assert "index_all_man_pages.py" not in caller, "knowledge_tasks names the indexer script again; it should delegate"

repo_tests/python_file_size_ratchet_baseline.py

@@ -392,7 +392,7 @@
     "autobot-backend/services/workflow_io.py": 693,
     "autobot-backend/slash_command_handler.py": 1663,
     "autobot-backend/takeover_manager.py": 1364,
-    "autobot-backend/tasks/knowledge_tasks.py": 912,
+    "autobot-backend/tasks/knowledge_tasks.py": 862,
     "autobot-backend/tests/agents/test_librarian_assistant_extract.py": 635,
     "autobot-backend/tests/api/test_canvas.py": 679,
     "autobot-backend/tests/api/test_knowledge_grounding.py": 929,

scripts/python_file_size_known_large.py

@@ -387,7 +387,7 @@
     "autobot-backend/services/workflow_io.py": 693,
     "autobot-backend/slash_command_handler.py": 1663,
     "autobot-backend/takeover_manager.py": 1364,
-    "autobot-backend/tasks/knowledge_tasks.py": 912,
+    "autobot-backend/tasks/knowledge_tasks.py": 862,
     "autobot-backend/tests/agents/test_librarian_assistant_extract.py": 635,
     "autobot-backend/tests/api/test_canvas.py": 679,
     "autobot-backend/tests/api/test_knowledge_grounding.py": 929,