fix(ci): triage every failure and error in the marker-excluded suite, and make it report an executed count (#14930)

.github/workflows/marker-tests.yml

@@ -30,6 +30,33 @@
 #     Redis, so one is provided below rather than letting the first night fail
 #     for infrastructure reasons and calling it "the point". #13286 stays open
 #     until a run is green or its remaining failures have been triaged.
+#
+#     TRIAGED under #14930. The run on 2026-08-24 read
+#     `19 failed, 35 passed, 12 skipped, 19 errors`. All 38 non-passes were one
+#     defect class in two disguises: a live service the runner does not have,
+#     reported as a hard result instead of a stated non-result.
+#       * 19 errors — `redis_service_api_integration_test.py` had no
+#         `backend_url` fixture on this tree at all, so every test in it errored
+#         at setup. The fixture is now defined in the module.
+#       * 16 failures — `infrastructure_integration_test.py` (6) and
+#         `voice_integration_test.py` (10) dial a backend on :8001 and a TTS
+#         worker on :8083 over real HTTP. Both now skip, naming the absent
+#         service, via `autobot_shared.live_service_probe`.
+#       * 2 failures — the two knowledge-base tests. NOT an infrastructure gap:
+#         the Redis below is healthy, but `autobot-backend/conftest.py` replaces
+#         `autobot_shared.redis_client` with a socket-free stand-in for every
+#         backend item, markers included, so `get_redis_client()` returns None
+#         whatever Redis is running. Tracked as #14932; they skip on that
+#         structural condition until it lands, then run for real.
+#       * 1 failure — `performance_benchmarks_test.py` asserted absolute process
+#         RSS under 500 MB. A worker sits at ~1.6 GB from imports alone before
+#         the first benchmark, so the budget was spent before the test started.
+#         Now measured as the RSS delta across the operation, which is what the
+#         assertion always meant.
+#
+#     A skip is a real outcome here, not a pass. "Report marker coverage" below
+#     prints the counts and fails the job if the selection collected nothing, so
+#     the difference is on the run page instead of inferred from a log.
 #   * No coverage gate. Coverage is measured by the PR gate; adding tracing here
 #     would roughly double the runtime for no added assertion.
 #   * Two invocations, same reason as ci.yml: autobot-backend/ and
@@ -48,17 +75,16 @@ on:
   # workflows named marker-tests.yml" because GitHub had never registered it.
   #
   # That changes the moment the integration branch is promoted to the default
-  # branch — the cron would start firing immediately. Its one observed run is
-  # RED (11 failed, 7 passed, 40 errors), so registering it as-is would install
-  # a nightly that fails every night against the default branch. A permanently
-  # red scheduled workflow is not a signal; it is noise that trains everyone to
+  # branch — the cron would start firing immediately. A permanently red
+  # scheduled workflow is not a signal; it is noise that trains everyone to
   # ignore the one place marker-excluded regressions would show up.
   #
   # So the cron is commented rather than deleted — it is correct, and it is what
-  # #13286 exists to deliver. Restore it as the closing step of that issue, once
-  # the 11 failures and 40 collection errors are triaged and the suite is green.
-  # `workflow_dispatch` below stays live, so the suite remains runnable on demand
-  # to verify exactly that.
+  # #13286 exists to deliver. #14930 triaged every failure and error the suite
+  # was carrying (see the block above); restore the cron as the closing step of
+  # #13286, and raise MARKER_MIN_PASSED to the figure the run then reports so a
+  # later collapse cannot pass unnoticed. `workflow_dispatch` below stays live,
+  # so the suite remains runnable on demand to verify exactly that.
   #
   # schedule:
   #   # 03:20 UTC daily — off the */15 dispatch-watchdog and */30 runner-health
@@ -118,6 +144,17 @@ jobs:
       # Single source for the marker expression: the schedule uses the default,
       # a manual dispatch may narrow it to one marker while triaging.
       MARKER_EXPRESSION: ${{ github.event.inputs.markers || 'integration or slow or distributed or performance' }}
+      # Floor for the number of marker-carrying tests that must PASS (#14930).
+      # This is a ratchet, not a target: raise it when the suite genuinely
+      # covers more, and lower it only with a stated reason for which tests
+      # stopped running. A silent drop is the failure mode it exists to catch —
+      # the suite ran 35 passing tests while reading as an undifferentiated red,
+      # so nobody would have noticed the number falling to 3.
+      #
+      # Kept at the presence floor (1) until the first run under this workflow
+      # reports a real post-fix figure; #13286 raises it to that number as part
+      # of turning the schedule on.
+      MARKER_MIN_PASSED: '1'
 
     steps:
       - uses: actions/checkout@v7
@@ -175,13 +212,17 @@ jobs:
       - name: Run marked tests — backend, shared, tts-worker, repo_tests
         run: |
           echo "Selecting: $MARKER_EXPRESSION"
-          # Exit 5 is pytest's "no tests collected". For a marker selection that
-          # is a legitimate outcome (no test currently carries the marker in this
-          # tree), not a failure — anything else propagates.
+          # Exit 5 is pytest's "no tests collected". Tolerated here ONLY so the
+          # second invocation still runs and the report step still gets both
+          # files — it is NOT accepted as an outcome: "Report marker coverage"
+          # below fails the job when the two invocations collected nothing
+          # between them. Before #14930 this line was the whole story, and an
+          # empty selection was indistinguishable from a passing one.
           python -m pytest \
             autobot-backend autobot_shared autobot-tts-worker repo_tests \
             -n auto --dist loadscope \
             -m "$MARKER_EXPRESSION" \
+            --junitxml=marker-report-backend.xml \
             --tb=short \
             --durations=25 \
             -q || [ $? -eq 5 ]
@@ -196,6 +237,37 @@ jobs:
             autobot-slm-backend \
             -n auto --dist loadscope \
             -m "$MARKER_EXPRESSION" \
+            --junitxml=marker-report-slm.xml \
             --tb=short \
             --durations=25 \
             -q || [ $? -eq 5 ]
+
+      # The one verdict pytest cannot give: "the selection still matches the
+      # tests it is supposed to match" (#14930).
+      #
+      # Runs on failure too — deliberately. A red pytest step that skipped this
+      # would hide exactly the case worth seeing: a suite whose coverage
+      # collapsed AND whose remainder failed. The counts are printed here and
+      # written to the run summary, so a drop from the recorded baseline is
+      # visible on the run page rather than buried in 4,000 lines of log.
+      #
+      # A missing report file is a hard failure inside the script, not a zero:
+      # a pytest that died before writing its XML must never read as "collected
+      # nothing", let alone as clean.
+      - name: Report marker coverage
+        if: ${{ !cancelled() }}
+        run: |
+          python pipeline-scripts/marker_suite_report.py \
+            backend=marker-report-backend.xml \
+            slm=marker-report-slm.xml \
+            --min-collected 1 \
+            --min-passed "$MARKER_MIN_PASSED"
+
+      - name: Upload marker reports
+        if: ${{ !cancelled() }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: marker-suite-reports
+          path: marker-report-*.xml
+          if-no-files-found: error
+          retention-days: 14

autobot-backend/api/infrastructure_integration_test.py

@@ -20,6 +20,7 @@
 # Add project root to path
 sys.path.insert(0, str(Path(__file__).parent.parent))
 
+from autobot_shared.live_service_probe import require_live_endpoint
 from autobot_shared.paths import project_root
 from constants.network_constants import ServiceURLs
 
@@ -28,6 +29,38 @@
 BASE_URL = f"{ServiceURLs.BACKEND_API}/api/iac"
 
 
+# ``test_celery_worker_status`` reads ``logs/celery-worker.log`` off disk and
+# issues no HTTP at all, so a backend that is down does not stop it — it was
+# passing before #14930 and must keep passing. A module-wide guard would have
+# traded a red result for lost coverage, which is the wrong trade and was caught
+# by comparing the skip count (7) against the number of tests that actually
+# failed on a refused connection (6).
+_NEEDS_NO_BACKEND = ("test_celery_worker_status",)
+
+
+@pytest.fixture(autouse=True)
+def _require_live_backend(request) -> None:
+    """Skip when no backend is listening, instead of failing on a refused socket (#14930).
+
+    The six checks that dial ``BASE_URL`` over real HTTP reported
+    ``ConnectionRefusedError`` as test failures on a runner with no backend up —
+    a red result that says nothing about the code under test and trained the
+    whole marker-excluded suite to be ignored. A skip naming the absent service
+    is the honest report; they still run, and still fail for real, wherever a
+    backend is actually up.
+    """
+    stranded = [name for name in _NEEDS_NO_BACKEND if name not in globals()]
+    assert not stranded, (
+        f"_NEEDS_NO_BACKEND names {stranded}, which no longer exist in this module. "
+        f"A rename stranded the exemption: it now exempts nothing, silently."
+    )
+
+    if request.node.name in _NEEDS_NO_BACKEND:
+        return
+
+    require_live_endpoint(ServiceURLs.BACKEND_API, what="the AutoBot backend API")
+
+
 def test_health():
     """Test 1: Health Check"""
     response = requests.get(f"{BASE_URL}/health")  # nosec B113

autobot-backend/api/voice_integration_test.py

@@ -19,6 +19,7 @@
 import pytest
 import requests
 
+from autobot_shared.live_service_probe import require_live_endpoint
 from autobot_shared.ssot_config import config
 
 # #12510: these tests hit a live backend + TTS worker over real HTTP, so they
@@ -33,6 +34,21 @@
 LATENCY_BUDGET_SEC = 5.0  # STT → TTS round-trip target
 
 
+@pytest.fixture(autouse=True)
+def _require_live_voice_stack() -> None:
+    """Skip when the backend or the TTS worker is absent (#14930).
+
+    All ten tests here drive the real pipeline over HTTP. On a GitHub-hosted
+    runner neither service exists, so each failed with a refused connection —
+    ten red results that measured the runner's inventory, not the voice
+    pipeline. Both endpoints are checked because the module drives both
+    directly, and a skip that named only one would misreport which half of the
+    stack was missing.
+    """
+    require_live_endpoint(BACKEND_URL, what="the AutoBot backend API")
+    require_live_endpoint(TTS_WORKER_URL, what="the AutoBot TTS worker")
+
+
 # ------------------------------------------------------------------ #
 # Helpers                                                              #
 # ------------------------------------------------------------------ #

autobot-backend/knowledge/gpu_kb_integration_test.py

@@ -23,8 +23,37 @@
 
 import pytest
 
+from autobot_shared.live_service_probe import require_real_redis_client
 from utils.semantic_chunker_gpu import get_gpu_semantic_chunker
 
+
+@pytest.fixture(autouse=True)
+def _require_real_redis(request) -> None:
+    """Skip while the backend conftest's Redis stand-in is installed (#14930, #14932).
+
+    ``get_knowledge_base()`` initialises Redis first (``knowledge/base.py``
+    step 1) and raises ``RuntimeError: Failed to initialize knowledge base``
+    when it cannot. In the marker-excluded run that is not an infrastructure
+    gap — the workflow provides a healthy Redis — it is
+    ``autobot-backend/conftest.py`` replacing ``autobot_shared.redis_client``
+    with a socket-free stand-in whose ``get_redis_client`` returns ``None`` for
+    every item under ``autobot-backend/``, markers included. #14932 owns that.
+
+    The condition checked is structural (is the real module installed?), not the
+    test's own failure, so this stops skipping by itself the moment #14932
+    lands — and a knowledge base that then fails to initialise fails the test,
+    which is the point.
+
+    Scoped to ``integration``-marked items only. Both files in this directory
+    pair the marked test with a hermetic one that needs no Redis at all and runs
+    on the PR unit gate; guarding those too would silently disable a passing
+    gate test, which is the same class of damage this change exists to undo.
+    """
+    if request.node.get_closest_marker("integration") is None:
+        return
+    require_real_redis_client("the knowledge base")
+
+
 # Long enough that chunking has something to divide, short enough to stay fast.
 SAMPLE_TEXT = (
     "AutoBot is an advanced Linux administration platform designed for intelligent automation. "

autobot-backend/knowledge/kb_optimization_test.py

@@ -13,13 +13,42 @@
 
 import pytest
 
+from autobot_shared.live_service_probe import require_real_redis_client
 from autobot_shared.ssot_config import config
 
 # Add AutoBot to path
 sys.path.insert(0, config.project_root)
 
 from knowledge_base import get_knowledge_base
 
+
+@pytest.fixture(autouse=True)
+def _require_real_redis(request) -> None:
+    """Skip while the backend conftest's Redis stand-in is installed (#14930, #14932).
+
+    ``get_knowledge_base()`` initialises Redis first (``knowledge/base.py``
+    step 1) and raises ``RuntimeError: Failed to initialize knowledge base``
+    when it cannot. In the marker-excluded run that is not an infrastructure
+    gap — the workflow provides a healthy Redis — it is
+    ``autobot-backend/conftest.py`` replacing ``autobot_shared.redis_client``
+    with a socket-free stand-in whose ``get_redis_client`` returns ``None`` for
+    every item under ``autobot-backend/``, markers included. #14932 owns that.
+
+    The condition checked is structural (is the real module installed?), not the
+    test's own failure, so this stops skipping by itself the moment #14932
+    lands — and a knowledge base that then fails to initialise fails the test,
+    which is the point.
+
+    Scoped to ``integration``-marked items only. Both files in this directory
+    pair the marked test with a hermetic one that needs no Redis at all and runs
+    on the PR unit gate; guarding those too would silently disable a passing
+    gate test, which is the same class of damage this change exists to undo.
+    """
+    if request.node.get_closest_marker("integration") is None:
+        return
+    require_real_redis_client("the knowledge base")
+
+
 # Test document content
 TEST_DOCUMENT_CONTENT = """
 AutoBot System Architecture and Design Philosophy

autobot-backend/performance_benchmarks_test.py

@@ -49,20 +49,42 @@ def __init__(self):
         self.metrics = []
         self.start_time = None
         self.end_time = None
+        self.start_rss_mb = None
 
     def start_measurement(self):
         """Start performance measurement"""
         self.start_time = time.perf_counter()
+        # #14930: record the baseline so end_measurement can report the memory
+        # THIS operation cost. See end_measurement for why the absolute figure
+        # was the wrong number.
+        self.start_rss_mb = psutil.Process().memory_info().rss / 1024 / 1024
 
     def end_measurement(self):
-        """End performance measurement and record metrics"""
+        """End performance measurement and record metrics.
+
+        ``memory_usage`` is the RSS *delta* across the measured region, not the
+        process's absolute RSS (#14930).
+
+        The absolute figure made ``assert stats["max_memory_mb"] < 500`` a
+        measurement of the interpreter's import footprint rather than of the
+        operation under test: importing the orchestrator, knowledge base,
+        memory manager and LLM service leaves a pytest-xdist worker at ~1.6 GB
+        before the first benchmark runs, so the assertion reported
+        ``1644.55 > 500`` and named it "Memory usage too high" on every run. It
+        could not have passed, and it could not have caught a regression in the
+        code it names — a budget that is spent before the test starts is not a
+        budget. The delta is the quantity the assertion always meant.
+        """
         self.end_time = time.perf_counter()
         duration = self.end_time - self.start_time
+        rss_mb = psutil.Process().memory_info().rss / 1024 / 1024
+        baseline_mb = getattr(self, "start_rss_mb", rss_mb)
         self.metrics.append(
             {
                 "duration": duration,
                 "timestamp": time.time(),
-                "memory_usage": psutil.Process().memory_info().rss / 1024 / 1024,  # MB
+                "memory_usage": rss_mb - baseline_mb,  # MB grown during this measurement
+                "rss_mb": rss_mb,  # absolute, kept for reporting only
                 "cpu_percent": psutil.Process().cpu_percent(),
             }
         )
@@ -145,7 +167,13 @@ async def test_llm_response_time_benchmark(self):
 
             # Performance assertions
             assert stats["avg_duration"] < 2.0, f"LLM response too slow for prompt length {len(prompt)}"
-            assert stats["max_memory_mb"] < 500, "Memory usage too high"
+            # #14930: a delta budget. 500 MB of GROWTH from one mocked chat()
+            # call would be a genuine leak; 500 MB of absolute RSS is just
+            # AutoBot being imported.
+            assert stats["max_memory_mb"] < 500, (
+                f"one measured operation grew RSS by {stats['max_memory_mb']:.1f} MB "
+                f"(budget 500 MB) for prompt length {len(prompt)}"
+            )
 
         # Log results for analysis
         print("\nLLM Performance Results:")  # noqa: print