[FIX] Incomplete runs must not report pass (#1177)

mod_api/routes/runs.py

@@ -28,7 +28,8 @@
                                   RunSchema, RunSummarySchema)
 from mod_api.services.error_service import derive_errors_for_run
 from mod_api.services.status import (batch_get_run_data, derive_run_status,
-                                     derive_sample_status)
+                                     derive_sample_status,
+                                     expected_regression_ids)
 from mod_api.utils import get_sort_column, paginated_response, single_response
 from mod_auth.models import Role
 from mod_customized.models import CustomizedTest
@@ -434,16 +435,13 @@ def get_run(run_id):
 def _run_regression_ids(test):
     """Regression test IDs that belong to this run.
 
-    Uses the customized selection when present; otherwise falls back to
-    every ACTIVE regression test, mirroring create_run's default. (The
-    model's get_customized_regressiontests() falls back to all tests
-    including inactive ones, which inflates total_samples/skipped_count
-    with tests the run could never execute.)
+    Delegates to expected_regression_ids so summary totals and run-status
+    completeness checks share one rule. (The model's
+    get_customized_regressiontests() falls back to all tests including
+    inactive ones, which inflates total_samples/skipped_count with tests
+    the run could never execute.)
     """
-    if test.customized_tests:
-        return [ct.regression_id for ct in test.customized_tests]
-    return [rt.id for rt in
-            RegressionTest.query.filter_by(active=True).all()]
+    return expected_regression_ids(test)
 
 
 def _aggregate_run_statistics(

mod_api/services/status.py

@@ -17,15 +17,27 @@
 """
 
 from collections import defaultdict
-from typing import List, Optional
+from typing import List, Optional, Set
 
 from sqlalchemy.orm import joinedload
 
-from mod_regression.models import RegressionTestOutput
+from mod_regression.models import RegressionTest, RegressionTestOutput
 from mod_test.models import (Test, TestProgress, TestResult, TestResultFile,
                              TestStatus)
 
 
+def expected_regression_ids(test: Test) -> List[int]:
+    """Regression test IDs this run was configured to execute.
+
+    Uses the customized selection when present; otherwise every ACTIVE
+    regression test — same rule as create_run / run summary totals.
+    """
+    if test.customized_tests:
+        return [ct.regression_id for ct in test.customized_tests]
+    return [rt.id for rt in
+            RegressionTest.query.filter_by(active=True).all()]
+
+
 def derive_run_status(test: Test) -> str:
     """
     Map the raw model state to one of the 7 normalized run statuses.
@@ -160,12 +172,20 @@
         t_id,
         results_by_test,
         files_by_test_and_rt,
-        expected_outputs_by_rt):
+        expected_outputs_by_rt,
+        expected_rt_ids: Optional[Set[int]] = None):
     results = results_by_test.get(t_id, [])
     if not results:
         # A run marked completed that produced zero TestResult rows is not
         # a pass — the worker finished without reporting anything.
         return 'error'
+    # Samples with no TestResult row are invisible to the loop below. If the
+    # run was configured for more samples than it reported, treat it as an
+    # error (same class as zero results) so a green fragment cannot pass.
+    if expected_rt_ids is not None:
+        reported_ids = {r.regression_test_id for r in results}
+        if not expected_rt_ids.issubset(reported_ids):
+            return 'error'
     for r in results:
         r_files = files_by_test_and_rt.get((t_id, r.regression_test_id), [])
         expected = expected_outputs_by_rt.get(
@@ -181,7 +201,8 @@
         results_by_test,
         files_by_test_and_rt,
         t_id,
-        expected_outputs_by_rt=None):
+        expected_outputs_by_rt=None,
+        expected_rt_ids: Optional[Set[int]] = None):
     if not t_prog:
         return 'queued'
 
@@ -196,11 +217,12 @@
             t_id,
             results_by_test,
             files_by_test_and_rt,
-            expected_outputs_by_rt)
+            expected_outputs_by_rt,
+            expected_rt_ids=expected_rt_ids)
     return 'incomplete'
 
 
 def batch_get_run_data(tests: list) -> tuple:
     """
     Batch compute derive_run_status and get_run_timestamps for a list of tests.
 
@@ -254,6 +276,19 @@
         for rto in all_expected:
             expected_outputs_by_rt[rto.regression_id].append(rto)
 
+    # Expected sample set per run (customized selection or all active RTs)
+    active_rt_ids = {
+        rt.id for rt in RegressionTest.query.filter_by(active=True).all()
+    }
+    expected_rt_ids_by_test = {}
+    for t in tests:
+        if t.customized_tests:
+            expected_rt_ids_by_test[t.id] = {
+                ct.regression_id for ct in t.customized_tests
+            }
+        else:
+            expected_rt_ids_by_test[t.id] = active_rt_ids
+
     statuses = {}
     timestamps_dict = {}
 
@@ -262,6 +297,7 @@
         timestamps_dict[t.id] = _compute_run_timestamps(t_prog)
         statuses[t.id] = _compute_run_status(
             t_prog, results_by_test, files_by_test_and_rt, t.id,
-            expected_outputs_by_rt=expected_outputs_by_rt)
+            expected_outputs_by_rt=expected_outputs_by_rt,
+            expected_rt_ids=expected_rt_ids_by_test[t.id])
 
     return statuses, timestamps_dict

tests/api/test_services_status.py

@@ -5,6 +5,7 @@
 from mod_api.services.status import (derive_output_status, derive_run_status,
                                      derive_sample_status, get_run_timestamps,
                                      is_dummy_row)
+from mod_customized.models import CustomizedTest
 from mod_regression.models import RegressionTestOutput
 from mod_regression.models import \
     RegressionTestOutputFiles as RegressionTestMultipleFiles
@@ -24,6 +25,12 @@ def setUp(self):
         g.db.add(self.test_obj)
         g.db.commit()
 
+    def _limit_run_to_regression(self, regression_id):
+        """Scope this run to one sample so partial-suite fixtures stay valid."""
+        g.db.add(CustomizedTest(self.test_obj.id, regression_id))
+        g.db.commit()
+        g.db.refresh(self.test_obj)
+
     def test_derive_run_status_queued(self):
         self.assertEqual(derive_run_status(self.test_obj), 'queued')
 
@@ -34,6 +41,7 @@ def test_derive_run_status_running(self):
         self.assertEqual(derive_run_status(self.test_obj), 'running')
 
     def test_derive_run_status_pass(self):
+        self._limit_run_to_regression(1)
         tp = TestProgress(self.test_obj.id, TestStatus.completed, 'done')
         # A passing result: exit code matches and the expected output for
         # regression test 1 was produced and matched (got=None).
@@ -51,7 +59,18 @@ def test_derive_run_status_completed_without_results_is_error(self):
         g.db.commit()
         self.assertEqual(derive_run_status(self.test_obj), 'error')
 
+    def test_derive_run_status_completed_partial_results_is_error(self):
+        # Base fixtures seed two active regression tests. Completing with a
+        # result for only one of them must not report pass (#1177).
+        tp = TestProgress(self.test_obj.id, TestStatus.completed, 'done')
+        tr = TestResult(self.test_obj.id, 1, 100, 0, 0)
+        rf = TestResultFile(self.test_obj.id, 1, 1, 'sample_out1')
+        g.db.add_all([tp, tr, rf])
+        g.db.commit()
+        self.assertEqual(derive_run_status(self.test_obj), 'error')
+
     def test_derive_run_status_fail(self):
+        self._limit_run_to_regression(1)
         tp = TestProgress(self.test_obj.id, TestStatus.completed, 'done')
         # runtime 100, exit_code 1, expected 0
         tr = TestResult(self.test_obj.id, 1, 100, 1, 0)