make client parsing better

modyn/protos/evaluator.proto

@@ -54,6 +54,8 @@ message EvaluateModelIntervalResponse {
 
 message EvaluateModelResponse {
   // only when all interval evaluations failed, this field will be set to false
+  // it is a field of convenience for the client to decide whether to wait for the evaluation completion.
+  // the client can always check the interval_responses
   bool evaluation_started = 1;
   int32 evaluation_id = 2;
   // always has the same size as the number of intervals

modyn/supervisor/internal/pipeline_executor/evaluation_executor.py

@@ -313,13 +313,12 @@
         # here we assume only one interval is evaluated
         eval_results: dict = {"dataset_size": response.interval_responses[0].dataset_size, "metrics": []}
 
-        for single_eval_data in eval_data:
-            if single_eval_data.interval_index == 0:
-                for metric in single_eval_data.evaluation_data:
-                    eval_results["metrics"].append({"name": metric.metric, "result": metric.result})
-                return None, eval_results
-        # this shouldn't happen
-        raise RuntimeError("No evaluation data found")
+        assert len(eval_data) == 1, "The evaluation request only contains one interval so we expect only one result."
+        assert eval_data[0].interval_index == 0
+        for metric in eval_data[0].evaluation_data:
+            eval_results["metrics"].append({"name": metric.metric, "result": metric.result})
+
+        return None, eval_results
 
 
 # ------------------------------------------------------------------------------------ #
@@ -343,5 +342,5 @@
         supervisor_logs=SupervisorLogs(),
     )
 
     logs_.supervisor_logs = ex.run_post_pipeline_evaluations()
     logs_.materialize(snapshot_path, mode="final")