feat: derive corrective turns from session judge

reflexio/models/api_schema/domain/entities.py

@@ -612,7 +612,14 @@ class AgentSuccessEvaluationResult(BaseModel):
     evaluation_name: str | None = None
     created_at: int = Field(default_factory=lambda: int(datetime.now(UTC).timestamp()))
     regular_vs_shadow: RegularVsShadow | None = None
-    number_of_correction_per_session: int = 0
+    number_of_correction_per_session: int = Field(
+        default=0,
+        ge=0,
+        description=(
+            "Number of user turns in the session that corrected or redirected an "
+            "earlier agent response or action."
+        ),
+    )
     user_turns_to_resolution: int | None = None
     is_escalated: bool = False
     embedding: EmbeddingVector = []

reflexio/models/api_schema/eval_overview_schema.py

@@ -22,10 +22,11 @@
 class HeroBucket(BaseModel):
     """One point on the trend chart in the hero block.
 
-    ``avg_corrections`` is the mean of ``number_of_correction_per_session``
-    across this bucket's evaluation results. Surfaced so the frontend can
-    plot a "corrections over time" line beside the success-rate trend.
-    Lower is better.
+    ``avg_corrections`` is the mean judge-derived count of user turns that
+    corrected or redirected an earlier agent response, across every evaluation
+    result in this bucket (including zero-count sessions). Surfaced so the
+    frontend can plot a "corrections over time" line beside the success-rate
+    trend. Lower is better.
 
     ``escalation_rate`` is the fraction of sessions in this bucket whose
     eval result had ``is_escalated=True``. Range 0.0 – 1.0. Surfaced so

reflexio/models/api_schema/ui/entities.py

@@ -124,7 +124,14 @@ class EvaluationResultView(BaseModel):
     evaluation_name: str | None = None
     created_at: int = Field(default_factory=lambda: int(datetime.now(UTC).timestamp()))
     regular_vs_shadow: RegularVsShadow | None = None
-    number_of_correction_per_session: int = 0
+    number_of_correction_per_session: int = Field(
+        default=0,
+        ge=0,
+        description=(
+            "Number of user turns in the session that corrected or redirected an "
+            "earlier agent response or action."
+        ),
+    )
     user_turns_to_resolution: int | None = None
     is_escalated: bool = False
 

reflexio/server/prompt/prompt_bank/agent_success_evaluation/v1.0.0.prompt.md

@@ -1,5 +1,5 @@
 ---
-active: true
+active: false
 description: "Evaluates agent success based on user interactions"
 variables:
   - agent_context_prompt

reflexio/server/prompt/prompt_bank/agent_success_evaluation/v1.1.0.prompt.md

@@ -0,0 +1,89 @@
+---
+active: true
+description: "Evaluates session success, escalation, and corrective user turns"
+changelog: "v1.1.0: adds a required count of corrective user turns, distinguishes revisions from new intents, and keeps correction count independent from final task success."
+variables:
+  - agent_context_prompt
+  - success_definition_prompt
+  - tool_can_use
+  - metadata_definition_prompt
+  - interactions
+---
+
+[Instruction]
+Given the interactions above, evaluate if the agent has successfully completed the user's primary requested task by the end of the session, based on the success definition below.
+The agent may make mistakes or take wrong actions during the session. The user may provide corrections or ask the agent to try differently. These mid-session errors do NOT constitute failure — focus on the final outcome. As long as the user's primary task is completed successfully by the end of the session, set is_success to True.
+
+Step 1: Identify the user's primary task from the interactions. Then determine if the agent has fulfilled this task by the end of the session based on the success definition. If so, set is_success to True. Otherwise set is_success to False and continue to the next step.
+
+Step 2: If the agent did not complete the primary task by the end of the session, determine the failure type and failure reason.
+There are four types of failures:
+- 'missing_tool': Agent does not have the right tool or action that it can take to get sufficient information to answer the user's request.
+- 'wrong_tool': Tools or actions are available, but the agent did not use the right tool or perform the right action.
+- 'insufficient_info_from_tool': Agent performed the right action and tool use, but the tool or action did not provide enough or correct information to answer successfully.
+- 'wrong_answer': Agent had enough information but still did not answer successfully.
+
+Step 3: Determine if the user was escalated (handed off to a human agent or another agent). Set is_escalated to True if so, False otherwise. Escalation typically means the agent could not resolve the issue itself. Escalated sessions are likely unsuccessful, but unsuccessful sessions are not necessarily escalated.
+
+Step 4: Count corrective user turns across the entire session.
+
+A user turn is corrective only when ALL of these are true:
+1. It refers to an earlier agent response or action.
+2. It indicates explicitly or by strong implication that the earlier response or action was incorrect, incomplete, insufficient, or misaligned.
+3. It steers the agent toward revising that earlier response or action, such as by supplying corrected information, identifying missing content, changing a constraint, rejecting an approach, requesting a redo, or specifying how the response should change.
+
+Count each qualifying user turn once, even if it corrects multiple issues. Count separate later corrective turns separately, including repeated corrections of the same issue. A correction may refer to an agent turn earlier than the immediately preceding turn.
+
+Do NOT count:
+- the user's initial request;
+- a genuinely different question or a request for a separate new deliverable, even when it concerns the same general topic;
+- an ordinary follow-up question that seeks additional information without indicating the earlier response should change;
+- an answer to a clarification question asked by the agent;
+- a confirmation, acknowledgment, approval, or request to continue;
+- information supplied before the agent made the relevant mistake;
+- an agent self-correction that was not prompted by a corrective user turn.
+
+To distinguish a corrective turn from a new intent, ask whether the user is evaluating, modifying, rejecting, completing, or replacing an earlier agent response. Topic continuity alone is not evidence of a correction. If the new intent can be fulfilled without revising the earlier response, do not count it.
+
+Set number_of_correction_per_session to the resulting non-negative integer. This count is independent of is_success: a session may contain corrective turns and still be successful if the primary task is completed by the end.
+
+[Context]
+{agent_context_prompt}
+
+[Success definition]
+{success_definition_prompt}
+
+[Interactions]
+User and agent interactions:
+{interactions}
+
+[Tools]
+{tool_can_use}
+
+[Metadata Definition]
+{metadata_definition_prompt}
+
+[Output]
+Generate the output in valid JSON format using the following schema.
+
+# success case example
+```json
+{{
+    "is_success": true,
+    "is_escalated": false,
+    "number_of_correction_per_session": 2
+}}
+```
+
+# failure case example
+`failure_type` must be one of `missing_tool`, `wrong_tool`, `insufficient_info_from_tool`, or `wrong_answer`; this example uses `missing_tool`.
+
+```json
+{{
+    "is_success": false,
+    "failure_type": "missing_tool",
+    "failure_reason": "explain the reason for failure and what the agent needs to do differently",
+    "is_escalated": true,
+    "number_of_correction_per_session": 1
+}}
+```

reflexio/server/services/agent_success_evaluation/agent_success_evaluation_constants.py

@@ -20,13 +20,14 @@ class AgentSuccessEvaluationOutput(StrictStructuredOutput):
     """
     Unified output schema for agent success evaluation.
 
-    For successful evaluations, only is_success=True is required.
-    For failed evaluations, all fields are required to provide failure details.
+    Every evaluation includes a corrective-user-turn count. Failed evaluations
+    additionally provide failure details.
 
     Attributes:
         is_success (bool): Indicates whether the agent successfully responded to the user
         failure_type (Optional[str]): Type of failure - 'missing_tool', 'wrong_tool', 'insufficient_info_from_tool', or 'wrong_answer'. Required when is_success=False
         failure_reason (Optional[str]): Explanation for the failure and what the agent needs to do differently. Required when is_success=False
+        number_of_correction_per_session (int): Number of user turns that corrected or redirected an earlier agent response.
     """
 
     is_success: bool = Field(
@@ -49,6 +50,14 @@ class AgentSuccessEvaluationOutput(StrictStructuredOutput):
         default=False,
         description="Whether the user was handed off to a human agent or another agent during the session.",
     )
+    number_of_correction_per_session: int = Field(
+        ge=0,
+        strict=True,
+        description=(
+            "Number of user turns in the session that corrected or redirected an "
+            "earlier agent response or action."
+        ),
+    )
     # OpenAI schema parsing requires explicitly forbidding additional properties
     model_config = ConfigDict(
         extra="allow",

reflexio/server/services/agent_success_evaluation/components/evaluator.py

@@ -282,7 +282,9 @@ def _build_evaluation_result(
             failure_type=evaluation_response.failure_type or "",
             failure_reason=evaluation_response.failure_reason or "",
             regular_vs_shadow=None,
-            number_of_correction_per_session=self._get_correction_count(),
+            number_of_correction_per_session=(
+                evaluation_response.number_of_correction_per_session
+            ),
             user_turns_to_resolution=(
                 self._count_user_turns(request_interaction_data_models)
                 if evaluation_response.is_success
@@ -335,25 +337,6 @@ def _count_user_turns(
                     count += 1
         return count
 
-    def _get_correction_count(self) -> int:
-        """
-        Count user playbooks linked to the current session.
-
-        Returns:
-            int: Number of user playbooks for the session, defaulting to 0 on error.
-        """
-        try:
-            count = self.request_context.storage.count_user_playbooks_by_session(  # type: ignore[reportOptionalMemberAccess]
-                self.service_config.session_id
-            )
-            return count if count is not None else 0
-        except Exception:
-            logger.warning(
-                "Failed to count user playbooks for session %s, defaulting to 0",
-                self.service_config.session_id,
-            )
-            return 0
-
     # F1 cleanup: ``_map_comparison_to_enum`` was retracted along with
     # ``_evaluate_with_shadow_comparison``. Per-turn shadow comparison has its
     # own mapping helpers in ``services/shadow_comparison/``.

reflexio/server/services/agent_success_evaluation/runner.py

@@ -112,12 +112,11 @@ def run_group_evaluation(
     2. Fetch all requests for the session
     3. Verify completion (latest request created_at >= delay ago; skipped when force_regenerate)
     4. Fetch interactions and build data models
-    5. Capture prior result_ids (when regenerating) so they
-       can be removed AFTER the new save lands
+    5. Capture prior result_ids (when regenerating) for post-save reconciliation
     6. Run evaluation service (which saves new rows)
-    7. On success, delete the captured prior rows by id — the new rows have
-       fresh auto-increment ids that do not overlap. A failure here leaves
-       the session in a consistent pre-regen state instead of zero rows.
+    7. On success, delete captured prior rows only when the backend inserted
+       fresh result_ids; preserve ids updated in place by an enterprise upsert.
+       A generation/save failure leaves the prior result untouched.
     8. Mark as evaluated in operation state
 
     Args:
@@ -252,19 +251,21 @@ def run_group_evaluation(
         )
         return GroupEvaluationOutcome("not_applicable", "skipped")
 
-    # 5. When regenerating, capture the prior result_ids
-    # so we can delete ONLY them AFTER the new rows have been saved. Doing
-    # the delete before the LLM call risks wiping the session's rows if the
-    # call fails (rate limit, network) and nothing replaces them. The new
-    # rows always get fresh auto-increment ids, so deleting the captured set
-    # afterwards cannot remove the new rows.
+    # 5. When regenerating, capture the prior result_ids so we can reconcile
+    # them AFTER the new rows have been saved. SQLite inserts a fresh row that
+    # needs the prior row cleaned up; enterprise storage upserts the prior row
+    # in place and therefore keeps its result_id. Doing any delete before the
+    # LLM call risks wiping the session's rows if the call fails (rate limit,
+    # network) and nothing replaces them.
     old_result_ids: list[int] = []
+    evaluation_name = ""
     if force_regenerate:
-        config = request_context.configurator.get_config()
+        root_config = request_context.configurator.get_config()
+        evaluation_name = get_extractor_name(root_config.agent_success_config)
         old_result_ids = storage.get_agent_success_evaluation_result_ids(  # type: ignore[reportOptionalMemberAccess]
             user_id=user_id,
             session_id=session_id,
-            evaluation_name=get_extractor_name(config),
+            evaluation_name=evaluation_name,
             agent_version=agent_version,
         )
 
@@ -310,20 +311,36 @@ def run_group_evaluation(
         )
         return GroupEvaluationOutcome("failed", "skipped")
 
-    # 6. New rows saved successfully — now safe to remove the captured prior
-    # rows. New rows have fresh auto-increment result_ids that do not overlap
-    # with old_result_ids, so this cannot delete the regenerated verdict.
+    # 7. New rows saved successfully. Delete captured prior rows only when the
+    # writer created a distinct new result_id (SQLite). Enterprise storage uses
+    # an in-place upsert, so its post-save ids are unchanged; deleting those ids
+    # would delete the regenerated verdict itself.
     if old_result_ids:
-        deleted = storage.delete_agent_success_evaluation_results_by_ids(  # type: ignore[reportOptionalMemberAccess]
-            old_result_ids
-        )
-        logger.info(
-            "Regenerate cleanup: deleted %d prior result row(s) for session=%s"
-            " (expected %d)",
-            deleted,
-            session_id,
-            len(old_result_ids),
+        saved_result_ids = storage.get_agent_success_evaluation_result_ids(  # type: ignore[reportOptionalMemberAccess]
+            user_id=user_id,
+            session_id=session_id,
+            evaluation_name=evaluation_name,
+            agent_version=agent_version,
         )
+        inserted_result_ids = set(saved_result_ids).difference(old_result_ids)
+        if inserted_result_ids:
+            deleted = storage.delete_agent_success_evaluation_results_by_ids(  # type: ignore[reportOptionalMemberAccess]
+                old_result_ids
+            )
+            logger.info(
+                "Regenerate cleanup: deleted %d prior result row(s) for session=%s"
+                " (expected %d)",
+                deleted,
+                session_id,
+                len(old_result_ids),
+            )
+        else:
+            logger.info(
+                "Regenerate cleanup: storage updated %d prior result row(s) in place"
+                " for session=%s",
+                len(old_result_ids),
+                session_id,
+            )
 
     # 7. Mark as evaluated
     evaluated_at = int(datetime.now(UTC).timestamp())

reflexio/test_support/llm_model_registry.py

@@ -117,6 +117,7 @@ def _build_registry() -> dict[str, ModelRegistryEntry]:
             minimal_valid={
                 "is_success": True,
                 "is_escalated": False,
+                "number_of_correction_per_session": 0,
             },
         ),
         "retrieved_learning_relevance": ModelRegistryEntry(

tests/fixtures/llm/agent_success_evaluation.json

@@ -2,7 +2,7 @@
   "choices": [
     {
       "message": {
-        "content": "{\"is_success\": true, \"is_escalated\": false}"
+        "content": "{\"is_success\": true, \"is_escalated\": false, \"number_of_correction_per_session\": 0}"
       },
       "finish_reason": "stop"
     }

tests/server/services/__snapshots__/test_llm_mock_schema_compliance/TestMockResponseSnapshots.test_recorded_fixture_content[agent_success_evaluation].json

@@ -1,4 +1,5 @@
 {
   "is_escalated": false,
-  "is_success": true
+  "is_success": true,
+  "number_of_correction_per_session": 0
 }