Merge pull request #1708 from gooddata/feat/chat-client-reasoning-steps

feat(gooddata-eval): capture agent reasoning steps in ChatResult

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -16,7 +16,7 @@
 from gooddata_eval.core.agentic.search_tool import evaluate_agentic_search_tool
 from gooddata_eval.core.agentic.visualization import evaluate_agentic_visualization
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import CreatedVisualization, DatasetItem
+from gooddata_eval.core.models import AgenticEvalOutcome, CreatedVisualization, DatasetItem
 from gooddata_eval.core.runner import EvalReport, ItemReport
 
 
@@ -86,8 +86,13 @@ def _dispatch_agentic(
     model_version_override: str | None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
-) -> None:
-    """Call the appropriate evaluate_agentic_* function for the item's test_kind."""
+) -> AgenticEvalOutcome | list[str] | None:
+    """Call the appropriate evaluate_agentic_* function for the item's test_kind.
+
+    Returns whatever that function returns -- alert_skill/metric_skill/conversation return
+    an AgenticEvalOutcome; the rest still return None
+    (unchanged).
+    """
     kind = item.test_kind
     eo = item.expected_output
     lf_kw: _LfKw = {
@@ -100,7 +105,7 @@ def _dispatch_agentic(
     }
 
     if kind in ("vis_agentic", "agentic_visualization"):
-        evaluate_agentic_visualization(
+        return evaluate_agentic_visualization(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -111,7 +116,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_metric_skill":
-        evaluate_agentic_metric_skill(
+        return evaluate_agentic_metric_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -122,7 +127,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_alert_skill":
-        evaluate_agentic_alert_skill(
+        return evaluate_agentic_alert_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -136,7 +141,7 @@ def _dispatch_agentic(
         eo_dict = eo if isinstance(eo, dict) else {}
         tool_call = eo_dict.get("tool_call", {})
         expected_args = tool_call.get("function_arguments", eo_dict)
-        evaluate_agentic_search_tool(
+        return evaluate_agentic_search_tool(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -147,7 +152,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_general_question":
-        evaluate_agentic_general_question(
+        return evaluate_agentic_general_question(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -158,7 +163,7 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_guardrail":
-        evaluate_agentic_guardrail(
+        return evaluate_agentic_guardrail(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -180,7 +185,7 @@ def _dispatch_agentic(
         )
     elif kind == "agentic_conversation":
         fixture_data = eo.get("fixture") or eo if isinstance(eo, dict) else {}
-        evaluate_agentic_conversation(
+        return evaluate_agentic_conversation(
             host=host,
             token=token,
             workspace_id=workspace_id,
@@ -228,14 +233,26 @@ def run_agentic_items(
         )
         t0 = time.perf_counter()
         try:
-            _dispatch_agentic(
+            outcome = _dispatch_agentic(
                 item, host, token, workspace_id, k, langfuse, run_ts, model_version, reasoning_effort, agent_id
             )
+            if isinstance(outcome, AgenticEvalOutcome):
+                reasoning_steps = outcome.reasoning_steps
+                conversation_id = outcome.conversation_id
+                response_id = outcome.response_id
+            else:
+                reasoning_steps, conversation_id, response_id = outcome, None, None
             item_report.pass_at_k = True
             item_report.runs = k
+            item_report.reasoning_steps = reasoning_steps or []
+            item_report.conversation_id = conversation_id
+            item_report.response_id = response_id
         except AssertionError as exc:
             item_report.pass_at_k = False
             item_report.runs = k
+            item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
+            item_report.conversation_id = getattr(exc, "conversation_id", None)
+            item_report.response_id = getattr(exc, "response_id", None)
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -6,15 +6,15 @@
 import json
 import os
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Any
 
 from gooddata_sdk import GoodDataSdk
 
 from gooddata_eval.core.agentic._catalog import CatalogMetricAlert
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import ToolCallEvent
+from gooddata_eval.core.models import AgenticEvalOutcome, ToolCallEvent
 
 try:
     from openai import OpenAI as _OpenAI
@@ -333,6 +333,8 @@ class AlertRunResult:
     alert_id: str | None
     eval: AlertEvaluation
     actual_alert_arguments: dict
+    reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 @dataclass
@@ -481,13 +483,17 @@ def _run_once(conv_id: str) -> AlertRunResult:
             alert_id: str | None = None
             actual_args: dict = {}
             tool_called = False
+            reasoning_steps: list[str] = []
+            response_id: str | None = None
             # conversation_history stores prior turns for GPT-4o context.
             # Roles follow GPT-4o's perspective: "assistant"=agent text, "user"=sim-user reply.
             conversation_history: list = []
             current_question = question
 
             for _iteration in range(max_iterations):
                 chat_result = client.send_message(conv_id, current_question)
+                reasoning_steps.extend(chat_result.reasoning_steps or [])
+                response_id = chat_result.response_id or response_id
                 alert_id, actual_args, tool_called = _extract_alert_call(chat_result.tool_call_events or [])
                 if tool_called:
                     alert_id_to_delete = alert_id
@@ -523,6 +529,8 @@ def _run_once(conv_id: str) -> AlertRunResult:
                 alert_id=alert_id,
                 eval=ev,
                 actual_alert_arguments=actual_args,
+                reasoning_steps=reasoning_steps,
+                response_id=response_id,
             )
         finally:
             if alert_id_to_delete:
@@ -573,6 +581,9 @@ class AlertSkillAssertionError(AssertionError):
     """Raised when an alert-skill evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
 
 
 def evaluate_agentic_alert_skill(
@@ -592,8 +603,16 @@ def evaluate_agentic_alert_skill(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
+) -> AgenticEvalOutcome:
+    """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure.
+
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as
+    ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
+    either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -667,11 +686,20 @@ def evaluate_agentic_alert_skill(
     if not summary.pass_at_k:
         best = summary.best
         ev = best.eval
-        raise AlertSkillAssertionError(
+        exc = AlertSkillAssertionError(
             f"Alert skill assertion failed. strict_pass={ev.strict_pass}. "
             f"alert_created={ev.alert_created}, operator_correct={ev.operator_correct}, "
             f"threshold_correct={ev.threshold_correct}, trigger_correct={ev.trigger_correct}, "
             f"filters_correct={ev.filters_correct}, metric_correct={ev.metric_correct}, "
             f"recipients_correct={ev.recipients_correct}. "
             f"Actual args: {best.actual_alert_arguments}"
         )
+        exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
+        raise exc
+    return AgenticEvalOutcome(
+        reasoning_steps=summary.best.reasoning_steps,
+        conversation_id=summary.best.conversation_id,
+        response_id=summary.best.response_id,
+    )

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -5,7 +5,7 @@
 
 import json
 import re
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import Literal
 
 from gooddata_sdk import GoodDataSdk
@@ -15,7 +15,7 @@
 from gooddata_eval.core.agentic.metric_skill import _delete_metric, _extract_created_metric_ids
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
-from gooddata_eval.core.models import ChatResult, ToolCallEvent
+from gooddata_eval.core.models import AgenticEvalOutcome, ChatResult, ToolCallEvent
 from gooddata_eval.core.scoring import (
     check_filters,
     check_viz_type,
@@ -265,6 +265,8 @@ class ConversationResult:
     full_skill_coverage: bool
     conversation_success: bool
     total_clarification_turns: int
+    reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 def run_agentic_conversation(
@@ -296,6 +298,8 @@ def run_agentic_conversation(
     # not persist in the (shared) workspace and get reused by a later test. Deferred to
     # the end — a later turn may $ref a metric an earlier turn created.
     created_metric_ids: list[str] = []
+    reasoning_steps: list[str] = []
+    response_id: str | None = None
 
     try:
         if initial_conversation_id is not None:
@@ -332,6 +336,8 @@ def run_agentic_conversation(
                 chat_result = client.send_message(conversation_id, current_message)
                 final_result = chat_result
                 all_tool_calls.extend(chat_result.tool_call_events or [])
+                reasoning_steps.extend(chat_result.reasoning_steps or [])
+                response_id = chat_result.response_id or response_id
 
                 if _check_output_present(resolved_turn, chat_result):
                     break
@@ -395,13 +401,18 @@ def run_agentic_conversation(
         full_skill_coverage=full_skill_coverage,
         conversation_success=conversation_success,
         total_clarification_turns=total_clarification_turns,
+        reasoning_steps=reasoning_steps,
+        response_id=response_id,
     )
 
 
 class ConversationAssertionError(AssertionError):
     """Raised when a conversation evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
 
 
 def evaluate_agentic_conversation(
@@ -419,8 +430,16 @@ def evaluate_agentic_conversation(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run conversation evaluation, log to Langfuse, and raise on failure."""
+) -> AgenticEvalOutcome:
+    """Run conversation evaluation, log to Langfuse, and raise on failure.
+
+    Returns the conversation's outcome (reasoning_steps, conversation_id, response_id) as
+    an AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as
+    ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors the
+    `conversation_id`-on-exception idiom in `ChatClient.ask()`) so callers can retrieve them
+    either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -497,8 +516,17 @@ def evaluate_agentic_conversation(
 
     if not result.conversation_success:
         failed_turns = [tr for tr in result.turn_results if not tr.skill_success]
-        raise ConversationAssertionError(
+        exc = ConversationAssertionError(
             f"Conversation assertion failed. "
             f"full_skill_coverage={result.full_skill_coverage}. "
             f"Failed turns: {[t.turn_id for t in failed_turns]}"
         )
+        exc.reasoning_steps = result.reasoning_steps
+        exc.conversation_id = result.conversation_id
+        exc.response_id = result.response_id
+        raise exc
+    return AgenticEvalOutcome(
+        reasoning_steps=result.reasoning_steps,
+        conversation_id=result.conversation_id,
+        response_id=result.response_id,
+    )