[bot] Merge master/bf0e8581 into rel/dev

packages/gooddata-eval/src/gooddata_eval/cli/agentic_runner.py

@@ -86,12 +86,12 @@ def _dispatch_agentic(
     model_version_override: str | None,
     reasoning_effort: ReasoningEffort | None = None,
     agent_id: str | None = None,
-) -> AgenticEvalOutcome | list[str] | None:
+) -> AgenticEvalOutcome:
     """Call the appropriate evaluate_agentic_* function for the item's test_kind.
 
-    Returns whatever that function returns -- alert_skill/metric_skill/conversation return
-    an AgenticEvalOutcome; the rest still return None
-    (unchanged).
+    Every evaluate_agentic_* function returns an AgenticEvalOutcome (reasoning_steps,
+    conversation_id, response_id, detail) on success and attaches the same four attributes
+    to its raised *AssertionError on failure -- no kind is exempt.
     """
     kind = item.test_kind
     eo = item.expected_output
@@ -174,13 +174,14 @@ def _dispatch_agentic(
             **lf_kw,
         )
     elif kind == "agentic_kda_skill":
-        evaluate_agentic_kda_skill(
+        return evaluate_agentic_kda_skill(
             host=host,
             token=token,
             workspace_id=workspace_id,
             question=item.question,
             expected_output=eo if isinstance(eo, dict) else {},
             k=k,
+            agent_id=agent_id,
             **lf_kw,
         )
     elif kind == "agentic_conversation":
@@ -240,19 +241,22 @@ def run_agentic_items(
                 reasoning_steps = outcome.reasoning_steps
                 conversation_id = outcome.conversation_id
                 response_id = outcome.response_id
+                detail = outcome.detail
             else:
-                reasoning_steps, conversation_id, response_id = outcome, None, None
+                reasoning_steps, conversation_id, response_id, detail = outcome, None, None, {}
             item_report.pass_at_k = True
             item_report.runs = k
             item_report.reasoning_steps = reasoning_steps or []
             item_report.conversation_id = conversation_id
             item_report.response_id = response_id
+            item_report.best_detail = detail or {}
         except AssertionError as exc:
             item_report.pass_at_k = False
             item_report.runs = k
             item_report.reasoning_steps = getattr(exc, "reasoning_steps", None) or []
             item_report.conversation_id = getattr(exc, "conversation_id", None)
             item_report.response_id = getattr(exc, "response_id", None)
+            item_report.best_detail = getattr(exc, "detail", None) or {}
             print(f"[agentic] {item.id} FAIL: {exc}", flush=True)
         except Exception as exc:
             item_report.error = f"{type(exc).__name__}: {exc}"

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -160,8 +160,18 @@ def _check_recipients(expected: CatalogMetricAlert, actual_args: dict, sdk: Good
         act_recip = []
     if set(expected.recipients) == set(act_recip or []):
         return True
-    act_internal = actual_args.get("internal_recipients")
-    if sdk is not None and isinstance(act_internal, list) and act_internal:
+    act_internal_raw = actual_args.get("internal_recipients")
+    # internal_recipients is declared `anyOf: [array of string, string, null]` in the
+    # create_metric_alert tool schema -- a single id as a bare string is schema-legal,
+    # not a malformed call, so it needs the same string/list normalization already
+    # applied to recipients/external_recipients above.
+    if isinstance(act_internal_raw, str):
+        act_internal = [act_internal_raw]
+    elif isinstance(act_internal_raw, list):
+        act_internal = act_internal_raw
+    else:
+        act_internal = []
+    if sdk is not None and act_internal:
         internal_recipient_ids = _resolve_internal_recipient_ids(sdk, expected.recipients)
         if internal_recipient_ids & set(act_internal):
             return True
@@ -584,6 +594,7 @@ class AlertSkillAssertionError(AssertionError):
     reasoning_steps: list[str]
     conversation_id: str
     response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_alert_skill(
@@ -697,9 +708,31 @@ def evaluate_agentic_alert_skill(
         exc.reasoning_steps = best.reasoning_steps
         exc.conversation_id = best.conversation_id
         exc.response_id = best.response_id
+        exc.detail = {
+            "alert_created": ev.alert_created,
+            "operator_correct": ev.operator_correct,
+            "threshold_correct": ev.threshold_correct,
+            "trigger_correct": ev.trigger_correct,
+            "filters_correct": ev.filters_correct,
+            "metric_correct": ev.metric_correct,
+            "recipients_correct": ev.recipients_correct,
+            "actual_alert_arguments": best.actual_alert_arguments,
+        }
         raise exc
+    best = summary.best
+    ev = best.eval
     return AgenticEvalOutcome(
-        reasoning_steps=summary.best.reasoning_steps,
-        conversation_id=summary.best.conversation_id,
-        response_id=summary.best.response_id,
+        reasoning_steps=best.reasoning_steps,
+        conversation_id=best.conversation_id,
+        response_id=best.response_id,
+        detail={
+            "alert_created": ev.alert_created,
+            "operator_correct": ev.operator_correct,
+            "threshold_correct": ev.threshold_correct,
+            "trigger_correct": ev.trigger_correct,
+            "filters_correct": ev.filters_correct,
+            "metric_correct": ev.metric_correct,
+            "recipients_correct": ev.recipients_correct,
+            "actual_alert_arguments": best.actual_alert_arguments,
+        },
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -393,13 +393,32 @@ def run_agentic_conversation(
     )
 
 
+def _conversation_detail(result: ConversationResult) -> dict:
+    return {
+        "full_skill_coverage": result.full_skill_coverage,
+        "total_clarification_turns": result.total_clarification_turns,
+        "turns": [
+            {
+                "turn_id": tr.turn_id,
+                "expected_skill": tr.expected_skill,
+                "skill_routing": tr.skill_routing,
+                "output_present": tr.output_present,
+                "output_correct": tr.output_correct,
+                "activated_skills": tr.activated_skills,
+            }
+            for tr in result.turn_results
+        ],
+    }
+
+
 class ConversationAssertionError(AssertionError):
     """Raised when a conversation evaluation fails."""
 
     __tracebackhide__ = True
     reasoning_steps: list[str]
     conversation_id: str
     response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_conversation(
@@ -511,9 +530,11 @@ def evaluate_agentic_conversation(
         exc.reasoning_steps = result.reasoning_steps
         exc.conversation_id = result.conversation_id
         exc.response_id = result.response_id
+        exc.detail = _conversation_detail(result)
         raise exc
     return AgenticEvalOutcome(
         reasoning_steps=result.reasoning_steps,
         conversation_id=result.conversation_id,
         response_id=result.response_id,
+        detail=_conversation_detail(result),
     )

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -3,11 +3,12 @@
 
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
+from gooddata_eval.core.models import AgenticEvalOutcome
 
 _DEFAULT_K = 1
 
@@ -52,6 +53,8 @@ class GeneralQuestionResult:
     passed: bool
     llm_judge_score: float
     reasoning: str
+    reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 @dataclass
@@ -98,6 +101,8 @@ def run_agentic_general_question(
                     passed=passed,
                     llm_judge_score=llm_judge_score,
                     reasoning=reasoning,
+                    reasoning_steps=list(chat_result.reasoning_steps or []),
+                    response_id=chat_result.response_id,
                 )
             )
         finally:
@@ -120,6 +125,8 @@ def run_agentic_general_question(
                         passed=passed,
                         llm_judge_score=llm_judge_score,
                         reasoning=reasoning,
+                        reasoning_steps=list(chat_result.reasoning_steps or []),
+                        response_id=chat_result.response_id,
                     )
                 )
             finally:
@@ -142,6 +149,10 @@ class GeneralQuestionAssertionError(AssertionError):
     """Raised when a general-question evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_general_question(
@@ -160,8 +171,13 @@ def evaluate_agentic_general_question(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run general-question evaluation, log to Langfuse, and raise on failure."""
+) -> AgenticEvalOutcome:
+    """Run general-question evaluation, log to Langfuse, and raise GeneralQuestionAssertionError on failure.
+
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id``.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -223,6 +239,26 @@ def evaluate_agentic_general_question(
 
     if not summary.pass_at_k:
         best = summary.best
-        raise GeneralQuestionAssertionError(
+        exc = GeneralQuestionAssertionError(
             f"General question assertion failed. passed={best.passed}. Reasoning: {best.reasoning}"
         )
+        exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
+        exc.detail = {
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+        }
+        raise exc
+    best = summary.best
+    return AgenticEvalOutcome(
+        reasoning_steps=best.reasoning_steps,
+        conversation_id=best.conversation_id,
+        response_id=best.response_id,
+        detail={
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+        },
+    )

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -3,11 +3,12 @@
 
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 
 from gooddata_eval.core.chat.sse_client import ChatClient
 from gooddata_eval.core.config import ReasoningEffort
 from gooddata_eval.core.evaluators._llm_judge import LLMJudge
+from gooddata_eval.core.models import AgenticEvalOutcome
 
 _DEFAULT_K = 1
 
@@ -49,6 +50,8 @@ class GuardrailResult:
     passed: bool
     llm_judge_score: float
     reasoning: str
+    reasoning_steps: list[str] = field(default_factory=list)
+    response_id: str | None = None
 
 
 @dataclass
@@ -95,6 +98,8 @@ def run_agentic_guardrail(
                     passed=passed,
                     llm_judge_score=llm_judge_score,
                     reasoning=reasoning,
+                    reasoning_steps=list(chat_result.reasoning_steps or []),
+                    response_id=chat_result.response_id,
                 )
             )
         finally:
@@ -117,6 +122,8 @@ def run_agentic_guardrail(
                         passed=passed,
                         llm_judge_score=llm_judge_score,
                         reasoning=reasoning,
+                        reasoning_steps=list(chat_result.reasoning_steps or []),
+                        response_id=chat_result.response_id,
                     )
                 )
             finally:
@@ -139,6 +146,10 @@ class GuardrailAssertionError(AssertionError):
     """Raised when a guardrail evaluation fails."""
 
     __tracebackhide__ = True
+    reasoning_steps: list[str]
+    conversation_id: str
+    response_id: str | None
+    detail: dict
 
 
 def evaluate_agentic_guardrail(
@@ -157,8 +168,14 @@ def evaluate_agentic_guardrail(
     model_version_override: str | None = None,
     run_metadata_extra: dict | None = None,
     reasoning_effort: ReasoningEffort | None = None,
-) -> None:
-    """Run guardrail evaluation, log to Langfuse, and raise on failure."""
+) -> AgenticEvalOutcome:
+    """Run guardrail evaluation, log to Langfuse, and raise GuardrailAssertionError on failure.
+
+    Returns the best run's outcome (reasoning_steps, conversation_id, response_id) as an
+    AgenticEvalOutcome on success; on failure the same three values are attached to the
+    raised exception as ``.reasoning_steps``/``.conversation_id``/``.response_id`` (mirrors
+    `evaluate_agentic_metric_skill`'s idiom) so callers can retrieve them either way.
+    """
     from datetime import datetime as _dt  # noqa: PLC0415
     from datetime import timezone as _tz  # noqa: PLC0415
 
@@ -220,4 +237,24 @@ def evaluate_agentic_guardrail(
 
     if not summary.pass_at_k:
         best = summary.best
-        raise GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}")
+        exc = GuardrailAssertionError(f"Guardrail assertion failed. passed={best.passed}. Reasoning: {best.reasoning}")
+        exc.reasoning_steps = best.reasoning_steps
+        exc.conversation_id = best.conversation_id
+        exc.response_id = best.response_id
+        exc.detail = {
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+        }
+        raise exc
+    best = summary.best
+    return AgenticEvalOutcome(
+        reasoning_steps=best.reasoning_steps,
+        conversation_id=best.conversation_id,
+        response_id=best.response_id,
+        detail={
+            "judge_passed": best.passed,
+            "judge_reasoning": best.reasoning,
+            "actual_output": best.actual_output,
+        },
+    )