Merge pull request #1686 from gooddata/qa-28482-run-metadata-extra

feat(gooddata-eval): optional run_metadata_extra on agentic evaluators

Author: myhoai

packages/gooddata-eval/src/gooddata_eval/core/agentic/_langfuse.py

@@ -305,7 +305,7 @@ def observe(
                 run_name=run_name,
                 dataset_item_id=dataset_item_id,
                 trace_id=trace_id,
-                metadata=run_metadata or {"testing_framework": "tavern-e2e"},
+                metadata=run_metadata or {},
                 run_description="",
             )
             _log.debug(
@@ -383,14 +383,31 @@ def build_run_context(
     dataset_name: str,
     run_timestamp: str | None,
     model_version_override: str | None,
+    run_metadata_extra: dict[str, Any] | None = None,
 ) -> tuple[str, dict[str, Any]]:
-    """Return (run_name_base, run_metadata) with model version resolved from workspace API."""
+    """Return (run_name_base, run_metadata) with model version resolved from workspace API.
+
+    Args:
+        host: GoodData host URL.
+        token: API token used to resolve the workspace model version.
+        workspace_id: Workspace whose active LLM provider is read when no override is given.
+        dataset_name: Langfuse dataset name; used as the run-name prefix.
+        run_timestamp: Shared run timestamp for the run name (falls back to the current time).
+        model_version_override: Explicit model-version tag; when set it wins over the
+            workspace lookup.
+        run_metadata_extra: Optional extra key/values added to the dataset-run metadata
+            (e.g. a testing-framework tag or a CI run id for scoping). Default None keeps
+            behavior unchanged. The SDK-derived model_version is applied last and cannot
+            be overwritten by this dict.
+    """
     model = get_model_version(host, token, workspace_id, model_version_override)
     ts = run_timestamp or datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
     base = f"{dataset_name}_{ts}"
     if model:
         base = f"{base}_{model}"
-    metadata: dict[str, Any] = {"testing_framework": "tavern-e2e"}
+    # Caller supplies its own run tags (e.g. testing_framework); model_version is applied
+    # last so the SDK-derived value cannot be overwritten by run_metadata_extra.
+    metadata: dict[str, Any] = dict(run_metadata_extra) if run_metadata_extra else {}
     if model:
         metadata["model_version"] = model
     return base, metadata

packages/gooddata-eval/src/gooddata_eval/core/agentic/alert_skill.py

@@ -438,6 +438,7 @@ def evaluate_agentic_alert_skill(
     dataset_name: str = "alert_skill",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run alert-skill evaluation, log to Langfuse, and raise AlertSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -475,6 +476,7 @@ def evaluate_agentic_alert_skill(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/conversation.py

@@ -384,6 +384,7 @@ def evaluate_agentic_conversation(
     dataset_name: str = "conversation",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run conversation evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -419,6 +420,7 @@ def evaluate_agentic_conversation(
             dataset_name or fixture.dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/general_question.py

@@ -152,6 +152,7 @@ def evaluate_agentic_general_question(
     dataset_name: str = "general_question",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run general-question evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -188,6 +189,7 @@ def evaluate_agentic_general_question(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/guardrail.py

@@ -149,6 +149,7 @@ def evaluate_agentic_guardrail(
     dataset_name: str = "guardrail",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run guardrail evaluation, log to Langfuse, and raise on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -185,6 +186,7 @@ def evaluate_agentic_guardrail(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -244,6 +244,7 @@ def evaluate_agentic_metric_skill(
     dataset_name: str = "metric_skill",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run metric-skill evaluation, log to Langfuse, and raise MetricSkillAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -281,6 +282,7 @@ def evaluate_agentic_metric_skill(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/search_tool.py

@@ -143,6 +143,7 @@ def evaluate_agentic_search_tool(
     dataset_name: str = "search",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
 ) -> None:
     """Run search-tool evaluation, log to Langfuse, and raise SearchToolAssertionError on failure."""
     from datetime import datetime as _dt  # noqa: PLC0415
@@ -179,6 +180,7 @@ def evaluate_agentic_search_tool(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         traces_by_conv = find_traces_per_conversation(
             langfuse,

packages/gooddata-eval/src/gooddata_eval/core/agentic/visualization.py

@@ -263,6 +263,7 @@ def evaluate_agentic_visualization(
     dataset_name: str = "visualization",
     run_timestamp: str | None = None,
     model_version_override: str | None = None,
+    run_metadata_extra: dict | None = None,
     record_output_path: str | None = None,
 ) -> None:
     """Run visualization evaluation, log to Langfuse, and raise VisualizationAssertionError on failure."""
@@ -302,6 +303,7 @@ def evaluate_agentic_visualization(
             dataset_name,
             run_timestamp,
             model_version_override,
+            run_metadata_extra,
         )
         K = len(summary.run_results)
         traces_by_conv = find_traces_per_conversation(

packages/gooddata-eval/tests/test_agentic_run_context.py

@@ -0,0 +1,32 @@
+# (C) 2026 GoodData Corporation
+from gooddata_eval.core.agentic._langfuse import build_run_context
+
+# model_version_override short-circuits get_model_version (no workspace API call).
+_COMMON = {
+    "host": "h",
+    "token": "t",
+    "workspace_id": "w",
+    "dataset_name": "ds",
+    "run_timestamp": "2026-07-10_00-00-00",
+    "model_version_override": "m",
+}
+
+
+def test_build_run_context_includes_caller_extra_keys():
+    base, metadata = build_run_context(
+        **_COMMON, run_metadata_extra={"testing_framework": "tavern-e2e", "github_run_id": "run-123"}
+    )
+    assert base == "ds_2026-07-10_00-00-00_m"
+    assert metadata["testing_framework"] == "tavern-e2e"
+    assert metadata["github_run_id"] == "run-123"
+    assert metadata["model_version"] == "m"
+
+
+def test_build_run_context_extra_cannot_override_model_version():
+    _, metadata = build_run_context(**_COMMON, run_metadata_extra={"model_version": "hack"})
+    assert metadata["model_version"] == "m"  # SDK-derived value wins
+
+
+def test_build_run_context_without_extra_has_only_model_version():
+    _, metadata = build_run_context(**_COMMON)
+    assert metadata == {"model_version": "m"}