Merge pull request #1719 from gooddata/fix/normalize-maql-case-insensitive

fix(gooddata-eval): make MAQL comparison case-insensitive for keywords

Author: Tomkess

packages/gooddata-eval/src/gooddata_eval/core/agentic/metric_skill.py

@@ -25,6 +25,12 @@
 _IFNULL_RE = re.compile(r"IFNULL\s*\([^,]+,\s*0\)", re.IGNORECASE)
 _SELECT_WRAP_RE = re.compile(r"^\s*\(\s*SELECT\s*\{([^}]+)\}\s*\)\s*$", re.IGNORECASE)
 _INNER_SELECT_RE = re.compile(r"\(\s*SELECT\s*\{([^}]+)\}\s*\)", re.IGNORECASE)
+# Matches whichever comes first: a {type/id} identifier reference or a quoted string
+# literal -- both are case-sensitive data and must survive casefolding untouched.
+# Everything else in MAQL (keywords, operators, numbers, punctuation) carries no
+# case-sensitive meaning, per the MAQL reference (SELECT/BY/WHERE/FOR PREVIOUS/etc.
+# are case-insensitive; only {..} identifiers and quoted literal values are not).
+_PROTECTED_RE = re.compile(r"\{[^}]*\}|\"[^\"]*\"|'[^']*'")
 
 
 def _strip_outer_parens(s: str) -> str:
@@ -42,8 +48,21 @@ def _strip_outer_parens(s: str) -> str:
     return s[1:-1].strip()
 
 
+def _casefold_outside_protected(s: str) -> str:
+    """Lowercase MAQL keywords/operators while preserving case-sensitive {type/id}
+    identifiers and quoted string literal values (e.g. WHERE {label/x} = "Active")."""
+    parts = []
+    last = 0
+    for m in _PROTECTED_RE.finditer(s):
+        parts.append(s[last : m.start()].lower())
+        parts.append(m.group(0))
+        last = m.end()
+    parts.append(s[last:].lower())
+    return "".join(parts)
+
+
 def _normalize_maql(maql: str) -> str:
-    """Semantic normalisation: strip whitespace, unwrap IFNULL/SELECT wrappers."""
+    """Semantic normalisation: strip whitespace, unwrap IFNULL/SELECT wrappers, casefold keywords."""
     if not maql:
         return ""
     m = maql.strip()
@@ -56,7 +75,7 @@ def _normalize_maql(maql: str) -> str:
     m = re.sub(r"\{\s+", "{", m)
     m = re.sub(r"\s+\}", "}", m)
     m = re.sub(r"\s+", " ", m)
-    return m.strip()
+    return _casefold_outside_protected(m.strip())
 
 
 def _best_maql_match(actual_maql: str, expected_outputs: list[dict]) -> tuple[bool, str]:

packages/gooddata-eval/tests/test_agentic_metric_skill.py

@@ -21,7 +21,7 @@
 
 
 def test_normalize_maql_strips_whitespace():
-    assert _normalize_maql("  SELECT  { metric/foo }  ") == "SELECT {metric/foo}"
+    assert _normalize_maql("  SELECT  { metric/foo }  ") == "select {metric/foo}"
 
 
 def test_normalize_maql_removes_select_wrapper():
@@ -61,6 +61,29 @@ def test_generate_simulated_response_prompt_preserves_maql_fidelity(monkeypatch)
     assert call_kwargs["max_tokens"] >= 300
 
 
+def test_normalize_maql_is_case_insensitive_for_keywords():
+    """Regression test for a live-reproduced bug: 'FOR PREVIOUS(...)' vs
+    'FOR Previous(...)' scored as a mismatch even though MAQL keywords are
+    case-insensitive -- a semantically identical agent answer failed the eval
+    purely on keyword casing."""
+    actual = "SELECT {metric/active_card_count_-_txn_-_cutcgco} FOR PREVIOUS({label/process_date.year})"
+    expected = "SELECT {metric/active_card_count_-_txn_-_cutcgco}\n  FOR Previous({label/process_date.year})"
+    assert _normalize_maql(actual) == _normalize_maql(expected)
+
+
+def test_normalize_maql_preserves_identifier_case():
+    # {type/id} references are real, case-sensitive ids -- must never be casefolded.
+    assert "Mixed_Case_Id" in _normalize_maql("SELECT {metric/Mixed_Case_Id}")
+
+
+def test_normalize_maql_preserves_quoted_literal_case():
+    """The bug this guards against: naively lowercasing everything outside {..}
+    would also lowercase quoted WHERE-clause literal values, which are real,
+    case-sensitive data -- not keywords. Two literals differing only in case
+    must NOT be treated as equal; that would be a false positive."""
+    assert _normalize_maql('WHERE {label/status} = "Active"') != _normalize_maql('WHERE {label/status} = "active"')
+
+
 def test_metric_run_result_fields():
     r = MetricRunResult(
         conversation_id="c1",