chore: evals

surfsense_evals/tests/suites/test_frames_grader.py

@@ -0,0 +1,160 @@
+"""Tests for the FRAMES grader's deterministic shortcut.
+
+The LLM-judge fallback is excluded here (network call); we just
+confirm the rule-based path picks up obvious correct/incorrect
+cases and routes the ambiguous ones to ``lexical_miss`` so the
+runner knows to consult the judge.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from surfsense_evals.suites.research.frames.grader import (
+    GradeResult,
+    _maybe_number,
+    _normalise,
+    _whole_word_substring,
+    grade_deterministic,
+)
+
+
+class TestNormalisation:
+    def test_lowercase_and_punct_stripped(self) -> None:
+        assert _normalise("Jane Ballou.") == "jane ballou"
+
+    def test_articles_removed(self) -> None:
+        assert _normalise("The Eiffel Tower") == "eiffel tower"
+
+    def test_whitespace_squashed(self) -> None:
+        assert _normalise("  multi   space\tinput  ") == "multi space input"
+
+    def test_empty_returns_empty(self) -> None:
+        assert _normalise("") == ""
+        assert _normalise(None) == ""  # type: ignore[arg-type]
+
+
+class TestNumericExtraction:
+    def test_simple_int(self) -> None:
+        assert _maybe_number("42") == 42.0
+
+    def test_int_with_commas(self) -> None:
+        assert _maybe_number("1,234") == 1234.0
+
+    def test_year_in_sentence(self) -> None:
+        assert _maybe_number("It was published in 1847.") == 1847.0
+
+    def test_word_number(self) -> None:
+        assert _maybe_number("five") == 5.0
+        assert _maybe_number("Twenty") == 20.0
+
+    def test_no_number_returns_none(self) -> None:
+        assert _maybe_number("Jane Ballou") is None
+        assert _maybe_number("") is None
+
+
+class TestWholeWordSubstring:
+    def test_phrase_match(self) -> None:
+        assert _whole_word_substring("president of the united states", "united states")
+
+    def test_word_boundary_required(self) -> None:
+        # "states" should NOT match inside "statesman"
+        assert not _whole_word_substring("the renowned statesman", "states")
+
+    def test_empty_needle(self) -> None:
+        assert not _whole_word_substring("anything", "")
+
+
+class TestExactMatch:
+    def test_identical(self) -> None:
+        r = grade_deterministic(pred="Jane Ballou", gold="Jane Ballou")
+        assert r.correct is True
+        assert r.method == "exact"
+
+    def test_case_insensitive(self) -> None:
+        r = grade_deterministic(pred="paris", gold="Paris")
+        assert r.correct is True
+        assert r.method == "exact"
+
+    def test_punctuation_ignored(self) -> None:
+        r = grade_deterministic(pred="Jane Ballou.", gold="Jane Ballou")
+        assert r.correct is True
+
+
+class TestNumericPath:
+    def test_int_match(self) -> None:
+        r = grade_deterministic(pred="The answer is 87", gold="87")
+        assert r.correct is True
+        assert r.method == "numeric"
+
+    def test_word_number_matches_digit(self) -> None:
+        r = grade_deterministic(pred="five", gold="5")
+        assert r.correct is True
+        assert r.method == "numeric"
+
+    def test_off_by_more_than_tolerance_fails(self) -> None:
+        r = grade_deterministic(pred="86", gold="87")
+        # 86 vs 87, abs diff = 1, tol = max(0.01*87, 0.5) = 0.87 → fails
+        assert r.correct is False
+        assert r.method == "numeric_miss"
+
+    def test_within_one_percent_passes(self) -> None:
+        r = grade_deterministic(pred="100", gold="101")
+        # 1.0 abs diff, tol = max(0.01*101, 0.5) = 1.01 → passes
+        assert r.correct is True
+
+
+class TestSubstringPath:
+    def test_pred_contains_gold(self) -> None:
+        r = grade_deterministic(
+            pred="The answer is Jane Ballou according to records",
+            gold="Jane Ballou",
+        )
+        assert r.correct is True
+        assert r.method == "substring"
+
+    def test_gold_contains_pred_with_minimum_length(self) -> None:
+        # Gold = "John F Kennedy", pred = "Kennedy" → reverse substring,
+        # ≥3 chars, but the FRAMES style usually accepts this.
+        r = grade_deterministic(pred="Kennedy", gold="John F. Kennedy")
+        assert r.correct is True
+        assert r.method == "substring_reverse"
+
+    def test_too_short_pred_no_reverse_credit(self) -> None:
+        r = grade_deterministic(pred="of", gold="World of Warcraft")
+        # "of" passes length but is a stopword; the article-stripping
+        # normaliser removes it from gold, so substring fails. Either
+        # way, the grader should NOT credit this.
+        assert r.correct is False
+
+
+class TestLexicalMiss:
+    def test_completely_different_pred_falls_through(self) -> None:
+        r = grade_deterministic(pred="London", gold="Paris")
+        assert r.correct is False
+        assert r.method == "lexical_miss"
+
+    def test_empty_pred(self) -> None:
+        r = grade_deterministic(pred="", gold="Paris")
+        assert r.correct is False
+        assert r.method == "empty_pred"
+
+    def test_empty_gold_defensive(self) -> None:
+        r = grade_deterministic(pred="something", gold="")
+        # Defensive guard — gold should never be empty in practice.
+        assert r.correct is False
+        assert r.method == "empty_gold"
+
+
+class TestGradeResultShape:
+    def test_dict_has_all_expected_keys(self) -> None:
+        r = grade_deterministic(pred="Paris", gold="Paris")
+        d = r.to_dict()
+        assert set(d) >= {
+            "correct",
+            "f1",
+            "method",
+            "normalised_pred",
+            "normalised_gold",
+            "judge_rationale",
+        }