feat: bumped version to 0.0.32

surfsense_evals/scripts/analyze_failures.py

@@ -12,12 +12,10 @@ Outputs (printed to stdout + written to `failures_n171.json`):
 from __future__ import annotations
 
 import json
-import re
 from collections import Counter, defaultdict
 from pathlib import Path
 from typing import Any
 
-
 REPO = Path(__file__).resolve().parents[1]
 RUN = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
 RAW = RUN / "raw.jsonl"

surfsense_evals/scripts/check_uploaded_status.py

@@ -15,7 +15,6 @@ from pathlib import Path
 import httpx
 from dotenv import load_dotenv
 
-
 REPO = Path(__file__).resolve().parents[1]
 PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"
 

surfsense_evals/scripts/compute_post_retry_accuracy.py

@@ -47,9 +47,7 @@ def _row_key(row: dict) -> tuple[str, str]:
 def _is_failure(row: dict) -> bool:
     if row.get("error"):
         return True
-    if not (row.get("raw_text") or "").strip():
-        return True
-    return False
+    return bool(not (row.get("raw_text") or "").strip())
 
 
 def _summarise(rows_by_arm: dict[str, list[dict]]) -> dict[str, dict]:

surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py

@@ -27,7 +27,6 @@ from __future__ import annotations
 import json
 from pathlib import Path
 
-
 REPO = Path(__file__).resolve().parents[1]
 MAP_PATH = REPO / "data" / "multimodal_doc" / "maps" / "mmlongbench_doc_map.jsonl"
 PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"

surfsense_evals/scripts/peek_crag_run.py

@@ -10,19 +10,20 @@ from collections import defaultdict
 def main() -> None:
     raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
     print(f"Reading: {raw_path}")
-    rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+    with open(raw_path, encoding="utf-8") as fh:
+        rows = [json.loads(line) for line in fh if line.strip()]
     by_q: dict[str, dict[str, dict]] = defaultdict(dict)
     for r in rows:
         by_q[r["qid"]][r["arm"]] = r
 
     for qid, arms in list(by_q.items()):
         b = arms.get("bare_llm", {})
-        l = arms.get("long_context", {})
+        lc = arms.get("long_context", {})
         s = arms.get("surfsense", {})
         print(f"\n=== {qid} ({b.get('domain')}/{b.get('question_type')}) ===")
         print(f"  question: {b.get('extra', {}).get('question', '?')!r}")
         print(f"  gold: {b.get('gold')!r}")
-        for arm_name, a in (("bare_llm", b), ("long_context", l), ("surfsense", s)):
+        for arm_name, a in (("bare_llm", b), ("long_context", lc), ("surfsense", s)):
             grade = a.get("graded", {})
             text = (a.get("raw_text") or "").strip()
             tail = text[-200:] if text else ""

surfsense_evals/scripts/peek_disagreements.py

@@ -10,7 +10,8 @@ from collections import defaultdict
 def main() -> None:
     raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
     print(f"Reading: {raw_path}")
-    rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+    with open(raw_path, encoding="utf-8") as fh:
+        rows = [json.loads(line) for line in fh if line.strip()]
     by_q: dict[str, dict[str, dict]] = defaultdict(dict)
     for r in rows:
         by_q[r["qid"]][r["arm"]] = r

surfsense_evals/scripts/retry_failed_questions.py

@@ -106,9 +106,7 @@ def _is_failure_row(row: dict[str, Any]) -> bool:
 
     if row.get("error"):
         return True
-    if not (row.get("raw_text") or "").strip():
-        return True
-    return False
+    return bool(not (row.get("raw_text") or "").strip())
 
 
 @dataclass
@@ -428,7 +426,7 @@ async def _run(args: argparse.Namespace) -> int:
 
         if f.arm == "native_pdf":
             pdf_path = Path(map_row["pdf_path"])
-            if not pdf_path.exists():
+            if not await asyncio.to_thread(pdf_path.exists):
                 logger.error("PDF missing on disk: %s — skipping", pdf_path)
                 continue
             request = _build_native_request(

surfsense_evals/scripts/summarise_crag_run.py

@@ -11,7 +11,8 @@ def main() -> None:
     if not runs:
         print("(no CRAG runs found)")
         return
-    m = json.load(open(runs[-1], encoding="utf-8"))
+    with open(runs[-1], encoding="utf-8") as fh:
+        m = json.load(fh)
     metrics = m["metrics"]
 
     print(f"Reading: {runs[-1]}")

surfsense_evals/scripts/summarise_parser_compare_run.py

@@ -16,7 +16,6 @@ import statistics
 from collections import defaultdict
 from pathlib import Path
 
-
 REPO = Path(__file__).resolve().parents[1]
 RUN_DIR = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
 RAW = RUN_DIR / "raw.jsonl"
@@ -91,7 +90,7 @@ def main() -> None:
 
     print()
     print("by answer_format (accuracy):")
-    formats = sorted({f for m in arm_metrics.values() for f in m["by_format"].keys()})
+    formats = sorted({f for m in arm_metrics.values() for f in m["by_format"]})
     header = f"{'arm':<25} " + " ".join(f"{f:>10}" for f in formats)
     print(header)
     print("-" * len(header))

surfsense_evals/src/surfsense_evals/core/cli.py

@@ -32,14 +32,13 @@ from __future__ import annotations
 
 import argparse
 import asyncio
+import contextlib
 import json
 import logging
 import sys
 from dataclasses import dataclass
 from typing import Any
 
-import sys
-
 import httpx
 from rich.console import Console
 from rich.table import Table
@@ -51,10 +50,8 @@ from rich.table import Table
 # Terminal, PowerShell, cmd) all interpret ANSI escapes natively.
 if sys.platform == "win32":
     for _stream in (sys.stdout, sys.stderr):
-        try:
+        with contextlib.suppress(AttributeError, ValueError):
             _stream.reconfigure(encoding="utf-8", errors="replace")
-        except (AttributeError, ValueError):
-            pass
 
 from . import registry
 from .auth import CredentialError, acquire_token, client_with_auth

surfsense_evals/src/surfsense_evals/core/clients/documents.py

@@ -18,6 +18,7 @@ Document processing is asynchronous:
 from __future__ import annotations
 
 import asyncio
+import contextlib
 import logging
 import mimetypes
 from collections.abc import Iterable, Sequence
@@ -157,10 +158,8 @@ class DocumentsClient:
             )
         finally:
             for _, (_, file_obj, _) in opened:
-                try:
+                with contextlib.suppress(Exception):
                     file_obj.close()
-                except Exception:  # noqa: BLE001
-                    pass
 
         response.raise_for_status()
         return FileUploadResult.from_payload(response.json())

surfsense_evals/src/surfsense_evals/core/metrics/comparison.py

@@ -71,8 +71,8 @@ def mcnemar_test(
             f"Length mismatch: arm_a={len(arm_a_correct)}, arm_b={len(arm_b_correct)}"
         )
     n = len(arm_a_correct)
-    b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct) if a and not c)
-    c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct) if (not a) and cc)
+    b = sum(1 for a, c in zip(arm_a_correct, arm_b_correct, strict=False) if a and not c)
+    c = sum(1 for a, cc in zip(arm_a_correct, arm_b_correct, strict=False) if (not a) and cc)
     discordant = b + c
     if discordant == 0:
         return McnemarResult(

surfsense_evals/src/surfsense_evals/core/parse/__init__.py

@@ -3,7 +3,7 @@
 from __future__ import annotations
 
 from .answer_letter import AnswerLetterResult, extract_answer_letter
-from .citations import CITATION_REGEX, CitationToken, ChunkCitation, UrlCitation, parse_citations
+from .citations import CITATION_REGEX, ChunkCitation, CitationToken, UrlCitation, parse_citations
 from .freeform_answer import extract_freeform_answer
 from .sse import SseEvent, iter_sse_events
 

surfsense_evals/src/surfsense_evals/core/parse/citations.py

@@ -15,7 +15,7 @@ from __future__ import annotations
 
 import re
 from dataclasses import dataclass
-from typing import Any, Union
+from typing import Any
 
 # Pattern preserves the TS source verbatim:
 #   /[\[【]\u200B?citation:\s*(https?:\/\/[^\]】\u200B]+|urlcite\d+|(?:doc-)?-?\d+(?:\s*,\s*(?:doc-)?-?\d+)*)\s*\u200B?[\]】]/g
@@ -64,7 +64,7 @@ class UrlCitation:
         return {"kind": "url", "url": self.url}
 
 
-CitationToken = Union[ChunkCitation, UrlCitation]
+CitationToken = ChunkCitation | UrlCitation
 
 
 def parse_citations(text: str, *, url_map: dict[str, str] | None = None) -> list[CitationToken]:

surfsense_evals/src/surfsense_evals/core/parsers/azure_di.py

@@ -25,6 +25,7 @@ import asyncio
 import logging
 import os
 import random
+from pathlib import Path
 
 logger = logging.getLogger(__name__)
 
@@ -82,7 +83,7 @@ async def parse_with_azure_di(
         ServiceResponseError,
     )
 
-    file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+    file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024)
     logger.info(
         "Azure DI parsing %s (mode=%s, model=%s, size=%.1fMB)",
         file_path, processing_mode, model_id, file_size_mb,
@@ -96,12 +97,12 @@ async def parse_with_azure_di(
                 credential=AzureKeyCredential(api_key),
             )
             async with client:
-                with open(file_path, "rb") as fh:
-                    poller = await client.begin_analyze_document(
-                        model_id,
-                        body=fh,
-                        output_content_format=DocumentContentFormat.MARKDOWN,
-                    )
+                body = await asyncio.to_thread(Path(file_path).read_bytes)
+                poller = await client.begin_analyze_document(
+                    model_id,
+                    body=body,
+                    output_content_format=DocumentContentFormat.MARKDOWN,
+                )
                 result = await poller.result()
             content = (result.content or "").strip()
             if not content:

surfsense_evals/src/surfsense_evals/core/parsers/llamacloud.py

@@ -98,7 +98,7 @@ async def parse_with_llamacloud(
     from llama_cloud_services.parse.base import JobFailedException
     from llama_cloud_services.parse.utils import ResultType
 
-    file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
+    file_size_mb = await asyncio.to_thread(os.path.getsize, file_path) / (1024 * 1024)
     # Match backend's per-page timeout heuristic so big PDFs don't drop
     # mid-job: 60s baseline + 30s/page (premium agent runs longer than
     # basic; both fit comfortably here).

surfsense_evals/src/surfsense_evals/core/providers/openrouter_chat.py

@@ -29,8 +29,8 @@ from typing import Any
 import httpx
 
 from .openrouter_pdf import (
-    OpenRouterResponse,
     _DEFAULT_HEADERS,
+    OpenRouterResponse,
     _parse_chat_completion,
 )
 

surfsense_evals/src/surfsense_evals/core/providers/openrouter_pdf.py

@@ -34,7 +34,7 @@ import base64
 import logging
 import time
 from dataclasses import dataclass
-from enum import Enum
+from enum import StrEnum
 from pathlib import Path
 from typing import Any
 
@@ -43,7 +43,7 @@ import httpx
 logger = logging.getLogger(__name__)
 
 
-class PdfEngine(str, Enum):
+class PdfEngine(StrEnum):
     NATIVE = "native"
     MISTRAL_OCR = "mistral-ocr"
     CLOUDFLARE_AI = "cloudflare-ai"

surfsense_evals/src/surfsense_evals/suites/__init__.py

@@ -20,7 +20,7 @@ from __future__ import annotations
 import importlib
 import logging
 import pkgutil
-from typing import Iterable
+from collections.abc import Iterable
 
 logger = logging.getLogger(__name__)
 

surfsense_evals/src/surfsense_evals/suites/_demo/hello/__init__.py

@@ -6,7 +6,6 @@ import argparse
 from typing import Any
 
 from ....core.registry import (
-    Benchmark,
     ReportSection,
     RunArtifact,
     RunContext,

surfsense_evals/src/surfsense_evals/suites/medical/cure/__init__.py

@@ -12,7 +12,7 @@ Recall@k / MRR / nDCG@10 against qrels.
 
 from __future__ import annotations
 
-from .runner import CureBenchmark
 from ....core import registry as _registry
+from .runner import CureBenchmark
 
 _registry.register(CureBenchmark())

surfsense_evals/src/surfsense_evals/suites/medical/cure/ingest.py

@@ -227,12 +227,10 @@ async def run_ingest(
 
 
 def _take(it: Iterable, n: int) -> Iterable:
-    yielded = 0
-    for x in it:
-        if yielded >= n:
+    for i, x in enumerate(it):
+        if i >= n:
             return
         yield x
-        yielded += 1
 
 
 __all__ = ["DISCIPLINES", "CorpusPassage", "PassageBatch", "run_ingest"]

surfsense_evals/src/surfsense_evals/suites/medical/cure/runner.py

@@ -34,7 +34,6 @@ from ....core.ingest_settings import (
 )
 from ....core.metrics.retrieval import score_run
 from ....core.registry import (
-    Benchmark,
     ReportSection,
     RunArtifact,
     RunContext,
@@ -276,7 +275,7 @@ class CureBenchmark:
         )
 
         per_query_retrieved: dict[str, list[str]] = {}
-        for q, res in zip(queries, results):
+        for q, res in zip(queries, results, strict=False):
             chunk_ids: list[int] = []
             seen: set[int] = set()
             for citation in res.citations:
@@ -311,7 +310,7 @@ class CureBenchmark:
         run_dir = ctx.runs_dir(run_timestamp=run_timestamp)
         raw_path = run_dir / "raw.jsonl"
         with raw_path.open("w", encoding="utf-8") as fh:
-            for q, res in zip(queries, results):
+            for q, res in zip(queries, results, strict=False):
                 fh.write(
                     json.dumps(
                         {

surfsense_evals/src/surfsense_evals/suites/medical/mirage/__init__.py

@@ -11,7 +11,7 @@ document — the corpus is millions of biomedical snippets.
 
 from __future__ import annotations
 
-from .runner import MirageBenchmark
 from ....core import registry as _registry
+from .runner import MirageBenchmark
 
 _registry.register(MirageBenchmark())

surfsense_evals/src/surfsense_evals/suites/medical/mirage/ingest.py

@@ -93,6 +93,25 @@ class SnippetRow:
 # ---------------------------------------------------------------------------
 
 
+def _reuse_cached_dest(dest: Path, *, expect_zip: bool, label: str) -> Path | None:
+    """Return ``dest`` if a usable cache hit, else ``None`` (and delete corrupt zips)."""
+
+    if not dest.exists():
+        return None
+    if expect_zip and not _is_valid_zip(dest):
+        logger.warning(
+            "Cached %s at %s failed ZIP validation (size=%d B); deleting "
+            "and re-downloading.",
+            label,
+            dest,
+            dest.stat().st_size,
+        )
+        dest.unlink(missing_ok=True)
+        return None
+    logger.info("Using cached %s at %s", label, dest)
+    return dest
+
+
 async def _fetch_to_path(
     url: str,
     *,
@@ -127,19 +146,9 @@ async def _fetch_to_path(
       surprise-grabbing 16 GB on a slow link.
     """
 
-    if dest.exists():
-        if expect_zip and not _is_valid_zip(dest):
-            logger.warning(
-                "Cached %s at %s failed ZIP validation (size=%d B); deleting "
-                "and re-downloading.",
-                label,
-                dest,
-                dest.stat().st_size,
-            )
-            dest.unlink(missing_ok=True)
-        else:
-            logger.info("Using cached %s at %s", label, dest)
-            return dest
+    cached = _reuse_cached_dest(dest, expect_zip=expect_zip, label=label)
+    if cached is not None:
+        return cached
 
     dest.parent.mkdir(parents=True, exist_ok=True)
     partial = dest.with_suffix(dest.suffix + ".partial")
@@ -170,39 +179,38 @@ async def _fetch_to_path(
             async with httpx.AsyncClient(
                 timeout=httpx.Timeout(timeout_s, connect=20.0),
                 follow_redirects=True,
-            ) as client:
-                async with client.stream("GET", url, headers=headers) as response:
-                    if existing_bytes and response.status_code == 200:
-                        logger.warning(
-                            "Server ignored Range header for %s; restarting from 0.",
-                            label,
-                        )
-                        partial.unlink(missing_ok=True)
-                        existing_bytes = 0
-                    elif response.status_code == 416:
-                        # Range not satisfiable — the .partial is at or
-                        # past the end. Treat as "already downloaded";
-                        # validate by closing and re-opening for atomic
-                        # rename below.
-                        logger.info(
-                            "Server reports %s already complete (HTTP 416).",
-                            label,
-                        )
-                    elif response.status_code not in (200, 206):
-                        response.raise_for_status()
+            ) as client, client.stream("GET", url, headers=headers) as response:
+                if existing_bytes and response.status_code == 200:
+                    logger.warning(
+                        "Server ignored Range header for %s; restarting from 0.",
+                        label,
+                    )
+                    partial.unlink(missing_ok=True)
+                    existing_bytes = 0
+                elif response.status_code == 416:
+                    # Range not satisfiable — the .partial is at or
+                    # past the end. Treat as "already downloaded";
+                    # validate by closing and re-opening for atomic
+                    # rename below.
+                    logger.info(
+                        "Server reports %s already complete (HTTP 416).",
+                        label,
+                    )
+                elif response.status_code not in (200, 206):
+                    response.raise_for_status()
 
-                    total_size = _planned_total_size(response, existing_bytes)
-                    if (
-                        total_size is not None
-                        and total_size > _LARGE_DOWNLOAD_BYTES
-                        and not allow_large_download
-                    ):
-                        raise _LargeDownloadAbort(label, total_size)
+                total_size = _planned_total_size(response, existing_bytes)
+                if (
+                    total_size is not None
+                    and total_size > _LARGE_DOWNLOAD_BYTES
+                    and not allow_large_download
+                ):
+                    raise _LargeDownloadAbort(label, total_size)
 
-                    mode = "ab" if existing_bytes else "wb"
-                    with partial.open(mode) as fh:
-                        async for chunk in response.aiter_bytes(chunk_size=1 << 18):
-                            fh.write(chunk)
+                mode = "ab" if existing_bytes else "wb"
+                with partial.open(mode) as fh:
+                    async for chunk in response.aiter_bytes(chunk_size=1 << 18):
+                        fh.write(chunk)
             # Optional content sanity check before promoting to dest.
             if expect_zip and not _is_valid_zip(partial):
                 raise zipfile.BadZipFile(

surfsense_evals/src/surfsense_evals/suites/medical/mirage/prompt.py

@@ -8,7 +8,6 @@ from __future__ import annotations
 
 from collections.abc import Mapping
 
-
 _PROMPT_TEMPLATE = """\
 You are a helpful medical expert. Answer the following multiple-choice
 question using the relevant medical knowledge available to you (and any

surfsense_evals/src/surfsense_evals/suites/medical/mirage/runner.py

@@ -29,7 +29,6 @@ from ....core.ingest_settings import (
 )
 from ....core.metrics.mc_accuracy import accuracy_with_wilson_ci, macro_accuracy
 from ....core.registry import (
-    Benchmark,
     ReportSection,
     RunArtifact,
     RunContext,
@@ -229,7 +228,7 @@ class MirageBenchmark:
         run_dir = ctx.runs_dir(run_timestamp=run_timestamp)
         raw_path = run_dir / "raw.jsonl"
         with raw_path.open("w", encoding="utf-8") as fh:
-            for q, res in zip(questions, results):
+            for q, res in zip(questions, results, strict=False):
                 fh.write(
                     json.dumps(
                         {
@@ -246,7 +245,7 @@ class MirageBenchmark:
         for task in tasks:
             n_correct = 0
             n_total = 0
-            for q, res in zip(questions, results):
+            for q, res in zip(questions, results, strict=False):
                 if q.task != task:
                     continue
                 n_total += 1

surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/ingest.py

@@ -41,8 +41,6 @@ from typing import Any
 
 from ....core.config import set_suite_state
 from ....core.parsers import (
-    AzureDIError,
-    LlamaCloudError,
     count_pdf_pages,
     parse_with_azure_di,
     parse_with_llamacloud,
@@ -131,7 +129,7 @@ async def _run_one_extraction(
     else:
         raise ValueError(f"Unknown parser {parser!r}")
     out_path.parent.mkdir(parents=True, exist_ok=True)
-    out_path.write_text(markdown, encoding="utf-8")
+    await asyncio.to_thread(out_path.write_text, markdown, encoding="utf-8")
     return markdown, time.monotonic() - started
 
 

surfsense_evals/src/surfsense_evals/suites/multimodal_doc/parser_compare/runner.py

@@ -603,8 +603,8 @@ class ParserCompareBenchmark:
             f"engine: `{extra.get('pdf_engine', 'native')}`)."
         )
         body.append(
-            f"- Preprocess tariff: basic = $1 / 1k pages, "
-            f"premium = $10 / 1k pages."
+            "- Preprocess tariff: basic = $1 / 1k pages, "
+            "premium = $10 / 1k pages."
         )
         body.append("")
         body.append("### Per-arm summary")

surfsense_evals/src/surfsense_evals/suites/research/crag/html_extract.py

@@ -177,10 +177,7 @@ def extract_main_content(
             # Prefer trafilatura output even if short, but only if it
             # contained any prose at all — empty trafilatura fall-through
             # to the stripped form.
-            if body and stripped and len(stripped) > len(body) * 1.5:
-                body = stripped
-                method = "fallback_strip"
-            elif not body and stripped:
+            if body and stripped and len(stripped) > len(body) * 1.5 or not body and stripped:
                 body = stripped
                 method = "fallback_strip"
             elif body:

surfsense_evals/src/surfsense_evals/suites/research/crag/prompt.py

@@ -28,7 +28,6 @@ in the runner, so the format is mandatory.
 
 from __future__ import annotations
 
-
 _BASE_INSTRUCTIONS = (
     "You are a careful question-answering assistant. The question is a "
     "real-world factual question that may be about finance, music, "

surfsense_evals/src/surfsense_evals/suites/research/crag/runner.py

@@ -830,11 +830,11 @@ def _per_facet_truthfulness(
     """Bucket truthfulness scores by ``key_fn(q)``."""
 
     buckets: dict[str, dict[str, list[CragGradeResult]]] = {}
-    for q, b, l, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False):
+    for q, b, lc, s in zip(questions, bare_grades, lc_grades, surf_grades, strict=False):
         key = key_fn(q)
         bucket = buckets.setdefault(key, {"bare_llm": [], "long_context": [], "surfsense": []})
         bucket["bare_llm"].append(b)
-        bucket["long_context"].append(l)
+        bucket["long_context"].append(lc)
         bucket["surfsense"].append(s)
     out: dict[str, Any] = {}
     for key, arms in buckets.items():

surfsense_evals/src/surfsense_evals/suites/research/frames/dataset.py

@@ -102,7 +102,7 @@ def _parse_wiki_links(raw: Any) -> list[str]:
     except (SyntaxError, ValueError):
         # Fall back: maybe it's a comma-separated string with no quotes.
         return [tok.strip() for tok in text.strip("[]").split(",") if tok.strip()]
-    if isinstance(parsed, (list, tuple)):
+    if isinstance(parsed, list | tuple):
         return [str(x).strip() for x in parsed if str(x).strip()]
     return [str(parsed).strip()]
 

surfsense_evals/src/surfsense_evals/suites/research/frames/ingest.py

@@ -34,7 +34,6 @@ filename (without extension), so we round-trip via
 
 from __future__ import annotations
 
-import asyncio
 import json
 import logging
 from dataclasses import dataclass

surfsense_evals/src/surfsense_evals/suites/research/frames/prompt.py

@@ -17,7 +17,6 @@ Format expectations (mirrors the FRAMES paper, section 4):
 
 from __future__ import annotations
 
-
 _BASE_INSTRUCTIONS = (
     "You are a careful question-answering assistant. The question may "
     "require combining facts from multiple sources, doing arithmetic, "

surfsense_evals/tests/suites/test_crag_dataset.py

@@ -11,8 +11,6 @@ import bz2
 import json
 from pathlib import Path
 
-import pytest
-
 from surfsense_evals.suites.research.crag.dataset import (
     CragPage,
     CragQuestion,

surfsense_evals/tests/suites/test_crag_grader.py

@@ -7,8 +7,6 @@ exercise the deterministic shortcut + the special-case routing for
 
 from __future__ import annotations
 
-import pytest
-
 from surfsense_evals.suites.research.crag.grader import (
     CragGradeResult,
     _flags_false_premise,

surfsense_evals/tests/suites/test_crag_html_extract.py

@@ -11,13 +11,10 @@ We don't network-fetch trafilatura; we just verify the wrapper:
 
 from __future__ import annotations
 
-import pytest
-
 from surfsense_evals.suites.research.crag.html_extract import (
     extract_main_content,
 )
 
-
 _RICH_HTML = """\
 <!DOCTYPE html>
 <html>

surfsense_evals/tests/suites/test_frames_dataset.py

@@ -16,8 +16,6 @@ from __future__ import annotations
 import textwrap
 from pathlib import Path
 
-import pytest
-
 from surfsense_evals.suites.research.frames.dataset import (
     FramesQuestion,
     _parse_reasoning_types,
@@ -25,7 +23,6 @@ from surfsense_evals.suites.research.frames.dataset import (
     load_questions,
 )
 
-
 # ---------------------------------------------------------------------------
 # Pure-function tests
 # ---------------------------------------------------------------------------

surfsense_evals/tests/suites/test_frames_grader.py

@@ -8,10 +8,7 @@ runner knows to consult the judge.
 
 from __future__ import annotations
 
-import pytest
-
 from surfsense_evals.suites.research.frames.grader import (
-    GradeResult,
     _maybe_number,
     _normalise,
     _whole_word_substring,