feat: bumped version to 0.0.32

surfsense_evals/scripts/analyze_failures.py

@@ -12,12 +12,10 @@ Outputs (printed to stdout + written to `failures_n171.json`):
 from __future__ import annotations
 
 import json
-import re
 from collections import Counter, defaultdict
 from pathlib import Path
 from typing import Any
 
-
 REPO = Path(__file__).resolve().parents[1]
 RUN = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
 RAW = RUN / "raw.jsonl"

surfsense_evals/scripts/check_uploaded_status.py

@@ -15,7 +15,6 @@ from pathlib import Path
 import httpx
 from dotenv import load_dotenv
 
-
 REPO = Path(__file__).resolve().parents[1]
 PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"
 

surfsense_evals/scripts/compute_post_retry_accuracy.py

@@ -47,9 +47,7 @@ def _row_key(row: dict) -> tuple[str, str]:
 def _is_failure(row: dict) -> bool:
     if row.get("error"):
         return True
-    if not (row.get("raw_text") or "").strip():
-        return True
-    return False
+    return bool(not (row.get("raw_text") or "").strip())
 
 
 def _summarise(rows_by_arm: dict[str, list[dict]]) -> dict[str, dict]:

surfsense_evals/scripts/patch_manifest_for_parallel_ingest.py

@@ -27,7 +27,6 @@ from __future__ import annotations
 import json
 from pathlib import Path
 
-
 REPO = Path(__file__).resolve().parents[1]
 MAP_PATH = REPO / "data" / "multimodal_doc" / "maps" / "mmlongbench_doc_map.jsonl"
 PDF_DIR = REPO / "data" / "multimodal_doc" / "mmlongbench" / "pdfs"

surfsense_evals/scripts/peek_crag_run.py

@@ -10,19 +10,20 @@ from collections import defaultdict
 def main() -> None:
     raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
     print(f"Reading: {raw_path}")
-    rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+    with open(raw_path, encoding="utf-8") as fh:
+        rows = [json.loads(line) for line in fh if line.strip()]
     by_q: dict[str, dict[str, dict]] = defaultdict(dict)
     for r in rows:
         by_q[r["qid"]][r["arm"]] = r
 
     for qid, arms in list(by_q.items()):
         b = arms.get("bare_llm", {})
-        l = arms.get("long_context", {})
+        lc = arms.get("long_context", {})
         s = arms.get("surfsense", {})
         print(f"\n=== {qid} ({b.get('domain')}/{b.get('question_type')}) ===")
         print(f"  question: {b.get('extra', {}).get('question', '?')!r}")
         print(f"  gold: {b.get('gold')!r}")
-        for arm_name, a in (("bare_llm", b), ("long_context", l), ("surfsense", s)):
+        for arm_name, a in (("bare_llm", b), ("long_context", lc), ("surfsense", s)):
             grade = a.get("graded", {})
             text = (a.get("raw_text") or "").strip()
             tail = text[-200:] if text else ""

surfsense_evals/scripts/peek_disagreements.py

@@ -10,7 +10,8 @@ from collections import defaultdict
 def main() -> None:
     raw_path = sorted(glob.glob("data/research/runs/*/crag/raw.jsonl"))[-1]
     print(f"Reading: {raw_path}")
-    rows = [json.loads(line) for line in open(raw_path, encoding="utf-8") if line.strip()]
+    with open(raw_path, encoding="utf-8") as fh:
+        rows = [json.loads(line) for line in fh if line.strip()]
     by_q: dict[str, dict[str, dict]] = defaultdict(dict)
     for r in rows:
         by_q[r["qid"]][r["arm"]] = r

surfsense_evals/scripts/retry_failed_questions.py

@@ -106,9 +106,7 @@ def _is_failure_row(row: dict[str, Any]) -> bool:
 
     if row.get("error"):
         return True
-    if not (row.get("raw_text") or "").strip():
-        return True
-    return False
+    return bool(not (row.get("raw_text") or "").strip())
 
 
 @dataclass
@@ -428,7 +426,7 @@ async def _run(args: argparse.Namespace) -> int:
 
         if f.arm == "native_pdf":
             pdf_path = Path(map_row["pdf_path"])
-            if not pdf_path.exists():
+            if not await asyncio.to_thread(pdf_path.exists):
                 logger.error("PDF missing on disk: %s — skipping", pdf_path)
                 continue
             request = _build_native_request(

surfsense_evals/scripts/summarise_crag_run.py

@@ -11,7 +11,8 @@ def main() -> None:
     if not runs:
         print("(no CRAG runs found)")
         return
-    m = json.load(open(runs[-1], encoding="utf-8"))
+    with open(runs[-1], encoding="utf-8") as fh:
+        m = json.load(fh)
     metrics = m["metrics"]
 
     print(f"Reading: {runs[-1]}")

surfsense_evals/scripts/summarise_parser_compare_run.py

@@ -16,7 +16,6 @@ import statistics
 from collections import defaultdict
 from pathlib import Path
 
-
 REPO = Path(__file__).resolve().parents[1]
 RUN_DIR = REPO / "data" / "multimodal_doc" / "runs" / "2026-05-14T00-53-19Z" / "parser_compare"
 RAW = RUN_DIR / "raw.jsonl"
@@ -91,7 +90,7 @@ def main() -> None:
 
     print()
     print("by answer_format (accuracy):")
-    formats = sorted({f for m in arm_metrics.values() for f in m["by_format"].keys()})
+    formats = sorted({f for m in arm_metrics.values() for f in m["by_format"]})
     header = f"{'arm':<25} " + " ".join(f"{f:>10}" for f in formats)
     print(header)
     print("-" * len(header))