fix: six P2 correctness/robustness cleanups from the SDK review

- legacy call_llm: open AsyncOpenAI via `async with` so the client (and
  its HTTP connection pool) is closed instead of leaked.
- LocalBackend.add_document: fail fast with CollectionNotFoundError when
  the collection doesn't exist, before the expensive parse + LLM index
  (previously the missing FK only tripped at save time, after paying for
  the LLM work). Also raise builtin FileNotFoundError for a missing path
  instead of FileTypeError (which now means only "unsupported extension").
- Collection.query(doc_ids=None): the empty-collection guard now always
  runs — previously it was skipped once PAGEINDEX_EXPERIMENTAL_MULTIDOC
  was set. A single list_documents call serves both the guard and the
  multi-doc warning (no separate call just to decide whether to warn).
- CloudBackend.query_stream: on early consumer break / GeneratorExit,
  signal the background SSE thread to stop and force-close the response,
  so it no longer drains the whole stream in the background.

Adds regression tests for each (client closed, fail-fast on unknown
collection, FileNotFoundError, empty-check under the multidoc env flag,
single list call, early-break thread stop).

Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
mountain 2026-07-07 11:34:13 +08:00
parent 6c948a332b
commit fe36e25773
8 changed files with 128 additions and 19 deletions

View file

@ -219,3 +219,32 @@ def test_query_uses_long_timeout_and_single_attempt(monkeypatch):
backend.query("col", "q", doc_ids=["d1"])
assert len(calls) == 1
assert calls[0]["timeout"] == 300
def test_query_stream_early_break_stops_background_thread(monkeypatch):
"""Consumer breaking early must signal the SSE thread to stop, not let it
drain the whole stream in the background."""
import threading
import time as _real_time # autouse fixture stubs cloud_mod.time.sleep, not this
backend = CloudBackend(api_key="pi-test")
drained_all = threading.Event()
class SlowResponse:
status_code = 200
text = ""
def iter_lines(self, decode_unicode=True):
for i in range(1000):
yield _sse("text", f"chunk{i} ")
_real_time.sleep(0.002) # pace so the consumer reliably breaks first
drained_all.set() # only reached if the thread was NOT stopped
def close(self):
pass
monkeypatch.setattr(cloud_mod.requests, "post", lambda *a, **k: SlowResponse())
async def _run():
async for _ in backend.query_stream("col", "q", doc_ids=["d1"]):
break # consume one event, then abandon
asyncio.run(_run())
assert not drained_all.is_set()

View file

@ -94,3 +94,26 @@ def test_query_accepts_str_doc_id(col):
def test_query_rejects_empty_list(col):
with pytest.raises(ValueError, match="cannot be empty"):
col.query("what?", doc_ids=[])
def test_empty_collection_check_runs_even_when_multidoc_acked(monkeypatch):
from unittest.mock import MagicMock
from pageindex.collection import Collection
monkeypatch.setenv("PAGEINDEX_EXPERIMENTAL_MULTIDOC", "1")
backend = MagicMock()
backend.list_documents.return_value = []
col = Collection(name="papers", backend=backend)
with pytest.raises(ValueError, match="empty"):
col.query("q") # doc_ids=None, collection empty -> must still raise
def test_whole_collection_query_lists_documents_once(monkeypatch):
from unittest.mock import MagicMock
from pageindex.collection import Collection
monkeypatch.setenv("PAGEINDEX_EXPERIMENTAL_MULTIDOC", "1") # silence warning path
backend = MagicMock()
backend.list_documents.return_value = [{"doc_id": "d1"}, {"doc_id": "d2"}]
backend.query.return_value = "ans"
col = Collection(name="papers", backend=backend)
col.query("q")
assert backend.list_documents.call_count == 1 # single call at the collection layer

View file

@ -75,6 +75,7 @@ def test_print_tree_keeps_legacy_exclude_fields(capsys):
def test_call_llm_keeps_legacy_async_openai_contract(monkeypatch):
calls = []
closed = []
class FakeCompletions:
async def create(self, **kwargs):
@ -88,6 +89,15 @@ def test_call_llm_keeps_legacy_async_openai_contract(monkeypatch):
self.api_key = api_key
self.chat = SimpleNamespace(completions=FakeCompletions())
# call_llm must open the client as an async context manager so it is
# closed (no leaked HTTP connection pool).
async def __aenter__(self):
return self
async def __aexit__(self, *exc):
closed.append(True)
return False
fake_openai = SimpleNamespace(AsyncOpenAI=FakeAsyncOpenAI)
monkeypatch.setitem(sys.modules, "openai", fake_openai)
@ -99,6 +109,7 @@ def test_call_llm_keeps_legacy_async_openai_contract(monkeypatch):
))
assert result == "answer"
assert closed == [True] # client was closed
assert calls == [{
"model": "gpt-test",
"messages": [{"role": "user", "content": "hello"}],

View file

@ -165,3 +165,18 @@ def test_delete_collection_rejects_path_traversal(backend, tmp_path):
with pytest.raises(PageIndexError, match="Invalid collection name"):
backend.delete_collection("../..")
assert canary.exists()
def test_add_document_missing_file_raises_file_not_found(backend, tmp_path):
backend.get_or_create_collection("papers")
with pytest.raises(FileNotFoundError):
backend.add_document("papers", str(tmp_path / "nope.pdf"))
def test_add_document_unknown_collection_fails_fast(backend, tmp_path):
from pageindex.errors import CollectionNotFoundError
pdf = tmp_path / "doc.pdf"
pdf.write_bytes(b"%PDF-1.4")
# Collection never created -> must raise before any parse/LLM work.
with pytest.raises(CollectionNotFoundError, match="does not exist"):
backend.add_document("ghost-collection", str(pdf))