mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-18 21:21:05 +02:00
fix: resolve five P1 defects from the SDK review
- AgentRunner.run: offload to a worker-thread event loop when called from inside a running loop (Jupyter, FastAPI handlers) — mirrors pipeline._run_async; Runner.run_sync raised RuntimeError there. - SQLiteStorage: create connections with check_same_thread=False so close() can actually close connections created by worker threads. Each thread still gets its own connection via threading.local; with the default True those closes raised ProgrammingError (silently swallowed) and leaked every worker connection. - CloudBackend.query: non-streaming chat completions now use a 300s timeout and a single attempt. The default 30s ReadTimeout fired before generation finished and the retry loop re-billed the full server-side retrieval + generation up to three times. _request gains retries/timeout overrides; the exhausted-retry path also no longer sleeps before raising. - MarkdownParser: content before the first heading (abstract/preamble) becomes a node instead of being silently dropped and unretrievable; a file with no headings at all yields a single document node instead of zero nodes (which pushed an empty page list into the pipeline). - LegacyCloudAPI.is_retrieval_ready: API failures (revoked key, network down) now propagate as PageIndexAPIError instead of reading as "not ready", which turned polling loops into infinite loops. Adds regression tests for each fix. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
parent
6a73279c0c
commit
956147d864
10 changed files with 171 additions and 16 deletions
|
|
@ -17,7 +17,7 @@ class MarkdownParser:
|
|||
|
||||
lines = content.split("\n")
|
||||
headers = self._extract_headers(lines)
|
||||
nodes = self._build_nodes(headers, lines, model)
|
||||
nodes = self._build_nodes(headers, lines, model, doc_title=path.stem)
|
||||
|
||||
return ParsedDocument(doc_name=path.stem, nodes=nodes)
|
||||
|
||||
|
|
@ -42,8 +42,37 @@ class MarkdownParser:
|
|||
})
|
||||
return headers
|
||||
|
||||
def _build_nodes(self, headers: list[dict], lines: list[str], model: str | None) -> list[ContentNode]:
|
||||
def _build_nodes(self, headers: list[dict], lines: list[str], model: str | None,
|
||||
doc_title: str = "Document") -> list[ContentNode]:
|
||||
nodes = []
|
||||
|
||||
# A file with no headings at all still has content — index it as a
|
||||
# single node instead of producing zero nodes (which would push an
|
||||
# empty page list into the LLM pipeline).
|
||||
if not headers:
|
||||
text = "\n".join(lines).strip()
|
||||
if text:
|
||||
nodes.append(ContentNode(
|
||||
content=text,
|
||||
tokens=count_tokens(text, model=model),
|
||||
title=doc_title,
|
||||
index=1,
|
||||
level=1,
|
||||
))
|
||||
return nodes
|
||||
|
||||
# Content before the first heading (abstract, preamble) would
|
||||
# otherwise be silently dropped and become unretrievable.
|
||||
preamble = "\n".join(lines[: headers[0]["line_num"] - 1]).strip()
|
||||
if preamble:
|
||||
nodes.append(ContentNode(
|
||||
content=preamble,
|
||||
tokens=count_tokens(preamble, model=model),
|
||||
title=doc_title,
|
||||
index=1,
|
||||
level=headers[0]["level"],
|
||||
))
|
||||
|
||||
for i, header in enumerate(headers):
|
||||
start = header["line_num"] - 1
|
||||
end = headers[i + 1]["line_num"] - 1 if i + 1 < len(headers) else len(lines)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue