mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
d97231b480
- local get_agent_tools: `set(doc_ids) if doc_ids is not None else None` so doc_ids=[] is a scope of nothing (reject all), not open mode. The public query path already guarded []; this hardens direct callers. - legacy get_tree: send summary=true/false (lowercase) instead of Python's capitalized True/False, matching the modern CloudBackend and the API. - markdown parser: track the opening fence character so a ```-fence isn't closed by a ~~~ line (CommonMark), keeping '#'-lines inside it out of headings. - dedupe the cloud base URL: single API_BASE in cloud_api, referenced by CloudBackend and PageIndexClient (was three independent copies). Regression tests for each.
104 lines
4.3 KiB
Python
104 lines
4.3 KiB
Python
import re
|
|
from pathlib import Path
|
|
from .protocol import ContentNode, ParsedDocument
|
|
from ..tokens import count_tokens
|
|
|
|
|
|
class MarkdownParser:
|
|
def supported_extensions(self) -> list[str]:
|
|
return [".md", ".markdown"]
|
|
|
|
def parse(self, file_path: str, **kwargs) -> ParsedDocument:
|
|
path = Path(file_path)
|
|
model = kwargs.get("model")
|
|
|
|
# utf-8-sig strips a leading BOM if present (common from Windows
|
|
# editors/exporters) and is otherwise identical to plain utf-8. Without
|
|
# it, a BOM-prefixed first line fails the header regex below (the BOM
|
|
# isn't whitespace, so .strip() doesn't remove it), misclassifying the
|
|
# document's first heading as unrecognized preamble text.
|
|
with open(path, "r", encoding="utf-8-sig") as f:
|
|
content = f.read()
|
|
|
|
lines = content.split("\n")
|
|
headers = self._extract_headers(lines)
|
|
nodes = self._build_nodes(headers, lines, model, doc_title=path.stem)
|
|
|
|
return ParsedDocument(doc_name=path.stem, nodes=nodes)
|
|
|
|
def _extract_headers(self, lines: list[str]) -> list[dict]:
|
|
header_pattern = r"^(#{1,6})\s+(.+)$"
|
|
# CommonMark allows both backtick and tilde fences, and a fence is
|
|
# closed only by one of the SAME character. Track which char opened the
|
|
# block so a ~~~ line inside a ```-fenced block (or vice versa) is
|
|
# treated as content, not a close — otherwise the block appears to end
|
|
# early and '#'-prefixed lines inside it get misparsed as headings.
|
|
fence_pattern = r"^(`{3,}|~{3,})"
|
|
headers = []
|
|
open_fence = None # the fence char ('`' or '~') of the open block, or None
|
|
|
|
for line_num, line in enumerate(lines, 1):
|
|
stripped = line.strip()
|
|
fence = re.match(fence_pattern, stripped)
|
|
if fence:
|
|
marker = fence.group(1)[0]
|
|
if open_fence is None:
|
|
open_fence = marker # open a block
|
|
elif open_fence == marker:
|
|
open_fence = None # matching char closes it
|
|
# a non-matching fence char while a block is open is content
|
|
continue
|
|
if open_fence is None and stripped:
|
|
match = re.match(header_pattern, stripped)
|
|
if match:
|
|
headers.append({
|
|
"title": match.group(2).strip(),
|
|
"level": len(match.group(1)),
|
|
"line_num": line_num,
|
|
})
|
|
return headers
|
|
|
|
def _build_nodes(self, headers: list[dict], lines: list[str], model: str | None,
|
|
doc_title: str = "Document") -> list[ContentNode]:
|
|
nodes = []
|
|
|
|
# A file with no headings at all still has content — index it as a
|
|
# single node instead of producing zero nodes (which would push an
|
|
# empty page list into the LLM pipeline).
|
|
if not headers:
|
|
text = "\n".join(lines).strip()
|
|
if text:
|
|
nodes.append(ContentNode(
|
|
content=text,
|
|
tokens=count_tokens(text, model=model),
|
|
title=doc_title,
|
|
index=1,
|
|
level=1,
|
|
))
|
|
return nodes
|
|
|
|
# Content before the first heading (abstract, preamble) would
|
|
# otherwise be silently dropped and become unretrievable.
|
|
preamble = "\n".join(lines[: headers[0]["line_num"] - 1]).strip()
|
|
if preamble:
|
|
nodes.append(ContentNode(
|
|
content=preamble,
|
|
tokens=count_tokens(preamble, model=model),
|
|
title=doc_title,
|
|
index=1,
|
|
level=headers[0]["level"],
|
|
))
|
|
|
|
for i, header in enumerate(headers):
|
|
start = header["line_num"] - 1
|
|
end = headers[i + 1]["line_num"] - 1 if i + 1 < len(headers) else len(lines)
|
|
text = "\n".join(lines[start:end]).strip()
|
|
tokens = count_tokens(text, model=model)
|
|
nodes.append(ContentNode(
|
|
content=text,
|
|
tokens=tokens,
|
|
title=header["title"],
|
|
index=header["line_num"],
|
|
level=header["level"],
|
|
))
|
|
return nodes
|