mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
fix: dedup race, cwd-relative image paths, unencoded legacy URLs
- SQLite dedup race: add UNIQUE(collection_name, file_hash) and switch save_document to a plain INSERT. add_document now catches the IntegrityError from a concurrent add of the same content, cleans up its managed files, and returns the winning doc_id — instead of two doc_ids for one file (each having paid for its own LLM indexing). - PDF image paths: store the absolute path to each extracted image instead of a path relative to the indexing process's cwd. The  references broke as soon as a query ran from a different directory. - LegacyCloudAPI: URL-encode doc_id / retrieval_id path segments (added _enc()), matching CloudBackend. An id containing '/', '?', '#' or a space previously hit the wrong endpoint or produced a malformed URL. Adds regression tests: UNIQUE enforcement, the add-race resolving to the winner, absolute image paths, and encoded legacy URLs. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
parent
fe36e25773
commit
b3616f76a2
8 changed files with 123 additions and 15 deletions
|
|
@ -27,3 +27,33 @@ def test_parse_nodes_are_flat_without_level():
|
|||
assert node.tokens >= 0
|
||||
assert node.index is not None
|
||||
assert node.level is None
|
||||
|
||||
|
||||
def test_image_paths_are_absolute(tmp_path):
|
||||
"""Image references must be absolute so they resolve regardless of cwd
|
||||
(cwd-relative paths broke after the query ran from another directory)."""
|
||||
import os
|
||||
import pymupdf
|
||||
from pageindex.parser.pdf import PdfParser
|
||||
|
||||
# Build a 1-page PDF with an embedded image (>= _MIN_IMAGE_SIZE).
|
||||
pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 64, 64), False)
|
||||
pix.clear_with(128)
|
||||
png = tmp_path / "img.png"
|
||||
pix.save(str(png))
|
||||
|
||||
doc = pymupdf.open()
|
||||
page = doc.new_page()
|
||||
page.insert_image(pymupdf.Rect(20, 20, 180, 180), filename=str(png))
|
||||
pdf_path = tmp_path / "withimg.pdf"
|
||||
doc.save(str(pdf_path))
|
||||
doc.close()
|
||||
|
||||
images_dir = tmp_path / "out" / "images"
|
||||
result = PdfParser().parse(str(pdf_path), images_dir=str(images_dir))
|
||||
|
||||
img_paths = [im["path"] for n in result.nodes if n.images for im in n.images]
|
||||
assert img_paths, "expected at least one extracted image"
|
||||
for p in img_paths:
|
||||
assert os.path.isabs(p), f"image path not absolute: {p}"
|
||||
assert os.path.exists(p), f"image path does not resolve: {p}"
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue