mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
b3616f76a2
- SQLite dedup race: add UNIQUE(collection_name, file_hash) and switch save_document to a plain INSERT. add_document now catches the IntegrityError from a concurrent add of the same content, cleans up its managed files, and returns the winning doc_id — instead of two doc_ids for one file (each having paid for its own LLM indexing). - PDF image paths: store the absolute path to each extracted image instead of a path relative to the indexing process's cwd. The  references broke as soon as a query ran from a different directory. - LegacyCloudAPI: URL-encode doc_id / retrieval_id path segments (added _enc()), matching CloudBackend. An id containing '/', '?', '#' or a space previously hit the wrong endpoint or produced a malformed URL. Adds regression tests: UNIQUE enforcement, the add-race resolving to the winner, absolute image paths, and encoded legacy URLs. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
59 lines
2.1 KiB
Python
59 lines
2.1 KiB
Python
import pytest
|
|
from pathlib import Path
|
|
from pageindex.parser.pdf import PdfParser
|
|
from pageindex.parser.protocol import ContentNode, ParsedDocument
|
|
|
|
TEST_PDF = Path("tests/pdfs/deepseek-r1.pdf")
|
|
|
|
def test_supported_extensions():
|
|
parser = PdfParser()
|
|
assert ".pdf" in parser.supported_extensions()
|
|
|
|
@pytest.mark.skipif(not TEST_PDF.exists(), reason="Test PDF not available")
|
|
def test_parse_returns_parsed_document():
|
|
parser = PdfParser()
|
|
result = parser.parse(str(TEST_PDF))
|
|
assert isinstance(result, ParsedDocument)
|
|
assert len(result.nodes) > 0
|
|
assert result.doc_name != ""
|
|
|
|
@pytest.mark.skipif(not TEST_PDF.exists(), reason="Test PDF not available")
|
|
def test_parse_nodes_are_flat_without_level():
|
|
parser = PdfParser()
|
|
result = parser.parse(str(TEST_PDF))
|
|
for node in result.nodes:
|
|
assert isinstance(node, ContentNode)
|
|
assert node.content is not None
|
|
assert node.tokens >= 0
|
|
assert node.index is not None
|
|
assert node.level is None
|
|
|
|
|
|
def test_image_paths_are_absolute(tmp_path):
|
|
"""Image references must be absolute so they resolve regardless of cwd
|
|
(cwd-relative paths broke after the query ran from another directory)."""
|
|
import os
|
|
import pymupdf
|
|
from pageindex.parser.pdf import PdfParser
|
|
|
|
# Build a 1-page PDF with an embedded image (>= _MIN_IMAGE_SIZE).
|
|
pix = pymupdf.Pixmap(pymupdf.csRGB, pymupdf.IRect(0, 0, 64, 64), False)
|
|
pix.clear_with(128)
|
|
png = tmp_path / "img.png"
|
|
pix.save(str(png))
|
|
|
|
doc = pymupdf.open()
|
|
page = doc.new_page()
|
|
page.insert_image(pymupdf.Rect(20, 20, 180, 180), filename=str(png))
|
|
pdf_path = tmp_path / "withimg.pdf"
|
|
doc.save(str(pdf_path))
|
|
doc.close()
|
|
|
|
images_dir = tmp_path / "out" / "images"
|
|
result = PdfParser().parse(str(pdf_path), images_dir=str(images_dir))
|
|
|
|
img_paths = [im["path"] for n in result.nodes if n.images for im in n.images]
|
|
assert img_paths, "expected at least one extracted image"
|
|
for p in img_paths:
|
|
assert os.path.isabs(p), f"image path not absolute: {p}"
|
|
assert os.path.exists(p), f"image path does not resolve: {p}"
|