mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
fix: dedup race, cwd-relative image paths, unencoded legacy URLs
- SQLite dedup race: add UNIQUE(collection_name, file_hash) and switch save_document to a plain INSERT. add_document now catches the IntegrityError from a concurrent add of the same content, cleans up its managed files, and returns the winning doc_id — instead of two doc_ids for one file (each having paid for its own LLM indexing). - PDF image paths: store the absolute path to each extracted image instead of a path relative to the indexing process's cwd. The  references broke as soon as a query ran from a different directory. - LegacyCloudAPI: URL-encode doc_id / retrieval_id path segments (added _enc()), matching CloudBackend. An id containing '/', '?', '#' or a space previously hit the wrong endpoint or produced a malformed URL. Adds regression tests: UNIQUE enforcement, the add-race resolving to the winner, absolute image paths, and encoded legacy URLs. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
parent
fe36e25773
commit
b3616f76a2
8 changed files with 123 additions and 15 deletions
|
|
@ -79,3 +79,16 @@ def test_close_closes_connections_created_in_other_threads(storage):
|
|||
storage.close() # main thread closes the worker's connection too
|
||||
with pytest.raises(sqlite3.ProgrammingError):
|
||||
conns["worker"].execute("SELECT 1")
|
||||
|
||||
|
||||
def test_duplicate_file_hash_in_collection_raises(storage):
|
||||
"""UNIQUE(collection_name, file_hash) guards the add-same-file race."""
|
||||
import sqlite3
|
||||
storage.create_collection("papers")
|
||||
doc = {"doc_name": "a", "doc_type": "pdf", "file_hash": "HASH1", "structure": []}
|
||||
storage.save_document("papers", "doc-1", doc)
|
||||
with pytest.raises(sqlite3.IntegrityError):
|
||||
storage.save_document("papers", "doc-2", {**doc, "doc_name": "b"})
|
||||
# same hash in a DIFFERENT collection is fine
|
||||
storage.create_collection("other")
|
||||
storage.save_document("other", "doc-3", {**doc})
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue