mirror of
https://github.com/VectifyAI/PageIndex.git
synced 2026-07-15 21:11:05 +02:00
fix(sqlite): make concurrent indexing writes robust (no "database is locked")
Real concurrency e2e (8 threads adding the same file) surfaced a bug the mocked unit tests missed: concurrent add_document calls failed with sqlite3.OperationalError "database is locked". Root cause — under WAL the dedup SELECT (find_document_by_hash) left a read snapshot on the connection, and the subsequent INSERT on that stale snapshot raised SQLITE_BUSY_SNAPSHOT, which busy_timeout does not retry. Fixes: - open connections in autocommit (isolation_level=None) so a SELECT never leaves a lingering read snapshot and each write is its own transaction - PRAGMA busy_timeout=10000 so concurrent writers wait for the WAL single-writer lock instead of failing immediately - an instance-level write lock serializing the fast write methods within the process (the expensive LLM indexing stays parallel) Now 8 concurrent adds of one file -> a single doc_id, zero errors. Adds a real-thread regression test. Claude-Session: https://claude.ai/code/session_01Kx5DgKbhK1N8autqXH8SmS
This commit is contained in:
parent
b64d717132
commit
890b520b1c
2 changed files with 73 additions and 27 deletions
|
|
@ -92,3 +92,30 @@ def test_duplicate_file_hash_in_collection_raises(storage):
|
|||
# same hash in a DIFFERENT collection is fine
|
||||
storage.create_collection("other")
|
||||
storage.save_document("other", "doc-3", {**doc})
|
||||
|
||||
|
||||
def test_concurrent_read_then_write_no_database_locked(storage):
|
||||
"""Regression: concurrent add (read hash -> write) hit 'database is locked'
|
||||
under WAL. Fixed via autocommit + busy_timeout + write lock. All writers
|
||||
must succeed (dedup via UNIQUE), none raise OperationalError."""
|
||||
import sqlite3, threading, uuid, time
|
||||
storage.create_collection("c")
|
||||
errs = []
|
||||
|
||||
def worker():
|
||||
try:
|
||||
storage.list_collections()
|
||||
storage.find_document_by_hash("c", "SAME") # read snapshot
|
||||
time.sleep(0.001) # widen the window
|
||||
try:
|
||||
storage.save_document("c", str(uuid.uuid4()),
|
||||
{"doc_name": "d", "doc_type": "pdf", "file_hash": "SAME", "structure": []})
|
||||
except sqlite3.IntegrityError:
|
||||
pass # expected: lost the dedup race
|
||||
except Exception as e:
|
||||
errs.append(f"{type(e).__name__}: {e}")
|
||||
|
||||
threads = [threading.Thread(target=worker) for _ in range(12)]
|
||||
[t.start() for t in threads]; [t.join() for t in threads]
|
||||
assert not errs, f"concurrent write errored: {errs}"
|
||||
assert len(storage.list_documents("c")) == 1 # dedup held
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue