mirror of
https://github.com/asg017/sqlite-vec.git
synced 2026-04-25 00:36:56 +02:00
3358e127f6
Add inverted file (IVF) index type: partitions vectors into clusters via k-means, quantizes to int8, and scans only the nearest nprobe partitions at query time. Includes shadow table management, insert/delete, KNN integration, compile flag (SQLITE_VEC_ENABLE_IVF), fuzz targets, and tests. Removes superseded ivf-benchmarks/ directory.
71 lines
2 KiB
Makefile
71 lines
2 KiB
Makefile
BENCH = python bench.py
|
|
BASE_DB = seed/base.db
|
|
EXT = ../dist/vec0
|
|
|
|
# --- Baseline (brute-force) configs ---
|
|
BASELINES = \
|
|
"brute-float:type=baseline,variant=float" \
|
|
"brute-int8:type=baseline,variant=int8" \
|
|
"brute-bit:type=baseline,variant=bit"
|
|
|
|
# --- IVF configs ---
|
|
IVF_CONFIGS = \
|
|
"ivf-n32-p8:type=ivf,nlist=32,nprobe=8" \
|
|
"ivf-n128-p16:type=ivf,nlist=128,nprobe=16" \
|
|
"ivf-n512-p32:type=ivf,nlist=512,nprobe=32"
|
|
|
|
RESCORE_CONFIGS = \
|
|
"rescore-bit-os8:type=rescore,quantizer=bit,oversample=8" \
|
|
"rescore-bit-os16:type=rescore,quantizer=bit,oversample=16" \
|
|
"rescore-int8-os8:type=rescore,quantizer=int8,oversample=8"
|
|
|
|
ALL_CONFIGS = $(BASELINES) $(RESCORE_CONFIGS) $(IVF_CONFIGS)
|
|
|
|
.PHONY: seed ground-truth bench-smoke bench-rescore bench-ivf bench-10k bench-50k bench-100k bench-all \
|
|
report clean
|
|
|
|
# --- Data preparation ---
|
|
seed:
|
|
$(MAKE) -C seed
|
|
|
|
ground-truth: seed
|
|
python ground_truth.py --subset-size 10000
|
|
python ground_truth.py --subset-size 50000
|
|
python ground_truth.py --subset-size 100000
|
|
|
|
# --- Quick smoke test ---
|
|
bench-smoke: seed
|
|
$(BENCH) --subset-size 5000 -k 10 -n 20 -o runs/smoke \
|
|
"brute-float:type=baseline,variant=float" \
|
|
"ivf-quick:type=ivf,nlist=16,nprobe=4"
|
|
|
|
bench-rescore: seed
|
|
$(BENCH) --subset-size 10000 -k 10 -o runs/rescore \
|
|
$(RESCORE_CONFIGS)
|
|
|
|
|
|
# --- Standard sizes ---
|
|
bench-10k: seed
|
|
$(BENCH) --subset-size 10000 -k 10 -o runs/10k $(ALL_CONFIGS)
|
|
|
|
bench-50k: seed
|
|
$(BENCH) --subset-size 50000 -k 10 -o runs/50k $(ALL_CONFIGS)
|
|
|
|
bench-100k: seed
|
|
$(BENCH) --subset-size 100000 -k 10 -o runs/100k $(ALL_CONFIGS)
|
|
|
|
bench-all: bench-10k bench-50k bench-100k
|
|
|
|
# --- IVF across sizes ---
|
|
bench-ivf: seed
|
|
$(BENCH) --subset-size 10000 -k 10 -o runs/ivf $(BASELINES) $(IVF_CONFIGS)
|
|
$(BENCH) --subset-size 50000 -k 10 -o runs/ivf $(BASELINES) $(IVF_CONFIGS)
|
|
$(BENCH) --subset-size 100000 -k 10 -o runs/ivf $(BASELINES) $(IVF_CONFIGS)
|
|
|
|
# --- Report ---
|
|
report:
|
|
@echo "Use: sqlite3 runs/<dir>/results.db 'SELECT * FROM bench_results ORDER BY recall DESC'"
|
|
|
|
# --- Cleanup ---
|
|
clean:
|
|
rm -rf runs/
|