apunkt/sqlite-vec
1
0
Fork 0
mirror of https://github.com/asg017/sqlite-vec.git synced 2026-04-25 00:36:56 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Hide IVF behind SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE, default off

Browse source 
Rename SQLITE_VEC_ENABLE_IVF to SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE and
flip the default from 1 to 0. IVF tests are automatically skipped when
the build flag is not set.

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
Alex Garcia 2026-03-30 23:17:30 -07:00
parent 3e26925ce0
commit bb3ef78f75
2 changed files with 45 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

42
sqlite-vec.c
View file

@ -93,8 +93,8 @@ typedef size_t usize;
#define COMPILER_SUPPORTS_VTAB_IN 1 #define COMPILER_SUPPORTS_VTAB_IN 1
#endif #endif
#ifndef SQLITE_VEC_ENABLE_IVF #ifndef SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
#define SQLITE_VEC_ENABLE_IVF 1 #define SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE 0
#endif #endif
#ifndef SQLITE_SUBTYPE #ifndef SQLITE_SUBTYPE
@ -2558,7 +2558,7 @@ struct Vec0RescoreConfig {
}; };
#endif #endif
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
enum Vec0IvfQuantizer { enum Vec0IvfQuantizer {
VEC0_IVF_QUANTIZER_NONE = 0, VEC0_IVF_QUANTIZER_NONE = 0,
VEC0_IVF_QUANTIZER_INT8 = 1, VEC0_IVF_QUANTIZER_INT8 = 1,
@ -2737,7 +2737,7 @@ static int vec0_parse_rescore_options(struct Vec0Scanner *scanner,
* @return int SQLITE_OK on success, SQLITE_EMPTY is it's not a vector column * @return int SQLITE_OK on success, SQLITE_EMPTY is it's not a vector column
* definition, SQLITE_ERROR on error. * definition, SQLITE_ERROR on error.
*/ */
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// Forward declaration — defined in sqlite-vec-ivf.c // Forward declaration — defined in sqlite-vec-ivf.c
static int vec0_parse_ivf_options(struct Vec0Scanner *scanner, static int vec0_parse_ivf_options(struct Vec0Scanner *scanner,
struct Vec0IvfConfig *config); struct Vec0IvfConfig *config);
@ -2922,7 +2922,7 @@ int vec0_parse_vector_column(const char *source, int source_length,
} }
#endif #endif
else if (sqlite3_strnicmp(token.start, "ivf", indexNameLen) == 0) { else if (sqlite3_strnicmp(token.start, "ivf", indexNameLen) == 0) {
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
indexType = VEC0_INDEX_TYPE_IVF; indexType = VEC0_INDEX_TYPE_IVF;
memset(&ivfConfig, 0, sizeof(ivfConfig)); memset(&ivfConfig, 0, sizeof(ivfConfig));
rc = vec0_parse_ivf_options(&scanner, &ivfConfig); rc = vec0_parse_ivf_options(&scanner, &ivfConfig);
@ -3321,7 +3321,7 @@ struct vec0_vtab {
int chunk_size; int chunk_size;
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// IVF cached state per vector column // IVF cached state per vector column
char *shadowIvfCellsNames[VEC0_MAX_VECTOR_COLUMNS]; // table name for blob_open char *shadowIvfCellsNames[VEC0_MAX_VECTOR_COLUMNS]; // table name for blob_open
int ivfTrainedCache[VEC0_MAX_VECTOR_COLUMNS]; // -1=unknown, 0=no, 1=yes int ivfTrainedCache[VEC0_MAX_VECTOR_COLUMNS]; // -1=unknown, 0=no, 1=yes
@ -3419,7 +3419,7 @@ void vec0_free_resources(vec0_vtab *p) {
sqlite3_finalize(p->stmtRowidsGetChunkPosition); sqlite3_finalize(p->stmtRowidsGetChunkPosition);
p->stmtRowidsGetChunkPosition = NULL; p->stmtRowidsGetChunkPosition = NULL;
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
for (int i = 0; i < VEC0_MAX_VECTOR_COLUMNS; i++) { for (int i = 0; i < VEC0_MAX_VECTOR_COLUMNS; i++) {
sqlite3_finalize(p->stmtIvfCellMeta[i]); p->stmtIvfCellMeta[i] = NULL; sqlite3_finalize(p->stmtIvfCellMeta[i]); p->stmtIvfCellMeta[i] = NULL;
sqlite3_finalize(p->stmtIvfCellUpdateN[i]); p->stmtIvfCellUpdateN[i] = NULL; sqlite3_finalize(p->stmtIvfCellUpdateN[i]); p->stmtIvfCellUpdateN[i] = NULL;
@ -3451,7 +3451,7 @@ void vec0_free(vec0_vtab *p) {
for (int i = 0; i < p->numVectorColumns; i++) { for (int i = 0; i < p->numVectorColumns; i++) {
sqlite3_free(p->shadowVectorChunksNames[i]); sqlite3_free(p->shadowVectorChunksNames[i]);
p->shadowVectorChunksNames[i] = NULL; p->shadowVectorChunksNames[i] = NULL;
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
sqlite3_free(p->shadowIvfCellsNames[i]); sqlite3_free(p->shadowIvfCellsNames[i]);
p->shadowIvfCellsNames[i] = NULL; p->shadowIvfCellsNames[i] = NULL;
#endif #endif
@ -3743,7 +3743,7 @@ int vec0_result_id(vec0_vtab *p, sqlite3_context *context, i64 rowid) {
* will be stored. * will be stored.
* @return int SQLITE_OK on success. * @return int SQLITE_OK on success.
*/ */
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// Forward declaration — defined in sqlite-vec-ivf.c (included later) // Forward declaration — defined in sqlite-vec-ivf.c (included later)
static int ivf_get_vector_data(vec0_vtab *p, i64 rowid, int col_idx, static int ivf_get_vector_data(vec0_vtab *p, i64 rowid, int col_idx,
void **outVector, int *outVectorSize); void **outVector, int *outVectorSize);
@ -3756,7 +3756,7 @@ int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx,
i64 chunk_id; i64 chunk_id;
i64 chunk_offset; i64 chunk_offset;
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// IVF-indexed columns store vectors in _ivf_cells, not _vector_chunks // IVF-indexed columns store vectors in _ivf_cells, not _vector_chunks
if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF) { if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF) {
return ivf_get_vector_data(p, rowid, vector_column_idx, outVector, outVectorSize); return ivf_get_vector_data(p, rowid, vector_column_idx, outVector, outVectorSize);
@ -4411,7 +4411,7 @@ int vec0_new_chunk(vec0_vtab *p, sqlite3_value ** partitionKeyValues, i64 *chunk
#if SQLITE_VEC_ENABLE_RESCORE #if SQLITE_VEC_ENABLE_RESCORE
// Rescore and IVF columns don't use _vector_chunks for float storage // Rescore and IVF columns don't use _vector_chunks for float storage
if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_RESCORE if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_RESCORE
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
|| p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF || p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF
#endif #endif
) { ) {
@ -4587,7 +4587,7 @@ void vec0_cursor_clear(vec0_cursor *pCur) {
} }
// IVF index implementation — #include'd here after all struct/helper definitions // IVF index implementation — #include'd here after all struct/helper definitions
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
#include "sqlite-vec-ivf-kmeans.c" #include "sqlite-vec-ivf-kmeans.c"
#include "sqlite-vec-ivf.c" #include "sqlite-vec-ivf.c"
#endif #endif
@ -4986,7 +4986,7 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv,
} }
#endif #endif
} }
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
for (int i = 0; i < pNew->numVectorColumns; i++) { for (int i = 0; i < pNew->numVectorColumns; i++) {
if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue;
pNew->shadowIvfCellsNames[i] = pNew->shadowIvfCellsNames[i] =
@ -5147,7 +5147,7 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv,
} }
#endif #endif
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// Create IVF shadow tables for IVF-indexed vector columns // Create IVF shadow tables for IVF-indexed vector columns
for (int i = 0; i < pNew->numVectorColumns; i++) { for (int i = 0; i < pNew->numVectorColumns; i++) {
if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue;
@ -5315,7 +5315,7 @@ static int vec0Destroy(sqlite3_vtab *pVtab) {
} }
#endif #endif
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// Drop IVF shadow tables // Drop IVF shadow tables
for (int i = 0; i < p->numVectorColumns; i++) { for (int i = 0; i < p->numVectorColumns; i++) {
if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue;
@ -7335,7 +7335,7 @@ int vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum,
} }
#endif #endif
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// IVF dispatch: if vector column has IVF, use IVF query instead of chunk scan // IVF dispatch: if vector column has IVF, use IVF query instead of chunk scan
if (vector_column->index_type == VEC0_INDEX_TYPE_IVF) { if (vector_column->index_type == VEC0_INDEX_TYPE_IVF) {
rc = ivf_query_knn(p, vectorColumnIdx, queryVector, rc = ivf_query_knn(p, vectorColumnIdx, queryVector,
@ -8177,7 +8177,7 @@ int vec0Update_InsertWriteFinalStep(vec0_vtab *p, i64 chunk_rowid,
#if SQLITE_VEC_ENABLE_RESCORE #if SQLITE_VEC_ENABLE_RESCORE
// Rescore and IVF columns don't use _vector_chunks // Rescore and IVF columns don't use _vector_chunks
if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_RESCORE if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_RESCORE
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
|| p->vector_columns[i].index_type == VEC0_INDEX_TYPE_IVF || p->vector_columns[i].index_type == VEC0_INDEX_TYPE_IVF
#endif #endif
) )
@ -8593,7 +8593,7 @@ int vec0Update_Insert(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,
} }
#endif #endif
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// Step #4: IVF index insert (if any vector column uses IVF) // Step #4: IVF index insert (if any vector column uses IVF)
for (int i = 0; i < p->numVectorColumns; i++) { for (int i = 0; i < p->numVectorColumns; i++) {
if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue;
@ -9189,7 +9189,7 @@ int vec0Update_Delete(sqlite3_vtab *pVTab, sqlite3_value *idValue) {
} }
} }
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// 7. delete from IVF index // 7. delete from IVF index
for (int i = 0; i < p->numVectorColumns; i++) { for (int i = 0; i < p->numVectorColumns; i++) {
if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue;
@ -9473,7 +9473,7 @@ static int vec0Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,
} }
// INSERT operation // INSERT operation
else if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL) { else if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL) {
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
// Check for IVF command inserts: INSERT INTO t(rowid) VALUES ('compute-centroids') // Check for IVF command inserts: INSERT INTO t(rowid) VALUES ('compute-centroids')
// The id column holds the command string. // The id column holds the command string.
sqlite3_value *idVal = argv[2 + VEC0_COLUMN_ID]; sqlite3_value *idVal = argv[2 + VEC0_COLUMN_ID];
@ -9632,7 +9632,7 @@ static sqlite3_module vec0Module = {
#define SQLITE_VEC_DEBUG_BUILD_RESCORE "" #define SQLITE_VEC_DEBUG_BUILD_RESCORE ""
#endif #endif
#if SQLITE_VEC_ENABLE_IVF #if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE
#define SQLITE_VEC_DEBUG_BUILD_IVF "ivf" #define SQLITE_VEC_DEBUG_BUILD_IVF "ivf"
#else #else
#define SQLITE_VEC_DEBUG_BUILD_IVF "" #define SQLITE_VEC_DEBUG_BUILD_IVF ""

24
tests/conftest.py
View file

@ -1,5 +1,29 @@
import pytest import pytest
import sqlite3 import sqlite3
import os
def _vec_debug():
db = sqlite3.connect(":memory:")
db.enable_load_extension(True)
db.load_extension("dist/vec0")
db.enable_load_extension(False)
return db.execute("SELECT vec_debug()").fetchone()[0]
def _has_build_flag(flag):
return flag in _vec_debug().split("Build flags:")[-1]
def pytest_collection_modifyitems(config, items):
has_ivf = _has_build_flag("ivf")
if has_ivf:
return
skip_ivf = pytest.mark.skip(reason="IVF not enabled (compile with -DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1)")
ivf_prefixes = ("test-ivf",)
for item in items:
if any(item.fspath.basename.startswith(p) for p in ivf_prefixes):
item.add_marker(skip_ivf)
@pytest.fixture() @pytest.fixture()