From bb3ef78f75cf72efb41f55ff297c186e37011329 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Mon, 30 Mar 2026 23:17:30 -0700 Subject: [PATCH] Hide IVF behind SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE, default off Rename SQLITE_VEC_ENABLE_IVF to SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE and flip the default from 1 to 0. IVF tests are automatically skipped when the build flag is not set. Co-Authored-By: Claude Opus 4.6 (1M context) --- sqlite-vec.c | 42 +++++++++++++++++++++--------------------- tests/conftest.py | 24 ++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 21 deletions(-) diff --git a/sqlite-vec.c b/sqlite-vec.c index 88f60b9..015792b 100644 --- a/sqlite-vec.c +++ b/sqlite-vec.c @@ -93,8 +93,8 @@ typedef size_t usize; #define COMPILER_SUPPORTS_VTAB_IN 1 #endif -#ifndef SQLITE_VEC_ENABLE_IVF -#define SQLITE_VEC_ENABLE_IVF 1 +#ifndef SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE +#define SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE 0 #endif #ifndef SQLITE_SUBTYPE @@ -2558,7 +2558,7 @@ struct Vec0RescoreConfig { }; #endif -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE enum Vec0IvfQuantizer { VEC0_IVF_QUANTIZER_NONE = 0, VEC0_IVF_QUANTIZER_INT8 = 1, @@ -2737,7 +2737,7 @@ static int vec0_parse_rescore_options(struct Vec0Scanner *scanner, * @return int SQLITE_OK on success, SQLITE_EMPTY is it's not a vector column * definition, SQLITE_ERROR on error. */ -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // Forward declaration — defined in sqlite-vec-ivf.c static int vec0_parse_ivf_options(struct Vec0Scanner *scanner, struct Vec0IvfConfig *config); @@ -2922,7 +2922,7 @@ int vec0_parse_vector_column(const char *source, int source_length, } #endif else if (sqlite3_strnicmp(token.start, "ivf", indexNameLen) == 0) { -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE indexType = VEC0_INDEX_TYPE_IVF; memset(&ivfConfig, 0, sizeof(ivfConfig)); rc = vec0_parse_ivf_options(&scanner, &ivfConfig); @@ -3321,7 +3321,7 @@ struct vec0_vtab { int chunk_size; -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // IVF cached state per vector column char *shadowIvfCellsNames[VEC0_MAX_VECTOR_COLUMNS]; // table name for blob_open int ivfTrainedCache[VEC0_MAX_VECTOR_COLUMNS]; // -1=unknown, 0=no, 1=yes @@ -3419,7 +3419,7 @@ void vec0_free_resources(vec0_vtab *p) { sqlite3_finalize(p->stmtRowidsGetChunkPosition); p->stmtRowidsGetChunkPosition = NULL; -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE for (int i = 0; i < VEC0_MAX_VECTOR_COLUMNS; i++) { sqlite3_finalize(p->stmtIvfCellMeta[i]); p->stmtIvfCellMeta[i] = NULL; sqlite3_finalize(p->stmtIvfCellUpdateN[i]); p->stmtIvfCellUpdateN[i] = NULL; @@ -3451,7 +3451,7 @@ void vec0_free(vec0_vtab *p) { for (int i = 0; i < p->numVectorColumns; i++) { sqlite3_free(p->shadowVectorChunksNames[i]); p->shadowVectorChunksNames[i] = NULL; -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE sqlite3_free(p->shadowIvfCellsNames[i]); p->shadowIvfCellsNames[i] = NULL; #endif @@ -3743,7 +3743,7 @@ int vec0_result_id(vec0_vtab *p, sqlite3_context *context, i64 rowid) { * will be stored. * @return int SQLITE_OK on success. */ -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // Forward declaration — defined in sqlite-vec-ivf.c (included later) static int ivf_get_vector_data(vec0_vtab *p, i64 rowid, int col_idx, void **outVector, int *outVectorSize); @@ -3756,7 +3756,7 @@ int vec0_get_vector_data(vec0_vtab *pVtab, i64 rowid, int vector_column_idx, i64 chunk_id; i64 chunk_offset; -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // IVF-indexed columns store vectors in _ivf_cells, not _vector_chunks if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF) { return ivf_get_vector_data(p, rowid, vector_column_idx, outVector, outVectorSize); @@ -4411,7 +4411,7 @@ int vec0_new_chunk(vec0_vtab *p, sqlite3_value ** partitionKeyValues, i64 *chunk #if SQLITE_VEC_ENABLE_RESCORE // Rescore and IVF columns don't use _vector_chunks for float storage if (p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_RESCORE -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE || p->vector_columns[vector_column_idx].index_type == VEC0_INDEX_TYPE_IVF #endif ) { @@ -4587,7 +4587,7 @@ void vec0_cursor_clear(vec0_cursor *pCur) { } // IVF index implementation — #include'd here after all struct/helper definitions -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE #include "sqlite-vec-ivf-kmeans.c" #include "sqlite-vec-ivf.c" #endif @@ -4986,7 +4986,7 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv, } #endif } -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE for (int i = 0; i < pNew->numVectorColumns; i++) { if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; pNew->shadowIvfCellsNames[i] = @@ -5147,7 +5147,7 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv, } #endif -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // Create IVF shadow tables for IVF-indexed vector columns for (int i = 0; i < pNew->numVectorColumns; i++) { if (pNew->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; @@ -5315,7 +5315,7 @@ static int vec0Destroy(sqlite3_vtab *pVtab) { } #endif -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // Drop IVF shadow tables for (int i = 0; i < p->numVectorColumns; i++) { if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; @@ -7335,7 +7335,7 @@ int vec0Filter_knn(vec0_cursor *pCur, vec0_vtab *p, int idxNum, } #endif -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // IVF dispatch: if vector column has IVF, use IVF query instead of chunk scan if (vector_column->index_type == VEC0_INDEX_TYPE_IVF) { rc = ivf_query_knn(p, vectorColumnIdx, queryVector, @@ -8177,7 +8177,7 @@ int vec0Update_InsertWriteFinalStep(vec0_vtab *p, i64 chunk_rowid, #if SQLITE_VEC_ENABLE_RESCORE // Rescore and IVF columns don't use _vector_chunks if (p->vector_columns[i].index_type == VEC0_INDEX_TYPE_RESCORE -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE || p->vector_columns[i].index_type == VEC0_INDEX_TYPE_IVF #endif ) @@ -8593,7 +8593,7 @@ int vec0Update_Insert(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, } #endif -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // Step #4: IVF index insert (if any vector column uses IVF) for (int i = 0; i < p->numVectorColumns; i++) { if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; @@ -9189,7 +9189,7 @@ int vec0Update_Delete(sqlite3_vtab *pVTab, sqlite3_value *idValue) { } } -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // 7. delete from IVF index for (int i = 0; i < p->numVectorColumns; i++) { if (p->vector_columns[i].index_type != VEC0_INDEX_TYPE_IVF) continue; @@ -9473,7 +9473,7 @@ static int vec0Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, } // INSERT operation else if (argc > 1 && sqlite3_value_type(argv[0]) == SQLITE_NULL) { -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE // Check for IVF command inserts: INSERT INTO t(rowid) VALUES ('compute-centroids') // The id column holds the command string. sqlite3_value *idVal = argv[2 + VEC0_COLUMN_ID]; @@ -9632,7 +9632,7 @@ static sqlite3_module vec0Module = { #define SQLITE_VEC_DEBUG_BUILD_RESCORE "" #endif -#if SQLITE_VEC_ENABLE_IVF +#if SQLITE_VEC_EXPERIMENTAL_IVF_ENABLE #define SQLITE_VEC_DEBUG_BUILD_IVF "ivf" #else #define SQLITE_VEC_DEBUG_BUILD_IVF "" diff --git a/tests/conftest.py b/tests/conftest.py index 9549d37..3a24468 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,5 +1,29 @@ import pytest import sqlite3 +import os + + +def _vec_debug(): + db = sqlite3.connect(":memory:") + db.enable_load_extension(True) + db.load_extension("dist/vec0") + db.enable_load_extension(False) + return db.execute("SELECT vec_debug()").fetchone()[0] + + +def _has_build_flag(flag): + return flag in _vec_debug().split("Build flags:")[-1] + + +def pytest_collection_modifyitems(config, items): + has_ivf = _has_build_flag("ivf") + if has_ivf: + return + skip_ivf = pytest.mark.skip(reason="IVF not enabled (compile with -DSQLITE_VEC_EXPERIMENTAL_IVF_ENABLE=1)") + ivf_prefixes = ("test-ivf",) + for item in items: + if any(item.fspath.basename.startswith(p) for p in ivf_prefixes): + item.add_marker(skip_ivf) @pytest.fixture()