Add IVF index for vec0 virtual table

Add inverted file (IVF) index type: partitions vectors into clusters via
k-means, quantizes to int8, and scans only the nearest nprobe partitions at
query time. Includes shadow table management, insert/delete, KNN integration,
compile flag (SQLITE_VEC_ENABLE_IVF), fuzz targets, and tests. Removes
superseded ivf-benchmarks/ directory.

tests/test-unit.c

@@ -577,6 +577,182 @@ void test_vec0_parse_vector_column() {
     assert(rc == SQLITE_ERROR);
   }
 
+#if SQLITE_VEC_ENABLE_IVF
+  // IVF: indexed by ivf() — defaults
+  {
+    const char *input = "v float[4] indexed by ivf()";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.dimensions == 4);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.ivf.nlist == 128);  // default
+    assert(col.ivf.nprobe == 10);  // default
+    sqlite3_free(col.name);
+  }
+
+  // IVF: indexed by ivf(nlist=8) — nprobe auto-clamped to 8
+  {
+    const char *input = "v float[4] indexed by ivf(nlist=8)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.ivf.nlist == 8);
+    assert(col.ivf.nprobe == 8);  // clamped from default 10
+    sqlite3_free(col.name);
+  }
+
+  // IVF: indexed by ivf(nlist=64, nprobe=8)
+  {
+    const char *input = "v float[4] indexed by ivf(nlist=64, nprobe=8)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.ivf.nlist == 64);
+    assert(col.ivf.nprobe == 8);
+    sqlite3_free(col.name);
+  }
+
+  // IVF: with distance_metric before indexed by
+  {
+    const char *input = "v float[4] distance_metric=cosine indexed by ivf(nlist=16)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.distance_metric == VEC0_DISTANCE_METRIC_COSINE);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.ivf.nlist == 16);
+    sqlite3_free(col.name);
+  }
+
+  // IVF: nlist=0 (deferred)
+  {
+    const char *input = "v float[4] indexed by ivf(nlist=0)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.nlist == 0);
+    sqlite3_free(col.name);
+  }
+
+  // IVF error: nprobe > nlist
+  {
+    const char *input = "v float[4] indexed by ivf(nlist=4, nprobe=10)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // IVF error: unknown key
+  {
+    const char *input = "v float[4] indexed by ivf(bogus=1)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // IVF error: unknown index type (hnsw not supported)
+  {
+    const char *input = "v float[4] indexed by hnsw()";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // Not IVF: no ivf config
+  {
+    const char *input = "v float[4]";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.index_type == VEC0_INDEX_TYPE_FLAT);
+    sqlite3_free(col.name);
+  }
+
+  // IVF: quantizer=binary
+  {
+    const char *input = "v float[768] indexed by ivf(nlist=128, quantizer=binary)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.ivf.nlist == 128);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_BINARY);
+    assert(col.ivf.oversample == 1);
+    sqlite3_free(col.name);
+  }
+
+  // IVF: quantizer=int8
+  {
+    const char *input = "v float[768] indexed by ivf(nlist=64, quantizer=int8)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_INT8);
+    sqlite3_free(col.name);
+  }
+
+  // IVF: quantizer=none (explicit)
+  {
+    const char *input = "v float[768] indexed by ivf(quantizer=none)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_NONE);
+    sqlite3_free(col.name);
+  }
+
+  // IVF: oversample=10 with quantizer
+  {
+    const char *input = "v float[768] indexed by ivf(nlist=128, quantizer=binary, oversample=10)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_BINARY);
+    assert(col.ivf.oversample == 10);
+    assert(col.ivf.nlist == 128);
+    sqlite3_free(col.name);
+  }
+
+  // IVF: all params
+  {
+    const char *input = "v float[768] distance_metric=cosine indexed by ivf(nlist=256, nprobe=16, quantizer=int8, oversample=4)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.distance_metric == VEC0_DISTANCE_METRIC_COSINE);
+    assert(col.ivf.nlist == 256);
+    assert(col.ivf.nprobe == 16);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_INT8);
+    assert(col.ivf.oversample == 4);
+    sqlite3_free(col.name);
+  }
+
+  // IVF error: oversample > 1 without quantizer
+  {
+    const char *input = "v float[768] indexed by ivf(oversample=10)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // IVF error: unknown quantizer value
+  {
+    const char *input = "v float[768] indexed by ivf(quantizer=pq)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // IVF: quantizer with defaults (nlist=128 default, nprobe=10 default)
+  {
+    const char *input = "v float[768] indexed by ivf(quantizer=binary, oversample=5)";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.nlist == 128);
+    assert(col.ivf.nprobe == 10);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_BINARY);
+    assert(col.ivf.oversample == 5);
+    sqlite3_free(col.name);
+  }
+#else
+  // When IVF is disabled, parsing "ivf" should fail
+  {
+    const char *input = "v float[4] indexed by ivf()";
+    rc = vec0_parse_vector_column(input, (int)strlen(input), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+#endif /* SQLITE_VEC_ENABLE_IVF */
+
   printf("  All vec0_parse_vector_column tests passed.\n");
 }
 
@@ -821,6 +997,38 @@ void test_rescore_quantize_float_to_int8() {
     float src[8] = {5.0f, 5.0f, 5.0f, 5.0f, 5.0f, 5.0f, 5.0f, 5.0f};
     _test_rescore_quantize_float_to_int8(src, dst, 8);
     for (int i = 0; i < 8; i++) {
+#if SQLITE_VEC_ENABLE_IVF
+void test_ivf_quantize_int8() {
+  printf("Starting %s...\n", __func__);
+
+  // Basic values in [-1, 1] range
+  {
+    float src[] = {0.0f, 1.0f, -1.0f, 0.5f};
+    int8_t dst[4];
+    ivf_quantize_int8(src, dst, 4);
+    assert(dst[0] == 0);
+    assert(dst[1] == 127);
+    assert(dst[2] == -127);
+    assert(dst[3] == 63);  // 0.5 * 127 = 63.5, truncated to 63
+  }
+
+  // Clamping: values beyond [-1, 1]
+  {
+    float src[] = {2.0f, -3.0f, 100.0f, -0.01f};
+    int8_t dst[4];
+    ivf_quantize_int8(src, dst, 4);
+    assert(dst[0] == 127);   // clamped to 1.0
+    assert(dst[1] == -127);  // clamped to -1.0
+    assert(dst[2] == 127);   // clamped to 1.0
+    assert(dst[3] == (int8_t)(-0.01f * 127.0f));
+  }
+
+  // Zero vector
+  {
+    float src[] = {0.0f, 0.0f, 0.0f, 0.0f};
+    int8_t dst[4];
+    ivf_quantize_int8(src, dst, 4);
+    for (int i = 0; i < 4; i++) {
       assert(dst[i] == 0);
     }
   }
@@ -882,6 +1090,103 @@ void test_rescore_quantized_byte_size() {
 }
 
 void test_vec0_parse_vector_column_rescore() {
+  // Negative zero
+  {
+    float src[] = {-0.0f};
+    int8_t dst[1];
+    ivf_quantize_int8(src, dst, 1);
+    assert(dst[0] == 0);
+  }
+
+  // Single element
+  {
+    float src[] = {0.75f};
+    int8_t dst[1];
+    ivf_quantize_int8(src, dst, 1);
+    assert(dst[0] == (int8_t)(0.75f * 127.0f));
+  }
+
+  // Boundary: exactly 1.0 and -1.0
+  {
+    float src[] = {1.0f, -1.0f};
+    int8_t dst[2];
+    ivf_quantize_int8(src, dst, 2);
+    assert(dst[0] == 127);
+    assert(dst[1] == -127);
+  }
+
+  printf("  All ivf_quantize_int8 tests passed.\n");
+}
+
+void test_ivf_quantize_binary() {
+  printf("Starting %s...\n", __func__);
+
+  // Basic sign-bit quantization: positive -> 1, negative/zero -> 0
+  {
+    float src[] = {1.0f, -1.0f, 0.5f, -0.5f, 0.0f, 0.1f, -0.1f, 2.0f};
+    uint8_t dst[1];
+    ivf_quantize_binary(src, dst, 8);
+    // bit 0: 1.0 > 0 -> 1  (LSB)
+    // bit 1: -1.0 -> 0
+    // bit 2: 0.5 > 0 -> 1
+    // bit 3: -0.5 -> 0
+    // bit 4: 0.0 -> 0 (not > 0)
+    // bit 5: 0.1 > 0 -> 1
+    // bit 6: -0.1 -> 0
+    // bit 7: 2.0 > 0 -> 1
+    // Expected: bits 0,2,5,7 = 0b10100101 = 0xA5
+    assert(dst[0] == 0xA5);
+  }
+
+  // All positive
+  {
+    float src[] = {1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f, 7.0f, 8.0f};
+    uint8_t dst[1];
+    ivf_quantize_binary(src, dst, 8);
+    assert(dst[0] == 0xFF);
+  }
+
+  // All negative
+  {
+    float src[] = {-1.0f, -2.0f, -3.0f, -4.0f, -5.0f, -6.0f, -7.0f, -8.0f};
+    uint8_t dst[1];
+    ivf_quantize_binary(src, dst, 8);
+    assert(dst[0] == 0x00);
+  }
+
+  // All zero (zero is NOT > 0, so all bits should be 0)
+  {
+    float src[] = {0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f, 0.0f};
+    uint8_t dst[1];
+    ivf_quantize_binary(src, dst, 8);
+    assert(dst[0] == 0x00);
+  }
+
+  // Multi-byte: 16 dimensions -> 2 bytes
+  {
+    float src[16];
+    for (int i = 0; i < 16; i++) src[i] = (i % 2 == 0) ? 1.0f : -1.0f;
+    uint8_t dst[2];
+    ivf_quantize_binary(src, dst, 16);
+    // Even indices are positive: bits 0,2,4,6 in each byte
+    // byte 0: bits 0,2,4,6 = 0b01010101 = 0x55
+    // byte 1: same pattern = 0x55
+    assert(dst[0] == 0x55);
+    assert(dst[1] == 0x55);
+  }
+
+  // Single byte, only first bit set
+  {
+    float src[] = {0.1f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f, -1.0f};
+    uint8_t dst[1];
+    ivf_quantize_binary(src, dst, 8);
+    assert(dst[0] == 0x01);
+  }
+
+  printf("  All ivf_quantize_binary tests passed.\n");
+}
+
+void test_ivf_config_parsing() {
   printf("Starting %s...\n", __func__);
   struct VectorColumnDefinition col;
   int rc;
@@ -955,6 +1260,116 @@ void test_vec0_parse_vector_column_rescore() {
 }
 
 #endif /* SQLITE_VEC_ENABLE_RESCORE */
+  // Default IVF config
+  {
+    const char *s = "v float[4] indexed by ivf()";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.index_type == VEC0_INDEX_TYPE_IVF);
+    assert(col.ivf.nlist == 128);   // default
+    assert(col.ivf.nprobe == 10);   // default
+    assert(col.ivf.quantizer == 0); // VEC0_IVF_QUANTIZER_NONE
+    sqlite3_free(col.name);
+  }
+
+  // Custom nlist and nprobe
+  {
+    const char *s = "v float[4] indexed by ivf(nlist=64, nprobe=8)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.nlist == 64);
+    assert(col.ivf.nprobe == 8);
+    sqlite3_free(col.name);
+  }
+
+  // nlist=0 (deferred)
+  {
+    const char *s = "v float[4] indexed by ivf(nlist=0)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.nlist == 0);
+    sqlite3_free(col.name);
+  }
+
+  // Quantizer options
+  {
+    const char *s = "v float[8] indexed by ivf(quantizer=int8)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_INT8);
+    sqlite3_free(col.name);
+  }
+
+  {
+    const char *s = "v float[8] indexed by ivf(quantizer=binary)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_BINARY);
+    sqlite3_free(col.name);
+  }
+
+  // nprobe > nlist (explicit) should fail
+  {
+    const char *s = "v float[4] indexed by ivf(nlist=4, nprobe=10)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // Unknown key
+  {
+    const char *s = "v float[4] indexed by ivf(bogus=1)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // nlist > max (65536) should fail
+  {
+    const char *s = "v float[4] indexed by ivf(nlist=65537)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // nlist at max boundary (65536) should succeed
+  {
+    const char *s = "v float[4] indexed by ivf(nlist=65536)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.nlist == 65536);
+    sqlite3_free(col.name);
+  }
+
+  // oversample > 1 without quantization should fail
+  {
+    const char *s = "v float[4] indexed by ivf(oversample=4)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_ERROR);
+  }
+
+  // oversample with quantizer should succeed
+  {
+    const char *s = "v float[8] indexed by ivf(quantizer=int8, oversample=4)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.oversample == 4);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_INT8);
+    sqlite3_free(col.name);
+  }
+
+  // All options combined
+  {
+    const char *s = "v float[8] indexed by ivf(nlist=32, nprobe=4, quantizer=int8, oversample=2)";
+    rc = vec0_parse_vector_column(s, (int)strlen(s), &col);
+    assert(rc == SQLITE_OK);
+    assert(col.ivf.nlist == 32);
+    assert(col.ivf.nprobe == 4);
+    assert(col.ivf.quantizer == VEC0_IVF_QUANTIZER_INT8);
+    assert(col.ivf.oversample == 2);
+    sqlite3_free(col.name);
+  }
+
+  printf("  All ivf_config_parsing tests passed.\n");
+}
+#endif /* SQLITE_VEC_ENABLE_IVF */
 
 int main() {
   printf("Starting unit tests...\n");
@@ -982,6 +1397,10 @@ int main() {
   test_rescore_quantize_float_to_int8();
   test_rescore_quantized_byte_size();
   test_vec0_parse_vector_column_rescore();
+#if SQLITE_VEC_ENABLE_IVF
+  test_ivf_quantize_int8();
+  test_ivf_quantize_binary();
+  test_ivf_config_parsing();
 #endif
   printf("All unit tests passed.\n");
 }