Merge branch 'main' of github.com:asg017/sqlite-vec into main

tests/fuzz/.gitignore

@@ -0,0 +1,2 @@
+*.dSYM
+targets/

tests/fuzz/Makefile

@@ -0,0 +1,48 @@
+
+TARGET_DIR=./targets
+
+$(TARGET_DIR):
+	mkdir -p $@
+
+# ASAN_OPTIONS=detect_leaks=1  ./fuzz_json -detect_leaks=1 '-trace_malloc=[12]' tmp
+$(TARGET_DIR)/json: json.c $(TARGET_DIR)
+	/opt/homebrew/opt/llvm/bin/clang \
+	-fsanitize=address,fuzzer \
+	-I ../../ -I ../../vendor -DSQLITE_CORE -g \
+	../../vendor/sqlite3.c \
+	../../sqlite-vec.c \
+	$< \
+	-o $@
+
+
+$(TARGET_DIR)/vec0_create: vec0-create.c ../../sqlite-vec.c $(TARGET_DIR)
+	/opt/homebrew/opt/llvm/bin/clang \
+	-fsanitize=address,fuzzer \
+	-I ../../ -I ../../vendor -DSQLITE_CORE -g \
+	../../vendor/sqlite3.c \
+	../../sqlite-vec.c \
+	$< \
+	-o $@
+
+$(TARGET_DIR)/numpy: numpy.c ../../sqlite-vec.c $(TARGET_DIR)
+	/opt/homebrew/opt/llvm/bin/clang \
+	-fsanitize=address,fuzzer \
+	-I ../../ -I ../../vendor -DSQLITE_CORE -g \
+	../../vendor/sqlite3.c \
+	../../sqlite-vec.c \
+	$< \
+	-o $@
+
+$(TARGET_DIR)/exec: exec.c ../../sqlite-vec.c $(TARGET_DIR)
+	/opt/homebrew/opt/llvm/bin/clang \
+	-fsanitize=address,fuzzer \
+	-I ../../ -I ../../vendor -DSQLITE_CORE -g \
+	../../vendor/sqlite3.c \
+	../../sqlite-vec.c \
+	$< \
+	-o $@
+
+all: $(TARGET_DIR)/json $(TARGET_DIR)/numpy $(TARGET_DIR)/json $(TARGET_DIR)/exec
+
+clean:
+	rm -rf $(TARGET_DIR)/*

tests/fuzz/README.md

@@ -0,0 +1,15 @@
+```
+ASAN_OPTIONS=detect_leaks=1 ./targets/vec0_create \
+  -dict=./vec0-create.dict -max_total_time=5 \
+  ./corpus/vec0-create
+```
+
+
+```
+export PATH="/opt/homebrew/opt/llvm/bin:$PATH"
+export LDFLAGS="-L/opt/homebrew/opt/llvm/lib"
+export CPPFLAGS="-I/opt/homebrew/opt/llvm/include"
+
+
+LDFLAGS="-L/opt/homebrew/opt/llvm/lib/c++ -Wl,-rpath,/opt/homebrew/opt/llvm/lib/c++"
+```

tests/fuzz/corpus/vec0-create/normal1

@@ -0,0 +1 @@
+aaa float[12]

tests/fuzz/corpus/vec0-create/normal2

@@ -0,0 +1 @@
+aaa float[12], bbb int8[6]

tests/fuzz/exec.c

@@ -0,0 +1,30 @@
+#include <stdint.h>
+#include <stddef.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite-vec.h"
+#include "sqlite3.h"
+#include <assert.h>
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  int rc = SQLITE_OK;
+  sqlite3 *db;
+  sqlite3_stmt *stmt;
+  if(size < 1) return 0;
+
+  rc = sqlite3_open(":memory:", &db);
+  assert(rc == SQLITE_OK);
+  rc = sqlite3_vec_init(db, NULL, NULL);
+  assert(rc == SQLITE_OK);
+
+  const char * zSrc = sqlite3_mprintf("%.*s", size, data);
+  assert(zSrc);
+
+  sqlite3_exec(db, zSrc, NULL, NULL, NULL);
+  sqlite3_free(zSrc);
+
+  sqlite3_close(db);
+  return 0;
+}

tests/fuzz/exec.dict

@@ -0,0 +1,21 @@
+select="select"
+from="from"
+cname1="aaa"
+cname1="bbb"
+cname1="ccc"
+type1="float"
+type2="int8"
+type3="bit"
+lparen="["
+rparen="]"
+pk="primary key"
+text="text"
+distance_metric="distance_metric"
+eq="="
+l1="l1"
+l2="l2"
+cosine="cosine"
+hamming="hamming"
+vec_distance_l2="vec_distance_l2"
+vec_distance_l1="vec_distance_l1"
+comma=","

tests/fuzz/json.c

@@ -0,0 +1,34 @@
+#include <stdint.h>
+#include <stddef.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite-vec.h"
+#include "sqlite3.h"
+#include <assert.h>
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  int rc = SQLITE_OK;
+  sqlite3 *db;
+  sqlite3_stmt *stmt;
+
+  //rc = sqlite3_auto_extension((void (*)())sqlite3_vec_init);
+  //assert(rc == SQLITE_OK);
+
+  rc = sqlite3_open(":memory:", &db);
+  assert(rc == SQLITE_OK);
+  rc = sqlite3_vec_init(db, NULL, NULL);
+  assert(rc == SQLITE_OK);
+
+  rc = sqlite3_prepare_v2(db, "SELECT vec_f32(cast(? as text))", -1, &stmt, NULL);
+  assert(rc == SQLITE_OK);
+
+  sqlite3_bind_blob(stmt, 1, data, size, SQLITE_STATIC);
+  sqlite3_step(stmt);
+
+  sqlite3_finalize(stmt);
+  sqlite3_close(db);
+  return 0;
+
+}

tests/fuzz/numpy.c

@@ -0,0 +1,42 @@
+#include <stdint.h>
+#include <stddef.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite-vec.h"
+#include "sqlite3.h"
+#include <assert.h>
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  int rc = SQLITE_OK;
+  sqlite3 *db;
+  sqlite3_stmt *stmt;
+
+  rc = sqlite3_open(":memory:", &db);
+  assert(rc == SQLITE_OK);
+  rc = sqlite3_vec_init(db, NULL, NULL);
+  assert(rc == SQLITE_OK);
+
+
+  rc = sqlite3_prepare_v2(db, "select * from vec_npy_each(?)", -1, &stmt, NULL);
+  assert(rc == SQLITE_OK);
+  sqlite3_bind_blob(stmt, 1, data, size, SQLITE_STATIC);
+  rc = sqlite3_step(stmt);
+  if(rc != SQLITE_DONE || rc != SQLITE_ROW) {
+    sqlite3_finalize(stmt);
+    sqlite3_close(db);
+    return -1;
+  }
+
+  while(1) {
+    if(rc == SQLITE_DONE) break;
+    if(rc == SQLITE_ROW) continue;
+    sqlite3_finalize(stmt);
+    sqlite3_close(db);
+    return 1;
+  }
+  sqlite3_finalize(stmt);
+  sqlite3_close(db);
+  return 0;
+}

tests/fuzz/numpy.dict

@@ -0,0 +1,7 @@
+magic="\x93NUMPY"
+lparen="("
+rparen=")"
+lbrace="{"
+rbrace="}"
+sq1="\""
+sq2="'"

tests/fuzz/vec0-create.c

@@ -0,0 +1,37 @@
+#include <stdint.h>
+#include <stddef.h>
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include "sqlite-vec.h"
+#include "sqlite3.h"
+#include <assert.h>
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  int rc = SQLITE_OK;
+  sqlite3 *db;
+  sqlite3_stmt *stmt;
+
+  rc = sqlite3_open(":memory:", &db);
+  assert(rc == SQLITE_OK);
+  rc = sqlite3_vec_init(db, NULL, NULL);
+  assert(rc == SQLITE_OK);
+
+  sqlite3_str * s = sqlite3_str_new(NULL);
+  assert(s);
+  sqlite3_str_appendall(s, "CREATE VIRTUAL TABLE v USING vec0(");
+  sqlite3_str_appendf(s, "%.*s", size, data);
+  sqlite3_str_appendall(s, ")");
+  const char * zSql = sqlite3_str_finish(s);
+  assert(zSql);
+
+  rc = sqlite3_prepare_v2(db, zSql, -1, &stmt, NULL);
+  sqlite3_free(zSql);
+  if(rc == SQLITE_OK) {
+    sqlite3_step(stmt);
+  }
+  sqlite3_finalize(stmt);
+  sqlite3_close(db);
+  return 0;
+}

tests/fuzz/vec0-create.dict

@@ -0,0 +1,16 @@
+cname1="aaa"
+cname1="bbb"
+cname1="ccc"
+type1="float"
+type2="int8"
+type3="bit"
+lparen="["
+rparen="]"
+pk="primary key"
+text="text"
+distance_metric="distance_metric"
+eq="="
+l1="l1"
+l2="l2"
+cosine="cosine"
+hamming="hamming"

tests/leak-fixtures/vec0-create.sql

@@ -0,0 +1,7 @@
+.load dist/vec0
+.mode box
+.header on
+.eqp on
+.echo on
+
+create virtual table v using vec0(y);