Update test-unit.c and unittest.rs functions to enforce pre-existing behavior

- Expand sqlite-vec-internal.h with scanner/tokenizer types, vector column
  definition types, and parser function declarations
- Fix min_idx declaration to match actual C signature (add candidates,
  bTaken, k_used params)
- Compile test-unit with -DSQLITE_CORE and link vendor/sqlite3.c so
  sqlite3 API functions (sqlite3_strnicmp, sqlite3_mprintf, etc.) resolve
- Add unit tests for vec0_token_next, Vec0Scanner, and
  vec0_parse_vector_column
- Fix Rust build.rs to define SQLITE_CORE and compile vendor/sqlite3.c
- Fix Rust min_idx FFI signature and wrapper to match actual C function

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

tests/sqlite-vec-internal.h

@@ -1,12 +1,78 @@
+#ifndef SQLITE_VEC_INTERNAL_H
+#define SQLITE_VEC_INTERNAL_H
+
 #include <stdlib.h>
+#include <stddef.h>
 
 int min_idx(
-  // list of distances, size n
   const float *distances,
-  // number of entries in distances
   int32_t n,
-  // output array of size k, the indicies of the lowest k values in distances
+  uint8_t *candidates,
   int32_t *out,
-  // output number of elements
-  int32_t k
+  int32_t k,
+  uint8_t *bTaken,
+  int32_t *k_used
 );
+
+// Scanner / tokenizer types and functions
+
+enum Vec0TokenType {
+  TOKEN_TYPE_IDENTIFIER = 0,
+  TOKEN_TYPE_DIGIT = 1,
+  TOKEN_TYPE_LBRACKET = 2,
+  TOKEN_TYPE_RBRACKET = 3,
+  TOKEN_TYPE_PLUS = 4,
+  TOKEN_TYPE_EQ = 5,
+};
+
+#define VEC0_TOKEN_RESULT_EOF   1
+#define VEC0_TOKEN_RESULT_SOME  2
+#define VEC0_TOKEN_RESULT_ERROR 3
+
+struct Vec0Token {
+  enum Vec0TokenType token_type;
+  char *start;
+  char *end;
+};
+
+struct Vec0Scanner {
+  char *start;
+  char *end;
+  char *ptr;
+};
+
+void vec0_scanner_init(struct Vec0Scanner *scanner, const char *source, int source_length);
+int vec0_scanner_next(struct Vec0Scanner *scanner, struct Vec0Token *out);
+int vec0_token_next(char *start, char *end, struct Vec0Token *out);
+
+// Vector column definition types and parser
+
+enum VectorElementType {
+  SQLITE_VEC_ELEMENT_TYPE_FLOAT32 = 223 + 0,
+  SQLITE_VEC_ELEMENT_TYPE_BIT     = 223 + 1,
+  SQLITE_VEC_ELEMENT_TYPE_INT8    = 223 + 2,
+};
+
+enum Vec0DistanceMetrics {
+  VEC0_DISTANCE_METRIC_L2 = 1,
+  VEC0_DISTANCE_METRIC_COSINE = 2,
+  VEC0_DISTANCE_METRIC_L1 = 3,
+};
+
+struct VectorColumnDefinition {
+  char *name;
+  int name_length;
+  size_t dimensions;
+  enum VectorElementType element_type;
+  enum Vec0DistanceMetrics distance_metric;
+};
+
+int vec0_parse_vector_column(const char *source, int source_length,
+                             struct VectorColumnDefinition *outColumn);
+
+int vec0_parse_partition_key_definition(const char *source, int source_length,
+                                        char **out_column_name,
+                                        int *out_column_name_length,
+                                        int *out_column_type);
+
+#endif /* SQLITE_VEC_INTERNAL_H */