apunkt/sqlite-vec
1
0
Fork 0
mirror of https://github.com/asg017/sqlite-vec.git synced 2026-04-25 00:36:56 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Add LPAREN, RPAREN, COMMA token types to the scanner

Browse source 
Extends the vec0 tokenizer to recognize '(', ')', and ',' as
single-character tokens, preparing for DiskANN index option parsing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Alex Garcia 2026-03-02 18:07:57 -08:00
parent aab9b37de2
commit 0bca960e9d
3 changed files with 96 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

21
sqlite-vec.c
View file

@ -1814,6 +1814,9 @@ enum Vec0TokenType {
TOKEN_TYPE_RBRACKET,
TOKEN_TYPE_PLUS,
TOKEN_TYPE_EQ,
TOKEN_TYPE_LPAREN,
TOKEN_TYPE_RPAREN,
TOKEN_TYPE_COMMA,
};
struct Vec0Token {
enum Vec0TokenType token_type;
@ -1864,6 +1867,24 @@ int vec0_token_next(char *start, char *end, struct Vec0Token *out) {
out->end = ptr;
out->token_type = TOKEN_TYPE_EQ;
return VEC0_TOKEN_RESULT_SOME;
} else if (curr == '(') {
ptr++;
out->start = ptr;
out->end = ptr;
out->token_type = TOKEN_TYPE_LPAREN;
return VEC0_TOKEN_RESULT_SOME;
} else if (curr == ')') {
ptr++;
out->start = ptr;
out->end = ptr;
out->token_type = TOKEN_TYPE_RPAREN;
return VEC0_TOKEN_RESULT_SOME;
} else if (curr == ',') {
ptr++;
out->start = ptr;
out->end = ptr;
out->token_type = TOKEN_TYPE_COMMA;
return VEC0_TOKEN_RESULT_SOME;
} else if (is_alpha(curr)) {
char *start = ptr;
while (ptr < end && (is_alpha(*ptr) || is_digit(*ptr) || *ptr == '_')) {