Add LPAREN, RPAREN, COMMA token types to the scanner

Extends the vec0 tokenizer to recognize '(', ')', and ',' as
single-character tokens, preparing for DiskANN index option parsing.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
Alex Garcia 2026-03-02 18:07:57 -08:00
parent aab9b37de2
commit 0bca960e9d
3 changed files with 96 additions and 0 deletions

View file

@ -23,6 +23,9 @@ enum Vec0TokenType {
TOKEN_TYPE_RBRACKET = 3,
TOKEN_TYPE_PLUS = 4,
TOKEN_TYPE_EQ = 5,
TOKEN_TYPE_LPAREN = 6,
TOKEN_TYPE_RPAREN = 7,
TOKEN_TYPE_COMMA = 8,
};
#define VEC0_TOKEN_RESULT_EOF 1