apunkt/sqlite-vec
1
0
Fork 0
mirror of https://github.com/asg017/sqlite-vec.git synced 2026-04-25 00:36:56 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

_info shadow table

Browse source
This commit is contained in:
Alex Garcia 2024-11-19 23:55:10 -08:00
parent a12c0d73e6
commit 672868d910
9 changed files with 195 additions and 40 deletions
Download patch file Download diff file Expand all files Collapse all files

71
sqlite-vec.c
View file

@ -3375,6 +3375,8 @@ static sqlite3_module vec_npy_eachModule = {
#define VEC0_COLUMN_OFFSET_DISTANCE 1
#define VEC0_COLUMN_OFFSET_K 2
#define VEC0_SHADOW_INFO_NAME "\"%w\".\"%w_info\""
#define VEC0_SHADOW_CHUNKS_NAME "\"%w\".\"%w_chunks\""
/// 1) schema, 2) original vtab table name
#define VEC0_SHADOW_CHUNKS_CREATE \
@ -4963,6 +4965,59 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv,
sqlite3_stmt *stmt;
int rc;
char * zCreateInfo = sqlite3_mprintf("CREATE TABLE "VEC0_SHADOW_INFO_NAME " (key text primary key, value any)", pNew->schemaName, pNew->tableName);
if(!zCreateInfo) {
goto error;
}
rc = sqlite3_prepare_v2(db, zCreateInfo, -1, &stmt, NULL);
sqlite3_free((void *) zCreateInfo);
if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {
// TODO(IMP)
sqlite3_finalize(stmt);
*pzErr = sqlite3_mprintf("Could not create '_info' shadow table: %s",
sqlite3_errmsg(db));
goto error;
}
sqlite3_finalize(stmt);
char * zSeedInfo = sqlite3_mprintf(
"INSERT INTO "VEC0_SHADOW_INFO_NAME "(key, value) VALUES "
"(?1, ?2), (?3, ?4), (?5, ?6), (?7, ?8) ",
pNew->schemaName, pNew->tableName
);
if(!zSeedInfo) {
goto error;
}
rc = sqlite3_prepare_v2(db, zSeedInfo, -1, &stmt, NULL);
sqlite3_free((void *) zSeedInfo);
if (rc != SQLITE_OK) {
// TODO(IMP)
sqlite3_finalize(stmt);
*pzErr = sqlite3_mprintf("Could not seed '_info' shadow table: %s",
sqlite3_errmsg(db));
goto error;
}
sqlite3_bind_text(stmt, 1, "CREATE_VERSION", -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 2, SQLITE_VEC_VERSION, -1, SQLITE_STATIC);
sqlite3_bind_text(stmt, 3, "CREATE_VERSION_MAJOR", -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 4, SQLITE_VEC_VERSION_MAJOR);
sqlite3_bind_text(stmt, 5, "CREATE_VERSION_MINOR", -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 6, SQLITE_VEC_VERSION_MINOR);
sqlite3_bind_text(stmt, 7, "CREATE_VERSION_PATCH", -1, SQLITE_STATIC);
sqlite3_bind_int(stmt, 8, SQLITE_VEC_VERSION_PATCH);
if(sqlite3_step(stmt) != SQLITE_DONE) {
// TODO(IMP)
sqlite3_finalize(stmt);
*pzErr = sqlite3_mprintf("Could not seed '_info' shadow table: %s",
sqlite3_errmsg(db));
goto error;
}
sqlite3_finalize(stmt);
// create the _chunks shadow table
char *zCreateShadowChunks = NULL;
if(pNew->numPartitionColumns) {
@ -5132,7 +5187,7 @@ static int vec0Destroy(sqlite3_vtab *pVtab) {
// Free up any sqlite3_stmt, otherwise DROPs on those tables will fail
vec0_free_resources(p);
// later: can't evidence-of here, bc always gives "SQL logic error" instead of
// TODO(test) later: can't evidence-of here, bc always gives "SQL logic error" instead of
// provided error
zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_CHUNKS_NAME, p->schemaName,
p->tableName);
@ -5145,6 +5200,17 @@ static int vec0Destroy(sqlite3_vtab *pVtab) {
}
sqlite3_finalize(stmt);
zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_INFO_NAME, p->schemaName,
p->tableName);
rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);
sqlite3_free((void *)zSql);
if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) {
rc = SQLITE_ERROR;
vtab_set_error(pVtab, "could not drop info shadow table");
goto done;
}
sqlite3_finalize(stmt);
zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_ROWIDS_NAME, p->schemaName,
p->tableName);
rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0);
@ -8759,7 +8825,8 @@ static int vec0Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv,
}
static int vec0ShadowName(const char *zName) {
static const char *azName[] = {"rowids", "chunks", "auxiliary",
static const char *azName[] = {
"rowids", "chunks", "auxiliary", "info",
// Up to VEC0_MAX_METADATA_COLUMNS
// TODO be smarter about this man

28
tests/__snapshots__/test-auxiliary.ambr
View file

@ -316,7 +316,7 @@
'type': 'table',
'name': 'sqlite_sequence',
'tbl_name': 'sqlite_sequence',
'rootpage': 3,
'rootpage': 5,
'sql': 'CREATE TABLE sqlite_sequence(name,seq)',
}),
]),
@ -326,18 +326,25 @@
OrderedDict({
'sql': 'select * from sqlite_master order by name',
'rows': list([
OrderedDict({
'type': 'index',
'name': 'sqlite_autoindex_v_info_1',
'tbl_name': 'v_info',
'rootpage': 3,
'sql': None,
}),
OrderedDict({
'type': 'index',
'name': 'sqlite_autoindex_v_vector_chunks00_1',
'tbl_name': 'v_vector_chunks00',
'rootpage': 6,
'rootpage': 8,
'sql': None,
}),
OrderedDict({
'type': 'table',
'name': 'sqlite_sequence',
'tbl_name': 'sqlite_sequence',
'rootpage': 3,
'rootpage': 5,
'sql': 'CREATE TABLE sqlite_sequence(name,seq)',
}),
OrderedDict({
@ -351,28 +358,35 @@
'type': 'table',
'name': 'v_auxiliary',
'tbl_name': 'v_auxiliary',
'rootpage': 7,
'rootpage': 9,
'sql': 'CREATE TABLE "v_auxiliary"( rowid integer PRIMARY KEY , value00)',
}),
OrderedDict({
'type': 'table',
'name': 'v_chunks',
'tbl_name': 'v_chunks',
'rootpage': 2,
'rootpage': 4,
'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_info',
'tbl_name': 'v_info',
'rootpage': 2,
'sql': 'CREATE TABLE "v_info" (key text primary key, value any)',
}),
OrderedDict({
'type': 'table',
'name': 'v_rowids',
'tbl_name': 'v_rowids',
'rootpage': 4,
'rootpage': 6,
'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',
}),
OrderedDict({
'type': 'table',
'name': 'v_vector_chunks00',
'tbl_name': 'v_vector_chunks00',
'rootpage': 5,
'rootpage': 7,
'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',
}),
]),

75
tests/__snapshots__/test-general.ambr
View file

@ -1,34 +1,64 @@
# serializer version: 1
# name: test_info
OrderedDict({
'sql': 'select key, typeof(value) from v_info order by 1',
'rows': list([
OrderedDict({
'key': 'CREATE_VERSION',
'typeof(value)': 'text',
}),
OrderedDict({
'key': 'CREATE_VERSION_MAJOR',
'typeof(value)': 'integer',
}),
OrderedDict({
'key': 'CREATE_VERSION_MINOR',
'typeof(value)': 'integer',
}),
OrderedDict({
'key': 'CREATE_VERSION_PATCH',
'typeof(value)': 'integer',
}),
]),
})
# ---
# name: test_shadow
OrderedDict({
'sql': 'select * from sqlite_master order by name',
'rows': list([
OrderedDict({
'type': 'index',
'name': 'sqlite_autoindex_v_info_1',
'tbl_name': 'v_info',
'rootpage': 3,
'sql': None,
}),
OrderedDict({
'type': 'index',
'name': 'sqlite_autoindex_v_metadatachunks00_1',
'tbl_name': 'v_metadatachunks00',
'rootpage': 8,
'rootpage': 10,
'sql': None,
}),
OrderedDict({
'type': 'index',
'name': 'sqlite_autoindex_v_metadatatext00_1',
'tbl_name': 'v_metadatatext00',
'rootpage': 10,
'rootpage': 12,
'sql': None,
}),
OrderedDict({
'type': 'index',
'name': 'sqlite_autoindex_v_vector_chunks00_1',
'tbl_name': 'v_vector_chunks00',
'rootpage': 6,
'rootpage': 8,
'sql': None,
}),
OrderedDict({
'type': 'table',
'name': 'sqlite_sequence',
'tbl_name': 'sqlite_sequence',
'rootpage': 3,
'rootpage': 5,
'sql': 'CREATE TABLE sqlite_sequence(name,seq)',
}),
OrderedDict({
@ -42,42 +72,49 @@
'type': 'table',
'name': 'v_auxiliary',
'tbl_name': 'v_auxiliary',
'rootpage': 11,
'rootpage': 13,
'sql': 'CREATE TABLE "v_auxiliary"( rowid integer PRIMARY KEY , value00)',
}),
OrderedDict({
'type': 'table',
'name': 'v_chunks',
'tbl_name': 'v_chunks',
'rootpage': 2,
'rootpage': 4,
'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_info',
'tbl_name': 'v_info',
'rootpage': 2,
'sql': 'CREATE TABLE "v_info" (key text primary key, value any)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatachunks00',
'tbl_name': 'v_metadatachunks00',
'rootpage': 7,
'rootpage': 9,
'sql': 'CREATE TABLE "v_metadatachunks00"(rowid PRIMARY KEY, data BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatatext00',
'tbl_name': 'v_metadatatext00',
'rootpage': 9,
'rootpage': 11,
'sql': 'CREATE TABLE "v_metadatatext00"(rowid PRIMARY KEY, data TEXT)',
}),
OrderedDict({
'type': 'table',
'name': 'v_rowids',
'tbl_name': 'v_rowids',
'rootpage': 4,
'rootpage': 6,
'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',
}),
OrderedDict({
'type': 'table',
'name': 'v_vector_chunks00',
'tbl_name': 'v_vector_chunks00',
'rootpage': 5,
'rootpage': 7,
'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',
}),
]),
@ -97,15 +134,15 @@
}),
OrderedDict({
'schema': 'main',
'name': 'v_metadatatext00',
'name': 'v_chunks',
'type': 'shadow',
'ncol': 2,
'ncol': 6,
'wr': 0,
'strict': 0,
}),
OrderedDict({
'schema': 'main',
'name': 'v_metadatachunks00',
'name': 'v_info',
'type': 'shadow',
'ncol': 2,
'wr': 0,
@ -121,9 +158,17 @@
}),
OrderedDict({
'schema': 'main',
'name': 'v_chunks',
'name': 'v_metadatachunks00',
'type': 'shadow',
'ncol': 6,
'ncol': 2,
'wr': 0,
'strict': 0,
}),
OrderedDict({
'schema': 'main',
'name': 'v_metadatatext00',
'type': 'shadow',
'ncol': 2,
'wr': 0,
'strict': 0,
}),

46
tests/__snapshots__/test-metadata.ambr
View file

@ -587,7 +587,7 @@
'type': 'table',
'name': 'sqlite_sequence',
'tbl_name': 'sqlite_sequence',
'rootpage': 3,
'rootpage': 5,
'sql': 'CREATE TABLE sqlite_sequence(name,seq)',
}),
OrderedDict({
@ -601,35 +601,42 @@
'type': 'table',
'name': 'v_chunks',
'tbl_name': 'v_chunks',
'rootpage': 2,
'rootpage': 4,
'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_info',
'tbl_name': 'v_info',
'rootpage': 2,
'sql': 'CREATE TABLE "v_info" (key text primary key, value any)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatachunks00',
'tbl_name': 'v_metadatachunks00',
'rootpage': 7,
'rootpage': 9,
'sql': 'CREATE TABLE "v_metadatachunks00"(rowid PRIMARY KEY, data BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatatext00',
'tbl_name': 'v_metadatatext00',
'rootpage': 9,
'rootpage': 11,
'sql': 'CREATE TABLE "v_metadatatext00"(rowid PRIMARY KEY, data TEXT)',
}),
OrderedDict({
'type': 'table',
'name': 'v_rowids',
'tbl_name': 'v_rowids',
'rootpage': 4,
'rootpage': 6,
'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',
}),
OrderedDict({
'type': 'table',
'name': 'v_vector_chunks00',
'tbl_name': 'v_vector_chunks00',
'rootpage': 5,
'rootpage': 7,
'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',
}),
]),
@ -1828,7 +1835,7 @@
'type': 'table',
'name': 'sqlite_sequence',
'tbl_name': 'sqlite_sequence',
'rootpage': 3,
'rootpage': 5,
'sql': 'CREATE TABLE sqlite_sequence(name,seq)',
}),
]),
@ -1842,7 +1849,7 @@
'type': 'table',
'name': 'sqlite_sequence',
'tbl_name': 'sqlite_sequence',
'rootpage': 3,
'rootpage': 5,
'sql': 'CREATE TABLE sqlite_sequence(name,seq)',
}),
OrderedDict({
@ -1856,56 +1863,63 @@
'type': 'table',
'name': 'v_chunks',
'tbl_name': 'v_chunks',
'rootpage': 2,
'rootpage': 4,
'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_info',
'tbl_name': 'v_info',
'rootpage': 2,
'sql': 'CREATE TABLE "v_info" (key text primary key, value any)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatachunks00',
'tbl_name': 'v_metadatachunks00',
'rootpage': 7,
'rootpage': 9,
'sql': 'CREATE TABLE "v_metadatachunks00"(rowid PRIMARY KEY, data BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatachunks01',
'tbl_name': 'v_metadatachunks01',
'rootpage': 9,
'rootpage': 11,
'sql': 'CREATE TABLE "v_metadatachunks01"(rowid PRIMARY KEY, data BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatachunks02',
'tbl_name': 'v_metadatachunks02',
'rootpage': 11,
'rootpage': 13,
'sql': 'CREATE TABLE "v_metadatachunks02"(rowid PRIMARY KEY, data BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatachunks03',
'tbl_name': 'v_metadatachunks03',
'rootpage': 13,
'rootpage': 15,
'sql': 'CREATE TABLE "v_metadatachunks03"(rowid PRIMARY KEY, data BLOB NOT NULL)',
}),
OrderedDict({
'type': 'table',
'name': 'v_metadatatext03',
'tbl_name': 'v_metadatatext03',
'rootpage': 15,
'rootpage': 17,
'sql': 'CREATE TABLE "v_metadatatext03"(rowid PRIMARY KEY, data TEXT)',
}),
OrderedDict({
'type': 'table',
'name': 'v_rowids',
'tbl_name': 'v_rowids',
'rootpage': 4,
'rootpage': 6,
'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)',
}),
OrderedDict({
'type': 'table',
'name': 'v_vector_chunks00',
'tbl_name': 'v_vector_chunks00',
'rootpage': 5,
'rootpage': 7,
'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)',
}),
]),

2
tests/test-auxiliary.py
View file

@ -154,5 +154,7 @@ def vec0_shadow_table_contents(db, v):
]
o = {}
for shadow_table in shadow_tables:
if shadow_table.endswith("_info"):
continue
o[shadow_table] = exec(db, f"select * from {shadow_table}")
return o

5
tests/test-general.py
View file

@ -22,6 +22,11 @@ def test_shadow(db, snapshot):
)
def test_info(db, snapshot):
db.execute("create virtual table v using vec0(a float[1])")
assert exec(db, "select key, typeof(value) from v_info order by 1") == snapshot()
def exec(db, sql, parameters=[]):
try:
rows = db.execute(sql, parameters).fetchall()

4
tests/test-loadable.py
View file

@ -1022,6 +1022,7 @@ def test_vec0_drops():
] == [
"t1",
"t1_chunks",
"t1_info",
"t1_rowids",
"t1_vector_chunks00",
"t1_vector_chunks01",
@ -2216,6 +2217,9 @@ def test_smoke():
{
"name": "vec_xyz_chunks",
},
{
"name": "vec_xyz_info",
},
{
"name": "vec_xyz_rowids",
},

2
tests/test-metadata.py
View file

@ -623,5 +623,7 @@ def vec0_shadow_table_contents(db, v):
]
o = {}
for shadow_table in shadow_tables:
if shadow_table.endswith("_info"):
continue
o[shadow_table] = exec(db, f"select * from {shadow_table}")
return o

2
tests/test-partition-keys.py
View file

@ -111,5 +111,7 @@ def vec0_shadow_table_contents(db, v):
]
o = {}
for shadow_table in shadow_tables:
if shadow_table.endswith("_info"):
continue
o[shadow_table] = exec(db, f"select * from {shadow_table}")
return o