From 672868d910a89d8e77a47cbe2787b4d0217b81fa Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Tue, 19 Nov 2024 23:55:10 -0800 Subject: [PATCH] _info shadow table --- sqlite-vec.c | 71 ++++++++++++++++++++++- tests/__snapshots__/test-auxiliary.ambr | 28 ++++++--- tests/__snapshots__/test-general.ambr | 75 ++++++++++++++++++++----- tests/__snapshots__/test-metadata.ambr | 46 +++++++++------ tests/test-auxiliary.py | 2 + tests/test-general.py | 5 ++ tests/test-loadable.py | 4 ++ tests/test-metadata.py | 2 + tests/test-partition-keys.py | 2 + 9 files changed, 195 insertions(+), 40 deletions(-) diff --git a/sqlite-vec.c b/sqlite-vec.c index 26f18e1..7c8768d 100644 --- a/sqlite-vec.c +++ b/sqlite-vec.c @@ -3375,6 +3375,8 @@ static sqlite3_module vec_npy_eachModule = { #define VEC0_COLUMN_OFFSET_DISTANCE 1 #define VEC0_COLUMN_OFFSET_K 2 +#define VEC0_SHADOW_INFO_NAME "\"%w\".\"%w_info\"" + #define VEC0_SHADOW_CHUNKS_NAME "\"%w\".\"%w_chunks\"" /// 1) schema, 2) original vtab table name #define VEC0_SHADOW_CHUNKS_CREATE \ @@ -4963,6 +4965,59 @@ static int vec0_init(sqlite3 *db, void *pAux, int argc, const char *const *argv, sqlite3_stmt *stmt; int rc; + char * zCreateInfo = sqlite3_mprintf("CREATE TABLE "VEC0_SHADOW_INFO_NAME " (key text primary key, value any)", pNew->schemaName, pNew->tableName); + if(!zCreateInfo) { + goto error; + } + rc = sqlite3_prepare_v2(db, zCreateInfo, -1, &stmt, NULL); + + sqlite3_free((void *) zCreateInfo); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + // TODO(IMP) + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not create '_info' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + char * zSeedInfo = sqlite3_mprintf( + "INSERT INTO "VEC0_SHADOW_INFO_NAME "(key, value) VALUES " + "(?1, ?2), (?3, ?4), (?5, ?6), (?7, ?8) ", + pNew->schemaName, pNew->tableName + ); + if(!zSeedInfo) { + goto error; + } + rc = sqlite3_prepare_v2(db, zSeedInfo, -1, &stmt, NULL); + sqlite3_free((void *) zSeedInfo); + if (rc != SQLITE_OK) { + // TODO(IMP) + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not seed '_info' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_bind_text(stmt, 1, "CREATE_VERSION", -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 2, SQLITE_VEC_VERSION, -1, SQLITE_STATIC); + sqlite3_bind_text(stmt, 3, "CREATE_VERSION_MAJOR", -1, SQLITE_STATIC); + sqlite3_bind_int(stmt, 4, SQLITE_VEC_VERSION_MAJOR); + sqlite3_bind_text(stmt, 5, "CREATE_VERSION_MINOR", -1, SQLITE_STATIC); + sqlite3_bind_int(stmt, 6, SQLITE_VEC_VERSION_MINOR); + sqlite3_bind_text(stmt, 7, "CREATE_VERSION_PATCH", -1, SQLITE_STATIC); + sqlite3_bind_int(stmt, 8, SQLITE_VEC_VERSION_PATCH); + + if(sqlite3_step(stmt) != SQLITE_DONE) { + // TODO(IMP) + sqlite3_finalize(stmt); + *pzErr = sqlite3_mprintf("Could not seed '_info' shadow table: %s", + sqlite3_errmsg(db)); + goto error; + } + sqlite3_finalize(stmt); + + + // create the _chunks shadow table char *zCreateShadowChunks = NULL; if(pNew->numPartitionColumns) { @@ -5132,7 +5187,7 @@ static int vec0Destroy(sqlite3_vtab *pVtab) { // Free up any sqlite3_stmt, otherwise DROPs on those tables will fail vec0_free_resources(p); - // later: can't evidence-of here, bc always gives "SQL logic error" instead of + // TODO(test) later: can't evidence-of here, bc always gives "SQL logic error" instead of // provided error zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_CHUNKS_NAME, p->schemaName, p->tableName); @@ -5145,6 +5200,17 @@ static int vec0Destroy(sqlite3_vtab *pVtab) { } sqlite3_finalize(stmt); + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_INFO_NAME, p->schemaName, + p->tableName); + rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); + sqlite3_free((void *)zSql); + if ((rc != SQLITE_OK) || (sqlite3_step(stmt) != SQLITE_DONE)) { + rc = SQLITE_ERROR; + vtab_set_error(pVtab, "could not drop info shadow table"); + goto done; + } + sqlite3_finalize(stmt); + zSql = sqlite3_mprintf("DROP TABLE " VEC0_SHADOW_ROWIDS_NAME, p->schemaName, p->tableName); rc = sqlite3_prepare_v2(p->db, zSql, -1, &stmt, 0); @@ -8759,7 +8825,8 @@ static int vec0Update(sqlite3_vtab *pVTab, int argc, sqlite3_value **argv, } static int vec0ShadowName(const char *zName) { - static const char *azName[] = {"rowids", "chunks", "auxiliary", + static const char *azName[] = { + "rowids", "chunks", "auxiliary", "info", // Up to VEC0_MAX_METADATA_COLUMNS // TODO be smarter about this man diff --git a/tests/__snapshots__/test-auxiliary.ambr b/tests/__snapshots__/test-auxiliary.ambr index 923e196..bfe3d2c 100644 --- a/tests/__snapshots__/test-auxiliary.ambr +++ b/tests/__snapshots__/test-auxiliary.ambr @@ -316,7 +316,7 @@ 'type': 'table', 'name': 'sqlite_sequence', 'tbl_name': 'sqlite_sequence', - 'rootpage': 3, + 'rootpage': 5, 'sql': 'CREATE TABLE sqlite_sequence(name,seq)', }), ]), @@ -326,18 +326,25 @@ OrderedDict({ 'sql': 'select * from sqlite_master order by name', 'rows': list([ + OrderedDict({ + 'type': 'index', + 'name': 'sqlite_autoindex_v_info_1', + 'tbl_name': 'v_info', + 'rootpage': 3, + 'sql': None, + }), OrderedDict({ 'type': 'index', 'name': 'sqlite_autoindex_v_vector_chunks00_1', 'tbl_name': 'v_vector_chunks00', - 'rootpage': 6, + 'rootpage': 8, 'sql': None, }), OrderedDict({ 'type': 'table', 'name': 'sqlite_sequence', 'tbl_name': 'sqlite_sequence', - 'rootpage': 3, + 'rootpage': 5, 'sql': 'CREATE TABLE sqlite_sequence(name,seq)', }), OrderedDict({ @@ -351,28 +358,35 @@ 'type': 'table', 'name': 'v_auxiliary', 'tbl_name': 'v_auxiliary', - 'rootpage': 7, + 'rootpage': 9, 'sql': 'CREATE TABLE "v_auxiliary"( rowid integer PRIMARY KEY , value00)', }), OrderedDict({ 'type': 'table', 'name': 'v_chunks', 'tbl_name': 'v_chunks', - 'rootpage': 2, + 'rootpage': 4, 'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)', }), + OrderedDict({ + 'type': 'table', + 'name': 'v_info', + 'tbl_name': 'v_info', + 'rootpage': 2, + 'sql': 'CREATE TABLE "v_info" (key text primary key, value any)', + }), OrderedDict({ 'type': 'table', 'name': 'v_rowids', 'tbl_name': 'v_rowids', - 'rootpage': 4, + 'rootpage': 6, 'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)', }), OrderedDict({ 'type': 'table', 'name': 'v_vector_chunks00', 'tbl_name': 'v_vector_chunks00', - 'rootpage': 5, + 'rootpage': 7, 'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)', }), ]), diff --git a/tests/__snapshots__/test-general.ambr b/tests/__snapshots__/test-general.ambr index 2e17192..0eac460 100644 --- a/tests/__snapshots__/test-general.ambr +++ b/tests/__snapshots__/test-general.ambr @@ -1,34 +1,64 @@ # serializer version: 1 +# name: test_info + OrderedDict({ + 'sql': 'select key, typeof(value) from v_info order by 1', + 'rows': list([ + OrderedDict({ + 'key': 'CREATE_VERSION', + 'typeof(value)': 'text', + }), + OrderedDict({ + 'key': 'CREATE_VERSION_MAJOR', + 'typeof(value)': 'integer', + }), + OrderedDict({ + 'key': 'CREATE_VERSION_MINOR', + 'typeof(value)': 'integer', + }), + OrderedDict({ + 'key': 'CREATE_VERSION_PATCH', + 'typeof(value)': 'integer', + }), + ]), + }) +# --- # name: test_shadow OrderedDict({ 'sql': 'select * from sqlite_master order by name', 'rows': list([ + OrderedDict({ + 'type': 'index', + 'name': 'sqlite_autoindex_v_info_1', + 'tbl_name': 'v_info', + 'rootpage': 3, + 'sql': None, + }), OrderedDict({ 'type': 'index', 'name': 'sqlite_autoindex_v_metadatachunks00_1', 'tbl_name': 'v_metadatachunks00', - 'rootpage': 8, + 'rootpage': 10, 'sql': None, }), OrderedDict({ 'type': 'index', 'name': 'sqlite_autoindex_v_metadatatext00_1', 'tbl_name': 'v_metadatatext00', - 'rootpage': 10, + 'rootpage': 12, 'sql': None, }), OrderedDict({ 'type': 'index', 'name': 'sqlite_autoindex_v_vector_chunks00_1', 'tbl_name': 'v_vector_chunks00', - 'rootpage': 6, + 'rootpage': 8, 'sql': None, }), OrderedDict({ 'type': 'table', 'name': 'sqlite_sequence', 'tbl_name': 'sqlite_sequence', - 'rootpage': 3, + 'rootpage': 5, 'sql': 'CREATE TABLE sqlite_sequence(name,seq)', }), OrderedDict({ @@ -42,42 +72,49 @@ 'type': 'table', 'name': 'v_auxiliary', 'tbl_name': 'v_auxiliary', - 'rootpage': 11, + 'rootpage': 13, 'sql': 'CREATE TABLE "v_auxiliary"( rowid integer PRIMARY KEY , value00)', }), OrderedDict({ 'type': 'table', 'name': 'v_chunks', 'tbl_name': 'v_chunks', - 'rootpage': 2, + 'rootpage': 4, 'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,sequence_id integer,partition00,validity BLOB NOT NULL, rowids BLOB NOT NULL)', }), + OrderedDict({ + 'type': 'table', + 'name': 'v_info', + 'tbl_name': 'v_info', + 'rootpage': 2, + 'sql': 'CREATE TABLE "v_info" (key text primary key, value any)', + }), OrderedDict({ 'type': 'table', 'name': 'v_metadatachunks00', 'tbl_name': 'v_metadatachunks00', - 'rootpage': 7, + 'rootpage': 9, 'sql': 'CREATE TABLE "v_metadatachunks00"(rowid PRIMARY KEY, data BLOB NOT NULL)', }), OrderedDict({ 'type': 'table', 'name': 'v_metadatatext00', 'tbl_name': 'v_metadatatext00', - 'rootpage': 9, + 'rootpage': 11, 'sql': 'CREATE TABLE "v_metadatatext00"(rowid PRIMARY KEY, data TEXT)', }), OrderedDict({ 'type': 'table', 'name': 'v_rowids', 'tbl_name': 'v_rowids', - 'rootpage': 4, + 'rootpage': 6, 'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)', }), OrderedDict({ 'type': 'table', 'name': 'v_vector_chunks00', 'tbl_name': 'v_vector_chunks00', - 'rootpage': 5, + 'rootpage': 7, 'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)', }), ]), @@ -97,15 +134,15 @@ }), OrderedDict({ 'schema': 'main', - 'name': 'v_metadatatext00', + 'name': 'v_chunks', 'type': 'shadow', - 'ncol': 2, + 'ncol': 6, 'wr': 0, 'strict': 0, }), OrderedDict({ 'schema': 'main', - 'name': 'v_metadatachunks00', + 'name': 'v_info', 'type': 'shadow', 'ncol': 2, 'wr': 0, @@ -121,9 +158,17 @@ }), OrderedDict({ 'schema': 'main', - 'name': 'v_chunks', + 'name': 'v_metadatachunks00', 'type': 'shadow', - 'ncol': 6, + 'ncol': 2, + 'wr': 0, + 'strict': 0, + }), + OrderedDict({ + 'schema': 'main', + 'name': 'v_metadatatext00', + 'type': 'shadow', + 'ncol': 2, 'wr': 0, 'strict': 0, }), diff --git a/tests/__snapshots__/test-metadata.ambr b/tests/__snapshots__/test-metadata.ambr index 75c6cc2..12212ff 100644 --- a/tests/__snapshots__/test-metadata.ambr +++ b/tests/__snapshots__/test-metadata.ambr @@ -587,7 +587,7 @@ 'type': 'table', 'name': 'sqlite_sequence', 'tbl_name': 'sqlite_sequence', - 'rootpage': 3, + 'rootpage': 5, 'sql': 'CREATE TABLE sqlite_sequence(name,seq)', }), OrderedDict({ @@ -601,35 +601,42 @@ 'type': 'table', 'name': 'v_chunks', 'tbl_name': 'v_chunks', - 'rootpage': 2, + 'rootpage': 4, 'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)', }), + OrderedDict({ + 'type': 'table', + 'name': 'v_info', + 'tbl_name': 'v_info', + 'rootpage': 2, + 'sql': 'CREATE TABLE "v_info" (key text primary key, value any)', + }), OrderedDict({ 'type': 'table', 'name': 'v_metadatachunks00', 'tbl_name': 'v_metadatachunks00', - 'rootpage': 7, + 'rootpage': 9, 'sql': 'CREATE TABLE "v_metadatachunks00"(rowid PRIMARY KEY, data BLOB NOT NULL)', }), OrderedDict({ 'type': 'table', 'name': 'v_metadatatext00', 'tbl_name': 'v_metadatatext00', - 'rootpage': 9, + 'rootpage': 11, 'sql': 'CREATE TABLE "v_metadatatext00"(rowid PRIMARY KEY, data TEXT)', }), OrderedDict({ 'type': 'table', 'name': 'v_rowids', 'tbl_name': 'v_rowids', - 'rootpage': 4, + 'rootpage': 6, 'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)', }), OrderedDict({ 'type': 'table', 'name': 'v_vector_chunks00', 'tbl_name': 'v_vector_chunks00', - 'rootpage': 5, + 'rootpage': 7, 'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)', }), ]), @@ -1828,7 +1835,7 @@ 'type': 'table', 'name': 'sqlite_sequence', 'tbl_name': 'sqlite_sequence', - 'rootpage': 3, + 'rootpage': 5, 'sql': 'CREATE TABLE sqlite_sequence(name,seq)', }), ]), @@ -1842,7 +1849,7 @@ 'type': 'table', 'name': 'sqlite_sequence', 'tbl_name': 'sqlite_sequence', - 'rootpage': 3, + 'rootpage': 5, 'sql': 'CREATE TABLE sqlite_sequence(name,seq)', }), OrderedDict({ @@ -1856,56 +1863,63 @@ 'type': 'table', 'name': 'v_chunks', 'tbl_name': 'v_chunks', - 'rootpage': 2, + 'rootpage': 4, 'sql': 'CREATE TABLE "v_chunks"(chunk_id INTEGER PRIMARY KEY AUTOINCREMENT,size INTEGER NOT NULL,validity BLOB NOT NULL,rowids BLOB NOT NULL)', }), + OrderedDict({ + 'type': 'table', + 'name': 'v_info', + 'tbl_name': 'v_info', + 'rootpage': 2, + 'sql': 'CREATE TABLE "v_info" (key text primary key, value any)', + }), OrderedDict({ 'type': 'table', 'name': 'v_metadatachunks00', 'tbl_name': 'v_metadatachunks00', - 'rootpage': 7, + 'rootpage': 9, 'sql': 'CREATE TABLE "v_metadatachunks00"(rowid PRIMARY KEY, data BLOB NOT NULL)', }), OrderedDict({ 'type': 'table', 'name': 'v_metadatachunks01', 'tbl_name': 'v_metadatachunks01', - 'rootpage': 9, + 'rootpage': 11, 'sql': 'CREATE TABLE "v_metadatachunks01"(rowid PRIMARY KEY, data BLOB NOT NULL)', }), OrderedDict({ 'type': 'table', 'name': 'v_metadatachunks02', 'tbl_name': 'v_metadatachunks02', - 'rootpage': 11, + 'rootpage': 13, 'sql': 'CREATE TABLE "v_metadatachunks02"(rowid PRIMARY KEY, data BLOB NOT NULL)', }), OrderedDict({ 'type': 'table', 'name': 'v_metadatachunks03', 'tbl_name': 'v_metadatachunks03', - 'rootpage': 13, + 'rootpage': 15, 'sql': 'CREATE TABLE "v_metadatachunks03"(rowid PRIMARY KEY, data BLOB NOT NULL)', }), OrderedDict({ 'type': 'table', 'name': 'v_metadatatext03', 'tbl_name': 'v_metadatatext03', - 'rootpage': 15, + 'rootpage': 17, 'sql': 'CREATE TABLE "v_metadatatext03"(rowid PRIMARY KEY, data TEXT)', }), OrderedDict({ 'type': 'table', 'name': 'v_rowids', 'tbl_name': 'v_rowids', - 'rootpage': 4, + 'rootpage': 6, 'sql': 'CREATE TABLE "v_rowids"(rowid INTEGER PRIMARY KEY AUTOINCREMENT,id,chunk_id INTEGER,chunk_offset INTEGER)', }), OrderedDict({ 'type': 'table', 'name': 'v_vector_chunks00', 'tbl_name': 'v_vector_chunks00', - 'rootpage': 5, + 'rootpage': 7, 'sql': 'CREATE TABLE "v_vector_chunks00"(rowid PRIMARY KEY,vectors BLOB NOT NULL)', }), ]), diff --git a/tests/test-auxiliary.py b/tests/test-auxiliary.py index 02e478a..39b69f3 100644 --- a/tests/test-auxiliary.py +++ b/tests/test-auxiliary.py @@ -154,5 +154,7 @@ def vec0_shadow_table_contents(db, v): ] o = {} for shadow_table in shadow_tables: + if shadow_table.endswith("_info"): + continue o[shadow_table] = exec(db, f"select * from {shadow_table}") return o diff --git a/tests/test-general.py b/tests/test-general.py index 294164b..2e14cae 100644 --- a/tests/test-general.py +++ b/tests/test-general.py @@ -22,6 +22,11 @@ def test_shadow(db, snapshot): ) +def test_info(db, snapshot): + db.execute("create virtual table v using vec0(a float[1])") + assert exec(db, "select key, typeof(value) from v_info order by 1") == snapshot() + + def exec(db, sql, parameters=[]): try: rows = db.execute(sql, parameters).fetchall() diff --git a/tests/test-loadable.py b/tests/test-loadable.py index 30171fe..b14902a 100644 --- a/tests/test-loadable.py +++ b/tests/test-loadable.py @@ -1022,6 +1022,7 @@ def test_vec0_drops(): ] == [ "t1", "t1_chunks", + "t1_info", "t1_rowids", "t1_vector_chunks00", "t1_vector_chunks01", @@ -2216,6 +2217,9 @@ def test_smoke(): { "name": "vec_xyz_chunks", }, + { + "name": "vec_xyz_info", + }, { "name": "vec_xyz_rowids", }, diff --git a/tests/test-metadata.py b/tests/test-metadata.py index 3774c80..3c2e542 100644 --- a/tests/test-metadata.py +++ b/tests/test-metadata.py @@ -623,5 +623,7 @@ def vec0_shadow_table_contents(db, v): ] o = {} for shadow_table in shadow_tables: + if shadow_table.endswith("_info"): + continue o[shadow_table] = exec(db, f"select * from {shadow_table}") return o diff --git a/tests/test-partition-keys.py b/tests/test-partition-keys.py index 41c7671..fee3560 100644 --- a/tests/test-partition-keys.py +++ b/tests/test-partition-keys.py @@ -111,5 +111,7 @@ def vec0_shadow_table_contents(db, v): ] o = {} for shadow_table in shadow_tables: + if shadow_table.endswith("_info"): + continue o[shadow_table] = exec(db, f"select * from {shadow_table}") return o