diff --git a/examples/nbc-headlines/.gitignore b/examples/nbc-headlines/.gitignore new file mode 100644 index 0000000..6848f87 --- /dev/null +++ b/examples/nbc-headlines/.gitignore @@ -0,0 +1,9 @@ +.venv/ +node_modules/ +*.gguf +*.dylib +llama.log +llamafile-0.8.11 + +2024-07-26.json +articles.json diff --git a/examples/nbc-headlines/Makefile b/examples/nbc-headlines/Makefile new file mode 100644 index 0000000..2f2d876 --- /dev/null +++ b/examples/nbc-headlines/Makefile @@ -0,0 +1,8 @@ +deps: + curl -L https://github.com/asg017/sqlite-rembed/releases/download/v0.0.1-alpha.9/install.sh | sh + curl -L https://github.com/asg017/sqlite-vec/releases/download/v0.0.1-alpha.37/install.sh | sh + wget https://github.com/Mozilla-Ocho/llamafile/releases/download/0.8.11/llamafile-0.8.11 + wget https://huggingface.co/leliuga/all-MiniLM-L6-v2-GGUF/resolve/main/all-MiniLM-L6-v2.F16.gguf + + +.PHONY: deps diff --git a/examples/nbc-headlines/README.md b/examples/nbc-headlines/README.md new file mode 100644 index 0000000..804b11a --- /dev/null +++ b/examples/nbc-headlines/README.md @@ -0,0 +1 @@ +https://www.nbcnews.com/archive/articles/last-seven-days diff --git a/examples/nbc-headlines/build.sql b/examples/nbc-headlines/build.sql new file mode 100644 index 0000000..b397798 --- /dev/null +++ b/examples/nbc-headlines/build.sql @@ -0,0 +1,31 @@ +.mode box +.header on +.bail on + +.load ./vec0 +.load ./rembed0 + +insert into rembed_clients(name, options) + values ('llamafile', 'llamafile'); + +create table articles as +select + value ->> 'url' as url, + value ->> 'headline' as headline, + rembed('llamafile', value ->> 'headline') as headline_embedding +from json_each( + readfile('2024-07-26.json') +); + +select writefile( + 'articles.json', + json_group_array( + json_object( + 'id', rowid, + 'url', url, + 'headline', headline, + 'headline_embedding', vec_to_json(headline_embedding) + ) + ) +) +from articles; diff --git a/examples/nbc-headlines/demo.mjs b/examples/nbc-headlines/demo.mjs new file mode 100644 index 0000000..5b861a5 --- /dev/null +++ b/examples/nbc-headlines/demo.mjs @@ -0,0 +1,34 @@ +import Database from "better-sqlite3"; +import * as sqliteVec from "sqlite-vec"; +import { pipeline } from "@xenova/transformers"; + +const db = new Database("articles.db"); +sqliteVec.load(db); + +const extractor = await pipeline( + "feature-extraction", + "Xenova/all-MiniLM-L6-v2" +); + +const query = "sports"; +const queryEmbedding = await extractor([query], { + pooling: "mean", + normalize: true, +}); + +const rows = db + .prepare( + ` + select + article_id, + headline, + distance + from vec_articles + left join articles on articles.id = vec_articles.article_id + where headline_embedding match ? + and k = 8; +` + ) + .all(queryEmbedding.data); + +console.log(rows); diff --git a/examples/nbc-headlines/demo.py b/examples/nbc-headlines/demo.py new file mode 100644 index 0000000..4b643d8 --- /dev/null +++ b/examples/nbc-headlines/demo.py @@ -0,0 +1,31 @@ +import sqlite3 +import sqlite_vec +from sentence_transformers import SentenceTransformer + +db = sqlite3.connect("articles.db") +db.enable_load_extension(True) +sqlite_vec.load(db) +db.enable_load_extension(False) + +model = SentenceTransformer("all-MiniLM-L6-v2") + + +query = "sports" +query_embedding = model.encode(query) + +results = db.execute( + """ + select + article_id, + headline, + distance + from vec_articles + left join articles on articles.id = vec_articles.article_id + where headline_embedding match ? + and k = 8; + """, + [query_embedding] +).fetchall() + +for (article_id, headline, distance) in results: + print(article_id, headline, distance) diff --git a/examples/nbc-headlines/demo.sql b/examples/nbc-headlines/demo.sql new file mode 100644 index 0000000..0ab0f55 --- /dev/null +++ b/examples/nbc-headlines/demo.sql @@ -0,0 +1,54 @@ +.load vec0 +.header on +.bail on +.timer on + +create temp table raw_articles as + select + value ->> 'id' as id, + value ->> 'url' as url, + value ->> 'headline' as headline, + value ->> 'headline_embedding' as headline_embedding + from json_each( + readfile('articles.json') + ); + +create table articles( + id integer primary key, + headline text, + url text +); + +insert into articles(id, headline, url) + select id, headline, url from temp.raw_articles; + +select * from articles limit 5; + +create virtual table vec_articles using vec0( + article_id integer primary key, + headline_embedding float[384] +); + +insert into vec_articles(article_id, headline_embedding) + select id, headline_embedding from temp.raw_articles; + + +select * from articles limit 5; +select article_id, vec_to_json(headline_embedding) from articles limit 5; + +.param set :query 'sports' + +.load ./rembed0 +insert into rembed_clients values ('all-MiniLM-L6-v2', 'llamafile'); + + +.mode qbox -ww + +select + article_id, + --headline, + distance +from vec_articles +--left join articles on articles.id = vec_articles.article_id +where headline_embedding match rembed('all-MiniLM-L6-v2', :query) + and k = 10; diff --git a/examples/nbc-headlines/index.html b/examples/nbc-headlines/index.html new file mode 100644 index 0000000..4c8fed9 --- /dev/null +++ b/examples/nbc-headlines/index.html @@ -0,0 +1,54 @@ + + +

sqlite-vec articles.db demo

+ + + + diff --git a/examples/nbc-headlines/package-lock.json b/examples/nbc-headlines/package-lock.json new file mode 100644 index 0000000..cb833d9 --- /dev/null +++ b/examples/nbc-headlines/package-lock.json @@ -0,0 +1,801 @@ +{ + "name": "nbc-headlines", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "nbc-headlines", + "version": "1.0.0", + "license": "ISC", + "dependencies": { + "@xenova/transformers": "^2.17.2", + "better-sqlite3": "^11.1.2", + "sqlite-vec": "^0.0.1-alpha.37" + } + }, + "node_modules/@huggingface/jinja": { + "version": "0.2.2", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/@protobufjs/aspromise": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/aspromise/-/aspromise-1.1.2.tgz", + "integrity": "sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/base64": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/base64/-/base64-1.1.2.tgz", + "integrity": "sha512-AZkcAA5vnN/v4PDqKyMR5lx7hZttPDgClv83E//FMNhR2TMcLUhfRUBHCmSl0oi9zMgDDqRUJkSxO3wm85+XLg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/codegen": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/@protobufjs/codegen/-/codegen-2.0.4.tgz", + "integrity": "sha512-YyFaikqM5sH0ziFZCN3xDC7zeGaB/d0IUb9CATugHWbd1FRFwWwt4ld4OYMPWu5a3Xe01mGAULCdqhMlPl29Jg==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/eventemitter": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/eventemitter/-/eventemitter-1.1.0.tgz", + "integrity": "sha512-j9ednRT81vYJ9OfVuXG6ERSTdEL1xVsNgqpkxMsbIabzSo3goCjDIveeGv5d03om39ML71RdmrGNjG5SReBP/Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/fetch": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/fetch/-/fetch-1.1.0.tgz", + "integrity": "sha512-lljVXpqXebpsijW71PZaCYeIcE5on1w5DlQy5WH6GLbFryLUrBD4932W/E2BSpfRJWseIL4v/KPgBFxDOIdKpQ==", + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.1", + "@protobufjs/inquire": "^1.1.0" + } + }, + "node_modules/@protobufjs/float": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/@protobufjs/float/-/float-1.0.2.tgz", + "integrity": "sha512-Ddb+kVXlXst9d+R9PfTIxh1EdNkgoRe5tOX6t01f1lYWOvJnSPDBlG241QLzcyPdoNTsblLUdujGSE4RzrTZGQ==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/inquire": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/inquire/-/inquire-1.1.0.tgz", + "integrity": "sha512-kdSefcPdruJiFMVSbn801t4vFK7KB/5gd2fYvrxhuJYg8ILrmn9SKSX2tZdV6V+ksulWqS7aXjBcRXl3wHoD9Q==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/path": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/@protobufjs/path/-/path-1.1.2.tgz", + "integrity": "sha512-6JOcJ5Tm08dOHAbdR3GrvP+yUUfkjG5ePsHYczMFLq3ZmMkAD98cDgcT2iA1lJ9NVwFd4tH/iSSoe44YWkltEA==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/pool": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/pool/-/pool-1.1.0.tgz", + "integrity": "sha512-0kELaGSIDBKvcgS4zkjz1PeddatrjYcmMWOlAuAPwAeccUrPHdUqo/J6LiymHHEiJT5NrF1UVwxY14f+fy4WQw==", + "license": "BSD-3-Clause" + }, + "node_modules/@protobufjs/utf8": { + "version": "1.1.0", + "resolved": "https://registry.npmjs.org/@protobufjs/utf8/-/utf8-1.1.0.tgz", + "integrity": "sha512-Vvn3zZrhQZkkBE8LSuW3em98c0FwgO4nxzv6OdSxPKJIEKY2bGbHn+mhGIPerzI4twdxaP8/0+06HBpwf345Lw==", + "license": "BSD-3-Clause" + }, + "node_modules/@types/long": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/@types/long/-/long-4.0.2.tgz", + "integrity": "sha512-MqTGEo5bj5t157U6fA/BiDynNkn0YknVdh48CMPkTSpFTVmvao5UQmm7uEF6xBEo7qIMAlY/JSleYaE6VOdpaA==", + "license": "MIT" + }, + "node_modules/@types/node": { + "version": "20.14.12", + "resolved": "https://registry.npmjs.org/@types/node/-/node-20.14.12.tgz", + "integrity": "sha512-r7wNXakLeSsGT0H1AU863vS2wa5wBOK4bWMjZz2wj+8nBx+m5PeIn0k8AloSLpRuiwdRQZwarZqHE4FNArPuJQ==", + "license": "MIT", + "dependencies": { + "undici-types": "~5.26.4" + } + }, + "node_modules/@xenova/transformers": { + "version": "2.17.2", + "resolved": "https://registry.npmjs.org/@xenova/transformers/-/transformers-2.17.2.tgz", + "integrity": "sha512-lZmHqzrVIkSvZdKZEx7IYY51TK0WDrC8eR0c5IMnBsO8di8are1zzw8BlLhyO2TklZKLN5UffNGs1IJwT6oOqQ==", + "license": "Apache-2.0", + "dependencies": { + "@huggingface/jinja": "^0.2.2", + "onnxruntime-web": "1.14.0", + "sharp": "^0.32.0" + }, + "optionalDependencies": { + "onnxruntime-node": "1.14.0" + } + }, + "node_modules/b4a": { + "version": "1.6.6", + "license": "Apache-2.0" + }, + "node_modules/bare-events": { + "version": "2.4.2", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-fs": { + "version": "2.3.1", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-events": "^2.0.0", + "bare-path": "^2.0.0", + "bare-stream": "^2.0.0" + } + }, + "node_modules/bare-os": { + "version": "2.4.0", + "license": "Apache-2.0", + "optional": true + }, + "node_modules/bare-path": { + "version": "2.1.3", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "bare-os": "^2.1.0" + } + }, + "node_modules/bare-stream": { + "version": "2.1.3", + "license": "Apache-2.0", + "optional": true, + "dependencies": { + "streamx": "^2.18.0" + } + }, + "node_modules/base64-js": { + "version": "1.5.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/better-sqlite3": { + "version": "11.1.2", + "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-11.1.2.tgz", + "integrity": "sha512-gujtFwavWU4MSPT+h9B+4pkvZdyOUkH54zgLdIrMmmmd4ZqiBIrRNBzNzYVFO417xo882uP5HBu4GjOfaSrIQw==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "bindings": "^1.5.0", + "prebuild-install": "^7.1.1" + } + }, + "node_modules/bindings": { + "version": "1.5.0", + "license": "MIT", + "dependencies": { + "file-uri-to-path": "1.0.0" + } + }, + "node_modules/bl": { + "version": "4.1.0", + "license": "MIT", + "dependencies": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + } + }, + "node_modules/buffer": { + "version": "5.7.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + }, + "node_modules/chownr": { + "version": "1.1.4", + "license": "ISC" + }, + "node_modules/color": { + "version": "4.2.3", + "license": "MIT", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, + "node_modules/color-convert": { + "version": "2.0.1", + "license": "MIT", + "dependencies": { + "color-name": "~1.1.4" + }, + "engines": { + "node": ">=7.0.0" + } + }, + "node_modules/color-name": { + "version": "1.1.4", + "license": "MIT" + }, + "node_modules/color-string": { + "version": "1.9.1", + "license": "MIT", + "dependencies": { + "color-name": "^1.0.0", + "simple-swizzle": "^0.2.2" + } + }, + "node_modules/decompress-response": { + "version": "6.0.0", + "license": "MIT", + "dependencies": { + "mimic-response": "^3.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/deep-extend": { + "version": "0.6.0", + "license": "MIT", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/detect-libc": { + "version": "2.0.3", + "license": "Apache-2.0", + "engines": { + "node": ">=8" + } + }, + "node_modules/end-of-stream": { + "version": "1.4.4", + "license": "MIT", + "dependencies": { + "once": "^1.4.0" + } + }, + "node_modules/expand-template": { + "version": "2.0.3", + "license": "(MIT OR WTFPL)", + "engines": { + "node": ">=6" + } + }, + "node_modules/fast-fifo": { + "version": "1.3.2", + "license": "MIT" + }, + "node_modules/file-uri-to-path": { + "version": "1.0.0", + "license": "MIT" + }, + "node_modules/flatbuffers": { + "version": "1.12.0", + "license": "SEE LICENSE IN LICENSE.txt" + }, + "node_modules/fs-constants": { + "version": "1.0.0", + "license": "MIT" + }, + "node_modules/github-from-package": { + "version": "0.0.0", + "license": "MIT" + }, + "node_modules/guid-typescript": { + "version": "1.0.9", + "license": "ISC" + }, + "node_modules/ieee754": { + "version": "1.2.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "BSD-3-Clause" + }, + "node_modules/inherits": { + "version": "2.0.4", + "license": "ISC" + }, + "node_modules/ini": { + "version": "1.3.8", + "license": "ISC" + }, + "node_modules/is-arrayish": { + "version": "0.3.2", + "license": "MIT" + }, + "node_modules/long": { + "version": "4.0.0", + "license": "Apache-2.0" + }, + "node_modules/mimic-response": { + "version": "3.1.0", + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/minimist": { + "version": "1.2.8", + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/mkdirp-classic": { + "version": "0.5.3", + "license": "MIT" + }, + "node_modules/napi-build-utils": { + "version": "1.0.2", + "license": "MIT" + }, + "node_modules/node-abi": { + "version": "3.65.0", + "license": "MIT", + "dependencies": { + "semver": "^7.3.5" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-addon-api": { + "version": "6.1.0", + "license": "MIT" + }, + "node_modules/once": { + "version": "1.4.0", + "license": "ISC", + "dependencies": { + "wrappy": "1" + } + }, + "node_modules/onnx-proto": { + "version": "4.0.4", + "license": "MIT", + "dependencies": { + "protobufjs": "^6.8.8" + } + }, + "node_modules/onnx-proto/node_modules/protobufjs": { + "version": "6.11.4", + "resolved": "https://registry.npmjs.org/protobufjs/-/protobufjs-6.11.4.tgz", + "integrity": "sha512-5kQWPaJHi1WoCpjTGszzQ32PG2F4+wRY6BmAT4Vfw56Q2FZ4YZzK20xUYQH4YkfehY1e6QSICrJquM6xXZNcrw==", + "hasInstallScript": true, + "license": "BSD-3-Clause", + "dependencies": { + "@protobufjs/aspromise": "^1.1.2", + "@protobufjs/base64": "^1.1.2", + "@protobufjs/codegen": "^2.0.4", + "@protobufjs/eventemitter": "^1.1.0", + "@protobufjs/fetch": "^1.1.0", + "@protobufjs/float": "^1.0.2", + "@protobufjs/inquire": "^1.1.0", + "@protobufjs/path": "^1.1.2", + "@protobufjs/pool": "^1.1.0", + "@protobufjs/utf8": "^1.1.0", + "@types/long": "^4.0.1", + "@types/node": ">=13.7.0", + "long": "^4.0.0" + }, + "bin": { + "pbjs": "bin/pbjs", + "pbts": "bin/pbts" + } + }, + "node_modules/onnxruntime-common": { + "version": "1.14.0", + "license": "MIT" + }, + "node_modules/onnxruntime-node": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-node/-/onnxruntime-node-1.14.0.tgz", + "integrity": "sha512-5ba7TWomIV/9b6NH/1x/8QEeowsb+jBEvFzU6z0T4mNsFwdPqXeFUM7uxC6QeSRkEbWu3qEB0VMjrvzN/0S9+w==", + "license": "MIT", + "optional": true, + "os": [ + "win32", + "darwin", + "linux" + ], + "dependencies": { + "onnxruntime-common": "~1.14.0" + } + }, + "node_modules/onnxruntime-web": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/onnxruntime-web/-/onnxruntime-web-1.14.0.tgz", + "integrity": "sha512-Kcqf43UMfW8mCydVGcX9OMXI2VN17c0p6XvR7IPSZzBf/6lteBzXHvcEVWDPmCKuGombl997HgLqj91F11DzXw==", + "license": "MIT", + "dependencies": { + "flatbuffers": "^1.12.0", + "guid-typescript": "^1.0.9", + "long": "^4.0.0", + "onnx-proto": "^4.0.4", + "onnxruntime-common": "~1.14.0", + "platform": "^1.3.6" + } + }, + "node_modules/platform": { + "version": "1.3.6", + "license": "MIT" + }, + "node_modules/prebuild-install": { + "version": "7.1.2", + "license": "MIT", + "dependencies": { + "detect-libc": "^2.0.0", + "expand-template": "^2.0.3", + "github-from-package": "0.0.0", + "minimist": "^1.2.3", + "mkdirp-classic": "^0.5.3", + "napi-build-utils": "^1.0.1", + "node-abi": "^3.3.0", + "pump": "^3.0.0", + "rc": "^1.2.7", + "simple-get": "^4.0.0", + "tar-fs": "^2.0.0", + "tunnel-agent": "^0.6.0" + }, + "bin": { + "prebuild-install": "bin.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/pump": { + "version": "3.0.0", + "license": "MIT", + "dependencies": { + "end-of-stream": "^1.1.0", + "once": "^1.3.1" + } + }, + "node_modules/queue-tick": { + "version": "1.0.1", + "license": "MIT" + }, + "node_modules/rc": { + "version": "1.2.8", + "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "dependencies": { + "deep-extend": "^0.6.0", + "ini": "~1.3.0", + "minimist": "^1.2.0", + "strip-json-comments": "~2.0.1" + }, + "bin": { + "rc": "cli.js" + } + }, + "node_modules/readable-stream": { + "version": "3.6.2", + "license": "MIT", + "dependencies": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + }, + "engines": { + "node": ">= 6" + } + }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/semver": { + "version": "7.6.3", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/sharp": { + "version": "0.32.6", + "hasInstallScript": true, + "license": "Apache-2.0", + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.2", + "node-addon-api": "^6.1.0", + "prebuild-install": "^7.1.1", + "semver": "^7.5.4", + "simple-get": "^4.0.1", + "tar-fs": "^3.0.4", + "tunnel-agent": "^0.6.0" + }, + "engines": { + "node": ">=14.15.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/sharp/node_modules/tar-fs": { + "version": "3.0.6", + "license": "MIT", + "dependencies": { + "pump": "^3.0.0", + "tar-stream": "^3.1.5" + }, + "optionalDependencies": { + "bare-fs": "^2.1.1", + "bare-path": "^2.1.0" + } + }, + "node_modules/sharp/node_modules/tar-stream": { + "version": "3.1.7", + "license": "MIT", + "dependencies": { + "b4a": "^1.6.4", + "fast-fifo": "^1.2.0", + "streamx": "^2.15.0" + } + }, + "node_modules/simple-concat": { + "version": "1.0.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, + "node_modules/simple-get": { + "version": "4.0.1", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT", + "dependencies": { + "decompress-response": "^6.0.0", + "once": "^1.3.1", + "simple-concat": "^1.0.0" + } + }, + "node_modules/simple-swizzle": { + "version": "0.2.2", + "license": "MIT", + "dependencies": { + "is-arrayish": "^0.3.1" + } + }, + "node_modules/sqlite-vec": { + "version": "0.0.1-alpha.37", + "resolved": "https://registry.npmjs.org/sqlite-vec/-/sqlite-vec-0.0.1-alpha.37.tgz", + "integrity": "sha512-ZbdIGPLXVr7cxy9pjwxT8MYBd342bLVbLdV3ZiZz0FpdT5yi0lXyYSh/BEtt0FNrRTwXSLDLL3saTKih6KYzaQ==", + "license": "MIT OR Apache", + "optionalDependencies": { + "sqlite-vec-darwin-arm64": "0.0.1-alpha.37", + "sqlite-vec-darwin-x64": "0.0.1-alpha.37", + "sqlite-vec-linux-x64": "0.0.1-alpha.37", + "sqlite-vec-windows-x64": "0.0.1-alpha.37" + } + }, + "node_modules/sqlite-vec-darwin-arm64": { + "version": "0.0.1-alpha.37", + "resolved": "https://registry.npmjs.org/sqlite-vec-darwin-arm64/-/sqlite-vec-darwin-arm64-0.0.1-alpha.37.tgz", + "integrity": "sha512-3cPdW8JbNVZ08bwsCMsAVv0atZsvRY6Co3bgbiOVjwduBYNd3cTCC6Q+ICjz+H4hTEMZNqGdIWEeCTLgbvnEPw==", + "cpu": [ + "arm64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sqlite-vec-darwin-x64": { + "version": "0.0.1-alpha.37", + "resolved": "https://registry.npmjs.org/sqlite-vec-darwin-x64/-/sqlite-vec-darwin-x64-0.0.1-alpha.37.tgz", + "integrity": "sha512-/56xiUbONFw+g3x/UlzGP2ykMDkrQ10trZVxk2Mmshd1y1QOqCoJoWzzcFm3OdaHI/Gz9vOJdCA8Py1QAr7Xqg==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/sqlite-vec-linux-x64": { + "version": "0.0.1-alpha.37", + "resolved": "https://registry.npmjs.org/sqlite-vec-linux-x64/-/sqlite-vec-linux-x64-0.0.1-alpha.37.tgz", + "integrity": "sha512-BEyesm7Vo4EzE+ZQR1CY4M+lHNQufG26q1rkT5LKv6bIyPpebd+zs+hC0ndJmiNjLDNqqsbCsLQoTfLpGG2Urg==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/sqlite-vec-windows-x64": { + "version": "0.0.1-alpha.37", + "resolved": "https://registry.npmjs.org/sqlite-vec-windows-x64/-/sqlite-vec-windows-x64-0.0.1-alpha.37.tgz", + "integrity": "sha512-jPSrUQNuFXy0Y6weSGS3ivU6pZp9nrvHHP3rseXFHuVj+pYK7gN6h1bC20J9Ch/BVz+9S2cVouoO/HvV90+E8w==", + "cpu": [ + "x64" + ], + "license": "MIT OR Apache", + "optional": true, + "os": [ + "windows" + ] + }, + "node_modules/streamx": { + "version": "2.18.0", + "license": "MIT", + "dependencies": { + "fast-fifo": "^1.3.2", + "queue-tick": "^1.0.1", + "text-decoder": "^1.1.0" + }, + "optionalDependencies": { + "bare-events": "^2.2.0" + } + }, + "node_modules/string_decoder": { + "version": "1.3.0", + "license": "MIT", + "dependencies": { + "safe-buffer": "~5.2.0" + } + }, + "node_modules/strip-json-comments": { + "version": "2.0.1", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/tar-fs": { + "version": "2.1.1", + "license": "MIT", + "dependencies": { + "chownr": "^1.1.1", + "mkdirp-classic": "^0.5.2", + "pump": "^3.0.0", + "tar-stream": "^2.1.4" + } + }, + "node_modules/tar-stream": { + "version": "2.2.0", + "license": "MIT", + "dependencies": { + "bl": "^4.0.3", + "end-of-stream": "^1.4.1", + "fs-constants": "^1.0.0", + "inherits": "^2.0.3", + "readable-stream": "^3.1.1" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/text-decoder": { + "version": "1.1.1", + "license": "Apache-2.0", + "dependencies": { + "b4a": "^1.6.4" + } + }, + "node_modules/tunnel-agent": { + "version": "0.6.0", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + }, + "engines": { + "node": "*" + } + }, + "node_modules/undici-types": { + "version": "5.26.5", + "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-5.26.5.tgz", + "integrity": "sha512-JlCMO+ehdEIKqlFxk6IfVoAUVmgz7cU7zD/h9XZ0qzeosSHmUJVOzSQvvYSYWXkFXC+IfLKSIffhv0sVZup6pA==", + "license": "MIT" + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "license": "MIT" + }, + "node_modules/wrappy": { + "version": "1.0.2", + "license": "ISC" + } + } +} diff --git a/examples/nbc-headlines/package.json b/examples/nbc-headlines/package.json new file mode 100644 index 0000000..6931ea6 --- /dev/null +++ b/examples/nbc-headlines/package.json @@ -0,0 +1,17 @@ +{ + "name": "nbc-headlines", + "version": "1.0.0", + "main": "demo.js", + "scripts": { + "test": "echo \"Error: no test specified\" && exit 1" + }, + "keywords": [], + "author": "", + "license": "ISC", + "description": "", + "dependencies": { + "@xenova/transformers": "^2.17.2", + "better-sqlite3": "^11.1.2", + "sqlite-vec": "^0.0.1-alpha.37" + } +} diff --git a/examples/simple-bun/demo.ts b/examples/simple-bun/demo.ts index 31595a7..0ea089c 100644 --- a/examples/simple-bun/demo.ts +++ b/examples/simple-bun/demo.ts @@ -7,7 +7,7 @@ db.loadExtension("../../dist/vec0"); const { sqlite_version, vec_version } = db .prepare( - "select sqlite_version() as sqlite_version, vec_version() as vec_version;" + "select sqlite_version() as sqlite_version, vec_version() as vec_version;", ) .get(); @@ -25,7 +25,7 @@ const query = [0.3, 0.3, 0.3, 0.3]; db.exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])"); const insertStmt = db.prepare( - "INSERT INTO vec_items(rowid, embedding) VALUES (?, vec_f32(?))" + "INSERT INTO vec_items(rowid, embedding) VALUES (?, vec_f32(?))", ); const insertVectors = db.transaction((items) => { @@ -46,7 +46,7 @@ const rows = db WHERE embedding MATCH ? ORDER BY distance LIMIT 3 -` +`, ) .all(new Float32Array(query)); diff --git a/examples/simple-deno/demo.ts b/examples/simple-deno/demo.ts index 376284b..92ff062 100644 --- a/examples/simple-deno/demo.ts +++ b/examples/simple-deno/demo.ts @@ -23,7 +23,7 @@ const query = [0.3, 0.3, 0.3, 0.3]; db.exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])"); const insertStmt = db.prepare( - "INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)" + "INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)", ); const insertVectors = db.transaction((items) => { @@ -44,7 +44,7 @@ const rows = db WHERE embedding MATCH ? ORDER BY distance LIMIT 5 -` +`, ) .all([new Uint8Array(new Float32Array(query).buffer)]); diff --git a/examples/simple-go/demo.go b/examples/simple-go/demo.go index 9527c9e..b4a7eaf 100644 --- a/examples/simple-go/demo.go +++ b/examples/simple-go/demo.go @@ -1,9 +1,7 @@ package main import ( - "bytes" "database/sql" - "encoding/binary" "fmt" "log" @@ -11,15 +9,6 @@ import ( _ "github.com/mattn/go-sqlite3" ) - -func serializeFloat32(vector []float32) ([]byte, error) { - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.LittleEndian, vector) - if err != nil { - return nil, err - } - return buf.Bytes(), nil -} func main() { sqlite_vec.Auto() db, err := sql.Open("sqlite3", ":memory:") @@ -51,7 +40,7 @@ func main() { q := []float32{0.3, 0.3, 0.3, 0.3} for id, values := range items { - v, err := serializeFloat32(values) + v, err := sqlite_vec.SerializeFloat32(values) if err != nil { log.Fatal(err) } @@ -61,7 +50,7 @@ func main() { } } - query, err := serializeFloat32(q) + query, err := sqlite_vec.SerializeFloat32(q) if err != nil { log.Fatal(err) } diff --git a/examples/simple-node/demo.mjs b/examples/simple-node/demo.mjs index 2d76d20..26f5e25 100644 --- a/examples/simple-node/demo.mjs +++ b/examples/simple-node/demo.mjs @@ -6,7 +6,7 @@ sqliteVec.load(db); const { sqlite_version, vec_version } = db .prepare( - "select sqlite_version() as sqlite_version, vec_version() as vec_version;" + "select sqlite_version() as sqlite_version, vec_version() as vec_version;", ) .get(); @@ -24,7 +24,7 @@ const query = [0.3, 0.3, 0.3, 0.3]; db.exec("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[4])"); const insertStmt = db.prepare( - "INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)" + "INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)", ); const insertVectors = db.transaction((items) => { @@ -45,7 +45,7 @@ const rows = db WHERE embedding MATCH ? ORDER BY distance LIMIT 3 -` +`, ) .all(new Float32Array(query)); diff --git a/examples/sqlite3-cli/README.md b/examples/sqlite3-cli/README.md index de7c92a..70c4274 100644 --- a/examples/sqlite3-cli/README.md +++ b/examples/sqlite3-cli/README.md @@ -1,5 +1,9 @@ # `sqlite-vec` statically compiled in the SQLite CLI -You can compile your own version of the `sqlite3` CLI with `sqlite-vec` builtin. The process is not well documented, but the special `SQLITE_EXTRA_INIT` compile option can be used to "inject" code at initialization time. See the `Makefile` at the root of this project for some more info. +You can compile your own version of the `sqlite3` CLI with `sqlite-vec` builtin. +The process is not well documented, but the special `SQLITE_EXTRA_INIT` compile +option can be used to "inject" code at initialization time. See the `Makefile` +at the root of this project for some more info. -The `core_init.c` file here demonstrates auto-loading the `sqlite-vec` entrypoints at startup. +The `core_init.c` file here demonstrates auto-loading the `sqlite-vec` +entrypoints at startup. diff --git a/examples/wasm/README.md b/examples/wasm/README.md index 5b3bcfe..d474518 100644 --- a/examples/wasm/README.md +++ b/examples/wasm/README.md @@ -1,5 +1,11 @@ # `sqlite-vec` statically compiled into WASM builds -You can compile your own version of SQLite's WASM build with `sqlite-vec` builtin. Dynamically loading SQLite extensions is not supported in the official WASM build yet, but you can statically compile extensions in. It's not well documented, but the `sqlite3_wasm_extra_init` option in the SQLite `ext/wasm` Makefile allows you to inject your own code at initialization time. See the `Makefile` at the room of the project for more info. +You can compile your own version of SQLite's WASM build with `sqlite-vec` +builtin. Dynamically loading SQLite extensions is not supported in the official +WASM build yet, but you can statically compile extensions in. It's not well +documented, but the `sqlite3_wasm_extra_init` option in the SQLite `ext/wasm` +Makefile allows you to inject your own code at initialization time. See the +`Makefile` at the room of the project for more info. -The `wasm.c` file here demonstrates auto-loading the `sqlite-vec` entrypoints at startup. +The `wasm.c` file here demonstrates auto-loading the `sqlite-vec` entrypoints at +startup.