Add SQLite storage foundation

Introduce a core-owned Kysely SQLite storage layer backed by
$WorkDir/db/rowboat.sqlite, with startup initialization, shutdown
cleanup, in-code migrations, and initial storage metadata schema.

Ignore database files in the workspace watcher, add focused
storage/watcher tests, and update Electron packaging to stage and rebuild
better-sqlite3 against Electron's native module ABI.
This commit is contained in:
Ramnique Singh 2026-06-09 22:03:17 +05:30
parent 1632b16dfc
commit 883872064f
14 changed files with 511 additions and 14 deletions

View file

@ -28,6 +28,7 @@
"@x/shared": "workspace:*",
"ai": "^5.0.133",
"awilix": "^12.0.5",
"better-sqlite3": "^12.10.0",
"chokidar": "^4.0.3",
"cors": "^2.8.6",
"cron-parser": "^5.5.0",
@ -36,6 +37,7 @@
"google-auth-library": "^10.5.0",
"googleapis": "^169.0.0",
"isomorphic-git": "^1.29.0",
"kysely": "^0.29.2",
"mammoth": "^1.11.0",
"node-html-markdown": "^2.0.0",
"ollama-ai-provider-v2": "^1.5.4",
@ -49,6 +51,7 @@
"zod": "^4.2.1"
},
"devDependencies": {
"@types/better-sqlite3": "^7.6.13",
"@types/cors": "^2.8.19",
"@types/express": "^5.0.6",
"@types/node": "^25.0.3",

View file

@ -12,3 +12,6 @@ export * as versionHistory from './knowledge/version_history.js';
// Voice mode (config + TTS)
export * as voice from './voice/voice.js';
// SQLite storage
export * as storage from './storage/index.js';

View file

@ -0,0 +1,82 @@
import fs from "node:fs";
import { createRequire } from "node:module";
import path from "node:path";
import { Kysely, SqliteDialect, type SqliteDatabase } from "kysely";
import { WorkDir } from "../config/config.js";
import type { Database } from "./schema.js";
import { migrateToLatest } from "./migrations.js";
type BetterSqliteDatabase = SqliteDatabase & {
pragma(source: string, options?: { simple?: boolean }): unknown;
};
type BetterSqliteConstructor = new (
filename?: string,
options?: { timeout?: number },
) => BetterSqliteDatabase;
const require = createRequire(import.meta.url);
const BetterSqlite = require("better-sqlite3") as BetterSqliteConstructor;
let db: Kysely<Database> | null = null;
let initPromise: Promise<void> | null = null;
export function getDatabasePath(): string {
return path.join(WorkDir, "db", "rowboat.sqlite");
}
function createDatabase(): Kysely<Database> {
const databasePath = getDatabasePath();
fs.mkdirSync(path.dirname(databasePath), { recursive: true });
const sqlite = new BetterSqlite(databasePath, { timeout: 5_000 });
sqlite.pragma("foreign_keys = ON");
sqlite.pragma("journal_mode = WAL");
sqlite.pragma("busy_timeout = 5000");
return new Kysely<Database>({
dialect: new SqliteDialect({
database: sqlite,
}),
});
}
export async function initStorage(): Promise<void> {
if (db) return;
if (initPromise) return initPromise;
initPromise = (async () => {
const nextDb = createDatabase();
try {
await migrateToLatest(nextDb);
db = nextDb;
} catch (error) {
await nextDb.destroy().catch((destroyError: unknown) => {
console.error("[storage] failed to close SQLite after init failure:", destroyError);
});
throw error;
} finally {
initPromise = null;
}
})();
return initPromise;
}
export function getDb(): Kysely<Database> {
if (!db) {
throw new Error("SQLite storage has not been initialized. Call initStorage() first.");
}
return db;
}
export async function shutdownStorage(): Promise<void> {
const currentDb = db;
db = null;
initPromise = null;
if (currentDb) {
await currentDb.destroy();
}
}

View file

@ -0,0 +1,2 @@
export { getDatabasePath, getDb, initStorage, shutdownStorage } from "./database.js";
export type { Database, StorageMetadataTable, TimestampColumn } from "./schema.js";

View file

@ -0,0 +1,50 @@
import type { Kysely } from "kysely";
import { Migrator, type Migration, type MigrationProvider } from "kysely/migration";
// Kysely migrations are intentionally schema-agnostic and frozen in time.
// eslint-disable-next-line @typescript-eslint/no-explicit-any
type MigrationDb = Kysely<any>;
const migrations: Record<string, Migration> = {
"2026-06-09_0001_initial_storage": {
async up(db: MigrationDb): Promise<void> {
await db.schema
.createTable("storage_metadata")
.ifNotExists()
.addColumn("key", "text", (col) => col.primaryKey())
.addColumn("value", "text", (col) => col.notNull())
.addColumn("updated_at", "text", (col) => col.notNull())
.execute();
},
async down(db: MigrationDb): Promise<void> {
await db.schema.dropTable("storage_metadata").ifExists().execute();
},
},
};
class InCodeMigrationProvider implements MigrationProvider {
async getMigrations(): Promise<Record<string, Migration>> {
return migrations;
}
}
export async function migrateToLatest(db: MigrationDb): Promise<void> {
const migrator = new Migrator({
db,
provider: new InCodeMigrationProvider(),
});
const { error, results } = await migrator.migrateToLatest();
for (const result of results ?? []) {
if (result.status === "Success") {
console.log(`[storage] migration applied: ${result.migrationName}`);
} else if (result.status === "Error") {
console.error(`[storage] migration failed: ${result.migrationName}`);
}
}
if (error) {
throw new Error("Failed to migrate SQLite storage", { cause: error });
}
}

View file

@ -0,0 +1,13 @@
import type { ColumnType } from "kysely";
export type TimestampColumn = ColumnType<string, string, string>;
export interface StorageMetadataTable {
key: string;
value: string;
updated_at: TimestampColumn;
}
export interface Database {
storage_metadata: StorageMetadataTable;
}

View file

@ -0,0 +1,90 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { sql } from "kysely";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
let tmpDir: string;
let workspaceDir: string;
let storageModule: typeof import("./index.js") | null = null;
beforeEach(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "rowboat-storage-test-"));
workspaceDir = path.join(tmpDir, "workspace");
process.env.ROWBOAT_WORKDIR = workspaceDir;
vi.resetModules();
vi.doMock("../knowledge/version_history.js", () => ({
initRepo: vi.fn(async () => undefined),
}));
vi.doMock("../knowledge/deprecate_today_note.js", () => ({
deprecateTodayNote: vi.fn(async () => undefined),
}));
});
afterEach(async () => {
if (storageModule) {
await storageModule.shutdownStorage().catch(() => undefined);
storageModule = null;
}
delete process.env.ROWBOAT_WORKDIR;
vi.doUnmock("../knowledge/version_history.js");
vi.doUnmock("../knowledge/deprecate_today_note.js");
vi.resetModules();
await fs.rm(tmpDir, { recursive: true, force: true });
});
async function loadStorage() {
storageModule = await import("./index.js");
return storageModule;
}
describe("SQLite storage", () => {
it("throws clearly when accessed before initialization", async () => {
const storage = await loadStorage();
expect(() => storage.getDb()).toThrow("SQLite storage has not been initialized");
});
it("creates the database under ROWBOAT_WORKDIR/db", async () => {
const storage = await loadStorage();
await storage.initStorage();
expect(storage.getDatabasePath()).toBe(path.join(workspaceDir, "db", "rowboat.sqlite"));
await expect(fs.access(storage.getDatabasePath())).resolves.toBeUndefined();
});
it("runs the initial migration", async () => {
const storage = await loadStorage();
await storage.initStorage();
const result = await sql<{ name: string }>`
select name
from sqlite_master
where type = 'table'
and name in ('storage_metadata', 'kysely_migration')
order by name
`.execute(storage.getDb());
expect(result.rows.map((row) => row.name)).toEqual(["kysely_migration", "storage_metadata"]);
});
it("is idempotent", async () => {
const storage = await loadStorage();
await storage.initStorage();
const firstDb = storage.getDb();
await storage.initStorage();
expect(storage.getDb()).toBe(firstDb);
});
it("resets the singleton on shutdown", async () => {
const storage = await loadStorage();
await storage.initStorage();
await storage.shutdownStorage();
expect(() => storage.getDb()).toThrow("SQLite storage has not been initialized");
});
});

View file

@ -0,0 +1,39 @@
import fs from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
let tmpDir: string;
let workspaceDir: string;
beforeEach(async () => {
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "rowboat-watcher-test-"));
workspaceDir = path.join(tmpDir, "workspace");
process.env.ROWBOAT_WORKDIR = workspaceDir;
vi.resetModules();
vi.doMock("../knowledge/version_history.js", () => ({
initRepo: vi.fn(async () => undefined),
}));
vi.doMock("../knowledge/deprecate_today_note.js", () => ({
deprecateTodayNote: vi.fn(async () => undefined),
}));
});
afterEach(async () => {
delete process.env.ROWBOAT_WORKDIR;
vi.doUnmock("../knowledge/version_history.js");
vi.doUnmock("../knowledge/deprecate_today_note.js");
vi.resetModules();
await fs.rm(tmpDir, { recursive: true, force: true });
});
describe("workspace watcher ignores", () => {
it("ignores SQLite storage files under db", async () => {
const watcher = await import("./watcher.js");
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "db"))).toBe(true);
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "db", "rowboat.sqlite"))).toBe(true);
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "db", "rowboat.sqlite-wal"))).toBe(true);
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "knowledge", "note.md"))).toBe(false);
});
});

View file

@ -9,6 +9,11 @@ import { Stats } from 'node:fs';
export type WorkspaceChangeCallback = (event: z.infer<typeof WorkspaceChangeEvent>) => void;
export function shouldIgnoreWorkspacePath(absPath: string): boolean {
const relPath = absToRelPosix(absPath);
return relPath === 'db' || relPath?.startsWith('db/') === true;
}
/**
* Create a workspace watcher
* Watches the configured workspace root recursively and emits change events via callback
@ -29,8 +34,12 @@ export async function createWorkspaceWatcher(
const codeModeDir = path.join(WorkDir, 'code-mode');
const watcher = chokidar.watch(WorkDir, {
ignoreInitial: true,
// Ignore the SQLite db dir (storage) AND code-section worktrees (full repo
// checkouts that would flood the event stream).
ignored: (watchedPath: string) =>
watchedPath === codeModeDir || watchedPath.startsWith(codeModeDir + path.sep),
shouldIgnoreWorkspacePath(watchedPath) ||
watchedPath === codeModeDir ||
watchedPath.startsWith(codeModeDir + path.sep),
awaitWriteFinish: {
stabilityThreshold: 150,
pollInterval: 50,