mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-06-24 20:28:16 +02:00
Add SQLite storage foundation
Introduce a core-owned Kysely SQLite storage layer backed by $WorkDir/db/rowboat.sqlite, with startup initialization, shutdown cleanup, in-code migrations, and initial storage metadata schema. Ignore database files in the workspace watcher, add focused storage/watcher tests, and update Electron packaging to stage and rebuild better-sqlite3 against Electron's native module ABI.
This commit is contained in:
parent
1632b16dfc
commit
883872064f
14 changed files with 511 additions and 14 deletions
|
|
@ -28,6 +28,7 @@
|
|||
"@x/shared": "workspace:*",
|
||||
"ai": "^5.0.133",
|
||||
"awilix": "^12.0.5",
|
||||
"better-sqlite3": "^12.10.0",
|
||||
"chokidar": "^4.0.3",
|
||||
"cors": "^2.8.6",
|
||||
"cron-parser": "^5.5.0",
|
||||
|
|
@ -36,6 +37,7 @@
|
|||
"google-auth-library": "^10.5.0",
|
||||
"googleapis": "^169.0.0",
|
||||
"isomorphic-git": "^1.29.0",
|
||||
"kysely": "^0.29.2",
|
||||
"mammoth": "^1.11.0",
|
||||
"node-html-markdown": "^2.0.0",
|
||||
"ollama-ai-provider-v2": "^1.5.4",
|
||||
|
|
@ -49,6 +51,7 @@
|
|||
"zod": "^4.2.1"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/better-sqlite3": "^7.6.13",
|
||||
"@types/cors": "^2.8.19",
|
||||
"@types/express": "^5.0.6",
|
||||
"@types/node": "^25.0.3",
|
||||
|
|
|
|||
|
|
@ -12,3 +12,6 @@ export * as versionHistory from './knowledge/version_history.js';
|
|||
|
||||
// Voice mode (config + TTS)
|
||||
export * as voice from './voice/voice.js';
|
||||
|
||||
// SQLite storage
|
||||
export * as storage from './storage/index.js';
|
||||
|
|
|
|||
82
apps/x/packages/core/src/storage/database.ts
Normal file
82
apps/x/packages/core/src/storage/database.ts
Normal file
|
|
@ -0,0 +1,82 @@
|
|||
import fs from "node:fs";
|
||||
import { createRequire } from "node:module";
|
||||
import path from "node:path";
|
||||
import { Kysely, SqliteDialect, type SqliteDatabase } from "kysely";
|
||||
import { WorkDir } from "../config/config.js";
|
||||
import type { Database } from "./schema.js";
|
||||
import { migrateToLatest } from "./migrations.js";
|
||||
|
||||
type BetterSqliteDatabase = SqliteDatabase & {
|
||||
pragma(source: string, options?: { simple?: boolean }): unknown;
|
||||
};
|
||||
|
||||
type BetterSqliteConstructor = new (
|
||||
filename?: string,
|
||||
options?: { timeout?: number },
|
||||
) => BetterSqliteDatabase;
|
||||
|
||||
const require = createRequire(import.meta.url);
|
||||
const BetterSqlite = require("better-sqlite3") as BetterSqliteConstructor;
|
||||
|
||||
let db: Kysely<Database> | null = null;
|
||||
let initPromise: Promise<void> | null = null;
|
||||
|
||||
export function getDatabasePath(): string {
|
||||
return path.join(WorkDir, "db", "rowboat.sqlite");
|
||||
}
|
||||
|
||||
function createDatabase(): Kysely<Database> {
|
||||
const databasePath = getDatabasePath();
|
||||
fs.mkdirSync(path.dirname(databasePath), { recursive: true });
|
||||
|
||||
const sqlite = new BetterSqlite(databasePath, { timeout: 5_000 });
|
||||
sqlite.pragma("foreign_keys = ON");
|
||||
sqlite.pragma("journal_mode = WAL");
|
||||
sqlite.pragma("busy_timeout = 5000");
|
||||
|
||||
return new Kysely<Database>({
|
||||
dialect: new SqliteDialect({
|
||||
database: sqlite,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
export async function initStorage(): Promise<void> {
|
||||
if (db) return;
|
||||
if (initPromise) return initPromise;
|
||||
|
||||
initPromise = (async () => {
|
||||
const nextDb = createDatabase();
|
||||
try {
|
||||
await migrateToLatest(nextDb);
|
||||
db = nextDb;
|
||||
} catch (error) {
|
||||
await nextDb.destroy().catch((destroyError: unknown) => {
|
||||
console.error("[storage] failed to close SQLite after init failure:", destroyError);
|
||||
});
|
||||
throw error;
|
||||
} finally {
|
||||
initPromise = null;
|
||||
}
|
||||
})();
|
||||
|
||||
return initPromise;
|
||||
}
|
||||
|
||||
export function getDb(): Kysely<Database> {
|
||||
if (!db) {
|
||||
throw new Error("SQLite storage has not been initialized. Call initStorage() first.");
|
||||
}
|
||||
|
||||
return db;
|
||||
}
|
||||
|
||||
export async function shutdownStorage(): Promise<void> {
|
||||
const currentDb = db;
|
||||
db = null;
|
||||
initPromise = null;
|
||||
|
||||
if (currentDb) {
|
||||
await currentDb.destroy();
|
||||
}
|
||||
}
|
||||
2
apps/x/packages/core/src/storage/index.ts
Normal file
2
apps/x/packages/core/src/storage/index.ts
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
export { getDatabasePath, getDb, initStorage, shutdownStorage } from "./database.js";
|
||||
export type { Database, StorageMetadataTable, TimestampColumn } from "./schema.js";
|
||||
50
apps/x/packages/core/src/storage/migrations.ts
Normal file
50
apps/x/packages/core/src/storage/migrations.ts
Normal file
|
|
@ -0,0 +1,50 @@
|
|||
import type { Kysely } from "kysely";
|
||||
import { Migrator, type Migration, type MigrationProvider } from "kysely/migration";
|
||||
|
||||
// Kysely migrations are intentionally schema-agnostic and frozen in time.
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
type MigrationDb = Kysely<any>;
|
||||
|
||||
const migrations: Record<string, Migration> = {
|
||||
"2026-06-09_0001_initial_storage": {
|
||||
async up(db: MigrationDb): Promise<void> {
|
||||
await db.schema
|
||||
.createTable("storage_metadata")
|
||||
.ifNotExists()
|
||||
.addColumn("key", "text", (col) => col.primaryKey())
|
||||
.addColumn("value", "text", (col) => col.notNull())
|
||||
.addColumn("updated_at", "text", (col) => col.notNull())
|
||||
.execute();
|
||||
},
|
||||
async down(db: MigrationDb): Promise<void> {
|
||||
await db.schema.dropTable("storage_metadata").ifExists().execute();
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
class InCodeMigrationProvider implements MigrationProvider {
|
||||
async getMigrations(): Promise<Record<string, Migration>> {
|
||||
return migrations;
|
||||
}
|
||||
}
|
||||
|
||||
export async function migrateToLatest(db: MigrationDb): Promise<void> {
|
||||
const migrator = new Migrator({
|
||||
db,
|
||||
provider: new InCodeMigrationProvider(),
|
||||
});
|
||||
|
||||
const { error, results } = await migrator.migrateToLatest();
|
||||
|
||||
for (const result of results ?? []) {
|
||||
if (result.status === "Success") {
|
||||
console.log(`[storage] migration applied: ${result.migrationName}`);
|
||||
} else if (result.status === "Error") {
|
||||
console.error(`[storage] migration failed: ${result.migrationName}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (error) {
|
||||
throw new Error("Failed to migrate SQLite storage", { cause: error });
|
||||
}
|
||||
}
|
||||
13
apps/x/packages/core/src/storage/schema.ts
Normal file
13
apps/x/packages/core/src/storage/schema.ts
Normal file
|
|
@ -0,0 +1,13 @@
|
|||
import type { ColumnType } from "kysely";
|
||||
|
||||
export type TimestampColumn = ColumnType<string, string, string>;
|
||||
|
||||
export interface StorageMetadataTable {
|
||||
key: string;
|
||||
value: string;
|
||||
updated_at: TimestampColumn;
|
||||
}
|
||||
|
||||
export interface Database {
|
||||
storage_metadata: StorageMetadataTable;
|
||||
}
|
||||
90
apps/x/packages/core/src/storage/storage.test.ts
Normal file
90
apps/x/packages/core/src/storage/storage.test.ts
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { sql } from "kysely";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
let tmpDir: string;
|
||||
let workspaceDir: string;
|
||||
let storageModule: typeof import("./index.js") | null = null;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "rowboat-storage-test-"));
|
||||
workspaceDir = path.join(tmpDir, "workspace");
|
||||
process.env.ROWBOAT_WORKDIR = workspaceDir;
|
||||
vi.resetModules();
|
||||
vi.doMock("../knowledge/version_history.js", () => ({
|
||||
initRepo: vi.fn(async () => undefined),
|
||||
}));
|
||||
vi.doMock("../knowledge/deprecate_today_note.js", () => ({
|
||||
deprecateTodayNote: vi.fn(async () => undefined),
|
||||
}));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
if (storageModule) {
|
||||
await storageModule.shutdownStorage().catch(() => undefined);
|
||||
storageModule = null;
|
||||
}
|
||||
delete process.env.ROWBOAT_WORKDIR;
|
||||
vi.doUnmock("../knowledge/version_history.js");
|
||||
vi.doUnmock("../knowledge/deprecate_today_note.js");
|
||||
vi.resetModules();
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function loadStorage() {
|
||||
storageModule = await import("./index.js");
|
||||
return storageModule;
|
||||
}
|
||||
|
||||
describe("SQLite storage", () => {
|
||||
it("throws clearly when accessed before initialization", async () => {
|
||||
const storage = await loadStorage();
|
||||
|
||||
expect(() => storage.getDb()).toThrow("SQLite storage has not been initialized");
|
||||
});
|
||||
|
||||
it("creates the database under ROWBOAT_WORKDIR/db", async () => {
|
||||
const storage = await loadStorage();
|
||||
|
||||
await storage.initStorage();
|
||||
|
||||
expect(storage.getDatabasePath()).toBe(path.join(workspaceDir, "db", "rowboat.sqlite"));
|
||||
await expect(fs.access(storage.getDatabasePath())).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("runs the initial migration", async () => {
|
||||
const storage = await loadStorage();
|
||||
await storage.initStorage();
|
||||
|
||||
const result = await sql<{ name: string }>`
|
||||
select name
|
||||
from sqlite_master
|
||||
where type = 'table'
|
||||
and name in ('storage_metadata', 'kysely_migration')
|
||||
order by name
|
||||
`.execute(storage.getDb());
|
||||
|
||||
expect(result.rows.map((row) => row.name)).toEqual(["kysely_migration", "storage_metadata"]);
|
||||
});
|
||||
|
||||
it("is idempotent", async () => {
|
||||
const storage = await loadStorage();
|
||||
|
||||
await storage.initStorage();
|
||||
const firstDb = storage.getDb();
|
||||
await storage.initStorage();
|
||||
|
||||
expect(storage.getDb()).toBe(firstDb);
|
||||
});
|
||||
|
||||
it("resets the singleton on shutdown", async () => {
|
||||
const storage = await loadStorage();
|
||||
|
||||
await storage.initStorage();
|
||||
await storage.shutdownStorage();
|
||||
|
||||
expect(() => storage.getDb()).toThrow("SQLite storage has not been initialized");
|
||||
});
|
||||
});
|
||||
39
apps/x/packages/core/src/workspace/watcher.test.ts
Normal file
39
apps/x/packages/core/src/workspace/watcher.test.ts
Normal file
|
|
@ -0,0 +1,39 @@
|
|||
import fs from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
|
||||
|
||||
let tmpDir: string;
|
||||
let workspaceDir: string;
|
||||
|
||||
beforeEach(async () => {
|
||||
tmpDir = await fs.mkdtemp(path.join(os.tmpdir(), "rowboat-watcher-test-"));
|
||||
workspaceDir = path.join(tmpDir, "workspace");
|
||||
process.env.ROWBOAT_WORKDIR = workspaceDir;
|
||||
vi.resetModules();
|
||||
vi.doMock("../knowledge/version_history.js", () => ({
|
||||
initRepo: vi.fn(async () => undefined),
|
||||
}));
|
||||
vi.doMock("../knowledge/deprecate_today_note.js", () => ({
|
||||
deprecateTodayNote: vi.fn(async () => undefined),
|
||||
}));
|
||||
});
|
||||
|
||||
afterEach(async () => {
|
||||
delete process.env.ROWBOAT_WORKDIR;
|
||||
vi.doUnmock("../knowledge/version_history.js");
|
||||
vi.doUnmock("../knowledge/deprecate_today_note.js");
|
||||
vi.resetModules();
|
||||
await fs.rm(tmpDir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
describe("workspace watcher ignores", () => {
|
||||
it("ignores SQLite storage files under db", async () => {
|
||||
const watcher = await import("./watcher.js");
|
||||
|
||||
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "db"))).toBe(true);
|
||||
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "db", "rowboat.sqlite"))).toBe(true);
|
||||
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "db", "rowboat.sqlite-wal"))).toBe(true);
|
||||
expect(watcher.shouldIgnoreWorkspacePath(path.join(workspaceDir, "knowledge", "note.md"))).toBe(false);
|
||||
});
|
||||
});
|
||||
|
|
@ -9,6 +9,11 @@ import { Stats } from 'node:fs';
|
|||
|
||||
export type WorkspaceChangeCallback = (event: z.infer<typeof WorkspaceChangeEvent>) => void;
|
||||
|
||||
export function shouldIgnoreWorkspacePath(absPath: string): boolean {
|
||||
const relPath = absToRelPosix(absPath);
|
||||
return relPath === 'db' || relPath?.startsWith('db/') === true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a workspace watcher
|
||||
* Watches the configured workspace root recursively and emits change events via callback
|
||||
|
|
@ -29,8 +34,12 @@ export async function createWorkspaceWatcher(
|
|||
const codeModeDir = path.join(WorkDir, 'code-mode');
|
||||
const watcher = chokidar.watch(WorkDir, {
|
||||
ignoreInitial: true,
|
||||
// Ignore the SQLite db dir (storage) AND code-section worktrees (full repo
|
||||
// checkouts that would flood the event stream).
|
||||
ignored: (watchedPath: string) =>
|
||||
watchedPath === codeModeDir || watchedPath.startsWith(codeModeDir + path.sep),
|
||||
shouldIgnoreWorkspacePath(watchedPath) ||
|
||||
watchedPath === codeModeDir ||
|
||||
watchedPath.startsWith(codeModeDir + path.sep),
|
||||
awaitWriteFinish: {
|
||||
stabilityThreshold: 150,
|
||||
pollInterval: 50,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue