From f01375b815bed00cd9e16e39bea9dbb79a57c9f3 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Mon, 20 Apr 2026 12:33:49 -0500 Subject: [PATCH] feat(v2.2-pulse): InsightToast + multi-process STORAGE docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two independent ship items landing together on the v2.2 branch ahead of the Tuesday launch — a new UI surface that makes Vestige's cognitive events visible in real time, and honest documentation of the multi-process safety story that underpins the Stigmergic Swarm narrative. **InsightToast** (apps/dashboard/src/lib/components/InsightToast.svelte, apps/dashboard/src/lib/stores/toast.ts): The dashboard already had a working WebSocket event stream on ws://localhost:3927/ws that broadcast every cognitive event (dream completions, consolidation sweeps, memory promotions/demotions, active- forgetting suppression and Rac1 cascades, bridge discoveries). None of that was surfaced to a user looking at anything other than the raw feed view. InsightToast subscribes to the existing eventFeed derived store, filters the spammy lifecycle events (Heartbeat, SearchPerformed, RetentionDecayed, ActivationSpread, ImportanceScored, MemoryCreated), and translates the narrative events into ephemeral toasts with a bioluminescent colored accent matching EVENT_TYPE_COLORS. Design notes: - Rate-limited ConnectionDiscovered at 1.5s intervals (dreams emit many). - Max 4 visible toasts, auto-dismiss at 4.5-7s depending on event weight. - Click or Enter/Space to dismiss early. - Bottom-right on desktop, top-banner on mobile. - Reduced-motion honored via prefers-reduced-motion. - Zero new websocket subscriptions — everything piggybacks on the existing derived store. Also added a "Preview Pulse" button to Settings -> Cognitive Operations that fires a synthetic sequence of four toasts (DreamCompleted, ConnectionDiscovered, MemorySuppressed, ConsolidationCompleted) so the animation is demoable without waiting for real cognitive activity. **Multi-Process Safety section in docs/STORAGE.md**: Grounds the Stigmergic Swarm story with concrete tables of what the current WAL + 5s busy_timeout configuration actually supports vs what remains experimental. Key honest points: - Shared --data-dir + ONE vestige-mcp + N clients is the shipping pattern for multi-agent coordination. - Two vestige-mcp processes writing the same file is experimental — documented with the lsof + pkill recovery path. - Roadmap lists the three items that would promote it to "supported": advisory file lock, retry-with-jitter on SQLITE_BUSY, and a concurrent-writer load test. Build + typecheck: - npm run check: 0 errors, 0 warnings across 583 files - npm run build: clean static build, adapter-static succeeds --- .../src/lib/components/InsightToast.svelte | 242 ++++++++++++++++ apps/dashboard/src/lib/stores/toast.ts | 270 ++++++++++++++++++ .../src/routes/(app)/settings/+page.svelte | 15 + apps/dashboard/src/routes/+layout.svelte | 4 + docs/STORAGE.md | 72 +++++ 5 files changed, 603 insertions(+) create mode 100644 apps/dashboard/src/lib/components/InsightToast.svelte create mode 100644 apps/dashboard/src/lib/stores/toast.ts diff --git a/apps/dashboard/src/lib/components/InsightToast.svelte b/apps/dashboard/src/lib/components/InsightToast.svelte new file mode 100644 index 0000000..eb55576 --- /dev/null +++ b/apps/dashboard/src/lib/components/InsightToast.svelte @@ -0,0 +1,242 @@ + + + +
+ {#each $toasts as t (t.id)} + + {/each} +
+ + diff --git a/apps/dashboard/src/lib/stores/toast.ts b/apps/dashboard/src/lib/stores/toast.ts new file mode 100644 index 0000000..c7e58ae --- /dev/null +++ b/apps/dashboard/src/lib/stores/toast.ts @@ -0,0 +1,270 @@ +// Pulse Toast — v2.2 +// Subscribes to the WebSocket event feed and surfaces meaningful cognitive +// events as ephemeral toast notifications. This is the "brain coming alive" +// moment — when the dashboard is open you SEE Vestige thinking. +// +// Design: +// - Filter the spammy events (Heartbeat, SearchPerformed, RetentionDecayed, +// ActivationSpread, ImportanceScored, MemoryCreated). Those fire during +// every ingest cycle and would flood the UI. +// - Surface the narrative events: Dreams completing, Consolidation sweeping, +// Memories being promoted/demoted/suppressed, Rac1 cascades, new bridges. +// - Rate-limit ConnectionDiscovered — dreams fire many of these in seconds. +// - Auto-dismiss after 5-6s. Max 4 on screen. + +import { writable, get } from 'svelte/store'; +import { eventFeed } from '$stores/websocket'; +import type { VestigeEvent, VestigeEventType } from '$types'; +import { EVENT_TYPE_COLORS } from '$types'; + +export interface Toast { + id: number; + type: VestigeEventType; + title: string; + body: string; + color: string; + dwellMs: number; + createdAt: number; +} + +const MAX_VISIBLE = 4; +const DEFAULT_DWELL_MS = 5500; +const CONNECTION_THROTTLE_MS = 1500; + +function createToastStore() { + const { subscribe, update } = writable([]); + let nextId = 1; + let lastConnectionAt = 0; + + function push(toast: Omit) { + const id = nextId++; + const createdAt = Date.now(); + const entry: Toast = { id, createdAt, ...toast }; + update(list => { + const next = [entry, ...list]; + if (next.length > MAX_VISIBLE) { + return next.slice(0, MAX_VISIBLE); + } + return next; + }); + setTimeout(() => dismiss(id), toast.dwellMs); + } + + function dismiss(id: number) { + update(list => list.filter(t => t.id !== id)); + } + + function clear() { + update(() => []); + } + + function translate(event: VestigeEvent): Omit | null { + const color = EVENT_TYPE_COLORS[event.type] ?? '#818CF8'; + const d = event.data as Record; + + switch (event.type) { + case 'DreamCompleted': { + const replayed = Number(d.memories_replayed ?? 0); + const found = Number(d.connections_found ?? 0); + const insights = Number(d.insights_generated ?? 0); + const ms = Number(d.duration_ms ?? 0); + const parts: string[] = []; + parts.push(`Replayed ${replayed} ${replayed === 1 ? 'memory' : 'memories'}`); + if (found > 0) parts.push(`${found} new connection${found === 1 ? '' : 's'}`); + if (insights > 0) parts.push(`${insights} insight${insights === 1 ? '' : 's'}`); + return { + type: event.type, + title: 'Dream consolidated', + body: `${parts.join(' · ')} in ${(ms / 1000).toFixed(1)}s`, + color, + dwellMs: 7000, + }; + } + + case 'ConsolidationCompleted': { + const nodes = Number(d.nodes_processed ?? 0); + const decay = Number(d.decay_applied ?? 0); + const embeds = Number(d.embeddings_generated ?? 0); + const ms = Number(d.duration_ms ?? 0); + const tail: string[] = []; + if (decay > 0) tail.push(`${decay} decayed`); + if (embeds > 0) tail.push(`${embeds} embedded`); + return { + type: event.type, + title: 'Consolidation swept', + body: `${nodes} node${nodes === 1 ? '' : 's'}${tail.length ? ' · ' + tail.join(' · ') : ''} in ${(ms / 1000).toFixed(1)}s`, + color, + dwellMs: 6000, + }; + } + + case 'ConnectionDiscovered': { + const now = Date.now(); + if (now - lastConnectionAt < CONNECTION_THROTTLE_MS) return null; + lastConnectionAt = now; + const kind = String(d.connection_type ?? 'link'); + const weight = Number(d.weight ?? 0); + return { + type: event.type, + title: 'Bridge discovered', + body: `${kind} · weight ${weight.toFixed(2)}`, + color, + dwellMs: 4500, + }; + } + + case 'MemoryPromoted': { + const r = Number(d.new_retention ?? 0); + return { + type: event.type, + title: 'Memory promoted', + body: `retention ${(r * 100).toFixed(0)}%`, + color, + dwellMs: 4500, + }; + } + + case 'MemoryDemoted': { + const r = Number(d.new_retention ?? 0); + return { + type: event.type, + title: 'Memory demoted', + body: `retention ${(r * 100).toFixed(0)}%`, + color, + dwellMs: 4500, + }; + } + + case 'MemorySuppressed': { + const count = Number(d.suppression_count ?? 0); + const cascade = Number(d.estimated_cascade ?? 0); + return { + type: event.type, + title: 'Forgetting', + body: cascade > 0 + ? `suppression #${count} · Rac1 cascade ~${cascade} neighbors` + : `suppression #${count}`, + color, + dwellMs: 5500, + }; + } + + case 'MemoryUnsuppressed': { + const remaining = Number(d.remaining_count ?? 0); + return { + type: event.type, + title: 'Recovered', + body: remaining > 0 ? `${remaining} suppression${remaining === 1 ? '' : 's'} remain` : 'fully unsuppressed', + color, + dwellMs: 5000, + }; + } + + case 'Rac1CascadeSwept': { + const seeds = Number(d.seeds ?? 0); + const neighbors = Number(d.neighbors_affected ?? 0); + return { + type: event.type, + title: 'Rac1 cascade', + body: `${seeds} seed${seeds === 1 ? '' : 's'} · ${neighbors} dendritic spine${neighbors === 1 ? '' : 's'} pruned`, + color, + dwellMs: 6000, + }; + } + + case 'MemoryDeleted': { + return { + type: event.type, + title: 'Memory deleted', + body: String(d.id ?? '').slice(0, 8), + color, + dwellMs: 4000, + }; + } + + // Noise — never toast + case 'Heartbeat': + case 'SearchPerformed': + case 'RetentionDecayed': + case 'ActivationSpread': + case 'ImportanceScored': + case 'MemoryCreated': + case 'MemoryUpdated': + case 'DreamStarted': + case 'DreamProgress': + case 'ConsolidationStarted': + case 'Connected': + return null; + + default: + return null; + } + } + + // Track the latest processed event by object identity. The websocket store + // prepends new events to its array, so when a new message arrives, the + // first element becomes a new object reference — no IDs or timestamps + // required to detect novelty. + let lastSeen: VestigeEvent | null = null; + + eventFeed.subscribe(events => { + if (events.length === 0) return; + const latest = events[0]; + if (latest === lastSeen) return; + lastSeen = latest; + const translated = translate(latest); + if (translated) push(translated); + }); + + return { + subscribe, + dismiss, + clear, + /** Manually fire a toast (test mode / demo button). */ + push, + }; +} + +export const toasts = createToastStore(); + +/** Fire a synthetic event sequence — used by the demo button in settings. */ +export function fireDemoSequence(): void { + const demos: Omit[] = [ + { + type: 'DreamCompleted', + title: 'Dream consolidated', + body: 'Replayed 127 memories · 43 new connections · 5 insights in 2.4s', + color: EVENT_TYPE_COLORS.DreamCompleted, + dwellMs: 7000, + }, + { + type: 'ConnectionDiscovered', + title: 'Bridge discovered', + body: 'semantic · weight 0.87', + color: EVENT_TYPE_COLORS.ConnectionDiscovered, + dwellMs: 4500, + }, + { + type: 'MemorySuppressed', + title: 'Forgetting', + body: 'suppression #2 · Rac1 cascade ~8 neighbors', + color: EVENT_TYPE_COLORS.MemorySuppressed, + dwellMs: 5500, + }, + { + type: 'ConsolidationCompleted', + title: 'Consolidation swept', + body: '892 nodes · 156 decayed · 48 embedded in 1.1s', + color: EVENT_TYPE_COLORS.ConsolidationCompleted, + dwellMs: 6000, + }, + ]; + demos.forEach((t, i) => { + setTimeout(() => { + // access the store getter for type safety — push is on the factory + (toasts as unknown as { push: typeof toasts.push }).push(t); + }, i * 800); + }); + // satisfy unused-import lint + void get(toasts); +} diff --git a/apps/dashboard/src/routes/(app)/settings/+page.svelte b/apps/dashboard/src/routes/(app)/settings/+page.svelte index 90d141e..9257ba2 100644 --- a/apps/dashboard/src/routes/(app)/settings/+page.svelte +++ b/apps/dashboard/src/routes/(app)/settings/+page.svelte @@ -2,6 +2,7 @@ import { onMount } from 'svelte'; import { api } from '$stores/api'; import { isConnected, memoryCount, avgRetention } from '$stores/websocket'; + import { fireDemoSequence } from '$stores/toast'; // Operation states let consolidating = $state(false); @@ -93,6 +94,20 @@ ◈ Cognitive Operations + +
+
+
+
Pulse Toast Preview
+
Fire a synthetic event sequence — useful for UI demos
+
+ +
+
+
diff --git a/apps/dashboard/src/routes/+layout.svelte b/apps/dashboard/src/routes/+layout.svelte index 48eb79a..52ea759 100644 --- a/apps/dashboard/src/routes/+layout.svelte +++ b/apps/dashboard/src/routes/+layout.svelte @@ -14,6 +14,7 @@ formatUptime, } from '$stores/websocket'; import ForgettingIndicator from '$lib/components/ForgettingIndicator.svelte'; + import InsightToast from '$lib/components/InsightToast.svelte'; let { children } = $props(); let showCommandPalette = $state(false); @@ -199,6 +200,9 @@
+ + + {#if showCommandPalette} diff --git a/docs/STORAGE.md b/docs/STORAGE.md index cefca33..8b3b8c5 100644 --- a/docs/STORAGE.md +++ b/docs/STORAGE.md @@ -169,3 +169,75 @@ SELECT COUNT(*) FROM knowledge_nodes WHERE retention_strength < 0.1; ``` **Caution**: Don't modify the database while Vestige is running. + +--- + +## Multi-Process Safety + +Vestige's SQLite configuration is tuned for **safe concurrent reads alongside a single writer**. Multiple `vestige-mcp` processes pointed at the same database file is a supported *read-heavy* pattern; concurrent heavy writes from multiple processes is **experimental** and documented here honestly. + +### What's shipped + +Every `Storage::new()` call executes these pragmas on both the reader and writer connection (`crates/vestige-core/src/storage/sqlite.rs`): + +```sql +PRAGMA journal_mode = WAL; -- readers don't block writers, writers don't block readers +PRAGMA synchronous = NORMAL; -- durable across app crashes, not across OS crashes +PRAGMA cache_size = -64000; -- 64 MiB page cache per connection +PRAGMA temp_store = MEMORY; +PRAGMA foreign_keys = ON; +PRAGMA busy_timeout = 5000; -- wait 5s on SQLITE_BUSY before surfacing the error +PRAGMA mmap_size = 268435456; -- 256 MiB memory-mapped I/O window +PRAGMA journal_size_limit = 67108864; +PRAGMA optimize = 0x10002; +``` + +Internally the `Storage` type holds **separate reader and writer connections**, each guarded by its own `Mutex`. Within a single process this means: + +- Any number of concurrent readers share the read connection lock. +- Writers serialize on the writer connection lock. +- WAL lets readers continue while a writer commits — they don't block each other at the SQLite level. + +### What works today + +| Pattern | Status | Notes | +|---------|--------|-------| +| One `vestige-mcp` + one Claude client | **Supported** | The default case. Zero contention. | +| Multiple Claude clients, separate `--data-dir` | **Supported** | Each process owns its own DB file. No shared state. | +| Multiple Claude clients, **shared** `--data-dir`, **one** `vestige-mcp` | **Supported** | Clients talk to a single MCP process that owns the DB. Recommended for multi-agent setups. | +| CLI (`vestige` binary) reading while `vestige-mcp` runs | **Supported** | WAL makes this safe — queries see a consistent snapshot. | +| Time Machine / `rsync` backup during writes | **Supported** | WAL journal gets copied with the main file; recovery handles it. | + +### What's experimental + +| Pattern | Status | Notes | +|---------|--------|-------| +| **Two `vestige-mcp` processes** writing the same DB concurrently | **Experimental** | SQLite serializes writers via a lock; if contention exceeds the 5s `busy_timeout`, writes surface `SQLITE_BUSY`. No exponential backoff or inter-process coordination layer beyond the pragma. | +| External writers (another SQLite client holding a write transaction open) | **Experimental** | Same concern as above — the 5s window is the only safety net. | +| Corrupted WAL recovery after hard-kill | **Supported by SQLite** | WAL is designed for crash recovery, but we do not explicitly test the `PRAGMA wal_checkpoint(RESTART)` path under load. | + +If you hit `database is locked` errors: + +```bash +# Identify the holder +lsof ~/Library/Application\ Support/com.vestige.core/vestige.db + +# Clean shutdown of all vestige processes +pkill -INT vestige-mcp +``` + +### Why the "Stigmergic Swarm" story is honest + +Multi-agent coordination through a shared memory graph — where agents alter the graph and other agents later *sense* those changes rather than passing explicit messages — is a first-class pattern on the **shared `--data-dir` + one `vestige-mcp`** setup above. In that configuration, every write flows through a single MCP process: WAL gives readers (agents querying state) a consistent view while the writer commits atomically, and the broadcast channel in `dashboard/events.rs` surfaces each cognitive event (dream, consolidation, promotion, suppression, Rac1 cascade) to every connected client in real time. No inter-process write coordination is required because there is one writer. + +Running two or more `vestige-mcp` processes against the same file is where "experimental" kicks in. For the swarm narrative, point every agent at one MCP instance — that's the shipping pattern. + +### Roadmap + +Things we haven't shipped yet, tracked for a future release: + +1. **File-based advisory lock** (`fs2` / `fcntl`) to detect and refuse startup when another `vestige-mcp` already owns the DB, instead of failing later with a lock error. +2. **Retry with jitter on `SQLITE_BUSY`** in addition to the pragma's blocking wait. +3. **Load test**: two `vestige-mcp` instances hammering the same file with mixed read/write traffic, verifying zero corruption and bounded write latency. + +Until those land, treat "two writer processes on one file" as experimental. For everything else on this page, WAL + the 5s busy timeout is the shipping story.