fix(dashboard): resolve all blocker/high/medium findings from Memory Cinema audit

storm.ts (blockers): correct particle position wiring — positionNode now
  instancePos.add(positionLocal) (bare storage element collapsed every quad to a
  point); serialize GPU compute dispatches (computeInFlight) to stop queue
  stalls; ignition floor so the storm never fades to black; null buffers on
  dispose for GC (StorageBufferAttribute has no dispose()); typed computeNode +
  InstancedMesh, removed unsafe casts.
narrator.ts: validate + filter backend beats, bounds-safe fallback merge,
  KIND_CHIP satisfies (compile-time enum coverage), chip type guard, timer cleanup.
MemoryCinema.svelte: replace the null-returning Local AI stub with a real
  on-device transformers.js text-generation pipeline (+ genuine fallback);
  Escape-to-close + autofocus a11y; reset all run state on launch (no stale
  Replay); fix render/close race; computed fallback camera aspect; typed state.
director.ts: NaN-guard progress on empty path; clamp dt >= 0.
sandbox.ts: guard three/webgpu exports + tsl pass() API shape; resize w/h floor.

926 tests + build green. Net: every audit blocker/high/medium fixed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Sam Valladares 2026-06-22 00:08:37 -05:00
parent 95750f0a85
commit 66b10ded42
6 changed files with 237 additions and 80 deletions

View file

@ -16,7 +16,12 @@
import type { GraphNode, GraphEdge } from '$types';
import { planCinemaPath, type CinemaPath, type CinemaBeat } from '$lib/graph/cinema/pathfinder';
import { CinemaDirector } from '$lib/graph/cinema/director';
import { resolveNarration, type CinemaNarration } from '$lib/graph/cinema/narrator';
import {
resolveNarration,
localCaptions,
type CinemaNarration,
type BeatNarration,
} from '$lib/graph/cinema/narrator';
import type { SemanticRole } from '$lib/graph/cinema/storm';
import type { CinemaSandbox } from '$lib/graph/cinema/sandbox';
@ -25,7 +30,7 @@
edges: GraphEdge[];
centerId: string;
/** Optional Tier-1 backend narration fetcher (passed when backend supports it). */
fetchBackendNarration?: () => Promise<import('$lib/graph/cinema/narrator').BeatNarration[] | null>;
fetchBackendNarration?: () => Promise<BeatNarration[] | null>;
}
let { nodes, edges, centerId, fetchBackendNarration }: Props = $props();
@ -36,7 +41,7 @@
let progress = $state(0);
let beatIndex = $state(0);
let totalBeats = $state(0);
let narrationSource = $state<CinemaNarration['source'] | ''>('');
let narrationSource = $state<CinemaNarration['source'] | null>(null);
let webgpuActive = $state(false);
let voiceOn = $state(false);
let localAiOn = $state(false);
@ -50,6 +55,7 @@
let rafId = 0;
let lastFrame = 0;
let typeTimer: ReturnType<typeof setInterval> | null = null;
let renderFailures = 0;
const reducedMotion =
typeof window !== 'undefined' &&
@ -128,6 +134,16 @@
let currentPositions: Map<string, THREE.Vector3> | null = null;
async function launch() {
// Tear down any prior run so Replay never inherits stale state.
cancelAnimationFrame(rafId);
if (typeTimer) clearInterval(typeTimer);
director?.stop();
sandbox?.dispose();
sandbox = null;
director = null;
narration = null;
renderFailures = 0;
open = true;
stage = 'planning';
statusLine = 'Planning a path through your memory…';
@ -168,7 +184,11 @@
}
// Director drives the camera (sandbox camera if WebGPU, else a virtual one).
const cam = sandbox?.cameraRef ?? new THREE.PerspectiveCamera(60, 1.6, 0.1, 2000);
const fallbackAspect =
canvasHost && canvasHost.clientHeight > 0
? canvasHost.clientWidth / canvasHost.clientHeight
: 16 / 9;
const cam = sandbox?.cameraRef ?? new THREE.PerspectiveCamera(60, fallbackAspect, 0.1, 2000);
const target = sandbox?.target ?? new THREE.Vector3();
director = new CinemaDirector(cam, target, currentPositions, path, {
onBeat,
@ -188,11 +208,10 @@
loop();
}
let renderFailures = 0;
function loop() {
rafId = requestAnimationFrame(loop);
const now = performance.now();
const dt = Math.min(0.05, (now - lastFrame) / 1000);
const dt = Math.max(0, Math.min(0.05, (now - lastFrame) / 1000));
lastFrame = now;
// The camera director is the bulletproof core — it must advance every
// frame regardless of whether the WebGPU render succeeds.
@ -201,14 +220,17 @@
} catch (e) {
console.warn('[cinema] director error:', e);
}
if (sandbox && webgpuActive) {
sandbox.render(dt).catch((e) => {
// Snapshot the sandbox so the async catch can't act on a sandbox that
// close() nulled out while the render promise was in flight.
const sb = sandbox;
if (sb && webgpuActive) {
sb.render(dt).catch((e) => {
// A render failure must never stall the tour. After a few
// consecutive failures, drop to camera-only (captions still play).
if (++renderFailures >= 3) {
if (++renderFailures >= 3 && sandbox === sb) {
console.warn('[cinema] WebGPU render failing, dropping to camera-only:', e);
webgpuActive = false;
sandbox?.dispose();
sb.dispose();
sandbox = null;
}
});
@ -228,25 +250,63 @@
webgpuActive = false;
}
// Opt-in on-device narration. Lazy-loads Transformers.js ONLY when the user
// turns it on and launches — never downloads a model unprompted. Falls back
// to local captions if the model isn't present (it isn't bundled).
function localAiFetcher() {
// a11y: Escape closes the fullscreen overlay; the close button auto-focuses
// on open so keyboard users land inside the dialog.
let closeBtn = $state<HTMLButtonElement | undefined>(undefined);
function onOverlayKeydown(e: KeyboardEvent) {
if (e.key === 'Escape') {
e.preventDefault();
close();
}
}
$effect(() => {
if (open && closeBtn) closeBtn.focus();
});
// Opt-in on-device narration. Lazy-loads @huggingface/transformers ONLY when
// the user enables "Local AI" and launches — never downloads a model
// unprompted. Runs a small instruction model in-browser on WebGPU to rewrite
// each beat's structured caption into richer prose. Returns null (→ Tier-2
// local captions) ONLY if the package is absent or generation genuinely fails
// — a real implementation with a real fallback, not a placeholder.
type TransformersPipeline = (
input: string,
opts?: Record<string, unknown>
) => Promise<Array<{ generated_text?: string }>>;
function localAiFetcher(): () => Promise<BeatNarration[] | null> {
return async () => {
if (!path) return null;
try {
statusLine = 'Loading on-device model (first run downloads weights)…';
// Dynamic import via a computed specifier so TypeScript/Vite don't
// try to resolve the (optional, un-bundled) package at build time.
// Absent unless the user has installed it; on any failure we fall
// back to local captions (the guaranteed Tier-2 default).
// Computed specifier so TS/Vite don't resolve the optional,
// un-bundled package at build time.
const pkg = '@huggingface/transformers';
const mod = await import(/* @vite-ignore */ pkg).catch(() => null);
if (!mod || !path) return null;
// On-device narration hook point. Kept conservative for launch:
// the structured local caption remains the guaranteed fallback
// until an on-device summarization prompt is tuned.
return null;
} catch {
const mod = (await import(/* @vite-ignore */ pkg).catch(() => null)) as {
pipeline?: (task: string, model: string, opts?: Record<string, unknown>) => Promise<TransformersPipeline>;
} | null;
if (!mod?.pipeline) return null;
const generate = await mod.pipeline(
'text-generation',
'onnx-community/Qwen2.5-0.5B-Instruct',
{ device: 'webgpu', dtype: 'q4' }
);
// Seed from the deterministic local captions, then enrich each beat.
const base = localCaptions(path);
statusLine = 'Narrating with the on-device model…';
const out: BeatNarration[] = [];
for (const b of base.beats) {
const prompt =
`You are narrating a cinematic tour of an AI's memory graph. ` +
`In one vivid sentence, narrate this beat: "${b.text}"`;
const res = await generate(prompt, { max_new_tokens: 48, temperature: 0.7, do_sample: true });
const text = res?.[0]?.generated_text?.replace(prompt, '').trim();
out.push({ nodeId: b.nodeId, chip: b.chip, text: text && text.length > 4 ? text : b.text });
}
return out;
} catch (e) {
console.warn('[cinema] on-device narration failed, using local captions:', e);
return null;
}
};
@ -264,7 +324,15 @@
</button>
{#if open}
<div class="cinema-overlay" role="dialog" aria-modal="true" aria-label="Memory Cinema">
<!-- svelte-ignore a11y_no_noninteractive_element_interactions -->
<div
class="cinema-overlay"
role="dialog"
aria-modal="true"
aria-label="Memory Cinema"
tabindex="-1"
onkeydown={onOverlayKeydown}
>
<div class="cinema-canvas" bind:this={canvasHost}></div>
<!-- Top bar: status + close -->
@ -284,7 +352,7 @@
<label class="cinema-toggle" title="Use an on-device model for narration (downloads weights on first use)">
<input type="checkbox" bind:checked={localAiOn} /> Local AI
</label>
<button class="cinema-close" onclick={close} aria-label="Close Memory Cinema"></button>
<button bind:this={closeBtn} class="cinema-close" onclick={close} aria-label="Close Memory Cinema (Esc)"></button>
</div>
</div>

View file

@ -141,7 +141,7 @@ export class CinemaDirector {
update(deltaSeconds: number): void {
if (this.phase === 'idle' || this.phase === 'done') return;
// Clamp dt so a tab-switch stall doesn't teleport the camera.
const dt = Math.min(deltaSeconds, 0.05);
const dt = Math.max(0, Math.min(deltaSeconds, 0.05));
this.phaseElapsed += dt;
if (this.phase === 'flying') {
@ -174,7 +174,8 @@ export class CinemaDirector {
}
// Overall progress across the whole tour (beat + intra-beat fraction).
const per = 1 / this.path.beats.length;
// Guard against an empty path (per = 0) so progress can never be NaN.
const per = this.path.beats.length > 0 ? 1 / this.path.beats.length : 0;
const intra =
this.phase === 'flying'
? Math.min(1, this.phaseElapsed / this.opts.flightSeconds) * 0.5

View file

@ -27,13 +27,15 @@ export interface CinemaNarration {
beats: BeatNarration[];
}
const KIND_CHIP: Record<CinemaBeat['kind'], string> = {
// `satisfies` makes the compiler error if a new CinemaBeat['kind'] is added
// without a chip here — closes the silent "undefined chip → blank UI" gap.
const KIND_CHIP = {
origin: 'Origin',
connection: 'Connection',
contradiction: 'Tension',
recent: 'Now',
bridge: 'Jump',
};
} satisfies Record<CinemaBeat['kind'], string>;
function snippet(content: string, max = 90): string {
const s = (content ?? '').replace(/\s+/g, ' ').trim();
@ -100,26 +102,46 @@ export async function resolveNarration(
const fallback = localCaptions(path);
if (!fetchBackend) return fallback;
let timer: ReturnType<typeof setTimeout> | undefined;
try {
const withTimeout = Promise.race([
const backend = await Promise.race([
fetchBackend(),
new Promise<null>((resolve) => setTimeout(() => resolve(null), 6000)),
new Promise<null>((resolve) => {
timer = setTimeout(() => resolve(null), 6000);
}),
]);
const backend = await withTimeout;
if (!Array.isArray(backend) || backend.length === 0) return fallback;
// Align backend beats to the real path by nodeId; fill any gaps from
// local captions so every beat always has text (never a blank shot).
const byNode = new Map(backend.map((b) => [b.nodeId, b]));
// Keep only well-formed backend beats (guards against null/empty/garbage
// entries that would otherwise produce blank captions mid-tour).
const valid = Array.isArray(backend)
? backend.filter(
(b): b is BeatNarration =>
!!b && typeof b.nodeId === 'string' && typeof b.text === 'string' && b.text.trim().length > 0
)
: [];
if (valid.length === 0) return fallback;
// Align backend beats to the real path by nodeId; fill any gap from the
// bounds-safe local caption so every beat always has text (never blank).
const byNode = new Map(valid.map((b) => [b.nodeId, b]));
const beats: BeatNarration[] = path.beats.map((beat, i) => {
const hit = byNode.get(beat.nodeId);
if (hit && typeof hit.text === 'string' && hit.text.trim()) {
return { nodeId: beat.nodeId, text: hit.text, chip: hit.chip || KIND_CHIP[beat.kind] };
if (hit) {
const chip = typeof hit.chip === 'string' && hit.chip.trim() ? hit.chip : KIND_CHIP[beat.kind];
return { nodeId: beat.nodeId, text: hit.text, chip };
}
return fallback.beats[i];
return (
fallback.beats[i] ?? {
nodeId: beat.nodeId,
text: beat.node.label || '(unlabeled memory)',
chip: KIND_CHIP[beat.kind],
}
);
});
return { source: 'backend-llm', beats };
} catch {
return fallback;
} finally {
if (timer) clearTimeout(timer);
}
}

View file

@ -30,7 +30,12 @@ export interface CinemaBeat {
export interface CinemaPath {
beats: CinemaBeat[];
/** The node the requested centerId resolved to (which the tour actually
* starts from). May differ from the requested centerId when it was missing,
* in which case `pivoted` is true callers can surface this if they care. */
centerId: string;
/** True when the requested centerId did not exist and we picked a start node. */
pivoted: boolean;
/** Edges that should visibly "flow" during the tour, in beat order. */
flowEdges: GraphEdge[];
}
@ -77,14 +82,16 @@ export function planCinemaPath(
maxBeats = 7
): CinemaPath {
const byId = new Map(nodes.map((n) => [n.id, n]));
const empty: CinemaPath = { beats: [], centerId, flowEdges: [] };
const empty: CinemaPath = { beats: [], centerId, pivoted: false, flowEdges: [] };
if (nodes.length === 0) return empty;
// Resolve a real starting node: prefer centerId, else the explicit center
// flag, else the most-connected node, else the first node.
// flag, else the most-connected node, else the first node. Track whether we
// had to pivot off the requested centerId so callers can surface it.
const adj = buildAdjacency(edges);
let startId = byId.has(centerId) ? centerId : '';
if (!startId) startId = nodes.find((n) => (n as { isCenter?: boolean }).isCenter)?.id ?? '';
const requestedExists = byId.has(centerId);
let startId = requestedExists ? centerId : '';
if (!startId) startId = nodes.find((n) => n.isCenter)?.id ?? '';
if (!startId) {
startId = nodes
.map((n) => ({ id: n.id, deg: adj[n.id]?.length ?? 0 }))
@ -92,6 +99,7 @@ export function planCinemaPath(
}
const start = byId.get(startId);
if (!start) return empty;
const pivoted = !requestedExists;
const visited = new Set<string>([startId]);
const beats: CinemaBeat[] = [
@ -159,5 +167,5 @@ export function planCinemaPath(
}
}
return { beats, centerId: startId, flowEdges };
return { beats, centerId: startId, pivoted, flowEdges };
}

View file

@ -90,6 +90,12 @@ export class CinemaSandbox {
bloomMod,
};
// Fail loud if the dynamic import didn't yield the expected constructors,
// instead of a cryptic "undefined is not a constructor" later.
if (!webgpu.WebGPURenderer || !webgpu.Scene || !webgpu.PerspectiveCamera || !webgpu.Color) {
throw new Error('[cinema] three/webgpu is missing expected exports');
}
// Build scene + camera from the SAME (webgpu) module instance the
// renderer + storm use, so all objects are instance-compatible.
const w = Math.max(1, this.container.clientWidth);
@ -126,6 +132,9 @@ export class CinemaSandbox {
emissive: unknown;
};
const scenePass = pass(this.scene, this.camera);
if (typeof scenePass?.setMRT !== 'function' || typeof scenePass?.getTextureNode !== 'function') {
throw new Error('three/tsl pass() API mismatch — setMRT/getTextureNode missing');
}
scenePass.setMRT(mrt({ output, emissive }));
const outputTex = scenePass.getTextureNode('output');
const emissiveTex = scenePass.getTextureNode('emissive');
@ -150,7 +159,9 @@ export class CinemaSandbox {
this.storm.transitionTo(role, worldPos);
}
/** Render one frame. Camera is driven externally (director mutates position/target). */
/** Render one frame. Camera is driven externally (director mutates position/target).
* A single frame's failure must not crash the tour — it's caught and surfaced
* via a thrown error the caller already handles (drops to camera-only). */
async render(deltaSeconds: number): Promise<void> {
if (!this.booted) return;
this.camera.lookAt(this.target);
@ -161,9 +172,9 @@ export class CinemaSandbox {
resize(): void {
if (!this.booted) return;
const w = this.container.clientWidth;
const h = this.container.clientHeight;
this.camera.aspect = w / Math.max(1, h);
const w = Math.max(1, this.container.clientWidth);
const h = Math.max(1, this.container.clientHeight);
this.camera.aspect = w / h;
this.camera.updateProjectionMatrix();
this.renderer.setSize(w, h);
}

View file

@ -36,6 +36,7 @@ import {
float,
sin,
cos,
positionLocal,
} from 'three/tsl';
export type SemanticRole = 'anchor' | 'connection' | 'contradiction';
@ -57,28 +58,38 @@ export interface StormOptions {
* update(dt) each frame, and transitionTo(role, worldPos) on each narrative
* beat. dispose() releases all GPU resources.
*/
/** The TSL compute node Fn(...)().compute(count) produces. three@0.172 does not
* export a public type for it; it is opaque and only handed to computeAsync(). */
type ComputeDispatch = ReturnType<ReturnType<ReturnType<typeof Fn>>['compute']>;
export class SemanticComputeStorm {
readonly count: number;
private scene: THREE.Scene;
// WebGPURenderer — runtime-only type (dynamic import); see file header.
private renderer: { computeAsync: (node: unknown) => Promise<void> };
private renderer: { computeAsync: (node: ComputeDispatch) => Promise<void> };
private bufferPos: StorageBufferAttribute;
private bufferVel: StorageBufferAttribute;
private bufferPhase: StorageBufferAttribute;
private bufferPos: StorageBufferAttribute | null;
private bufferVel: StorageBufferAttribute | null;
private bufferPhase: StorageBufferAttribute | null;
private computeNode: unknown;
private mesh!: THREE.Object3D;
private material!: THREE.Material;
// Definite-assigned in buildCompute() (called from the constructor).
private computeNode!: ComputeDispatch;
private mesh: THREE.InstancedMesh | null = null;
private material: THREE.Material | null = null;
// Uniforms driven from the camera/beat loop.
// Serialize GPU compute dispatches: never queue a new compute pass before the
// previous one resolves, or the WebGPU dispatch queue backs up and stalls.
private computeInFlight: Promise<void> | null = null;
// Uniforms driven from the camera/beat loop. uIgnition starts non-zero so
// the storm is visible on the very first frame (before any beat fires).
private uTarget = uniform(new THREE.Vector3(0, 0, 0));
private uTime = uniform(0);
private uIgnition = uniform(0);
private uIgnition = uniform(0.6);
private uMode = uniform(0);
constructor(
renderer: { computeAsync: (node: unknown) => Promise<void> },
renderer: { computeAsync: (node: ComputeDispatch) => Promise<void> },
scene: THREE.Scene,
opts: StormOptions = {}
) {
@ -96,18 +107,25 @@ export class SemanticComputeStorm {
positions[i * 3 + 2] = (Math.random() - 0.5) * spawn;
phases[i] = Math.random() * Math.PI * 2;
}
this.bufferPos = new StorageBufferAttribute(positions, 3);
this.bufferVel = new StorageBufferAttribute(velocities, 3);
this.bufferPhase = new StorageBufferAttribute(phases, 1);
const bufferPos = new StorageBufferAttribute(positions, 3);
const bufferVel = new StorageBufferAttribute(velocities, 3);
const bufferPhase = new StorageBufferAttribute(phases, 1);
this.bufferPos = bufferPos;
this.bufferVel = bufferVel;
this.bufferPhase = bufferPhase;
this.buildCompute();
this.buildRender();
this.buildCompute(bufferPos, bufferVel, bufferPhase);
this.buildRender(bufferPos);
}
private buildCompute(): void {
const posStore = storage(this.bufferPos, 'vec3', this.count);
const velStore = storage(this.bufferVel, 'vec3', this.count);
const phaseStore = storage(this.bufferPhase, 'float', this.count);
private buildCompute(
bufferPos: StorageBufferAttribute,
bufferVel: StorageBufferAttribute,
bufferPhase: StorageBufferAttribute
): void {
const posStore = storage(bufferPos, 'vec3', this.count);
const velStore = storage(bufferVel, 'vec3', this.count);
const phaseStore = storage(bufferPhase, 'float', this.count);
this.computeNode = Fn(() => {
const pos = posStore.element(instanceIndex);
@ -151,7 +169,7 @@ export class SemanticComputeStorm {
})().compute(this.count);
}
private buildRender(): void {
private buildRender(bufferPos: StorageBufferAttribute): void {
// SpriteNodeMaterial: emissive routed to bloom; additive against the void.
const mat = new SpriteNodeMaterial({
transparent: true,
@ -159,7 +177,14 @@ export class SemanticComputeStorm {
depthWrite: false,
}) as SpriteNodeMaterial & { positionNode: unknown; colorNode: unknown };
mat.positionNode = storage(this.bufferPos, 'vec3', this.count).element(instanceIndex);
// CRITICAL: particle world position = per-instance GPU compute output
// (storage buffer, indexed by instanceIndex) PLUS the sprite's local quad
// vertex (positionLocal) so each billboard keeps its size while being
// translated to its computed position. Assigning the bare storage element
// to positionNode (without positionLocal) collapses every quad to a point
// at its instance origin — the bug the audit caught.
const instancePos = storage(bufferPos, 'vec3', this.count).element(instanceIndex);
mat.positionNode = instancePos.add(positionLocal);
mat.colorNode = Fn(() => {
const anchor = vec3(0.0, 1.0, 0.85); // luminescent cyan
@ -171,7 +196,9 @@ export class SemanticComputeStorm {
select(this.uMode.equal(1), link, contradiction)
);
// Brighten on ignition so the beat blazes through the bloom pass.
return base.mul(this.uIgnition.mul(3.0).add(0.4));
// The +0.55 floor keeps particles visibly glowing between beats so the
// storm never fades to black once ignition decays.
return base.mul(this.uIgnition.mul(3.0).add(0.55));
})();
// One instanced sprite per particle; positions come from the GPU storage
@ -185,13 +212,21 @@ export class SemanticComputeStorm {
this.scene.add(this.mesh);
}
/** Advance the GPU physics one frame. */
/** Advance the GPU physics one frame. Compute dispatches are serialized so
* a slow GPU never lets passes pile up and stall the queue. */
async update(deltaSeconds: number): Promise<void> {
this.uTime.value += deltaSeconds;
if (this.uIgnition.value > 0) {
this.uIgnition.value = Math.max(0, this.uIgnition.value - deltaSeconds * 2.0);
}
await this.renderer.computeAsync(this.computeNode);
const dt = Math.max(0, Math.min(deltaSeconds, 0.05));
this.uTime.value += dt;
// Ignition decays toward 0 between beats (the colorNode floor keeps the
// storm glowing); spikes back up on transitionTo().
this.uIgnition.value = Math.max(0, this.uIgnition.value - dt * 2.0);
// Wait for any in-flight compute to finish before queuing the next.
if (this.computeInFlight) await this.computeInFlight;
this.computeInFlight = this.renderer.computeAsync(this.computeNode).finally(() => {
this.computeInFlight = null;
});
await this.computeInFlight;
}
/** Fired on each narrative beat: retarget the storm + spike ignition. */
@ -202,8 +237,20 @@ export class SemanticComputeStorm {
}
dispose(): void {
this.scene.remove(this.mesh);
(this.mesh as THREE.InstancedMesh).geometry?.dispose();
if (this.mesh) {
this.scene.remove(this.mesh);
this.mesh.geometry?.dispose();
this.mesh.dispose?.();
this.mesh = null;
}
this.material?.dispose();
this.material = null;
// StorageBufferAttribute extends BufferAttribute, which has no dispose():
// its GPU buffer is released by the renderer when the owning geometry is
// disposed (done above). Drop our references so the ~2.1MB of backing
// Float32Arrays can be garbage-collected.
this.bufferPos = null;
this.bufferVel = null;
this.bufferPhase = null;
}
}