From 4e3542eecb500205154c765affc636d902566bd6 Mon Sep 17 00:00:00 2001 From: Sam Valladares Date: Mon, 22 Jun 2026 14:35:32 -0500 Subject: [PATCH] feat(cinema): flythrough streaks + interactive parallax (immersion steps 5+6/6) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Step 5 — VELOCITY-STRETCH FLYTHROUGH: sandbox derives camera velocity per frame (one Vector3, zero compute) and pushes view-space apparent velocity to the storm; flythrough relaxes the camera clamp floor (lerp 30→6) so the camera plunges inside the shell. Storm stretches each sprite along screen-space velocity via rotationNode + scaleNode (clamped streak), separate output graph (no extra positionView read). Defaults 0 → no-op until wired. Step 6 — INTERACTIVE PARALLAX: pointer orbits / scroll + pinch zoom the camera with frame-rate-independent damping, composed onto the director's base pose in loop() (after director.update, before render); idle >2.5s eases back to 0 so it's a toy when touched and a film when left alone. sandbox.render re-clamps so the user can't break framing. Per-beat flythrough strength wired from shot.tension; dream mode flies through at 0.6. Fully gated off under reduced-motion (no listeners, flythrough 0). The 4-feature immersion stack (infinite zoom + flythrough + parallax + DOF/fog) now composes. Gate: svelte-check 0/0, 937 tests, build green, verified live (all 4 compose, no white-out, no recursion, parallax responds without breaking framing). Co-Authored-By: Claude Opus 4.8 --- .../src/lib/components/MemoryCinema.svelte | 102 ++++++++++++++++++ .../dashboard/src/lib/graph/cinema/sandbox.ts | 51 ++++++++- apps/dashboard/src/lib/graph/cinema/storm.ts | 46 ++++++++ 3 files changed, 196 insertions(+), 3 deletions(-) diff --git a/apps/dashboard/src/lib/components/MemoryCinema.svelte b/apps/dashboard/src/lib/components/MemoryCinema.svelte index 7ae4084..8e71552 100644 --- a/apps/dashboard/src/lib/components/MemoryCinema.svelte +++ b/apps/dashboard/src/lib/components/MemoryCinema.svelte @@ -83,6 +83,25 @@ typeof window !== 'undefined' && window.matchMedia('(prefers-reduced-motion: reduce)').matches; + // ── INTERACTIVE PARALLAX ──────────────────────────────────────────────── + // Pointer/scroll/touch nudge the camera around the storm so the overlay feels + // like a live toy, then converge back to 0 when idle so the scripted film is + // never broken. None of this state is read in the template, so plain let/const + // is correct (no $state needed). Frame-rate-independent easing via `damp`. + const damp = (cur: number, tgt: number, lambda: number, dt: number) => + cur + (tgt - cur) * (1 - Math.exp(-lambda * dt)); + const pointer = { x: 0, y: 0 }; + const camOff = { yaw: 0, pitch: 0 }; + let zoomTarget = 0, + zoomLive = 0; + let lastInputAt = 0; // performance.now() of the last pointer/scroll/touch input + const IDLE_MS = 2500, + MAX_YAW = 0.35, + MAX_PITCH = 0.22; + const markInput = () => { + lastInputAt = performance.now(); + }; + // Deterministic layout: spread path nodes on a gentle spiral so the camera // has distinct world positions to fly between (independent of the WebGL // graph's internal coordinates — keeps the sandbox isolated). @@ -173,6 +192,10 @@ // fades in extra-soft on beats 0/1 (which otherwise wash to white). sandbox.transitionTo(stormRole(mode), wp, shot?.act ?? 'I', index); } + // Drive flythrough strength from beat energy: high-tension beats fly + // THROUGH the storm (relaxed clamp + streak); reduced-motion = no streak. + const energy = shot?.tension ?? 0; + sandbox.setFlythrough(reducedMotion ? 0 : energy * 0.8); } } @@ -284,6 +307,30 @@ } catch (e) { console.warn('[cinema] director error:', e); } + // Compose interactive parallax ON TOP of the director's camera. The user + // nudges yaw/pitch/zoom; we ease toward 0 when idle so the scripted film is + // seamless. sandbox.render() re-clamps distance + lookAt(origin), so the + // user can never break the framing — this is purely additive sugar. + if (!reducedMotion && sandbox && webgpuActive) { + const cam = sandbox.cameraRef; + const idle = performance.now() - lastInputAt > IDLE_MS; + const yawT = idle ? 0 : pointer.x * MAX_YAW; + const pitchT = idle ? 0 : pointer.y * MAX_PITCH; + const lam = idle ? 1.5 : 3.5; + camOff.yaw = damp(camOff.yaw, yawT, lam, dt); + camOff.pitch = damp(camOff.pitch, pitchT, lam, dt); + if (idle) zoomTarget = damp(zoomTarget, 0, 1.2, dt); + zoomLive = damp(zoomLive, zoomTarget, 5.5, dt); + // Spherical offset around the director's current camera position + // (target = origin). + const dir = cam.position.clone(); + const sph = new THREE.Spherical().setFromVector3(dir); + sph.theta += camOff.yaw; + sph.phi = THREE.MathUtils.clamp(sph.phi + camOff.pitch, 0.2, Math.PI - 0.2); + sph.radius *= 1 - zoomLive * 0.35; + dir.setFromSpherical(sph); + cam.position.copy(dir); + } // Snapshot the sandbox so the async catch can't act on a sandbox that // close() nulled out while the render promise was in flight. const sb = sandbox; @@ -310,6 +357,8 @@ function startDreamMode() { if (reducedMotion || !sandbox || !webgpuActive) return; // honor reduced-motion stopDreamMode(); + // The endless dream gently flies THROUGH the procedural figures. + if (!reducedMotion) sandbox?.setFlythrough(0.6); // Fire the first wild figure immediately, then keep going forever. sandbox?.dreamBeat(); caption = ''; @@ -337,6 +386,10 @@ stopDreamMode(); if (typeTimer) clearInterval(typeTimer); if (typeof speechSynthesis !== 'undefined') speechSynthesis.cancel(); + // Reset flythrough + parallax so a Replay never inherits stale offsets. + if (sandbox) sandbox.setFlythrough?.(0); + camOff.yaw = camOff.pitch = 0; + zoomTarget = zoomLive = 0; director?.stop(); sandbox?.dispose(); sandbox = null; @@ -372,6 +425,55 @@ return () => document.body.classList.remove('cinema-open'); }); + // Interactive parallax listeners — only while the overlay is open, motion is + // allowed, and the canvas host exists. Reduced-motion fully disables them + // (no listeners attached at all). Cleanup removes everything on close/unmount. + $effect(() => { + if (!open || reducedMotion || !canvasHost) return; + const host = canvasHost; + const clamp = (v: number, lo: number, hi: number) => Math.min(hi, Math.max(lo, v)); + host.style.touchAction = 'none'; + + const onPointerMove = (e: PointerEvent) => { + pointer.x = (e.clientX / window.innerWidth) * 2 - 1; + pointer.y = -(e.clientY / window.innerHeight) * 2 + 1; + markInput(); + }; + const onWheel = (e: WheelEvent) => { + e.preventDefault(); + zoomTarget = clamp(zoomTarget + e.deltaY * 0.0008, -1, 1); + markInput(); + }; + let pinchPrev: number | null = null; + const onTouchMove = (e: TouchEvent) => { + if (e.touches.length === 2) { + const dx = e.touches[0].clientX - e.touches[1].clientX; + const dy = e.touches[0].clientY - e.touches[1].clientY; + const dist = Math.hypot(dx, dy); + if (pinchPrev !== null) { + zoomTarget = clamp(zoomTarget + (dist - pinchPrev) * 0.002, -1, 1); + } + pinchPrev = dist; + markInput(); + } + }; + const onTouchEnd = () => { + pinchPrev = null; + }; + + host.addEventListener('pointermove', onPointerMove, { passive: true }); + host.addEventListener('wheel', onWheel, { passive: false }); + host.addEventListener('touchmove', onTouchMove, { passive: true }); + host.addEventListener('touchend', onTouchEnd); + + return () => { + host.removeEventListener('pointermove', onPointerMove); + host.removeEventListener('wheel', onWheel); + host.removeEventListener('touchmove', onTouchMove); + host.removeEventListener('touchend', onTouchEnd); + }; + }); + // Opt-in on-device narration. Lazy-loads @huggingface/transformers ONLY when // the user enables "Local AI" and launches — never downloads a model // unprompted. Runs a small instruction model in-browser on WebGPU to rewrite diff --git a/apps/dashboard/src/lib/graph/cinema/sandbox.ts b/apps/dashboard/src/lib/graph/cinema/sandbox.ts index 19a837a..fbc7fe1 100644 --- a/apps/dashboard/src/lib/graph/cinema/sandbox.ts +++ b/apps/dashboard/src/lib/graph/cinema/sandbox.ts @@ -60,6 +60,14 @@ export class CinemaSandbox { /** Camera target the director drives; mirrored into camera.lookAt each frame. */ readonly target = new THREE.Vector3(0, 0, 0); + // FLYTHROUGH — when >0, relaxes the camera-distance clamp floor so the camera + // can plunge inside the shell, and the storm stretches sprites along the + // apparent motion vector. Camera velocity is derived per-frame from the + // position delta (one Vector3, no compute). 0 = no streak (reduced-motion). + private flythrough = 0; + private prevCamPos = new THREE.Vector3(); + private camVel = new THREE.Vector3(); + constructor(container: HTMLElement) { this.container = container; } @@ -185,6 +193,24 @@ export class CinemaSandbox { this.storm.dreamBeat(); } + /** Flythrough strength 0..1. Relaxes the clamp floor so the camera can dive + * inside the shell, and drives the storm's velocity-stretch streak. Set to 0 + * for reduced-motion (no streak, normal clamp). */ + setFlythrough(s: number): void { + this.flythrough = THREE.MathUtils.clamp(s, 0, 1); + if (this.booted) this.storm.setStreak(s); + } + + /** Pass-through: set the storm streak strength directly. */ + setStreak(s: number): void { + if (this.booted) this.storm.setStreak(s); + } + + /** Pass-through: push view-space camera velocity to the storm. */ + setCameraVel(v: THREE.Vector3): void { + if (this.booted) this.storm.setCameraVel(v); + } + /** Render one frame. The storm is pinned to the origin and the camera always * looks at the origin, so the storm CANNOT leave the frame. The director * varies only the camera's orbital position/angle (set via cameraRef), and we @@ -192,17 +218,36 @@ export class CinemaSandbox { async render(deltaSeconds: number): Promise { if (!this.booted) return; + // Camera velocity (world units / sec) from the position delta this frame — + // one Vector3 subtract, no compute. Captured BEFORE the clamp so it tracks + // the director's intended move (the clamp only rescues runaway distances). + this.camVel.copy(this.camera.position).sub(this.prevCamPos).divideScalar(Math.max(deltaSeconds, 1e-3)); + this.prevCamPos.copy(this.camera.position); + // Hard guarantee: clamp the camera into a distance band from origin so a // runaway director move can never push the subject out of view, then look - // dead at the origin where the storm lives. + // dead at the origin where the storm lives. Flythrough relaxes the floor + // toward 6 so the camera can plunge inside the shell; the MAX clamp stands. + const minDist = THREE.MathUtils.lerp(MIN_CAM_DIST, 6, this.flythrough); const distToOrigin = this.camera.position.length(); - if (distToOrigin < MIN_CAM_DIST || distToOrigin > MAX_CAM_DIST || !Number.isFinite(distToOrigin)) { - const d = Math.min(MAX_CAM_DIST, Math.max(MIN_CAM_DIST, distToOrigin || MAX_CAM_DIST)); + if (distToOrigin < minDist || distToOrigin > MAX_CAM_DIST || !Number.isFinite(distToOrigin)) { + const d = Math.min(MAX_CAM_DIST, Math.max(minDist, distToOrigin || MAX_CAM_DIST)); if (distToOrigin > 1e-3) this.camera.position.setLength(d); else this.camera.position.set(0, 12, d); } this.camera.lookAt(ORIGIN); + // Push view-space apparent particle velocity to the storm (negated world + // camera velocity transformed into view space → the direction sprites + // appear to streak). matrixWorldInverse is the previous frame's (the + // renderer refreshes it during renderAsync below) — a one-frame lag that is + // imperceptible for a streak direction. + const camVelView = this.camVel + .clone() + .applyMatrix3(new THREE.Matrix3().setFromMatrix4(this.camera.matrixWorldInverse)) + .negate(); + this.storm.setCameraVel(camVelView); + // Size the containment sphere to the camera's VERTICAL FOV at the origin // (the limiting dimension on a landscape frame). 0.82 lets the storm fill // most of the frame; the storm's internal shell sits well inside this and diff --git a/apps/dashboard/src/lib/graph/cinema/storm.ts b/apps/dashboard/src/lib/graph/cinema/storm.ts index 41336cf..0cc36b6 100644 --- a/apps/dashboard/src/lib/graph/cinema/storm.ts +++ b/apps/dashboard/src/lib/graph/cinema/storm.ts @@ -169,6 +169,15 @@ export class SemanticComputeStorm { private uZoomPeriod = uniform(9.0); // T: one promotion every 9s private uLambda = uniform(1.923); // 1 / 0.52 self-similar ratio private uZoomOn = uniform(0); // 0 = off (beats 0/1, reduced-motion), 1 = diving + // VELOCITY-STRETCH FLYTHROUGH STREAK — when the camera plunges through the + // shell, sprites elongate along the screen-space apparent motion vector (a + // motion-streak look). Pure scaleNode/rotationNode (a SEPARATE output graph + // from color/emissive) + camera-velocity uniforms → zero per-frame compute, + // no positionView read in color/emissive. Strength is gated to 0 from JS for + // reduced-motion, so this is a no-op until the director drives uStreak. + private uCamVelView = uniform(new THREE.Vector3(0, 0, 0)); // view-space apparent particle velocity + private uStreak = uniform(0); // 0..1 flythrough strength + private uMaxStretch = uniform(7.0); // JS-side dream state (not uniforms): which figure is live + how many fired. private dreamCount = 0; @@ -568,6 +577,32 @@ export class SemanticComputeStorm { const instancePos = storage(bufferPos, 'vec3', this.count).element(instanceIndex); mat.positionNode = instancePos; + // ── VELOCITY-STRETCH STREAK (scaleNode + rotationNode) ── a SEPARATE output + // graph from color/emissive: it reads only camera-velocity uniforms + + // phaseStore, NEVER positionView (a 2nd positionView read in a material + // output would stack-overflow three@0.172's node builder). Zero per-frame + // compute — the camera velocity is one uniform pushed from the sandbox. + const matStretch = mat as typeof mat & { scaleNode: unknown; rotationNode: unknown }; + { + // Screen-plane apparent particle velocity (view space x/y). speed≥ε so + // length()/atan() stay finite when the camera is still. + const velView = vec3(this.uCamVelView); + const vScreen = vec2(velView.x, velView.y); + const speed = length(vScreen).max(1e-4); + // Per-particle jitter (0.75..1.25) so streaks don't all stretch identically. + const jit = fract(phaseStore.element(instanceIndex).mul(13.17)).mul(0.5).add(0.75); + // Orient the quad's long axis along the motion vector. atan(y, x) is the + // 2-arg form (full -π..π range), NOT atan2. + matStretch.rotationNode = atan(vScreen.y, vScreen.x); + // X-stretch: 1 (no streak) → uMaxStretch, scaled by speed × strength × jitter. + const stretch = clamp(speed.mul(this.uStreak).mul(jit).mul(0.6).add(1.0), float(1.0), this.uMaxStretch); + // bloat = DOF circle-of-confusion base (DOF brightness is in depthFade; + // the sprite-scale bloat is the neutral 1.0 here). Streak elongates X only; + // bloat stays on both axes so the DOF circle is preserved. + const bloat = float(1.0); + matStretch.scaleNode = vec2(bloat.mul(stretch), bloat); + } + // ── SHARED RAINBOW COLOR ── // One Fn produces the pure iridescent color for a particle; we feed it to // BOTH colorNode (the lit/additive surface color) AND emissiveNode (the @@ -938,6 +973,17 @@ export class SemanticComputeStorm { this.uZoomOn.value = on ? 1 : 0; } + /** View-space apparent particle velocity (the negated, view-transformed camera + * velocity). Drives the streak orientation + length. */ + setCameraVel(v: THREE.Vector3): void { + this.uCamVelView.value.copy(v); + } + + /** Flythrough streak strength 0..1. Set to 0 for reduced-motion (no streak). */ + setStreak(s: number): void { + this.uStreak.value = s; + } + dispose(): void { if (this.mesh) { this.scene.remove(this.mesh);