mirror of
https://github.com/rowboatlabs/rowboat.git
synced 2026-07-12 21:02:17 +02:00
ship pre-recorded tour narration + new default voice
Bundle the 11 tour narration clips as renderer assets so the tour never calls ElevenLabs (works offline and signed-out). useVoiceTTS gains speakUrl() which plays a ready URL through the same queue/analyser path, keeping lip-sync and cancellation intact. Clips are regenerated with scripts/generate-tour-audio.mjs, which parses TOUR_STEPS from product-tour.tsx and synthesizes via @x/core. Default TTS voice is now s3TPKV1kjDlVtZbl4Ksh. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
0a76a1ecfc
commit
b028e56e44
16 changed files with 115 additions and 16 deletions
49
apps/x/apps/renderer/scripts/generate-tour-audio.mjs
Normal file
49
apps/x/apps/renderer/scripts/generate-tour-audio.mjs
Normal file
|
|
@ -0,0 +1,49 @@
|
|||
/**
|
||||
* Regenerates the bundled product-tour narration clips.
|
||||
*
|
||||
* Parses the TOUR_STEPS texts straight out of product-tour.tsx (so the code
|
||||
* stays the single source of truth), synthesizes each one via @x/core's
|
||||
* synthesizeSpeech (Rowboat proxy when signed in, direct ElevenLabs otherwise,
|
||||
* using the voice id configured there / in ~/.rowboat/config/elevenlabs.json),
|
||||
* and writes MP3s to src/assets/tour/<step-id>.mp3.
|
||||
*
|
||||
* Run whenever a step's narration text or the tour voice changes:
|
||||
* cd apps/x && npm run deps # script imports core's built output
|
||||
* node apps/renderer/scripts/generate-tour-audio.mjs
|
||||
*/
|
||||
import { mkdir, readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url))
|
||||
const tourSource = path.join(here, '../src/components/product-tour.tsx')
|
||||
const outDir = path.join(here, '../src/assets/tour')
|
||||
const corePath = path.join(here, '../../../packages/core/dist/voice/voice.js')
|
||||
|
||||
const { synthesizeSpeech } = await import(corePath)
|
||||
|
||||
const src = await readFile(tourSource, 'utf8')
|
||||
const start = src.indexOf('const TOUR_STEPS')
|
||||
const end = src.indexOf('\n]', start)
|
||||
if (start === -1 || end === -1) throw new Error('Could not locate TOUR_STEPS in product-tour.tsx')
|
||||
const block = src.slice(start, end)
|
||||
|
||||
const steps = []
|
||||
const re = /id: '([^']+)'[\s\S]*?text:\s*("[^"]*"|'[^']*')/g
|
||||
for (let m; (m = re.exec(block)); ) {
|
||||
// The capture is a JS string literal from our own source; evaluate it to
|
||||
// resolve the quoting.
|
||||
steps.push({ id: m[1], text: new Function(`return ${m[2]}`)() })
|
||||
}
|
||||
if (steps.length === 0) throw new Error('Parsed zero tour steps — regex out of sync with product-tour.tsx?')
|
||||
console.log(`Parsed ${steps.length} tour steps`)
|
||||
|
||||
await mkdir(outDir, { recursive: true })
|
||||
for (const step of steps) {
|
||||
process.stdout.write(`synthesizing ${step.id}... `)
|
||||
const { audioBase64 } = await synthesizeSpeech(step.text)
|
||||
const file = path.join(outDir, `${step.id}.mp3`)
|
||||
await writeFile(file, Buffer.from(audioBase64, 'base64'))
|
||||
console.log(`${(audioBase64.length * 0.75 / 1024).toFixed(0)} KB`)
|
||||
}
|
||||
console.log(`Done — ${steps.length} clips in ${path.relative(process.cwd(), outDir)}`)
|
||||
|
|
@ -6470,6 +6470,7 @@ function App() {
|
|||
ttsAvailable={ttsAvailable}
|
||||
ttsState={tts.state}
|
||||
speak={tts.speak}
|
||||
speakUrl={tts.speakUrl}
|
||||
cancelSpeech={tts.cancel}
|
||||
getLevel={tts.getLevel}
|
||||
/>
|
||||
|
|
|
|||
BIN
apps/x/apps/renderer/src/assets/tour/agents.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/agents.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/chats.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/chats.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/code.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/code.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/composer.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/composer.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/done.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/done.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/email.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/email.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/home.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/home.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/knowledge.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/knowledge.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/meetings.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/meetings.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/welcome.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/welcome.mp3
Normal file
Binary file not shown.
BIN
apps/x/apps/renderer/src/assets/tour/workspaces.mp3
Normal file
BIN
apps/x/apps/renderer/src/assets/tour/workspaces.mp3
Normal file
Binary file not shown.
|
|
@ -7,6 +7,17 @@ import { AgentsFleet, MascotVignette, type TourVignetteKind } from '@/components
|
|||
import { TourSounds } from '@/lib/tour-sounds'
|
||||
import type { TTSState } from '@/hooks/useVoiceTTS'
|
||||
import { cn } from '@/lib/utils'
|
||||
import tourClipWelcome from '@/assets/tour/welcome.mp3'
|
||||
import tourClipHome from '@/assets/tour/home.mp3'
|
||||
import tourClipEmail from '@/assets/tour/email.mp3'
|
||||
import tourClipMeetings from '@/assets/tour/meetings.mp3'
|
||||
import tourClipCode from '@/assets/tour/code.mp3'
|
||||
import tourClipKnowledge from '@/assets/tour/knowledge.mp3'
|
||||
import tourClipAgents from '@/assets/tour/agents.mp3'
|
||||
import tourClipWorkspaces from '@/assets/tour/workspaces.mp3'
|
||||
import tourClipChats from '@/assets/tour/chats.mp3'
|
||||
import tourClipComposer from '@/assets/tour/composer.mp3'
|
||||
import tourClipDone from '@/assets/tour/done.mp3'
|
||||
|
||||
export type TourNavTarget =
|
||||
| 'home'
|
||||
|
|
@ -116,6 +127,23 @@ const TOUR_STEPS: TourStep[] = [
|
|||
},
|
||||
]
|
||||
|
||||
// Pre-recorded narration bundled with the app (no TTS API call, works
|
||||
// offline/signed-out). Regenerate with scripts/generate-tour-audio.mjs after
|
||||
// editing any step's text. Steps without a clip fall back to live TTS.
|
||||
const TOUR_CLIPS: Record<string, string> = {
|
||||
welcome: tourClipWelcome,
|
||||
home: tourClipHome,
|
||||
email: tourClipEmail,
|
||||
meetings: tourClipMeetings,
|
||||
code: tourClipCode,
|
||||
knowledge: tourClipKnowledge,
|
||||
agents: tourClipAgents,
|
||||
workspaces: tourClipWorkspaces,
|
||||
chats: tourClipChats,
|
||||
composer: tourClipComposer,
|
||||
done: tourClipDone,
|
||||
}
|
||||
|
||||
const MASCOT_SIZE = 120
|
||||
const VIEWPORT_MARGIN = 16
|
||||
const BUBBLE_WIDTH = 288
|
||||
|
|
@ -214,6 +242,7 @@ type ProductTourProps = {
|
|||
ttsAvailable: boolean
|
||||
ttsState: TTSState
|
||||
speak: (text: string) => void
|
||||
speakUrl: (url: string) => void
|
||||
cancelSpeech: () => void
|
||||
getLevel: () => number
|
||||
}
|
||||
|
|
@ -234,6 +263,7 @@ export function ProductTour({
|
|||
ttsAvailable,
|
||||
ttsState,
|
||||
speak,
|
||||
speakUrl,
|
||||
cancelSpeech,
|
||||
getLevel,
|
||||
}: ProductTourProps) {
|
||||
|
|
@ -279,6 +309,7 @@ export function ProductTour({
|
|||
const onCloseRef = useRef(onClose)
|
||||
const onNavigateRef = useRef(onNavigate)
|
||||
const speakRef = useRef(speak)
|
||||
const speakUrlRef = useRef(speakUrl)
|
||||
const cancelSpeechRef = useRef(cancelSpeech)
|
||||
const ttsAvailableRef = useRef(ttsAvailable)
|
||||
|
||||
|
|
@ -289,6 +320,7 @@ export function ProductTour({
|
|||
onCloseRef.current = onClose
|
||||
onNavigateRef.current = onNavigate
|
||||
speakRef.current = speak
|
||||
speakUrlRef.current = speakUrl
|
||||
cancelSpeechRef.current = cancelSpeech
|
||||
ttsAvailableRef.current = ttsAvailable
|
||||
})
|
||||
|
|
@ -482,7 +514,10 @@ export function ProductTour({
|
|||
setArrived(true)
|
||||
soundsRef.current?.bump()
|
||||
cancelSpeechRef.current()
|
||||
if (ttsAvailableRef.current) {
|
||||
const clip = TOUR_CLIPS[step.id]
|
||||
if (clip) {
|
||||
speakUrlRef.current(clip)
|
||||
} else if (ttsAvailableRef.current) {
|
||||
speakRef.current(step.text)
|
||||
}
|
||||
if (stepIndex === TOUR_STEPS.length - 1) {
|
||||
|
|
|
|||
|
|
@ -44,10 +44,13 @@ function playAudio(
|
|||
});
|
||||
}
|
||||
|
||||
/** A queue entry: text to synthesize, or a ready-to-play audio URL (e.g. a bundled clip). */
|
||||
type QueueItem = { text: string } | { url: string };
|
||||
|
||||
export function useVoiceTTS() {
|
||||
const [state, setState] = useState<TTSState>('idle');
|
||||
const audioRef = useRef<HTMLAudioElement | null>(null);
|
||||
const queueRef = useRef<string[]>([]);
|
||||
const queueRef = useRef<QueueItem[]>([]);
|
||||
const processingRef = useRef(false);
|
||||
// Pre-fetched audio ready to play immediately
|
||||
const prefetchedRef = useRef<Promise<SynthesizedAudio> | null>(null);
|
||||
|
|
@ -128,20 +131,23 @@ export function useVoiceTTS() {
|
|||
const gen = generationRef.current;
|
||||
|
||||
while (queueRef.current.length > 0) {
|
||||
const text = queueRef.current.shift()!;
|
||||
if (!text.trim()) continue;
|
||||
const item = queueRef.current.shift()!;
|
||||
if ('text' in item && !item.text.trim()) continue;
|
||||
|
||||
try {
|
||||
// Use pre-fetched result if available, otherwise synthesize now
|
||||
// Pre-recorded URL plays as-is; text uses the pre-fetched
|
||||
// result if available, otherwise synthesizes now.
|
||||
let audioPromise: Promise<SynthesizedAudio>;
|
||||
if (prefetchedRef.current) {
|
||||
if ('url' in item) {
|
||||
audioPromise = Promise.resolve({ dataUrl: item.url });
|
||||
} else if (prefetchedRef.current) {
|
||||
console.log('[tts] using pre-fetched audio');
|
||||
audioPromise = prefetchedRef.current;
|
||||
prefetchedRef.current = null;
|
||||
} else {
|
||||
setState('synthesizing');
|
||||
console.log('[tts] synthesizing:', text.substring(0, 80));
|
||||
audioPromise = synthesize(text);
|
||||
console.log('[tts] synthesizing:', item.text.substring(0, 80));
|
||||
audioPromise = synthesize(item.text);
|
||||
}
|
||||
|
||||
const audio = await audioPromise;
|
||||
|
|
@ -151,10 +157,10 @@ export function useVoiceTTS() {
|
|||
setState('speaking');
|
||||
|
||||
// Kick off pre-fetch for next chunk while this one plays
|
||||
const nextText = queueRef.current[0];
|
||||
if (nextText?.trim()) {
|
||||
console.log('[tts] pre-fetching next:', nextText.substring(0, 80));
|
||||
prefetchedRef.current = synthesize(nextText);
|
||||
const next = queueRef.current[0];
|
||||
if (next && 'text' in next && next.text.trim()) {
|
||||
console.log('[tts] pre-fetching next:', next.text.substring(0, 80));
|
||||
prefetchedRef.current = synthesize(next.text);
|
||||
}
|
||||
|
||||
await playAudio(audio.dataUrl, audioRef, connectAnalyser);
|
||||
|
|
@ -174,7 +180,15 @@ export function useVoiceTTS() {
|
|||
|
||||
const speak = useCallback((text: string) => {
|
||||
console.log('[tts] speak() called:', text.substring(0, 80));
|
||||
queueRef.current.push(text);
|
||||
queueRef.current.push({ text });
|
||||
processQueue();
|
||||
}, [processQueue]);
|
||||
|
||||
// Play a pre-recorded clip (e.g. bundled tour narration) through the same
|
||||
// queue, so lip-sync levels, state, and cancel() all work unchanged.
|
||||
const speakUrl = useCallback((url: string) => {
|
||||
console.log('[tts] speakUrl() called:', url.substring(0, 120));
|
||||
queueRef.current.push({ url });
|
||||
processQueue();
|
||||
}, [processQueue]);
|
||||
|
||||
|
|
@ -190,5 +204,5 @@ export function useVoiceTTS() {
|
|||
setState('idle');
|
||||
}, []);
|
||||
|
||||
return { state, speak, cancel, getLevel };
|
||||
return { state, speak, speakUrl, cancel, getLevel };
|
||||
}
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ export async function synthesizeSpeech(text: string): Promise<{ audioBase64: str
|
|||
let headers: Record<string, string>;
|
||||
|
||||
if (signedIn) {
|
||||
const voiceId = config.elevenlabs?.voiceId || 'UgBBYS2sOqTuMpoF3BR0';
|
||||
const voiceId = config.elevenlabs?.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
|
||||
const accessToken = await getAccessToken();
|
||||
url = `${API_URL}/v1/voice/text-to-speech/${voiceId}`;
|
||||
headers = {
|
||||
|
|
@ -52,7 +52,7 @@ export async function synthesizeSpeech(text: string): Promise<{ audioBase64: str
|
|||
if (!config.elevenlabs) {
|
||||
throw new Error(`ElevenLabs not configured. Create ${path.join(WorkDir, 'config', 'elevenlabs.json')} with { "apiKey": "<your-key>" }`);
|
||||
}
|
||||
const voiceId = config.elevenlabs.voiceId || 'UgBBYS2sOqTuMpoF3BR0';
|
||||
const voiceId = config.elevenlabs.voiceId || 's3TPKV1kjDlVtZbl4Ksh';
|
||||
url = `https://api.elevenlabs.io/v1/text-to-speech/${voiceId}`;
|
||||
headers = {
|
||||
'xi-api-key': config.elevenlabs.apiKey,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue