apunkt/rowboat
1
0
Fork 0
mirror of https://github.com/rowboatlabs/rowboat.git synced 2026-07-12 21:02:17 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

(1) add running transcript to voice input (2) capture till end of speech when mic is used (3) notify only on important emails

Browse source
This commit is contained in:
Arjun 2026-06-30 18:40:50 +05:30
parent 753e3448f0
commit f2b5c6b1ab
7 changed files with 147 additions and 41 deletions
Download patch file Download diff file Expand all files Collapse all files

9
apps/x/apps/renderer/src/App.tsx
View file

@ -1040,8 +1040,9 @@ function App() {
const editorRefsByTabId = useRef<Map<string, MarkdownEditorHandle>>(new Map())
const [pendingPaletteSubmit, setPendingPaletteSubmit] = useState<{ text: string; mention: CommandPaletteMention | null } | null>(null)
const handleSubmitRecording = useCallback(() => {
const text = voice.submit()
const handleSubmitRecording = useCallback(async () => {
if (!isRecordingRef.current) return
const text = await voice.submit()
setIsRecording(false)
isRecordingRef.current = false
if (text) {
@ -6377,7 +6378,7 @@ function App() {
onWorkDirChange={(v) => setTabWorkDir(tab.id, v)}
isRecording={isActive && isRecording}
recordingText={isActive ? voice.interimText : undefined}
recordingState={isActive ? (voice.state === 'connecting' ? 'connecting' : 'listening') : undefined}
recordingState={isActive ? (voice.state === 'submitting' ? 'stopping' : voice.state === 'connecting' ? 'connecting' : 'listening') : undefined}
audioLevelsRef={voice.audioLevelsRef}
onStartRecording={isActive ? handleStartRecording : undefined}
onSubmitRecording={isActive ? handleSubmitRecording : undefined}
@ -6486,7 +6487,7 @@ function App() {
collapsedLeftPaddingPx={collapsedLeftPaddingPx}
isRecording={isRecording}
recordingText={voice.interimText}
recordingState={voice.state === 'connecting' ? 'connecting' : 'listening'}
recordingState={voice.state === 'submitting' ? 'stopping' : voice.state === 'connecting' ? 'connecting' : 'listening'}
audioLevelsRef={voice.audioLevelsRef}
onStartRecording={handleStartRecording}
onSubmitRecording={handleSubmitRecording}

31
apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx
View file

@ -223,11 +223,11 @@ interface ChatInputInnerProps {
onDraftChange?: (text: string) => void
isRecording?: boolean
recordingText?: string
recordingState?: 'connecting' | 'listening'
recordingState?: 'connecting' | 'listening' | 'stopping'
/** Live mic amplitude history (RMS per frame) driving the recording waveform. */
audioLevelsRef?: React.MutableRefObject<number[]>
onStartRecording?: () => void
onSubmitRecording?: () => void
onSubmitRecording?: () => void | Promise<void>
onCancelRecording?: () => void
voiceAvailable?: boolean
ttsAvailable?: boolean
@ -262,6 +262,7 @@ function ChatInputInner({
onDraftChange,
isRecording,
recordingText,
recordingState,
audioLevelsRef,
onStartRecording,
onSubmitRecording,
@ -829,23 +830,33 @@ function ChatInputInner({
>
<X className="h-4 w-4" />
</button>
{/* Audio-reactive waveform only — the transcribed words are intentionally
not shown while recording; they're still captured and submitted. */}
<div className="flex flex-1 items-center overflow-hidden">
<div className="flex min-w-0 flex-1 flex-col gap-1 overflow-hidden">
<VoiceWaveform audioLevelsRef={audioLevelsRef} />
<div
className={cn(
'min-h-5 truncate text-sm leading-5',
recordingText?.trim() ? 'text-foreground' : 'text-muted-foreground'
)}
>
{recordingText?.trim() || (recordingState === 'stopping' ? 'Finalizing...' : 'Listening...')}
</div>
</div>
<Button
size="icon"
onClick={onSubmitRecording}
disabled={!recordingText?.trim()}
disabled={recordingState === 'stopping'}
className={cn(
'h-7 w-7 shrink-0 rounded-full transition-all',
recordingText?.trim()
recordingState !== 'stopping'
? 'bg-primary text-primary-foreground hover:bg-primary/90'
: 'bg-muted text-muted-foreground'
)}
>
<ArrowUp className="h-4 w-4" />
{recordingState === 'stopping' ? (
<LoaderIcon className="h-4 w-4 animate-spin" />
) : (
<ArrowUp className="h-4 w-4" />
)}
</Button>
</div>
) : (
@ -1477,10 +1488,10 @@ export interface ChatInputWithMentionsProps {
onDraftChange?: (text: string) => void
isRecording?: boolean
recordingText?: string
recordingState?: 'connecting' | 'listening'
recordingState?: 'connecting' | 'listening' | 'stopping'
audioLevelsRef?: React.MutableRefObject<number[]>
onStartRecording?: () => void
onSubmitRecording?: () => void
onSubmitRecording?: () => void | Promise<void>
onCancelRecording?: () => void
voiceAvailable?: boolean
ttsAvailable?: boolean

4
apps/x/apps/renderer/src/components/chat-sidebar.tsx
View file

@ -177,10 +177,10 @@ interface ChatSidebarProps {
// Voice / TTS props
isRecording?: boolean
recordingText?: string
recordingState?: 'connecting' | 'listening'
recordingState?: 'connecting' | 'listening' | 'stopping'
audioLevelsRef?: React.MutableRefObject<number[]>
onStartRecording?: () => void
onSubmitRecording?: () => void
onSubmitRecording?: () => void | Promise<void>
onCancelRecording?: () => void
voiceAvailable?: boolean
ttsAvailable?: boolean

45
apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts
View file

@ -1,6 +1,7 @@
import { useCallback, useRef, useState } from 'react';
import { toast } from 'sonner';
import { buildDeepgramListenUrl } from '@/lib/deepgram-listen-url';
import { finalizeDeepgramStream } from '@/lib/deepgram-finalize';
import { useRowboatAccount } from '@/hooks/useRowboatAccount';
export type MeetingTranscriptionState = 'idle' | 'connecting' | 'recording' | 'stopping';
@ -196,6 +197,25 @@ export function useMeetingTranscription(onAutoStop?: () => void) {
}, 1000);
}, [writeTranscriptToFile]);
const stopInputCapture = useCallback(() => {
if (processorRef.current) {
processorRef.current.disconnect();
processorRef.current = null;
}
if (audioCtxRef.current) {
audioCtxRef.current.close();
audioCtxRef.current = null;
}
if (micStreamRef.current) {
micStreamRef.current.getTracks().forEach(t => t.stop());
micStreamRef.current = null;
}
if (systemStreamRef.current) {
systemStreamRef.current.getTracks().forEach(t => t.stop());
systemStreamRef.current = null;
}
}, []);
const cleanup = useCallback(() => {
if (writeTimerRef.current) {
clearTimeout(writeTimerRef.current);
@ -213,28 +233,13 @@ export function useMeetingTranscription(onAutoStop?: () => void) {
clearInterval(trackPollingRef.current);
trackPollingRef.current = null;
}
if (processorRef.current) {
processorRef.current.disconnect();
processorRef.current = null;
}
if (audioCtxRef.current) {
audioCtxRef.current.close();
audioCtxRef.current = null;
}
if (micStreamRef.current) {
micStreamRef.current.getTracks().forEach(t => t.stop());
micStreamRef.current = null;
}
if (systemStreamRef.current) {
systemStreamRef.current.getTracks().forEach(t => t.stop());
systemStreamRef.current = null;
}
stopInputCapture();
if (wsRef.current) {
wsRef.current.onclose = null;
wsRef.current.close();
wsRef.current = null;
}
}, []);
}, [stopInputCapture]);
const start = useCallback(async (calendarEvent?: CalendarEventMeta): Promise<string | null> => {
if (state !== 'idle') return null;
@ -551,12 +556,14 @@ export function useMeetingTranscription(onAutoStop?: () => void) {
if (state !== 'recording') return;
setState('stopping');
stopInputCapture();
await finalizeDeepgramStream(wsRef.current, 2200);
cleanup();
interimRef.current = new Map();
await writeTranscriptToFile();
interimRef.current = new Map();
setState('idle');
}, [state, cleanup, writeTranscriptToFile]);
}, [state, cleanup, stopInputCapture, writeTranscriptToFile]);
return { state, start, stop };
}

65
apps/x/apps/renderer/src/hooks/useVoiceMode.ts
View file

@ -1,10 +1,11 @@
import { useCallback, useRef, useState } from 'react';
import { buildDeepgramListenUrl } from '@/lib/deepgram-listen-url';
import { finalizeDeepgramStream } from '@/lib/deepgram-finalize';
import { useRowboatAccount } from '@/hooks/useRowboatAccount';
import posthog from 'posthog-js';
import * as analytics from '@/lib/analytics';
export type VoiceState = 'idle' | 'connecting' | 'listening';
export type VoiceState = 'idle' | 'connecting' | 'listening' | 'submitting';
const DEEPGRAM_PARAMS = new URLSearchParams({
model: 'nova-3',
@ -129,8 +130,45 @@ export function useVoiceMode() {
};
}, [refreshAuth]);
// Stop audio capture and close WS
const stopAudioCapture = useCallback(() => {
const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise<boolean> => {
const ws = wsRef.current;
if (!ws) return false;
if (ws.readyState === WebSocket.OPEN) return true;
if (ws.readyState !== WebSocket.CONNECTING) return false;
return new Promise<boolean>((resolve) => {
let done = false;
let timeout: ReturnType<typeof setTimeout>;
const finish = (ok: boolean) => {
if (done) return;
done = true;
clearTimeout(timeout);
ws.removeEventListener('open', onOpen);
ws.removeEventListener('error', onError);
ws.removeEventListener('close', onClose);
resolve(ok);
};
const onOpen = () => finish(true);
const onError = () => finish(false);
const onClose = () => finish(false);
timeout = setTimeout(() => finish(false), timeoutMs);
ws.addEventListener('open', onOpen);
ws.addEventListener('error', onError);
ws.addEventListener('close', onClose);
});
}, []);
const flushBufferedAudio = useCallback(() => {
const ws = wsRef.current;
if (!ws || ws.readyState !== WebSocket.OPEN) return;
const buffered = audioBufferRef.current;
audioBufferRef.current = [];
for (const chunk of buffered) {
ws.send(chunk);
}
}, []);
const stopInputCapture = useCallback(() => {
if (processorRef.current) {
processorRef.current.disconnect();
processorRef.current = null;
@ -143,6 +181,11 @@ export function useVoiceMode() {
mediaStreamRef.current.getTracks().forEach(t => t.stop());
mediaStreamRef.current = null;
}
}, []);
// Stop audio capture and close WS
const stopAudioCapture = useCallback(() => {
stopInputCapture();
if (wsRef.current) {
wsRef.current.onclose = null;
wsRef.current.close();
@ -155,7 +198,7 @@ export function useVoiceMode() {
transcriptBufferRef.current = '';
interimRef.current = '';
setState('idle');
}, []);
}, [stopInputCapture]);
const start = useCallback(async () => {
if (state !== 'idle') return;
@ -250,15 +293,25 @@ export function useVoiceMode() {
}, [state, connectWs, stopAudioCapture]);
/** Stop recording and return the full transcript (finalized + any current interim) */
const submit = useCallback((): string => {
const submit = useCallback(async (): Promise<string> => {
setState('submitting');
stopInputCapture();
if (wsRef.current?.readyState === WebSocket.CONNECTING) {
await waitForWsOpen();
}
flushBufferedAudio();
await finalizeDeepgramStream(wsRef.current);
let text = transcriptBufferRef.current;
if (interimRef.current) {
text += (text ? ' ' : '') + interimRef.current;
}
text = text.trim();
stopAudioCapture();
return text;
}, [stopAudioCapture]);
}, [flushBufferedAudio, stopAudioCapture, stopInputCapture, waitForWsOpen]);
/** Cancel recording without returning transcript */
const cancel = useCallback(() => {

33
apps/x/apps/renderer/src/lib/deepgram-finalize.ts Normal file
View file

@ -0,0 +1,33 @@
export async function finalizeDeepgramStream(ws: WebSocket | null, timeoutMs = 1800): Promise<void> {
if (!ws || ws.readyState !== WebSocket.OPEN) return;
await new Promise<void>((resolve) => {
let done = false;
let timeout: ReturnType<typeof setTimeout>;
const finish = () => {
if (done) return;
done = true;
clearTimeout(timeout);
ws.removeEventListener('message', onMessage);
resolve();
};
timeout = setTimeout(finish, timeoutMs);
const onMessage = (event: MessageEvent) => {
try {
const data = JSON.parse(event.data);
if (data?.is_final || data?.speech_final || data?.type === 'UtteranceEnd') {
finish();
}
} catch {
// Ignore non-JSON control frames.
}
};
ws.addEventListener('message', onMessage);
try {
ws.send(JSON.stringify({ type: 'Finalize' }));
} catch {
finish();
}
});
}