From f2b5c6b1ab8b1f67c2556a4ef3dce26cd34b6aba Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 30 Jun 2026 18:40:50 +0530 Subject: [PATCH 01/57] (1) add running transcript to voice input (2) capture till end of speech when mic is used (3) notify only on important emails --- apps/x/apps/renderer/src/App.tsx | 9 +-- .../components/chat-input-with-mentions.tsx | 31 ++++++--- .../renderer/src/components/chat-sidebar.tsx | 4 +- .../src/hooks/useMeetingTranscription.ts | 45 +++++++------ .../x/apps/renderer/src/hooks/useVoiceMode.ts | 65 +++++++++++++++++-- .../renderer/src/lib/deepgram-finalize.ts | 33 ++++++++++ .../packages/core/src/knowledge/sync_gmail.ts | 1 + 7 files changed, 147 insertions(+), 41 deletions(-) create mode 100644 apps/x/apps/renderer/src/lib/deepgram-finalize.ts diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index bfe71c85..7ccb56e3 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -1040,8 +1040,9 @@ function App() { const editorRefsByTabId = useRef>(new Map()) const [pendingPaletteSubmit, setPendingPaletteSubmit] = useState<{ text: string; mention: CommandPaletteMention | null } | null>(null) - const handleSubmitRecording = useCallback(() => { - const text = voice.submit() + const handleSubmitRecording = useCallback(async () => { + if (!isRecordingRef.current) return + const text = await voice.submit() setIsRecording(false) isRecordingRef.current = false if (text) { @@ -6377,7 +6378,7 @@ function App() { onWorkDirChange={(v) => setTabWorkDir(tab.id, v)} isRecording={isActive && isRecording} recordingText={isActive ? voice.interimText : undefined} - recordingState={isActive ? (voice.state === 'connecting' ? 'connecting' : 'listening') : undefined} + recordingState={isActive ? (voice.state === 'submitting' ? 'stopping' : voice.state === 'connecting' ? 'connecting' : 'listening') : undefined} audioLevelsRef={voice.audioLevelsRef} onStartRecording={isActive ? handleStartRecording : undefined} onSubmitRecording={isActive ? handleSubmitRecording : undefined} @@ -6486,7 +6487,7 @@ function App() { collapsedLeftPaddingPx={collapsedLeftPaddingPx} isRecording={isRecording} recordingText={voice.interimText} - recordingState={voice.state === 'connecting' ? 'connecting' : 'listening'} + recordingState={voice.state === 'submitting' ? 'stopping' : voice.state === 'connecting' ? 'connecting' : 'listening'} audioLevelsRef={voice.audioLevelsRef} onStartRecording={handleStartRecording} onSubmitRecording={handleSubmitRecording} diff --git a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx index f3609212..182ed20a 100644 --- a/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx +++ b/apps/x/apps/renderer/src/components/chat-input-with-mentions.tsx @@ -223,11 +223,11 @@ interface ChatInputInnerProps { onDraftChange?: (text: string) => void isRecording?: boolean recordingText?: string - recordingState?: 'connecting' | 'listening' + recordingState?: 'connecting' | 'listening' | 'stopping' /** Live mic amplitude history (RMS per frame) driving the recording waveform. */ audioLevelsRef?: React.MutableRefObject onStartRecording?: () => void - onSubmitRecording?: () => void + onSubmitRecording?: () => void | Promise onCancelRecording?: () => void voiceAvailable?: boolean ttsAvailable?: boolean @@ -262,6 +262,7 @@ function ChatInputInner({ onDraftChange, isRecording, recordingText, + recordingState, audioLevelsRef, onStartRecording, onSubmitRecording, @@ -829,23 +830,33 @@ function ChatInputInner({ > - {/* Audio-reactive waveform only — the transcribed words are intentionally - not shown while recording; they're still captured and submitted. */} -
+
+
+ {recordingText?.trim() || (recordingState === 'stopping' ? 'Finalizing...' : 'Listening...')} +
) : ( @@ -1477,10 +1488,10 @@ export interface ChatInputWithMentionsProps { onDraftChange?: (text: string) => void isRecording?: boolean recordingText?: string - recordingState?: 'connecting' | 'listening' + recordingState?: 'connecting' | 'listening' | 'stopping' audioLevelsRef?: React.MutableRefObject onStartRecording?: () => void - onSubmitRecording?: () => void + onSubmitRecording?: () => void | Promise onCancelRecording?: () => void voiceAvailable?: boolean ttsAvailable?: boolean diff --git a/apps/x/apps/renderer/src/components/chat-sidebar.tsx b/apps/x/apps/renderer/src/components/chat-sidebar.tsx index a463b080..a9d6d9f7 100644 --- a/apps/x/apps/renderer/src/components/chat-sidebar.tsx +++ b/apps/x/apps/renderer/src/components/chat-sidebar.tsx @@ -177,10 +177,10 @@ interface ChatSidebarProps { // Voice / TTS props isRecording?: boolean recordingText?: string - recordingState?: 'connecting' | 'listening' + recordingState?: 'connecting' | 'listening' | 'stopping' audioLevelsRef?: React.MutableRefObject onStartRecording?: () => void - onSubmitRecording?: () => void + onSubmitRecording?: () => void | Promise onCancelRecording?: () => void voiceAvailable?: boolean ttsAvailable?: boolean diff --git a/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts b/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts index 96f42412..75b4d91f 100644 --- a/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts +++ b/apps/x/apps/renderer/src/hooks/useMeetingTranscription.ts @@ -1,6 +1,7 @@ import { useCallback, useRef, useState } from 'react'; import { toast } from 'sonner'; import { buildDeepgramListenUrl } from '@/lib/deepgram-listen-url'; +import { finalizeDeepgramStream } from '@/lib/deepgram-finalize'; import { useRowboatAccount } from '@/hooks/useRowboatAccount'; export type MeetingTranscriptionState = 'idle' | 'connecting' | 'recording' | 'stopping'; @@ -196,6 +197,25 @@ export function useMeetingTranscription(onAutoStop?: () => void) { }, 1000); }, [writeTranscriptToFile]); + const stopInputCapture = useCallback(() => { + if (processorRef.current) { + processorRef.current.disconnect(); + processorRef.current = null; + } + if (audioCtxRef.current) { + audioCtxRef.current.close(); + audioCtxRef.current = null; + } + if (micStreamRef.current) { + micStreamRef.current.getTracks().forEach(t => t.stop()); + micStreamRef.current = null; + } + if (systemStreamRef.current) { + systemStreamRef.current.getTracks().forEach(t => t.stop()); + systemStreamRef.current = null; + } + }, []); + const cleanup = useCallback(() => { if (writeTimerRef.current) { clearTimeout(writeTimerRef.current); @@ -213,28 +233,13 @@ export function useMeetingTranscription(onAutoStop?: () => void) { clearInterval(trackPollingRef.current); trackPollingRef.current = null; } - if (processorRef.current) { - processorRef.current.disconnect(); - processorRef.current = null; - } - if (audioCtxRef.current) { - audioCtxRef.current.close(); - audioCtxRef.current = null; - } - if (micStreamRef.current) { - micStreamRef.current.getTracks().forEach(t => t.stop()); - micStreamRef.current = null; - } - if (systemStreamRef.current) { - systemStreamRef.current.getTracks().forEach(t => t.stop()); - systemStreamRef.current = null; - } + stopInputCapture(); if (wsRef.current) { wsRef.current.onclose = null; wsRef.current.close(); wsRef.current = null; } - }, []); + }, [stopInputCapture]); const start = useCallback(async (calendarEvent?: CalendarEventMeta): Promise => { if (state !== 'idle') return null; @@ -551,12 +556,14 @@ export function useMeetingTranscription(onAutoStop?: () => void) { if (state !== 'recording') return; setState('stopping'); + stopInputCapture(); + await finalizeDeepgramStream(wsRef.current, 2200); cleanup(); - interimRef.current = new Map(); await writeTranscriptToFile(); + interimRef.current = new Map(); setState('idle'); - }, [state, cleanup, writeTranscriptToFile]); + }, [state, cleanup, stopInputCapture, writeTranscriptToFile]); return { state, start, stop }; } diff --git a/apps/x/apps/renderer/src/hooks/useVoiceMode.ts b/apps/x/apps/renderer/src/hooks/useVoiceMode.ts index 8263e45d..a94a7eb8 100644 --- a/apps/x/apps/renderer/src/hooks/useVoiceMode.ts +++ b/apps/x/apps/renderer/src/hooks/useVoiceMode.ts @@ -1,10 +1,11 @@ import { useCallback, useRef, useState } from 'react'; import { buildDeepgramListenUrl } from '@/lib/deepgram-listen-url'; +import { finalizeDeepgramStream } from '@/lib/deepgram-finalize'; import { useRowboatAccount } from '@/hooks/useRowboatAccount'; import posthog from 'posthog-js'; import * as analytics from '@/lib/analytics'; -export type VoiceState = 'idle' | 'connecting' | 'listening'; +export type VoiceState = 'idle' | 'connecting' | 'listening' | 'submitting'; const DEEPGRAM_PARAMS = new URLSearchParams({ model: 'nova-3', @@ -129,8 +130,45 @@ export function useVoiceMode() { }; }, [refreshAuth]); - // Stop audio capture and close WS - const stopAudioCapture = useCallback(() => { + const waitForWsOpen = useCallback(async (timeoutMs = 1500): Promise => { + const ws = wsRef.current; + if (!ws) return false; + if (ws.readyState === WebSocket.OPEN) return true; + if (ws.readyState !== WebSocket.CONNECTING) return false; + + return new Promise((resolve) => { + let done = false; + let timeout: ReturnType; + const finish = (ok: boolean) => { + if (done) return; + done = true; + clearTimeout(timeout); + ws.removeEventListener('open', onOpen); + ws.removeEventListener('error', onError); + ws.removeEventListener('close', onClose); + resolve(ok); + }; + const onOpen = () => finish(true); + const onError = () => finish(false); + const onClose = () => finish(false); + timeout = setTimeout(() => finish(false), timeoutMs); + ws.addEventListener('open', onOpen); + ws.addEventListener('error', onError); + ws.addEventListener('close', onClose); + }); + }, []); + + const flushBufferedAudio = useCallback(() => { + const ws = wsRef.current; + if (!ws || ws.readyState !== WebSocket.OPEN) return; + const buffered = audioBufferRef.current; + audioBufferRef.current = []; + for (const chunk of buffered) { + ws.send(chunk); + } + }, []); + + const stopInputCapture = useCallback(() => { if (processorRef.current) { processorRef.current.disconnect(); processorRef.current = null; @@ -143,6 +181,11 @@ export function useVoiceMode() { mediaStreamRef.current.getTracks().forEach(t => t.stop()); mediaStreamRef.current = null; } + }, []); + + // Stop audio capture and close WS + const stopAudioCapture = useCallback(() => { + stopInputCapture(); if (wsRef.current) { wsRef.current.onclose = null; wsRef.current.close(); @@ -155,7 +198,7 @@ export function useVoiceMode() { transcriptBufferRef.current = ''; interimRef.current = ''; setState('idle'); - }, []); + }, [stopInputCapture]); const start = useCallback(async () => { if (state !== 'idle') return; @@ -250,15 +293,25 @@ export function useVoiceMode() { }, [state, connectWs, stopAudioCapture]); /** Stop recording and return the full transcript (finalized + any current interim) */ - const submit = useCallback((): string => { + const submit = useCallback(async (): Promise => { + setState('submitting'); + stopInputCapture(); + + if (wsRef.current?.readyState === WebSocket.CONNECTING) { + await waitForWsOpen(); + } + flushBufferedAudio(); + await finalizeDeepgramStream(wsRef.current); + let text = transcriptBufferRef.current; if (interimRef.current) { text += (text ? ' ' : '') + interimRef.current; } text = text.trim(); + stopAudioCapture(); return text; - }, [stopAudioCapture]); + }, [flushBufferedAudio, stopAudioCapture, stopInputCapture, waitForWsOpen]); /** Cancel recording without returning transcript */ const cancel = useCallback(() => { diff --git a/apps/x/apps/renderer/src/lib/deepgram-finalize.ts b/apps/x/apps/renderer/src/lib/deepgram-finalize.ts new file mode 100644 index 00000000..91c3e37f --- /dev/null +++ b/apps/x/apps/renderer/src/lib/deepgram-finalize.ts @@ -0,0 +1,33 @@ +export async function finalizeDeepgramStream(ws: WebSocket | null, timeoutMs = 1800): Promise { + if (!ws || ws.readyState !== WebSocket.OPEN) return; + + await new Promise((resolve) => { + let done = false; + let timeout: ReturnType; + const finish = () => { + if (done) return; + done = true; + clearTimeout(timeout); + ws.removeEventListener('message', onMessage); + resolve(); + }; + timeout = setTimeout(finish, timeoutMs); + const onMessage = (event: MessageEvent) => { + try { + const data = JSON.parse(event.data); + if (data?.is_final || data?.speech_final || data?.type === 'UtteranceEnd') { + finish(); + } + } catch { + // Ignore non-JSON control frames. + } + }; + + ws.addEventListener('message', onMessage); + try { + ws.send(JSON.stringify({ type: 'Finalize' })); + } catch { + finish(); + } + }); +} diff --git a/apps/x/packages/core/src/knowledge/sync_gmail.ts b/apps/x/packages/core/src/knowledge/sync_gmail.ts index e4d478e9..6b4018ec 100644 --- a/apps/x/packages/core/src/knowledge/sync_gmail.ts +++ b/apps/x/packages/core/src/knowledge/sync_gmail.ts @@ -248,6 +248,7 @@ function notifyNewEmails(threads: SyncedThread[]): void { const now = Date.now(); for (const { threadId } of threads) { const snapshot = readCachedSnapshot(threadId)?.snapshot; + if (snapshot?.importance !== 'important') continue; if (snapshot && isEmailTooOldToNotify(snapshotDateMs(snapshot), now)) continue; const subject = snapshot?.subject?.trim() || '(no subject)'; const from = snapshot?.from?.trim(); From e6ff631191325c9b420b0020ee8fc623c3ea88a7 Mon Sep 17 00:00:00 2001 From: Arjun <6592213+arkml@users.noreply.github.com> Date: Tue, 30 Jun 2026 21:57:05 +0530 Subject: [PATCH 02/57] fix codex unresponsive --- .../src/components/code/code-chat.tsx | 27 +++++++-- .../src/components/code/use-code-chat.ts | 38 +++++++++++++ .../renderer/src/components/coding-run.tsx | 6 +- .../packages/core/src/code-mode/acp/client.ts | 2 + apps/x/packages/shared/src/code-mode.ts | 5 ++ ...gentclientprotocol__codex-acp@0.0.44.patch | 57 +++++++++++++++++++ apps/x/pnpm-lock.yaml | 9 ++- apps/x/pnpm-workspace.yaml | 1 + 8 files changed, 135 insertions(+), 10 deletions(-) create mode 100644 apps/x/patches/@agentclientprotocol__codex-acp@0.0.44.patch diff --git a/apps/x/apps/renderer/src/components/code/code-chat.tsx b/apps/x/apps/renderer/src/components/code/code-chat.tsx index 7b1656f3..6c34c295 100644 --- a/apps/x/apps/renderer/src/components/code/code-chat.tsx +++ b/apps/x/apps/renderer/src/components/code/code-chat.tsx @@ -103,7 +103,8 @@ export function CodeChat({ onOpenDiff: (path: string) => void }) { const { - items, liveText, isProcessing, pendingPermission, pendingToolPermissions, pendingAskHumans, + items, liveText, isProcessing, compactionStatus, contextUsage, + pendingPermission, pendingToolPermissions, pendingAskHumans, loading, send, stop, resolvePermission, respondToToolPermission, respondToAskHuman, } = useCodeChat(session) const [draft, setDraft] = useState('') @@ -111,6 +112,9 @@ export function CodeChat({ const textareaRef = useRef(null) const busy = isProcessing || status === 'working' || status === 'needs-you' + const contextUsedPercent = contextUsage + ? Math.min(100, Math.round((contextUsage.used / contextUsage.size) * 100)) + : null // Attached file PATHS — like dragging a file into the Claude Code CLI, the // agent receives paths and reads the files itself with its own tools. const [attachments, setAttachments] = useState([]) @@ -188,7 +192,10 @@ export function CodeChat({
{session.title}
-
{AGENT_LABEL[session.agent]} — direct
+
+ {AGENT_LABEL[session.agent]} — direct + {contextUsedPercent != null ? ` · ${contextUsedPercent}% context used` : ''} +
@@ -239,9 +246,19 @@ export function CodeChat({ /> ))} {busy && !pendingPermission && pendingToolPermissions.size === 0 && pendingAskHumans.size === 0 && ( - - {stopping ? 'Stopping…' : `${AGENT_LABEL[session.agent]} is working…`} - + compactionStatus === 'stalled' ? ( +
+ Context compaction is taking longer than expected. You can stop and retry in a fresh session. +
+ ) : ( + + {stopping + ? 'Stopping…' + : compactionStatus === 'running' + ? 'Compacting context…' + : `${AGENT_LABEL[session.agent]} is working…`} + + ) )} diff --git a/apps/x/apps/renderer/src/components/code/use-code-chat.ts b/apps/x/apps/renderer/src/components/code/use-code-chat.ts index 08468c49..ce8973c0 100644 --- a/apps/x/apps/renderer/src/components/code/use-code-chat.ts +++ b/apps/x/apps/renderer/src/components/code/use-code-chat.ts @@ -41,6 +41,10 @@ export interface PendingCodePermission { const DIRECT_PREFIX = 'direct-' const STRUCTURAL_EVENTS = new Set(['tool_call', 'tool_call_update', 'plan', 'permission']) +const COMPACTION_TITLE = 'Compacting context' +const COMPACTION_STALLED_MS = 90_000 + +export type CompactionStatus = 'idle' | 'running' | 'stalled' function messageText(content: unknown): string { if (typeof content === 'string') return content @@ -62,6 +66,8 @@ export function useCodeChat(session: CodeSession | null) { const [items, setItems] = useState([]) const [liveText, setLiveText] = useState('') const [isProcessing, setIsProcessing] = useState(false) + const [compactionStatus, setCompactionStatus] = useState('idle') + const [contextUsage, setContextUsage] = useState<{ used: number; size: number } | null>(null) const [pendingPermission, setPendingPermission] = useState(null) // Rowboat-mode copilot gates, same as the main chat: pre-tool-call permission // requests and ask-human questions. Keyed by toolCallId. @@ -69,6 +75,7 @@ export function useCodeChat(session: CodeSession | null) { const [pendingAskHumans, setPendingAskHumans] = useState>>(new Map()) const [loading, setLoading] = useState(false) const seenMessageIdsRef = useRef>(new Set()) + const compactionToolIdRef = useRef(null) const applyCodeRunEvent = useCallback((toolCallId: string, event: CodeRunEvent) => { if (toolCallId.startsWith(DIRECT_PREFIX)) { @@ -99,6 +106,9 @@ export function useCodeChat(session: CodeSession | null) { if (!sessionId) { setItems([]) setLiveText('') + setCompactionStatus('idle') + setContextUsage(null) + compactionToolIdRef.current = null setPendingPermission(null) return } @@ -106,6 +116,9 @@ export function useCodeChat(session: CodeSession | null) { setLoading(true) setItems([]) setLiveText('') + setCompactionStatus('idle') + setContextUsage(null) + compactionToolIdRef.current = null setPendingPermission(null) setPendingToolPermissions(new Map()) setPendingAskHumans(new Map()) @@ -217,6 +230,12 @@ export function useCodeChat(session: CodeSession | null) { return () => { cancelled = true } }, [sessionId]) + useEffect(() => { + if (compactionStatus !== 'running') return + const timer = window.setTimeout(() => setCompactionStatus('stalled'), COMPACTION_STALLED_MS) + return () => window.clearTimeout(timer) + }, [compactionStatus]) + // Live event stream. useEffect(() => { if (!sessionId) return @@ -230,6 +249,8 @@ export function useCodeChat(session: CodeSession | null) { break case 'run-processing-end': setIsProcessing(false) + setCompactionStatus('idle') + compactionToolIdRef.current = null setPendingPermission(null) // Anything still streaming that never landed as a message (e.g. the // turn errored) is flushed so the text isn't lost. @@ -247,6 +268,8 @@ export function useCodeChat(session: CodeSession | null) { break case 'run-stopped': setIsProcessing(false) + setCompactionStatus('idle') + compactionToolIdRef.current = null setPendingPermission(null) setPendingToolPermissions(new Map()) setPendingAskHumans(new Map()) @@ -348,6 +371,19 @@ export function useCodeChat(session: CodeSession | null) { break case 'code-run-event': { setIsProcessing(true) + if (event.event.type === 'usage') { + setContextUsage({ used: event.event.used, size: event.event.size }) + } + if (event.event.type === 'tool_call' && event.event.title === COMPACTION_TITLE) { + compactionToolIdRef.current = event.event.id ?? null + setCompactionStatus('running') + } + if (event.event.type === 'tool_call_update' + && event.event.id != null + && event.event.id === compactionToolIdRef.current) { + compactionToolIdRef.current = null + setCompactionStatus('idle') + } if (event.event.type === 'message' && event.event.role === 'agent' && event.toolCallId.startsWith(DIRECT_PREFIX)) { const text = event.event.text setLiveText((prev) => prev + text) @@ -460,6 +496,8 @@ export function useCodeChat(session: CodeSession | null) { items, liveText, isProcessing, + compactionStatus, + contextUsage, pendingPermission, pendingToolPermissions, pendingAskHumans, diff --git a/apps/x/apps/renderer/src/components/coding-run.tsx b/apps/x/apps/renderer/src/components/coding-run.tsx index ea620dc9..916aad37 100644 --- a/apps/x/apps/renderer/src/components/coding-run.tsx +++ b/apps/x/apps/renderer/src/components/coding-run.tsx @@ -9,6 +9,7 @@ import { Pencil, Search, ShieldQuestion, + Sparkles, Terminal, Trash2, Wrench, @@ -87,7 +88,8 @@ export function reduceEvents(events: CodeRunEvent[]): Row[] { return rows } -function toolKindIcon(kind?: string) { +function toolKindIcon(kind?: string, title?: string) { + if (title === 'Compacting context') return switch (kind) { case 'read': return case 'edit': return @@ -137,7 +139,7 @@ export function CodingRunTimeline({ {running ? : } - {toolKindIcon(row.toolKind)} + {toolKindIcon(row.toolKind, row.title)} {row.title ?? row.toolKind ?? 'Tool call'} {row.diffs.length > 0 && ( diff --git a/apps/x/packages/core/src/code-mode/acp/client.ts b/apps/x/packages/core/src/code-mode/acp/client.ts index b78c5619..f7351604 100644 --- a/apps/x/packages/core/src/code-mode/acp/client.ts +++ b/apps/x/packages/core/src/code-mode/acp/client.ts @@ -139,6 +139,8 @@ function toEvent(update: SessionUpdate): CodeRunEvent { priority: e.priority ?? undefined, })), }; + case 'usage_update': + return { type: 'usage', used: update.used, size: update.size }; default: return { type: 'other', sessionUpdate: update.sessionUpdate }; } diff --git a/apps/x/packages/shared/src/code-mode.ts b/apps/x/packages/shared/src/code-mode.ts index a3bd46a7..8a85f62f 100644 --- a/apps/x/packages/shared/src/code-mode.ts +++ b/apps/x/packages/shared/src/code-mode.ts @@ -53,6 +53,11 @@ export const CodeRunEvent = z.discriminatedUnion("type", [ priority: z.string().optional(), })), }), + z.object({ + type: z.literal("usage"), + used: z.number().nonnegative(), + size: z.number().positive(), + }), z.object({ type: z.literal("permission"), ask: PermissionAsk, diff --git a/apps/x/patches/@agentclientprotocol__codex-acp@0.0.44.patch b/apps/x/patches/@agentclientprotocol__codex-acp@0.0.44.patch new file mode 100644 index 00000000..79ab1623 --- /dev/null +++ b/apps/x/patches/@agentclientprotocol__codex-acp@0.0.44.patch @@ -0,0 +1,57 @@ +diff --git a/dist/index.js b/dist/index.js +--- a/dist/index.js ++++ b/dist/index.js +@@ -17678,15 +17678,22 @@ + case "dynamicToolCall": + return await createDynamicToolCallUpdate(event.item); ++ case "contextCompaction": ++ return { ++ sessionUpdate: "tool_call", ++ toolCallId: event.item.id, ++ kind: "other", ++ title: "Compacting context", ++ status: "in_progress" ++ }; + case "collabAgentToolCall": + case "userMessage": + case "hookPrompt": + case "agentMessage": + case "reasoning": + case "webSearch": + case "imageView": + case "imageGeneration": + case "enteredReviewMode": + case "exitedReviewMode": +- case "contextCompaction": + case "plan": + return null; +@@ -17713,23 +17720,28 @@ + return this.completeCommandExecutionEvent(event.item); + case "reasoning": + const summary = event.item.summary[0]; + if (!summary) return null; + return { + sessionUpdate: "agent_thought_chunk", + content: { + type: "text", + text: summary + } + }; ++ case "contextCompaction": ++ return { ++ sessionUpdate: "tool_call_update", ++ toolCallId: event.item.id, ++ status: "completed" ++ }; + case "collabAgentToolCall": + case "userMessage": + case "hookPrompt": + case "agentMessage": + case "webSearch": + case "imageView": + case "imageGeneration": + case "enteredReviewMode": + case "exitedReviewMode": +- case "contextCompaction": + case "plan": + return null; diff --git a/apps/x/pnpm-lock.yaml b/apps/x/pnpm-lock.yaml index dc18481a..5ad372b2 100644 --- a/apps/x/pnpm-lock.yaml +++ b/apps/x/pnpm-lock.yaml @@ -11,6 +11,9 @@ catalogs: version: 4.1.7 patchedDependencies: + '@agentclientprotocol/codex-acp@0.0.44': + hash: 0079b1ab3870451b47fe5ea5ecaca697e17139cff50fd2ee9c296b172aba525e + path: patches/@agentclientprotocol__codex-acp@0.0.44.patch '@openai/codex@0.128.0': hash: 9edd926108a95aaa788aa93870fd6b16d70eeccdf5740b503af5d34cc9f25e86 path: patches/@openai__codex@0.128.0.patch @@ -57,7 +60,7 @@ importers: version: 0.39.0(@anthropic-ai/sdk@0.100.1(zod@4.2.1))(@modelcontextprotocol/sdk@1.25.1(hono@4.11.3)(zod@4.2.1)) '@agentclientprotocol/codex-acp': specifier: ^0.0.44 - version: 0.0.44(zod@4.2.1) + version: 0.0.44(patch_hash=0079b1ab3870451b47fe5ea5ecaca697e17139cff50fd2ee9c296b172aba525e)(zod@4.2.1) '@x/core': specifier: workspace:* version: link:../../packages/core @@ -414,7 +417,7 @@ importers: version: 0.39.0(@anthropic-ai/sdk@0.100.1(zod@4.2.1))(@modelcontextprotocol/sdk@1.25.1(hono@4.11.3)(zod@4.2.1)) '@agentclientprotocol/codex-acp': specifier: ^0.0.44 - version: 0.0.44(zod@4.2.1) + version: 0.0.44(patch_hash=0079b1ab3870451b47fe5ea5ecaca697e17139cff50fd2ee9c296b172aba525e)(zod@4.2.1) '@agentclientprotocol/sdk': specifier: ^0.22.1 version: 0.22.1(zod@4.2.1) @@ -8512,7 +8515,7 @@ snapshots: - '@anthropic-ai/sdk' - '@modelcontextprotocol/sdk' - '@agentclientprotocol/codex-acp@0.0.44(zod@4.2.1)': + '@agentclientprotocol/codex-acp@0.0.44(patch_hash=0079b1ab3870451b47fe5ea5ecaca697e17139cff50fd2ee9c296b172aba525e)(zod@4.2.1)': dependencies: '@agentclientprotocol/sdk': 0.21.1(zod@4.2.1) '@openai/codex': 0.128.0(patch_hash=9edd926108a95aaa788aa93870fd6b16d70eeccdf5740b503af5d34cc9f25e86) diff --git a/apps/x/pnpm-workspace.yaml b/apps/x/pnpm-workspace.yaml index 3f3b314a..c588042e 100644 --- a/apps/x/pnpm-workspace.yaml +++ b/apps/x/pnpm-workspace.yaml @@ -24,4 +24,5 @@ onlyBuiltDependencies: - macos-alias - protobufjs patchedDependencies: + '@agentclientprotocol/codex-acp@0.0.44': patches/@agentclientprotocol__codex-acp@0.0.44.patch '@openai/codex@0.128.0': patches/@openai__codex@0.128.0.patch From 470b75e1b6a8766fb5701f7aa9fb8dbf36aa60ce Mon Sep 17 00:00:00 2001 From: arkml <6592213+arkml@users.noreply.github.com> Date: Tue, 30 Jun 2026 23:05:39 +0530 Subject: [PATCH 03/57] code mode output sticks to bottom; mic input in direct drive; fix dark mode and minor margins (#649) --- apps/x/apps/renderer/src/App.css | 11 ++ apps/x/apps/renderer/src/App.tsx | 1 + .../components/ai-elements/conversation.tsx | 11 +- .../x/apps/renderer/src/components/code/cm.ts | 58 +++++- .../src/components/code/code-chat.tsx | 171 +++++++++++++++--- .../src/components/code/code-view.tsx | 14 +- .../src/components/code/terminal-pane.tsx | 10 +- 7 files changed, 242 insertions(+), 34 deletions(-) diff --git a/apps/x/apps/renderer/src/App.css b/apps/x/apps/renderer/src/App.css index bdb1430f..1fd8c287 100644 --- a/apps/x/apps/renderer/src/App.css +++ b/apps/x/apps/renderer/src/App.css @@ -1017,6 +1017,17 @@ var(--rowboat-shadow); } +.dark .rowboat-code-chat-input { + border-color: color-mix(in oklab, var(--border) 76%, transparent); + background: #000000; + box-shadow: none; +} + +.dark .rowboat-code-chat-input:focus-within { + border-color: color-mix(in oklab, var(--ring) 70%, var(--border) 30%); + box-shadow: 0 0 0 1px color-mix(in oklab, var(--ring) 30%, transparent); +} + [data-slot="message-content"] { line-height: 1.62; } diff --git a/apps/x/apps/renderer/src/App.tsx b/apps/x/apps/renderer/src/App.tsx index 7ccb56e3..cf1d3d8f 100644 --- a/apps/x/apps/renderer/src/App.tsx +++ b/apps/x/apps/renderer/src/App.tsx @@ -6413,6 +6413,7 @@ function App() { session={activeCodeSession.session} status={activeCodeSession.status} onOpenDiff={setCodeDiffPath} + voiceAvailable={voiceAvailable} /> ) : isRightPaneContext && ( diff --git a/apps/x/apps/renderer/src/components/ai-elements/conversation.tsx b/apps/x/apps/renderer/src/components/ai-elements/conversation.tsx index 7a3f8836..6dfeedaf 100644 --- a/apps/x/apps/renderer/src/components/ai-elements/conversation.tsx +++ b/apps/x/apps/renderer/src/components/ai-elements/conversation.tsx @@ -43,6 +43,7 @@ export const Conversation = ({ const contentRef = useRef(null); const scrollRef = useRef(null); const spacerRef = useRef(null); + const stickToBottomRef = useRef(true); const [isAtBottom, setIsAtBottom] = useState(true); const updateBottomState = useCallback(() => { @@ -50,7 +51,9 @@ export const Conversation = ({ if (!container) return; const distanceFromBottom = container.scrollHeight - container.scrollTop - container.clientHeight; - setIsAtBottom(distanceFromBottom <= BOTTOM_THRESHOLD_PX); + const atBottom = distanceFromBottom <= BOTTOM_THRESHOLD_PX; + stickToBottomRef.current = atBottom; + setIsAtBottom(atBottom); }, []); const applyAnchorLayout = useCallback( @@ -131,7 +134,12 @@ export const Conversation = ({ cancelAnimationFrame(rafId); } rafId = requestAnimationFrame(() => { + const shouldStick = !anchorMessageId && stickToBottomRef.current; applyAnchorLayout(false); + if (shouldStick) { + container.scrollTop = container.scrollHeight; + updateBottomState(); + } }); }; @@ -178,6 +186,7 @@ export const Conversation = ({ const container = scrollRef.current; if (!container) return; container.scrollTop = container.scrollHeight; + stickToBottomRef.current = true; updateBottomState(); }, [updateBottomState]); diff --git a/apps/x/apps/renderer/src/components/code/cm.ts b/apps/x/apps/renderer/src/components/code/cm.ts index 4b75c1b2..8fa523f7 100644 --- a/apps/x/apps/renderer/src/components/code/cm.ts +++ b/apps/x/apps/renderer/src/components/code/cm.ts @@ -29,6 +29,12 @@ const darkHighlight = HighlightStyle.define([ ]) export function cmBaseExtensions(isDark: boolean): Extension[] { + const bg = isDark ? '#0f1117' : '#ffffff' + const panelBg = isDark ? '#151821' : '#f6f8fa' + const text = isDark ? '#d4d4d8' : '#24292f' + const muted = isDark ? '#7d8590' : '#6e7781' + const border = isDark ? '#2f3542' : '#d0d7de' + return [ lineNumbers(), bracketMatching(), @@ -39,18 +45,62 @@ export function cmBaseExtensions(isDark: boolean): Extension[] { EditorView.theme( { '&': { - backgroundColor: 'transparent', + backgroundColor: bg, + color: text, fontSize: '12px', height: '100%', }, + '.cm-editor': { + backgroundColor: bg, + color: text, + }, + '.cm-content': { + caretColor: text, + }, '.cm-scroller': { fontFamily: 'ui-monospace, SFMono-Regular, Menlo, monospace', overflow: 'auto', }, + '.cm-line': { + color: text, + }, '.cm-gutters': { - backgroundColor: 'transparent', - border: 'none', - color: isDark ? '#6b7280' : '#9ca3af', + backgroundColor: panelBg, + borderRight: `1px solid ${border}`, + color: muted, + }, + '.cm-activeLine': { + backgroundColor: isDark ? 'rgba(110, 118, 129, 0.16)' : 'rgba(175, 184, 193, 0.18)', + }, + '.cm-activeLineGutter': { + backgroundColor: isDark ? 'rgba(110, 118, 129, 0.16)' : 'rgba(175, 184, 193, 0.18)', + }, + '.cm-selectionBackground, &.cm-focused .cm-selectionBackground, .cm-content ::selection': { + backgroundColor: isDark ? 'rgba(88, 166, 255, 0.32)' : 'rgba(9, 105, 218, 0.22)', + }, + '.cm-panels, .cm-panel': { + backgroundColor: panelBg, + color: text, + borderColor: border, + }, + '.cm-mergeView': { + backgroundColor: bg, + color: text, + }, + '.cm-mergeViewEditors': { + backgroundColor: bg, + }, + '.cm-mergeView .cm-editor': { + borderColor: border, + }, + '.cm-changedLine': { + backgroundColor: isDark ? 'rgba(56, 139, 253, 0.14)' : 'rgba(9, 105, 218, 0.08)', + }, + '.cm-deletedChunk': { + backgroundColor: isDark ? 'rgba(248, 81, 73, 0.14)' : 'rgba(255, 235, 233, 0.95)', + }, + '.cm-insertedLine, .cm-insertedChunk': { + backgroundColor: isDark ? 'rgba(63, 185, 80, 0.14)' : 'rgba(234, 255, 234, 0.95)', }, '&.cm-focused': { outline: 'none' }, // GitHub-style expander bar for folded unchanged regions (@codemirror/merge). diff --git a/apps/x/apps/renderer/src/components/code/code-chat.tsx b/apps/x/apps/renderer/src/components/code/code-chat.tsx index 6c34c295..28a55db5 100644 --- a/apps/x/apps/renderer/src/components/code/code-chat.tsx +++ b/apps/x/apps/renderer/src/components/code/code-chat.tsx @@ -1,5 +1,5 @@ -import { useEffect, useRef, useState } from 'react' -import { ArrowUp, FileText, Loader2, LoaderIcon, Plus, Square, Terminal, X } from 'lucide-react' +import { useEffect, useRef, useState, type DragEvent, type MutableRefObject } from 'react' +import { ArrowUp, FileText, Loader2, LoaderIcon, Mic, Plus, Square, Terminal, X } from 'lucide-react' import type { CodeSession, CodeSessionStatus } from '@x/shared/src/code-sessions.js' import { cn } from '@/lib/utils' import { toast } from 'sonner' @@ -15,9 +15,54 @@ import { CodeRunPermissionRequest, CodingRunTimeline } from '@/components/coding import { PermissionRequest } from '@/components/ai-elements/permission-request' import { AskHumanRequest } from '@/components/ai-elements/ask-human-request' import { WebSearchResult } from '@/components/ai-elements/web-search-result' +import { useVoiceMode } from '@/hooks/useVoiceMode' import { useCodeChat, isDirectTurn, isChatToolCall, isChatErrorMessage, type CodeChatItem } from './use-code-chat' const AGENT_LABEL: Record = { claude: 'Claude Code', codex: 'Codex' } +const WAVE_BAR_WIDTH = 3 +const WAVE_BAR_GAP = 2 +const WAVE_BAR_MIN = 1.5 +const WAVE_BAR_MAX = 18 + +function VoiceWaveform({ audioLevelsRef }: { audioLevelsRef: MutableRefObject }) { + const [bars, setBars] = useState([]) + + useEffect(() => { + let raf = 0 + let lastSig = '' + const tick = () => { + const levels = audioLevelsRef.current + const next = levels.length > 48 ? levels.slice(levels.length - 48) : levels + const sig = `${next.length}:${next.length ? next[next.length - 1] : 0}` + if (sig !== lastSig) { + lastSig = sig + setBars(next.slice()) + } + raf = requestAnimationFrame(tick) + } + raf = requestAnimationFrame(tick) + return () => cancelAnimationFrame(raf) + }, [audioLevelsRef]) + + return ( +
+ {bars.map((level, i) => { + const amp = Math.min(1, Math.max(0, level)) ** 0.8 + return ( + + ) + })} +
+ ) +} function RowboatToolCall({ item, onOpenDiff }: { item: ToolCall; onOpenDiff: (path: string) => void }) { const [open, setOpen] = useState(false) @@ -97,10 +142,12 @@ export function CodeChat({ session, status, onOpenDiff, + voiceAvailable = false, }: { session: CodeSession status: CodeSessionStatus onOpenDiff: (path: string) => void + voiceAvailable?: boolean }) { const { items, liveText, isProcessing, compactionStatus, contextUsage, @@ -110,8 +157,12 @@ export function CodeChat({ const [draft, setDraft] = useState('') const [stopping, setStopping] = useState(false) const textareaRef = useRef(null) + const voice = useVoiceMode() + const voiceWarmup = voice.warmup const busy = isProcessing || status === 'working' || status === 'needs-you' + const recording = voice.state !== 'idle' + const recordingStopping = voice.state === 'submitting' const contextUsedPercent = contextUsage ? Math.min(100, Math.round((contextUsage.used / contextUsage.size) * 100)) : null @@ -123,6 +174,7 @@ export function CodeChat({ setDraft('') setAttachments([]) setStopping(false) + voice.cancel() textareaRef.current?.focus() }, [session.id]) @@ -130,6 +182,10 @@ export function CodeChat({ if (!busy) setStopping(false) }, [busy]) + useEffect(() => { + if (voiceAvailable) voiceWarmup() + }, [voiceAvailable, voiceWarmup]) + const addAttachments = (paths: string[]) => { const cleaned = paths.filter(Boolean) if (cleaned.length === 0) return @@ -145,7 +201,7 @@ export function CodeChat({ textareaRef.current?.focus() } - const handleDrop = (e: React.DragEvent) => { + const handleDrop = (e: DragEvent) => { if (!e.dataTransfer?.files?.length) return e.preventDefault() const paths = Array.from(e.dataTransfer.files) @@ -179,6 +235,27 @@ export function CodeChat({ await stop() } + const handleStartRecording = () => { + if (busy) return + void voice.start() + } + + const handleSubmitRecording = async () => { + if (!recording || recordingStopping) return + const text = await voice.submit() + if (!text) return + const result = await send(text) + if (!result.ok && result.error) { + toast.error(result.error) + setDraft(text) + } + } + + const handleCancelRecording = () => { + voice.cancel() + textareaRef.current?.focus() + } + const basename = (p: string) => p.split(/[\\/]/).pop() || p return ( @@ -266,8 +343,8 @@ export function CodeChat({ {/* Composer — mirrors the assistant chat input's look (rounded card, borderless textarea, round primary send / destructive stop). */} -
-
+
+
{attachments.length > 0 && (
{attachments.map((p) => ( @@ -290,22 +367,58 @@ export function CodeChat({ ))}
)} -
-